From bbefd7431cbd05a88690344d19c103372c47bfb4 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 01:33:46 +0000 Subject: [PATCH 1/3] feat(describe): describe pictures on this machine with Florence-2 Settings > LLM provider > Describing pictures can now have Microsoft's Florence-2 (MIT) describe each page and shot, instead of the provider's model that can see images (llm.describe_engine "florence-2"; the default stays "llm"). It runs the model's 8-bit ONNX export on ONNX Runtime on the CPU, so no picture leaves the machine and no provider is needed; the files are a component fetched on first use, pinned to a revision and checked against their SHA-256: base (about 280 MB) or large (about 830 MB), llm.florence_model. Generation follows Florence2ForConditionalGeneration: 768 x 768 picture, the prompt, forced BOS, greedy with no repeated trigrams, and the merged decoder's attention cache. Each picture is a model.describe row in the activity ledger. Agents read and change it with the MCP tools picture_descriptions and set_picture_descriptions (admins). Docs, changelog, devlog and threat model updated. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_013kCtgLNMjPQYdZpV5gbwUW --- CHANGELOG.md | 8 + DEVLOG.md | 18 ++ docs/activity.md | 6 +- docs/api.md | 16 +- docs/components.md | 3 +- docs/configuration.md | 31 ++- docs/mcp.md | 5 +- fastapi_backend/app/api/mcp_tools.py | 58 ++++ fastapi_backend/app/domain/components.py | 58 ++++ fastapi_backend/app/domain/descriptions.py | 86 +++++- fastapi_backend/app/domain/florence.py | 163 +++++++++++ fastapi_backend/app/domain/settings.py | 2 + fastapi_backend/app/domain/store.py | 3 + fastapi_backend/pyproject.toml | 2 + fastapi_backend/tests/api/test_florence.py | 255 ++++++++++++++++++ fastapi_backend/tests/api/test_mcp.py | 1 + fastapi_backend/uv.lock | 8 +- .../__tests__/settings-llm-presets.test.tsx | 18 ++ .../components/settings/bodies.tsx | 19 +- nextjs-frontend/components/settings/model.ts | 23 +- threat-model.json | 2 +- threat-model.md | 1 + threat-model.yaml | 2 +- 23 files changed, 751 insertions(+), 37 deletions(-) create mode 100644 fastapi_backend/app/domain/florence.py create mode 100644 fastapi_backend/tests/api/test_florence.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e85d00782..9d031787a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Describe pictures on this machine with Florence-2.** Settings → LLM provider → Describing pictures can now have + Microsoft's Florence-2 (MIT) describe each page and shot instead of the provider's model that can see images + (`llm.describe_engine` `florence-2`). It runs on the CPU through ONNX Runtime, so no picture leaves the machine and + no provider is needed; its 8-bit export is fetched the first time a job waits to describe something, pinned and + checked against its SHA-256: base (about 280 MB, `llm.florence_model`) or large (about 830 MB). Each picture is a + `model.describe` row in the activity ledger. Agents read and change it with the MCP tools `picture_descriptions` and + `set_picture_descriptions`. The default stays the provider's model ([Descriptions](docs/configuration.md#descriptions)). + - **Automations run.** An automation saved over the API (`/api/v1/automations`) now reacts to new assets in its namespaces: when its If passes, it runs its workflow as the person who made it, within its limits and budgets, and each run shows what its actions did. Try one on an activity without keeping or sending anything, or run it now; diff --git a/DEVLOG.md b/DEVLOG.md index 1b28864f9..d974035ac 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -2,6 +2,24 @@ Plans and progress for work in flight. Newest first. +## 2026-10-17 · Describing pictures with Florence-2 on this machine + +Adeel: use Moondream 3 and/or Microsoft Florence for visual descriptions. Florence-2 (MIT) is now a describe engine +(`llm.describe_engine` `florence-2`): its 8-bit ONNX export runs on ONNX Runtime on the CPU, fetched on first use, +pinned and hashed, base or large. It runs a Raspberry Pi's way (no PyTorch, about 280 MB for base) and needs no model +provider. Checked on two photos here: base takes about 3 s a picture on four CPU cores, large about 9 s, and both say +plainly what each shows. + +Moondream 3 comes next as an opt-in engine. Its licences aren't open: Moondream 3 Preview's weights are BSL 1.1 (with +a production use grant; Apache-2.0 two years after release) and Moondream 3.1's are the Moondream Model License; 3.1 +runs only on Photon, whose kernels (`kestrel-kernels`) are proprietary and licensed only under a separate agreement, so +Lens won't install Photon. Moondream 3 Preview runs through Hugging Face transformers (permissive) on an NVIDIA GPU or +a Mac with enough memory (about 19 GB of weights): not on a Pi. + +Refine later: beam search (the model's own default is three beams; greedy with no repeated trigrams matches it on the +photos tried); Florence-2's OCR task beside the OCR step; a GPU (CUDA or CoreML execution provider) when there is one; +the Q4 exports for a smaller download; describing only the pictures NSFW classification didn't flag. + ## 2026-10-14 · Long recordings in pieces a speech provider takes Adeel: large audio files are segmented automatically to each transcription provider's limits. `speech.Engine` cuts a diff --git a/docs/activity.md b/docs/activity.md index 0da5a9487..74a62ecd0 100644 --- a/docs/activity.md +++ b/docs/activity.md @@ -23,9 +23,9 @@ its namespace, its pipeline and the routine. A request counts for the resources /api/v1/routines/3` is `routine:3`), for what a `POST` made (from the `id` in its reply), and for who sent it. Besides the resources, a row has its kind (`in`, `out` or `run`), what it was (`POST /api/v1/routines/{rid}`, -`model.chat`, `embeddings`, `decision`, `notify.webhook`, `tool.http`, `web.search`, `web.read`, `google.token`, -`gmail.list`, `job.succeeded`, `routine.done`), the model, -tokens in and out, the estimated cost in USD, how long it took, and how it ended (an error's type or HTTP status, +`model.chat`, `model.describe` (a picture Florence-2 described on this server), `embeddings`, `decision`, +`notify.webhook`, `tool.http`, `web.search`, `web.read`, `google.token`, `gmail.list`, `job.succeeded`, +`routine.done`), the model, tokens in and out, the estimated cost in USD, how long it took, and how it ended (an error's type or HTTP status, never its message). It never holds prompt, reply, asset text or file content. ## Costs diff --git a/docs/api.md b/docs/api.md index 6e8f3bec8..2590c2f63 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1309,14 +1309,16 @@ photo search results carry `nsfw: true` for the ones the web app blurs. ### Descriptions -The `describe` step has a model that can see images, the one an admin chose as `llm.vision_model`, say in a few -sentences what each page of a document or an image shows, and each shot of a video (from its keyframe): the setting, -the people (by what they do and wear, never who they might be), the things and any text in it. It comes after -`nsfw` in the standard pipeline, and describes at most `llm.describe_max` pages or shots of a resource. Without an -LLM, or with no `llm.vision_model`, it's skipped, saying why; a model that can't see makes it fail with what the server -said ([Configuration](configuration.md#descriptions)). The player's `descriptions` lists them in order: +The `describe` step has a model that can see images say in a few sentences what each page of a document or an image +shows, and each shot of a video (from its keyframe): the setting, the people (by what they do and wear, never who they +might be), the things and any text in it. The model is the one an admin chose as `llm.vision_model` on the LLM +provider, or, with `llm.describe_engine` `florence-2`, Florence-2 on the server itself, which needs no provider. It +comes after `nsfw` in the standard pipeline, and describes at most `llm.describe_max` pages or shots of a resource. +Without an LLM, with no `llm.vision_model`, or before Florence-2 is fetched, it's skipped, saying why; a model that +can't see makes it fail with what the server said ([Configuration](configuration.md#descriptions)). The player's `descriptions` lists them in order: `{idx, t0, t1, text, model, frame, paged}`: the shot or page, where it starts and ends (ms; on pages, the page from 0 -and the next), what the model said, which model, and a signed link to the picture it was shown. Search finds them +and the next), what the model said, which model (such as `Florence-2-base`), and a signed link to the picture it was +shown. Search finds them (`source: described`, [search](#search)). Moving or deleting a resource moves or deletes them; the step run again replaces them. diff --git a/docs/components.md b/docs/components.md index c57cb1d7b..e1f4b005c 100644 --- a/docs/components.md +++ b/docs/components.md @@ -20,6 +20,7 @@ never fetch a speech or voice model. | Face models | `video.face_engine` is `anytopdf` | SFace, for anytopdf to describe faces with (anytopdf too) | | Object model | `video.object_engine` is `yolox` (the default) or `anytopdf` | YOLOX-s, and ONNX Runtime for `yolox` | | NSFW model | [NSFW classification](nsfw.md) is on for the archive or a namespace (off by default) | the 8-bit ONNX export of AdamCodd/vit-base-nsfw-detector (Apache-2.0, 88 MB), and ONNX Runtime | +| Florence-2 | `llm.describe_engine` is `florence-2` | Florence-2-base-ft or -large-ft (MIT, `llm.florence_model`; about 280 or 830 MB), their 8-bit ONNX exports, the tokenizers library and ONNX Runtime, to [describe pictures](configuration.md#descriptions) on this machine | | Prompt-injection classifier | `ai.injection_check` is `classifier` | ProtectAI's deberta-v3-base-prompt-injection-v2 (Apache-2.0, about 740 MB), the tokenizers library and ONNX Runtime ([third-party content](assistant.md#third-party-content-and-prompt-injection)) | | DuckDuckGo search | `web.search` is `duckduckgo` | ddgs (MIT, about 30 MB with lxml and primp), the client the assistant's [web search](assistant.md#web-search-and-reading-pages) reads DuckDuckGo's results with | | yt-dlp | `documents.video_urls` is on | the yt-dlp program, release 2026.08.19 (Unlicense), to download videos and podcasts imported by address | @@ -29,7 +30,7 @@ never fetch a speech or voice model. - Python packages go into `data_dir/python`, at the versions in `uv.lock`. Packages the image already has win. - Models go into `data_dir/models`: the Hugging Face, ModelScope and PyTorch caches point there (unless `HF_HOME`, `MODELSCOPE_CACHE` or `TORCH_HOME` say otherwise). They outlast container rebuilds and upgrades. -- Model files fetched by URL (YOLOX, YuNet, SFace, the NSFW model) are checked against their SHA-256. +- Model files fetched by URL (YOLOX, YuNet, SFace, the NSFW model, Florence-2) are checked against their SHA-256. - A job step that needs something still being fetched waits for it instead of falling back or failing. If a fetch fails, the step runs as before (for transcription, falling back to another engine), and the fetch is tried again an hour later, when the settings change, or on **Check again now**. diff --git a/docs/configuration.md b/docs/configuration.md index 4a92591ad..90d61accb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -217,18 +217,35 @@ Settings → AI assistant → Web search; the setup chat always asks before it c ## Descriptions -The describe step ([API](api.md#descriptions)) sends each page of a document or an image, and each shot's keyframe of -a video (at most 1024 pixels on a side), to a model that can see images, and keeps what it says each shows, for -search and for people who can't see them. It uses the LLM provider's server (Settings → LLM provider) with a model you -choose there as one that can see images: OpenAI's, or a vision model on LM Studio, Ollama or vLLM (LLaVA, Qwen-VL, -Llama 3.2 Vision, Gemma 3 …). Lens can't tell which models see, so none is chosen until you choose one. Each page or -shot is a request, so a long document costs as many: `llm.describe_max` caps how many of a resource are described. +The describe step ([API](api.md#descriptions)) shows each page of a document or an image, and each shot's keyframe of +a video, to a model that can see images, and keeps what it says each shows, for search and for people who can't see +them. Settings → LLM provider → Describing pictures chooses the model (`llm.describe_engine`): + +- **The provider's model that can see images** (`llm`, the default). Each picture (at most 1024 pixels on a side) is a + request to the LLM provider's server, with a model you choose as one that can see images: OpenAI's, or a vision + model on LM Studio, Ollama or vLLM (LLaVA, Qwen-VL, Llama 3.2 Vision, Gemma 3 …). Lens can't tell which models see, + so none is chosen until you choose one. +- **Florence-2 on this server** (`florence-2`). Microsoft's Florence-2 (MIT), fine-tuned, describes each picture in a + paragraph on the worker's CPU, through ONNX Runtime: no picture leaves the machine, no provider is needed and nothing + is billed. Its 8-bit ONNX export is fetched the first time a job waits to describe something + ([Components](components.md)), pinned to a revision and checked against its SHA-256: base (0.23B parameters, about + 280 MB; a few seconds a picture on a laptop's CPU, longer on a Raspberry Pi) or large (0.77B, about 830 MB, more + detail and about three times slower). It says what it sees in its own words: it reads no instructions, so it isn't + told what the picture is, and it describes, rather than transcribes, the text in it (the OCR step reads that). + +Each page or shot is a call to the model, so a long document costs as many: `llm.describe_max` caps how many of a +resource are described. Each call is in the [activity ledger](activity.md) (`model.chat` for the provider's model, +`model.describe` for Florence-2, with the tokens read and written and the time it took). | Setting | Default | | |---|---|---| -| `llm.vision_model` | none | the model that describes pages and shots; none: the describe step is skipped | +| `llm.describe_engine` | `llm` | what describes pages and shots: `llm` (the provider's model below) or `florence-2` (on this server) | +| `llm.vision_model` | none | the provider's model that describes pages and shots; none (with `llm`): the describe step is skipped | +| `llm.florence_model` | `base` | Florence-2's size: `base` or `large` | | `llm.describe_max` | 50 | pages or shots of a resource described at most, 1–1000 (in order: the rest aren't) | +Agents read and change these with the MCP tools `picture_descriptions` and `set_picture_descriptions` (admins). + ## Uploads Audio, video, documents (PDF, Office and OpenDocument files, text, Markdown, saved web pages and emails) and images diff --git a/docs/mcp.md b/docs/mcp.md index a92688f1b..c6163efe3 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -115,6 +115,8 @@ For an app that speaks OpenAI's API rather than MCP, the namespace can also answ | `preview_template` | what a template in a drafted automation gives before it's saved, against an activity or the trigger kind's sample; a prompt shows outside text marked as the model sees it; an error names the missing variable and its nearest name | | `file_storage` | where Lens keeps its own files (notes' images and attachments) and the connections it could use; `check` tests it. Admins only | | `set_file_storage` | keep new files on this machine or a storage connection (folder, rclone crypt); checked before it's saved. Admins only, with a write-scope token | +| `picture_descriptions` | what describes the pages and shots of resources ([Descriptions](configuration.md#descriptions)): the LLM provider's model that can see images or Florence-2 on this server, its model, whether it's ready and why not, and the engines there are. Admins only | +| `set_picture_descriptions` | choose what describes pictures (`engine` `llm` or `florence-2`), the provider's `vision_model`, Florence-2's size (`florence_model`) and `describe_max`. Admins only, with a write-scope token | | `install_settings` | whether other devices on the network reach Lens and its edition (full or lean), whether the machine's upgrade agent can change them, and the install line to run when it can't. Admins only | | `set_install_settings` | open Lens to the network or keep it to this machine, or change the edition: the machine sets itself up again and Lens restarts ([Updates](updates.md#setting-the-machine-up-again)). Admins only, with a write-scope token | | `propose_graph_change` | ask for two entities to be merged or linked; it waits in Proposed changes unless `apply` makes it at once (it can be undone). Needs a write-scope token and editor access | @@ -136,7 +138,8 @@ which records an undoable change (proposed by default), `undo_graph_change`, `ed `merge_topics`, `delete_topic` and `undo_topic_change`, which curate entities and topics as an editor does in the web app (merges and deletes can be undone), `write_note`, which adds a note, `create_podcast`, which queues an episode, `set_nsfw` and `mark_nsfw`, which change [NSFW classification](nsfw.md) (owners or admins, and -editors), `set_namespace_integrations`, which turns a namespace's own MCP server or chat completions on or off (owners), `set_file_storage`, which changes where new files are kept (admins), and `set_monitoring` and +editors), `set_namespace_integrations`, which turns a namespace's own MCP server or chat completions on or off (owners), `set_picture_descriptions`, which changes what describes pictures (admins), `set_file_storage`, which changes where new +files are kept (admins), and `set_monitoring` and `set_install_settings`, which have the machine change how Lens runs (admins). Importing stays in the web app and the API. diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index 4c6acdcc1..b7e68102a 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -43,9 +43,11 @@ approvals, auth, blobs, + descriptions, events, feeds, filetree, + florence, graph_history, integrations, library, @@ -1899,6 +1901,62 @@ def read_file(ctx: Context, path: str, offset: int) -> dict[str, Any]: return out +# ---------- describing pictures (admins) ---------- +@tool( + "picture_descriptions", + "Describing pictures", + "What describes the pages of documents and images and the shots of videos (the describe step; search finds what " + "it says): the LLM provider's model that can see images, or Florence-2 on this server. Its model, whether it's " + "ready and why not, how many pages or shots of a resource are described at most, and the engines there are. " + "Admins only.", + admin=True, +) +def picture_descriptions(ctx: Context) -> dict[str, Any]: + if not ctx.user.admin: + raise ToolError("only admins see how pictures are described") + return {**descriptions.status(ctx.cfg), "settings": f"{ctx.web}/settings/llm"} + + +@tool( + "set_picture_descriptions", + "Change how pictures are described", + "Choose what describes the pages and shots of new and reprocessed resources: engine llm, the LLM provider's model " + "that can see images (vision_model), or florence-2, Florence-2 (MIT) on this server's CPU, so no picture leaves it " + "(florence_model base, about 280 MB, or large, about 830 MB, fetched the first time a job needs it); and " + "describe_max, how many pages or shots of a resource are described at most. Admins only, with a token that has " + "the write scope.", + Arg("engine", "string", "what describes pictures", enum=tuple(descriptions.ENGINES)), + Arg("vision_model", "string", "the LLM provider's model that can see images, for engine llm; empty for none", max_length=200), + Arg("florence_model", "string", "Florence-2's size, for engine florence-2", enum=florence.SIZES), + Arg("describe_max", "integer", "pages or shots of a resource described at most, 1 to 1000"), + writes=True, + admin=True, +) +def set_picture_descriptions( + ctx: Context, + engine: str | None = None, + vision_model: str | None = None, + florence_model: str | None = None, + describe_max: int | None = None, +) -> dict[str, Any]: + if not ctx.user.admin: + raise ToolError("only admins change how pictures are described") + if not ctx.user.can_write: + raise ToolError("this token is read-only: ask for a token with the write scope to change how pictures are described") + given = ( + ("describe_engine", engine), + ("vision_model", vision_model), + ("florence_model", florence_model), + ("describe_max", describe_max), + ) + changes = {k: v for k, v in given if v is not None} + if not changes: + raise ToolError("say what to change: engine, vision_model, florence_model or describe_max") + settings.save(ctx.db, ctx.request.app.state.archive.base, "llm", changes, ctx.user.email) + auth.audit(ctx.db, ctx.user.as_audit(), "settings.save", "llm", sorted(changes) + ["mcp"]) + return {"saved": changes, **descriptions.status(ctx.request.app.state.settings.current())} + + # ---------- where files are kept (admins) ---------- def _admin(ctx: Context) -> None: if not ctx.user.admin: diff --git a/fastapi_backend/app/domain/components.py b/fastapi_backend/app/domain/components.py index 4c2941a60..0161ad3d3 100644 --- a/fastapi_backend/app/domain/components.py +++ b/fastapi_backend/app/domain/components.py @@ -76,6 +76,34 @@ 89, ), } +# Florence-2-base-ft and -large-ft (MIT), the 8-bit ONNX exports at onnx-community, at fixed revisions (florence.py): +# {size: (revision, {file: (sha256, size in MB)})} +FLORENCE = { + "base": ( + "e88a44eaf3791a35eae0c5a47b3dbcd36e67eb6f", + { + "onnx/vision_encoder_int8.onnx": ("d7876c1ab0f7ec11998942ca189e99a775c5a4a912b813c7745d0f6fa9343487", 94), + "onnx/embed_tokens_int8.onnx": ("6b2258db1c8ee9b160576ccde3cd3814d83a2edaed0dd1c6ca9ff3c38fa62214", 40), + "onnx/encoder_model_int8.onnx": ("f4ad7a68f1fb875d3bcf735ea14a7021b7ba7e83baf7cf10289881b4ed6d9b85", 44), + "onnx/decoder_model_merged_int8.onnx": ("f22f52f980c33df0efa15932c2f3db6d9d3595ce6387eca938b8cfe23dc4c641", 99), + "tokenizer.json": ("d69dcdb2323e124ac4f800cb9863ddccea0d7bb11e16125e8df3bd60f2f8aeac", 3), + }, + ), + "large": ( + "04ace74913c28d7ec94af32cee5c111a10126b6f", + { + "onnx/vision_encoder_int8.onnx": ("51d21e33e3b8cd34a9882e0611a2e4e03fb2b26daecea8accd1d4769ea11ba72", 367), + "onnx/embed_tokens_int8.onnx": ("dab1d09ab449f1302df6e439b35ec9e93fb037134bf6bed9ffeaa93a5b35d43c", 53), + "onnx/encoder_model_int8.onnx": ("9fe30b02cc8c2247631f8f0b9bc4d6613df1c33da0afe4087abff2f940bc2fe0", 153), + "onnx/decoder_model_merged_int8.onnx": ("167e7807fb1e5b8d693138f84bc81e1368ea11833264131c1ea96eafc9d13222", 258), + "tokenizer.json": ("d69dcdb2323e124ac4f800cb9863ddccea0d7bb11e16125e8df3bd60f2f8aeac", 3), + }, + ), +} +for _size, (_rev, _files) in FLORENCE.items(): + for _path, (_sha, _mb) in _files.items(): + _url = f"https://huggingface.co/onnx-community/Florence-2-{_size}-ft/resolve/{_rev}/{_path}" + FILES[f"florence-2-{_size}-ft/{_path.rsplit('/', 1)[-1]}"] = (_url, _sha, _mb) class Unavailable(RuntimeError): @@ -474,6 +502,28 @@ def fetch(self, cfg, m, say): fetch_file(cfg, "vit-base-nsfw-detector-int8.onnx", say) +class Florence(Component): + """Florence-2 and ONNX Runtime (florence.py), when Settings → LLM provider has it describe pictures.""" + + kind = "model" + + def needed(self, cfg, m): + return (cfg.get("llm") or {}).get("describe_engine") == "florence-2" + + def present(self, cfg): + from . import florence + + return florence.ready(cfg) + + def fetch(self, cfg, m, say): + from . import florence + + if not importable("onnxruntime", "tokenizers"): + pip_install(cfg, "florence", say) + for n in florence.files(florence.size(cfg)): + fetch_file(cfg, n, say) + + class InjectionModel(Component): """The prompt-injection classifier (injection_model.py), fetched when ai.injection_check asks for it.""" @@ -757,6 +807,14 @@ def detail(self, cfg): size_mb=105, license="Apache-2.0", ), + Florence( + "florence", + "Florence-2", + "describes pages and shots on this machine (llm.describe_engine florence-2; base about 280 MB, large about 830 MB)", + steps={"describe"}, + size_mb=280, + license="MIT", + ), LAYA, InjectionModel( "injection-model", diff --git a/fastapi_backend/app/domain/descriptions.py b/fastapi_backend/app/domain/descriptions.py index fed6508b2..94adb78fa 100644 --- a/fastapi_backend/app/domain/descriptions.py +++ b/fastapi_backend/app/domain/descriptions.py @@ -1,7 +1,8 @@ -"""Descriptions of what's seen (docs/api.md#descriptions): a model that can see images, the one an admin chose as -llm.vision_model, says in a few sentences what each of a document's or an image's pages shows, and each of a video's -shots (from its keyframe), so that they can be searched and read by people who can't see them. Without such a model -the describe step is skipped, saying why.""" +"""Descriptions of what's seen (docs/api.md#descriptions): a model that can see images says in a few sentences what +each of a document's or an image's pages shows, and each of a video's shots (from its keyframe), so that they can be +searched and read by people who can't see them. Which model is llm.describe_engine: "llm", the LLM provider's model an +admin chose as llm.vision_model (the default), or "florence-2", Florence-2 on this machine (florence.py), which needs +no provider and sends no picture anywhere. Without one the describe step is skipped, saying why.""" from __future__ import annotations @@ -22,6 +23,18 @@ MAX_CHARS = 1500 # of a description kept +# what can describe pictures (llm.describe_engine), as agents are told +ENGINES = { + "llm": "the LLM provider's model that can see images (llm.vision_model): each picture is a request to it", + "florence-2": "Florence-2 (MIT) on this server's CPU: no picture leaves it and no provider is called; base is about " + "280 MB, large about 830 MB, fetched the first time a job needs it", +} + + +class Failed(RuntimeError): + """A model on this machine couldn't describe a picture.""" + + def model_why(cfg): """The model that describes pictures, and None; or None, and why there's none.""" if not llm.configured(cfg): @@ -47,10 +60,61 @@ def picture(path, longest=LONGEST): def describe(cfg, model, path, about): """What the model says the picture at `path` shows; `about` says what it is (page 3 of …).""" content = [{"type": "text", "text": about}, {"type": "image_url", "image_url": {"url": picture(path)}}] - text = llm.chat(cfg, [{"role": "system", "content": PROMPT}, {"role": "user", "content": content}], model=model) + return llm.chat(cfg, [{"role": "system", "content": PROMPT}, {"role": "user", "content": content}], model=model) + + +def _tidy(text): + """A description as it's kept: its spaces made single, at most MAX_CHARS.""" return re.sub(r"\s+", " ", text or "").strip()[:MAX_CHARS] +class Provider: + """The LLM provider's model that can see images (llm.vision_model).""" + + def __init__(self, cfg, model): + self.cfg, self.name = cfg, model + + def describe(self, path, about): + return describe(self.cfg, self.name, path, about) + + +def engine(cfg): + """(what describes pictures, None), or (None, why there's none): its `name` and `describe(path, about)`.""" + if (cfg["llm"].get("describe_engine") or "llm") == "florence-2": + from . import florence + + try: + return florence.Florence(cfg), None + except RuntimeError as e: + return None, str(e) + model, why = model_why(cfg) + return (Provider(cfg, model), None) if model else (None, why) + + +def status(cfg): + """What describes pictures: the engine and its model, whether it's ready (and why not), the settings, and the + engines there are. Nothing is loaded to say so.""" + from . import florence + + l = cfg["llm"] + which = l.get("describe_engine") or "llm" + if which == "florence-2": + model = florence.name(florence.size(cfg)) + why = None if florence.ready(cfg) else "it isn't here yet: it's fetched the first time a job waits to describe something" + else: + model, why = model_why(cfg) + return { + "engine": which, + "model": model, + "ready": why is None, + "why_not": why, + "vision_model": l.get("vision_model"), + "florence_model": florence.size(cfg), + "describe_max": int(l.get("describe_max") or 50), + "engines": ENGINES, + } + + def _clock(ms): s = int(ms or 0) // 1000 return f"{s // 3600}:{s // 60 % 60:02d}:{s % 60:02d}" if s >= 3600 else f"{s // 60}:{s % 60:02d}" @@ -75,9 +139,10 @@ def step_describe(db, cfg, rid, say): items = [{"idx": s["idx"], "t0": s["t0"], "t1": s["t1"], "frame": s["frame"]} for s in shots if s.get("frame")] if not items: raise jobs.Skip("it has no shots yet (the shots step finds them)") - model, why = model_why(cfg) - if not model: + eye, why = engine(cfg) + if not eye: raise jobs.Skip(why) + model = eye.name most = int(cfg["llm"].get("describe_max") or 50) title, d, rows, failed = rec.get("title") or "untitled", video.frames_dir(cfg, rid), [], [] for it in items[:most]: @@ -89,14 +154,15 @@ def step_describe(db, cfg, rid, say): about = f"The first frame of a shot in the video “{title}”, at {_clock(it['t0'])}." try: with keyring.plain_picture(db, cfg, d / it["frame"]) as pic: - text = describe(cfg, model, pic, about) - except (llm.LLMError, OSError) as e: + text = _tidy(eye.describe(pic, about)) + except (llm.LLMError, OSError, Failed) as e: failed.append(str(e)) continue if text: rows.append(store.clean({"recording": rid, "space": rec["space"], **it, "paged": paged or None, "text": text, "model": model})) if failed and not rows: - raise llm.LLMError(f"{model} couldn't describe its {'pages' if paged else 'shots'}: {failed[0]}") + why = f"{model} couldn't describe its {'pages' if paged else 'shots'}: {failed[0]}" + raise llm.LLMError(why) if isinstance(eye, Provider) else Failed(why) db.run(["DELETE description WHERE recording = $r", *(["INSERT INTO description $rows"] if rows else [])], r=rid, rows=rows) what = "page(s)" if paged else "shot(s)" note = f"{len(rows)} {what} described ({model})" diff --git a/fastapi_backend/app/domain/florence.py b/fastapi_backend/app/domain/florence.py new file mode 100644 index 000000000..f88f86a02 --- /dev/null +++ b/fastapi_backend/app/domain/florence.py @@ -0,0 +1,163 @@ +"""Florence-2 describes pictures on this machine (docs/configuration.md#descriptions): with llm.describe_engine +"florence-2", Microsoft's Florence-2 (MIT), fine-tuned, says in a paragraph what each page or shot shows, for the +describe step, instead of a model provider's model that can see images. No picture leaves the machine and no model +provider is called. + +It runs on ONNX Runtime on the CPU (the `florence` extra: ONNX Runtime and the tokenizers library), from the 8-bit +ONNX exports onnx-community published, pinned to a revision and checked against their SHA-256: base (0.23B parameters, +about 275 MB) or large (0.77B, about 830 MB), llm.florence_model. components.py fetches them the first time a job +waits at the describe step with Florence-2 chosen, so an install that doesn't use it fetches nothing. A picture takes +a few seconds on a laptop's CPU, longer on a Raspberry Pi; base is the one for small machines. + +It runs the way Florence2ForConditionalGeneration does: the picture, resized to 768 x 768 and normalised, becomes 577 +image features; the task's prompt (: "Describe with a paragraph what is shown in the image.") +is embedded after them; the encoder reads both; and the decoder writes the description a token at a time, the likeliest +each time, never writing three tokens in a row it already wrote (the model's no_repeat_ngram_size), with its attention +cache. Each picture is a row in the activity ledger (`model.describe`), with the tokens read and written and the time +it took. Florence-2 reads no instructions, so what a picture is ("page 3 of …") isn't given to it. +""" + +from __future__ import annotations + +import threading + +from . import activity, components +from .descriptions import Failed + +SIZES = ("base", "large") +PARTS = ("vision_encoder", "embed_tokens", "encoder_model", "decoder_model_merged") +PROMPT = "Describe with a paragraph what is shown in the image." # the model's task +SIDE = 768 # pixels on each side, as the model reads a picture +MEAN, STD = (0.485, 0.456, 0.406), (0.229, 0.224, 0.225) +BOS, EOS = 0, 2 # the decoder starts from EOS (decoder_start_token_id) and its first token is BOS (forced_bos_token_id) +NO_REPEAT = 3 +MOST = 256 # tokens written at most: a long paragraph + +_lock = threading.Lock() +_loaded: dict = {} + + +def size(cfg): + s = (cfg.get("llm") or {}).get("florence_model") or "base" + return s if s in SIZES else "base" + + +def name(s): + return f"Florence-2-{s}" + + +def files(s): + """Its files, as components.FILES names them: the four ONNX models, then the tokenizer.""" + return [f"florence-2-{s}-ft/{p}_int8.onnx" for p in PARTS] + [f"florence-2-{s}-ft/tokenizer.json"] + + +def ready(cfg): + """Whether the chosen model, its tokenizer and ONNX Runtime are here.""" + components.activate(cfg) + return components.importable("onnxruntime", "tokenizers") and all(components.model_file(cfg, n) for n in files(size(cfg))) + + +def _load(cfg, s): + with _lock: # one per process: loading hundreds of MB for every picture would take longer than describing it + if _loaded.get("size") != s: + import onnxruntime + from tokenizers import Tokenizer + + paths = [components.model_file(cfg, n) for n in files(s)] + _loaded.clear() # the other size's sessions go: one is enough memory + sessions = {p: onnxruntime.InferenceSession(path, providers=["CPUExecutionProvider"]) for p, path in zip(PARTS, paths)} + tok = Tokenizer.from_file(paths[-1]) + _loaded.update(size=s, sessions=sessions, tok=tok, prompt=tok.encode(PROMPT).ids) + return _loaded + + +def pixels(path): + """The picture as the model reads it: RGB, 768 x 768 (bicubic), scaled to [0, 1], normalised, channels first.""" + import numpy as np + from PIL import Image + + with Image.open(path) as im: + x = np.asarray(im.convert("RGB").resize((SIDE, SIDE), Image.BICUBIC), dtype=np.float32) / 255.0 + return ((x - np.array(MEAN, dtype=np.float32)) / np.array(STD, dtype=np.float32)).transpose(2, 0, 1)[None].astype(np.float32) + + +def _banned(seq, n=NO_REPEAT): + """The tokens that would repeat an n-gram already in seq.""" + if len(seq) < n: + return set() + tail = tuple(seq[len(seq) - n + 1 :]) + return {seq[i + n - 1] for i in range(len(seq) - n + 1) if tuple(seq[i : i + n - 1]) == tail} + + +def generate(sessions, prompt, picture, most=MOST): + """The tokens the model writes about a picture (its pixels), and how many it read (the picture's features and the + prompt's tokens). `sessions` are the four ONNX models, by PARTS.""" + import numpy as np + + def embed(ids): + return sessions["embed_tokens"].run(None, {"input_ids": np.array([ids], dtype=np.int64)})[0] + + image = sessions["vision_encoder"].run(None, {"pixel_values": picture})[0] + x = np.concatenate([image, embed(prompt)], axis=1) + mask = np.ones(x.shape[:2], dtype=np.int64) + hidden = sessions["encoder_model"].run(None, {"inputs_embeds": x, "attention_mask": mask})[0] + dec = sessions["decoder_model_merged"] + outs = [o.name for o in dec.get_outputs()] + # the attention cache: empty at first; the encoder's part is kept from the first step, the decoder's grows + cache = { + i.name: np.zeros((1, i.shape[1], 0, i.shape[3]), dtype=np.float32) + for i in dec.get_inputs() + if i.name.startswith("past_key_values.") + } + seq = [EOS] + for step in range(most): + feed = { + "encoder_attention_mask": mask, + "encoder_hidden_states": hidden, + "inputs_embeds": embed(seq[-1:]), + "use_cache_branch": np.array([step > 0]), + **cache, + } + got = dict(zip(outs, dec.run(None, feed))) + if step == 0: + token = BOS + else: + logits = np.array(got["logits"][0, -1], dtype=np.float64) + for b in _banned(seq): + logits[b] = -np.inf + token = int(np.argmax(logits)) + for k in cache: + if step == 0 or ".decoder." in k: + cache[k] = got["present." + k[len("past_key_values.") :]] + seq.append(token) + if token == EOS: + break + return seq[1:], int(image.shape[1]) + len(prompt) + + +class Florence: + """Florence-2, base or large (llm.florence_model), on ONNX Runtime.""" + + def __init__(self, cfg): + self.cfg, self.size = cfg, size(cfg) + self.name = name(self.size) + components.activate(cfg) + if not components.importable("onnxruntime", "tokenizers"): + raise RuntimeError("ONNX Runtime isn't installed (it's fetched on first use, or pip install \"lens[florence]\")") + if not all(components.model_file(cfg, n) for n in files(self.size)): + raise RuntimeError(f"the {self.name} model isn't here yet (it's fetched the first time a job needs it)") + self.m = _load(cfg, self.size) + + def describe(self, path, about=None): + """What the model says the picture at `path` shows (`about`, what it is, isn't read: Florence-2 takes no + instructions).""" + x = pixels(path) + with activity.call( + "model.describe", self.cfg, self.name, detail={"engine": "florence-2"}, price=activity.model_cost(self.cfg) + ) as ledger: + try: + ids, read = generate(self.m["sessions"], self.m["prompt"], x) + except Exception as e: # noqa: BLE001 - ONNX Runtime's errors aren't one type + raise Failed(f"{self.name} couldn't read it: {e}") from e + ledger.usage(tokens_in=read, tokens_out=len(ids)) + return self.m["tok"].decode(ids, skip_special_tokens=True).strip() diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index f0d246015..90c9e157c 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -160,6 +160,8 @@ ("search", "stemming"): {"english", "none"}, ("search", "engine"): set(store.SEARCH_ENGINES), ("reports", "audio"): {"link", "embed", "none"}, + ("llm", "describe_engine"): {"llm", "florence-2"}, + ("llm", "florence_model"): {"base", "large"}, ("video", "ocr_engine"): {"auto", "tesseract", "apple-vision", "rapidocr", "doctr", "none"}, ("video", "face_engine"): {"opencv", "insightface", "anytopdf", "none"}, ("video", "object_engine"): {"yolox", "ultralytics", "anytopdf", "off"}, diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index a08cead08..2ebff7f00 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -166,6 +166,9 @@ def labels(): # a model the admin knows can see images: it describes pages and shots (the describe step); none, none are "vision_model": None, "describe_max": 50, # pages or shots of a resource described at most + # what describes them: "llm" (vision_model, above) or "florence-2" (on this machine, florence.py) + "describe_engine": "llm", + "florence_model": "base", # Florence-2 base (about 280 MB) or large (about 830 MB) }, "graph": {"max_nodes": 150, "min_edge_weight": 2}, "search": { diff --git a/fastapi_backend/pyproject.toml b/fastapi_backend/pyproject.toml index b8a3fe256..39153254a 100644 --- a/fastapi_backend/pyproject.toml +++ b/fastapi_backend/pyproject.toml @@ -47,6 +47,8 @@ insightface = ["insightface", "onnxruntime"] objects = ["onnxruntime"] # A local prompt-injection classifier (ai.injection_check "classifier"): ProtectAI's DeBERTa model on ONNX Runtime injection = ["onnxruntime", "tokenizers"] +# Florence-2 (MIT) describing pictures on this machine (llm.describe_engine "florence-2"): its ONNX export on ONNX Runtime +florence = ["onnxruntime", "tokenizers"] # Checking IIIF manifests against the official schema validate = ["jsonschema"] # the SMB share of the file tree (smb_share.py): impacket's SMB server (a modified Apache 1.1 license); fetched when diff --git a/fastapi_backend/tests/api/test_florence.py b/fastapi_backend/tests/api/test_florence.py new file mode 100644 index 000000000..cf04443f1 --- /dev/null +++ b/fastapi_backend/tests/api/test_florence.py @@ -0,0 +1,255 @@ +"""Florence-2 describing pictures on this machine (docs/configuration.md#descriptions): with llm.describe_engine +"florence-2" the describe step needs no LLM provider; the picture is read the way the model was trained (768 x 768, +ImageNet's mean and spread); the decoder starts from EOS with BOS forced, keeps the encoder's attention cache from its +first step, never repeats three tokens and stops at EOS; each picture is a ledger row; the model is fetched as a +component, pinned and hashed; and admins and agents choose it.""" + +from __future__ import annotations + +import os +import shutil + +import numpy as np +import pytest +from PIL import Image +from tokenizers import Tokenizer, models, pre_tokenizers, processors + +from app.domain import components, descriptions, florence, machine, store +from tests.api.test_documents import _png, _upload +from tests.api.test_mcp import tool, tool_error +from tests.helpers import drain, login, make_user + +R = store.R +WORDS = ["", "", "", "", "A", "red", "blue", "square", "on", "white", "Describe", "with", "a"] +WORDS += ["paragraph", "what", "is", "shown", "in", "the", "image."] +ID = {w: i for i, w in enumerate(WORDS)} +DIM, HEADS, HEAD = 4, 2, 3 + + +def tokenizer(): + """A word-level tokenizer that adds and the way Florence-2's (BART's) does.""" + tok = Tokenizer(models.WordLevel(ID, unk_token="")) + tok.pre_tokenizer = pre_tokenizers.WhitespaceSplit() + tok.add_special_tokens(["", "", "", ""]) + tok.post_processor = processors.TemplateProcessing(single=" $A ", special_tokens=[("", 0), ("", 2)]) + return tok + + +class Port: + """An ONNX Runtime input or output: its name and shape.""" + + def __init__(self, name, shape=None): + self.name, self.shape = name, shape + + +class Vision: + """577 features that say whether the picture is redder than it is blue.""" + + def run(self, _, feed): + x = feed["pixel_values"] + assert x.shape == (1, 3, 768, 768) and x.dtype == np.float32 + return [np.full((1, 577, DIM), float(x[0, 0].mean() > x[0, 2].mean()), dtype=np.float32)] + + +class Embed: + def run(self, _, feed): + return [np.repeat(feed["input_ids"][..., None], DIM, axis=2).astype(np.float32)] + + +class Encoder: + seen: list = [] + + def run(self, _, feed): + x = feed["inputs_embeds"] + assert feed["attention_mask"].shape == x.shape[:2] + Encoder.seen.append(x[0, 577:, 0].astype(int).tolist()) # the prompt's tokens, after the picture's features + return [x] + + +class Decoder: + """Writes `red` after a red picture, else `other`, a token a step (it counts the tokens in its cache), each + followed by `then` as the next likeliest; past the end, EOS. Its cached branch hands back no encoder cache, as the + merged ONNX decoder's does, so it must be kept from the first step.""" + + def __init__(self, red, other, then=None): + self.red, self.other, self.then, self.steps = red, other, then, 0 + + def get_inputs(self): + cache = [ + Port(f"past_key_values.0.{p}.{kv}", ["batch_size", HEADS, "past", HEAD]) + for p in ("decoder", "encoder") + for kv in ("key", "value") + ] + return [Port("encoder_attention_mask"), Port("encoder_hidden_states"), Port("inputs_embeds"), *cache, Port("use_cache_branch")] + + def get_outputs(self): + return [Port("logits"), *(Port(f"present.0.{p}.{kv}") for p in ("decoder", "encoder") for kv in ("key", "value"))] + + def run(self, _, feed): + self.steps += 1 + seen = feed["past_key_values.0.decoder.key"].shape[2] + enc = feed["encoder_hidden_states"] + if feed["use_cache_branch"][0]: + assert feed["past_key_values.0.encoder.key"].shape == (1, HEADS, enc.shape[1], HEAD) # kept from the first step + else: + assert seen == 0 + script = self.red if enc[0, 0, 0] == 1.0 else self.other + logits = np.zeros((1, 1, len(WORDS)), dtype=np.float32) + if 0 < seen <= len(script): + logits[0, 0, ID[script[seen - 1]]] = 2.0 + if self.then: + logits[0, 0, ID[self.then]] = 1.0 + else: + logits[0, 0, ID[""]] = 2.0 + grown = np.zeros((1, HEADS, seen + 1, HEAD), dtype=np.float32) + kept = ( + np.zeros((1, HEADS, enc.shape[1], HEAD), dtype=np.float32) if not feed["use_cache_branch"][0] else np.zeros((0, HEADS, 0, HEAD)) + ) + return [logits, grown, grown, kept, kept] + + +def sessions(decoder): + return {"vision_encoder": Vision(), "embed_tokens": Embed(), "encoder_model": Encoder(), "decoder_model_merged": decoder} + + +def test_the_decoder_starts_with_bos_keeps_its_cache_and_never_repeats_three_tokens(): + tok = tokenizer() + prompt = tok.encode(florence.PROMPT).ids + assert prompt[0] == 0 and prompt[-1] == 2 # … , as BART's tokenizer adds them + red = np.full((1, 3, 768, 768), 0.0, dtype=np.float32) + red[0, 0] = 1.0 + ids, read = florence.generate(sessions(Decoder(["A", "red", "square"], ["A", "blue", "square"])), prompt, red) + assert tok.decode(ids, skip_special_tokens=True) == "A red square" and ids[0] == 0 and ids[-1] == 2 + assert read == 577 + len(prompt) and Encoder.seen[-1] == prompt + + # it wants "red square on red square on …": the second "red square on" is never written, the next likeliest is + dec = Decoder(["red", "square", "on", "red", "square", "on"], [], then="white") + ids, _ = florence.generate(sessions(dec), prompt, red) + assert tok.decode(ids, skip_special_tokens=True) == "red square on red square white" + # and it stops at the most it may write + ids, _ = florence.generate(sessions(Decoder(["A"] * 10, [], then="red")), prompt, red, most=4) + assert len(ids) == 4 + + +def test_a_picture_is_read_as_the_model_was_trained(folder): + path = folder / "blue.png" + Image.new("RGB", (40, 20), (0, 0, 255)).save(path) + x = florence.pixels(path) + assert x.shape == (1, 3, 768, 768) and x.dtype == np.float32 + assert np.allclose(x[0, :, 5, 5], [(0 - 0.485) / 0.229, (0 - 0.456) / 0.224, (1 - 0.406) / 0.225], atol=1e-5) + + +@pytest.fixture +def model(cfg, monkeypatch): + """Florence-2 base "fetched": its files in the models folder, ONNX Runtime handing back the fakes for them.""" + import onnxruntime + + for n in florence.files("base"): + p = components.models_dir(cfg) / n + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(b"onnx") + tokenizer().save(str(components.models_dir(cfg) / florence.files("base")[-1])) + fakes = sessions(Decoder(["A", "red", "square", "on", "white"], ["A", "blue", "square"])) + opened = [] + monkeypatch.setattr( + onnxruntime, "InferenceSession", lambda path, providers: opened.append(path) or fakes[next(p for p in fakes if p in path)] + ) + monkeypatch.setattr(florence, "_loaded", {}) + cfg["llm"].update(base_url=None, model=None, vision_model=None, describe_engine="florence-2") + return opened + + +@pytest.fixture +def people(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + return {"ha": login(client, "root@x.io", "root password 1"), "he": login(client, "ed@x.io", "editor password 1")} + + +def test_florence_describes_pictures_without_a_model_provider(client, people, db, cfg, model): + red = _upload(client, people["he"], _png(Image.new("RGB", (64, 48), (220, 20, 20))), "red.png")["recording"] + blue = _upload(client, people["he"], _png(Image.new("RGB", (64, 48), (20, 20, 220))), "blue.png")["recording"] + drain(db, cfg) + assert len(model) == 4 # the four ONNX models, opened once for both pictures + got = {rid: descriptions.for_recording(db, rid) for rid in (red, blue)} + assert [(d["text"], d["model"], d["paged"]) for d in got[red]] == [("A red square on white", "Florence-2-base", True)] + assert got[blue][0]["text"] == "A blue square" + log = " ".join(db.one("SELECT log FROM job WHERE recording = $r", r=red)["log"]) + assert "1 page(s) described (Florence-2-base)" in log + # each picture is a row in the activity ledger, for its resource, with what it read and wrote + rows = db.rows("SELECT model, tokens_in, tokens_out, resources, detail FROM activity WHERE action = 'model.describe'") + assert len(rows) == 2 and {r["model"] for r in rows} == {"Florence-2-base"} + mine = next(r for r in rows if f"recording:{red}" in r["resources"]) + prompt = tokenizer().encode(florence.PROMPT).ids + assert (mine["tokens_in"], mine["tokens_out"], mine["detail"]) == (577 + len(prompt), 7, {"engine": "florence-2"}) + + +def test_without_its_model_the_step_says_so(client, people, db, cfg): + cfg["llm"]["describe_engine"] = "florence-2" + rid = _upload(client, people["he"], _png(Image.new("RGB", (64, 48), "white")), "white.png")["recording"] + drain(db, cfg) + log = " ".join(db.one("SELECT log FROM job WHERE recording = $r", r=rid)["log"]) + assert "describe skipped: the Florence-2-base model isn't here yet (it's fetched the first time a job needs it)" in log + + +def test_florence_is_fetched_pinned_and_checked(cfg, monkeypatch): + c = components.BY_ID["florence"] + m = machine.probe(cfg["data_dir"]) + assert not c.needed(cfg, m) and c.steps == {"describe"} and c.license == "MIT" + cfg["llm"]["describe_engine"] = "florence-2" + cfg["llm"]["florence_model"] = "large" + assert c.needed(cfg, m) and not c.present(cfg) + got = [] + monkeypatch.setattr(components, "importable", lambda *mods: False) + monkeypatch.setattr(components, "pip_install", lambda cfg, extra, say=None: got.append(extra)) + monkeypatch.setattr(components, "fetch_file", lambda cfg, name, say=None: got.append(name)) + c.fetch(cfg, m, None) + assert got == ["florence", *florence.files("large")] + url, sha, mb = components.FILES["florence-2-large-ft/vision_encoder_int8.onnx"] + assert url == ( + "https://huggingface.co/onnx-community/Florence-2-large-ft/resolve/04ace74913c28d7ec94af32cee5c111a10126b6f/onnx/vision_encoder_int8.onnx" + ) + assert (sha, mb) == ("51d21e33e3b8cd34a9882e0611a2e4e03fb2b26daecea8accd1d4769ea11ba72", 367) + assert all(n in components.FILES for s in florence.SIZES for n in florence.files(s)) + + +def test_admins_and_agents_choose_what_describes_pictures(client, people, db): + ha = people["ha"] + assert client.get("/api/v1/settings", headers=ha).json()["llm"]["values"]["describe_engine"] == "llm" # the default + r = client.put("/api/v1/settings/llm", json={"describe_engine": "moondream-9"}, headers=ha) + assert r.status_code == 400 + assert client.put("/api/v1/settings/llm", json={"florence_model": "huge"}, headers=ha).status_code == 400 + assert client.put("/api/v1/settings/llm", json={"describe_engine": "florence-2"}, headers=people["he"]).status_code == 403 + + # agents, over MCP: admins only + assert "only admins" in tool_error(client, people["he"], "picture_descriptions") + got = tool(client, ha, "picture_descriptions") + assert (got["engine"], got["ready"], got["why_not"]) == ("llm", False, "no LLM is configured") + assert set(got["engines"]) == {"llm", "florence-2"} and got["settings"].endswith("/settings/llm") + assert "say what to change" in tool_error(client, ha, "set_picture_descriptions") + assert tool_error(client, ha, "set_picture_descriptions", describe_max=0) == "llm.describe_max is a whole number from 1 to 1000" + out = tool(client, ha, "set_picture_descriptions", engine="florence-2", florence_model="large") + assert out["saved"] == {"describe_engine": "florence-2", "florence_model": "large"} + assert (out["engine"], out["model"], out["ready"]) == ("florence-2", "Florence-2-large", False) + assert "fetched the first time a job waits" in out["why_not"] + assert client.get("/api/v1/settings", headers=ha).json()["llm"]["values"]["florence_model"] == "large" + audit = db.rows("SELECT target, detail FROM audit_log WHERE action = 'settings.save'") + assert {"target": "llm", "detail": ["describe_engine", "florence_model", "mcp"]} in audit + assert "only admins" in tool_error(client, people["he"], "set_picture_descriptions", engine="llm") + + +@pytest.mark.skipif(not os.environ.get("LENS_TEST_FLORENCE_DIR"), reason="LENS_TEST_FLORENCE_DIR: the real base model's files") +def test_the_real_model(cfg, folder, monkeypatch): + from PIL import ImageDraw + + for n in florence.files("base"): + (components.models_dir(cfg) / n).parent.mkdir(parents=True, exist_ok=True) + shutil.copy(os.path.join(os.environ["LENS_TEST_FLORENCE_DIR"], n.rsplit("/", 1)[-1]), components.models_dir(cfg) / n) + monkeypatch.setattr(florence, "_loaded", {}) + img = Image.new("RGB", (640, 480), "white") + ImageDraw.Draw(img).ellipse((220, 140, 420, 340), fill=(220, 20, 20)) + img.save(folder / "dot.png") + cfg["llm"]["describe_engine"] = "florence-2" + eye, why = descriptions.engine(cfg) + assert why is None and eye.name == "Florence-2-base" + assert "red" in eye.describe(folder / "dot.png", "The image “dot”.").lower() diff --git a/fastapi_backend/tests/api/test_mcp.py b/fastapi_backend/tests/api/test_mcp.py index 6e720b9df..cff00d809 100644 --- a/fastapi_backend/tests/api/test_mcp.py +++ b/fastapi_backend/tests/api/test_mcp.py @@ -86,6 +86,7 @@ def test_clients_are_sent_to_sign_in(client, app, env): "set_nsfw", "mark_nsfw", "set_namespace_integrations", + "set_picture_descriptions", "set_file_storage", "set_monitoring", "set_install_settings", diff --git a/fastapi_backend/uv.lock b/fastapi_backend/uv.lock index bffd74334..f2aefad3b 100644 --- a/fastapi_backend/uv.lock +++ b/fastapi_backend/uv.lock @@ -1848,6 +1848,10 @@ doctr = [ faces = [ { name = "opencv-python-headless" }, ] +florence = [ + { name = "onnxruntime" }, + { name = "tokenizers" }, +] injection = [ { name = "onnxruntime" }, { name = "tokenizers" }, @@ -1944,6 +1948,7 @@ requires-dist = [ { name = "mlx-whisper", marker = "extra == 'mlx'" }, { name = "modelscope", marker = "extra == 'sensevoice'" }, { name = "numpy", specifier = ">=1.26" }, + { name = "onnxruntime", marker = "extra == 'florence'" }, { name = "onnxruntime", marker = "extra == 'injection'" }, { name = "onnxruntime", marker = "extra == 'insightface'" }, { name = "onnxruntime", marker = "extra == 'objects'" }, @@ -1965,6 +1970,7 @@ requires-dist = [ { name = "speechbrain", marker = "extra == 'voices'" }, { name = "starlette", specifier = ">=1.3.1" }, { name = "surrealdb", specifier = ">=2.0,<3" }, + { name = "tokenizers", marker = "extra == 'florence'" }, { name = "tokenizers", marker = "extra == 'injection'" }, { name = "torch", marker = "extra == 'sensevoice'" }, { name = "torch", marker = "extra == 'voices'" }, @@ -1972,7 +1978,7 @@ requires-dist = [ { name = "torchaudio", marker = "extra == 'voices'" }, { name = "webauthn", specifier = ">=3.0.1,<4" }, ] -provides-extras = ["sensevoice", "whisper", "mlx", "laya", "voices", "pyannote", "faces", "mac-ocr", "rapidocr", "doctr", "insightface", "objects", "injection", "validate", "smb", "web", "msg"] +provides-extras = ["sensevoice", "whisper", "mlx", "laya", "voices", "pyannote", "faces", "mac-ocr", "rapidocr", "doctr", "insightface", "objects", "injection", "florence", "validate", "smb", "web", "msg"] [package.metadata.requires-dev] dev = [ diff --git a/nextjs-frontend/__tests__/settings-llm-presets.test.tsx b/nextjs-frontend/__tests__/settings-llm-presets.test.tsx index 7dd6220ae..da1654a0b 100644 --- a/nextjs-frontend/__tests__/settings-llm-presets.test.tsx +++ b/nextjs-frontend/__tests__/settings-llm-presets.test.tsx @@ -29,6 +29,8 @@ const view = (keySet: boolean, locked: string[] = []) => chat_models: [], vision_model: null, describe_max: 20, + describe_engine: "llm", + florence_model: "base", api_key: { secret: true, set: keySet }, api_key_env: null, max_chars: 24000, @@ -99,6 +101,22 @@ describe("Settings › LLM provider presets", () => { await waitFor(() => expect(screen.getByPlaceholderText("Paste the new key")).toHaveFocus()); }); + it("has Florence-2 describe pictures on this server in one pick", async () => { + show(view(true)); + expect(screen.getByText("Describing pictures")).toBeInTheDocument(); + expect(screen.getByLabelText("Model that can see images")).toBeInTheDocument(); + expect(screen.queryByLabelText("Florence-2 size")).not.toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("Describe pictures with"), { target: { value: "florence-2" } }); + expect(screen.getByLabelText("Florence-2 size")).toHaveValue("base"); // the small one, for a Raspberry Pi + expect(screen.queryByLabelText("Model that can see images")).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Save" })); + await waitFor(() => expect(Admin.updateSettings).toHaveBeenCalledTimes(1)); + expect(m(Admin.updateSettings).mock.calls[0][0]).toMatchObject({ + path: { section: "llm" }, + body: { describe_engine: "florence-2" }, + }); + }); + it("offers no presets when .env sets the address", async () => { show(view(false, ["base_url"])); expect(screen.getByLabelText("Base URL (OpenAI-compatible)")).toHaveAttribute("readonly"); diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index 0632f1f63..fd16dbcf8 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -633,15 +633,26 @@ function LlmBody({ ctx }: { ctx: BodyCtx }) { + + + +
- + {ctx.state("llm.describe_engine").value === "florence-2" ? ( + + ) : ( + + )}
-