From 37e44f11e997118c2a5e4cb344d8a9f6f4550c0a Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 01:44:54 +0000 Subject: [PATCH 1/3] feat(describe): describe pictures on this machine with Moondream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings → LLM provider → Describing pictures can also have M87 Labs' Moondream caption each page and shot (llm.describe_engine moondream) through transformers and PyTorch, the moondream extra fetched on first use: Moondream 3 Preview (BSL 1.1, about 19 GB, an NVIDIA GPU or a Mac with 32 GB) or Moondream 2 (Apache-2.0, about 3.7 GB, on a CPU, slowly), pinned to a revision. moondream.cannot_run keeps the model from being fetched where it can't run and the step says why. Agents choose it with set_picture_descriptions. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_013kCtgLNMjPQYdZpV5gbwUW --- CHANGELOG.md | 7 + DEVLOG.md | 21 ++ README.md | 3 +- docs/activity.md | 2 +- docs/api.md | 9 +- docs/components.md | 1 + docs/configuration.md | 19 +- docs/mcp.md | 4 +- docs/requirements.md | 1 + fastapi_backend/app/api/mcp_tools.py | 16 +- fastapi_backend/app/domain/components.py | 33 +++ fastapi_backend/app/domain/descriptions.py | 24 +- fastapi_backend/app/domain/moondream.py | 177 ++++++++++++++ fastapi_backend/app/domain/settings.py | 3 +- fastapi_backend/app/domain/store.py | 4 +- fastapi_backend/pyproject.toml | 3 + fastapi_backend/tests/api/test_florence.py | 2 +- fastapi_backend/tests/api/test_moondream.py | 220 ++++++++++++++++++ fastapi_backend/uv.lock | 8 +- .../__tests__/settings-llm-presets.test.tsx | 16 ++ .../components/settings/bodies.tsx | 4 +- nextjs-frontend/components/settings/model.ts | 13 +- threat-model.json | 2 +- threat-model.md | 1 + threat-model.yaml | 2 +- 25 files changed, 566 insertions(+), 29 deletions(-) create mode 100644 fastapi_backend/app/domain/moondream.py create mode 100644 fastapi_backend/tests/api/test_moondream.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f901c06c..e16084746 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version fixed, by rules, on English lines. Every changed line is in Change history with Revert, and lines someone corrected are left alone. Reprocess, pipelines, `POST /api/v1/recordings//reprocess` and `lens clean` run it ([Cleaning up transcripts](docs/processing.md#cleaning-up-transcripts)). +- **Describe pictures on this machine with Moondream.** Settings → LLM provider → Describing pictures can also have + M87 Labs' Moondream caption each page and shot (`llm.describe_engine` `moondream`), through transformers and + PyTorch, so no picture leaves the machine: Moondream 3 Preview (`llm.moondream_model` `moondream3-preview`, about + 19 GB, an NVIDIA GPU or a Mac with 32 GB; BSL 1.1, not open source) or Moondream 2 (`moondream2`, Apache-2.0, about + 3.7 GB, on a CPU, slowly). It's fetched, pinned to a revision, the first time a job waits to describe something, and + only where the chosen one can run; elsewhere the step says why. Agents choose it with `set_picture_descriptions` + ([Descriptions](docs/configuration.md#descriptions)). - **Describe pictures on this machine with Florence-2.** Settings → LLM provider → Describing pictures can now have Microsoft's Florence-2 (MIT) describe each page and shot instead of the provider's model that can see images (`llm.describe_engine` `florence-2`). It runs on the CPU through ONNX Runtime, so no picture leaves the machine and diff --git a/DEVLOG.md b/DEVLOG.md index 62f67c08f..cca5b8485 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -25,6 +25,27 @@ from a LanguageTool server, spaCy fetched for named entities, and dates found an Refine later: revert a whole cleanup at once; cleanup rules for other languages' fillers; per-namespace switch. +## 2026-10-17 · Describing pictures with Moondream on this machine + +Moondream is the third describe engine (`llm.describe_engine` `moondream`), opt-in, through Hugging Face transformers +(Apache-2.0) and PyTorch (BSD), the `moondream` extra fetched on first use. Both models' code comes with their +weights (`trust_remote_code`), pinned to a revision; both caption with the same call (`caption(image, +length="normal")`). `llm.moondream_model` picks Moondream 3 Preview (the default when Moondream is chosen, since that's +what was asked for) or Moondream 2. `moondream.cannot_run` keeps 19 GB from being fetched where Moondream 3 can't run +(no NVIDIA GPU with 20 GB, no Mac with 32 GB) and says so in the step's log and in `picture_descriptions`. + +Checked here: Moondream 2 at the pinned revision loads with the locked transformers and describes a photo well ("A +vintage light blue Volkswagen Beetle is parked parallel to a yellow building with a wooden door …"), but took 3 to 6 +minutes a picture on four shared CPU cores in float32: Florence-2 base took 3 s. Moondream 3 Preview wasn't +run here (no GPU); its code at the pinned revision was read: the same caption call, with flex attention. The opt-in +test (`LENS_TEST_MOONDREAM=1`) fetches PyTorch's CPU build, transformers and Moondream 2 through the component and +describes a red dot. It first wrote "and, (, (, (…": with `OMP_THREAD_LIMIT` below PyTorch's thread count (the test +suite sets it to 1), PyTorch's CPU kernels write nonsense, so `_load` now caps PyTorch's threads at that limit. + +Refine later: Moondream's query skill with what the picture is ("page 3 of …"), as the provider's model is told; its +`compile()` on a GPU; bfloat16 on CPUs that have it; Moondream 3 on a GPU in CI or on a maintainer's machine; a +quantised Moondream 2 for small machines. + ## 2026-10-17 · Describing pictures with Florence-2 on this machine Adeel: use Moondream 3 and/or Microsoft Florence for visual descriptions. Florence-2 (MIT) is now a describe engine diff --git a/README.md b/README.md index bd05eb16d..d4e568a07 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,8 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p asset, with Undo instead of a confirm. Names get their real types (people, places, organisations, events, works) by rules, without a model. Find what is most like a passage, an asset, a topic or an entity, in a Similar tab on every asset or by selecting text anywhere, and, with anytopdf, photos by what they show. Pages and shots are - described in words by the model provider's vision model or, off by default, by Florence-2 on your own server. + described in words by the model provider's vision model or, off by default, by Florence-2 or Moondream on your own + server. ([Graph](docs/graph.md), [entity pages](docs/notes.md#entity-pages), [similar content](docs/processing.md#similar-content), [graph history](docs/graph-history.md), [topics](docs/topics.md), [video](docs/video.md), [NSFW classification](docs/nsfw.md), diff --git a/docs/activity.md b/docs/activity.md index 74a62ecd0..02a681f66 100644 --- a/docs/activity.md +++ b/docs/activity.md @@ -23,7 +23,7 @@ its namespace, its pipeline and the routine. A request counts for the resources /api/v1/routines/3` is `routine:3`), for what a `POST` made (from the `id` in its reply), and for who sent it. Besides the resources, a row has its kind (`in`, `out` or `run`), what it was (`POST /api/v1/routines/{rid}`, -`model.chat`, `model.describe` (a picture Florence-2 described on this server), `embeddings`, `decision`, +`model.chat`, `model.describe` (a picture Florence-2 or Moondream described on this server), `embeddings`, `decision`, `notify.webhook`, `tool.http`, `web.search`, `web.read`, `google.token`, `gmail.list`, `job.succeeded`, `routine.done`), the model, tokens in and out, the estimated cost in USD, how long it took, and how it ended (an error's type or HTTP status, never its message). It never holds prompt, reply, asset text or file content. diff --git a/docs/api.md b/docs/api.md index f7865bac3..cfeaf5669 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1369,12 +1369,15 @@ photo search results carry `nsfw: true` for the ones the web app blurs. The `describe` step has a model that can see images say in a few sentences what each page of a document or an image shows, and each shot of a video (from its keyframe): the setting, the people (by what they do and wear, never who they might be), the things and any text in it. The model is the one an admin chose as `llm.vision_model` on the LLM -provider, or, with `llm.describe_engine` `florence-2`, Florence-2 on the server itself, which needs no provider. It +provider, or, with `llm.describe_engine` `florence-2` or `moondream`, Florence-2 or Moondream on the server itself, +which needs no provider. It comes after `nsfw` in the standard pipeline, and describes at most `llm.describe_max` pages or shots of a resource. -Without an LLM, with no `llm.vision_model`, or before Florence-2 is fetched, it's skipped, saying why; a model that +Without an LLM, with no `llm.vision_model`, before Florence-2 or Moondream is fetched, or where the chosen Moondream +can't run, it's skipped, saying why; a model that can't see makes it fail with what the server said ([Configuration](configuration.md#descriptions)). The player's `descriptions` lists them in order: `{idx, t0, t1, text, model, frame, paged}`: the shot or page, where it starts and ends (ms; on pages, the page from 0 -and the next), what the model said, which model (such as `Florence-2-base`), and a signed link to the picture it was +and the next), what the model said, which model (such as `Florence-2-base` or `Moondream-2`), and a signed link to +the picture it was shown. Search finds them (`source: described`, [search](#search)). Moving or deleting a resource moves or deletes them; the step run again replaces them. diff --git a/docs/components.md b/docs/components.md index 074cf8efc..89aca762e 100644 --- a/docs/components.md +++ b/docs/components.md @@ -21,6 +21,7 @@ never fetch a speech or voice model. | Object model | `video.object_engine` is `yolox` (the default) or `anytopdf` | YOLOX-s, and ONNX Runtime for `yolox` | | NSFW model | [NSFW classification](nsfw.md) is on for the archive or a namespace (off by default) | the 8-bit ONNX export of AdamCodd/vit-base-nsfw-detector (Apache-2.0, 88 MB), and ONNX Runtime | | Florence-2 | `llm.describe_engine` is `florence-2` | Florence-2-base-ft or -large-ft (MIT, `llm.florence_model`; about 280 or 830 MB), their 8-bit ONNX exports, the tokenizers library and ONNX Runtime, to [describe pictures](configuration.md#descriptions) on this machine | +| Moondream | `llm.describe_engine` is `moondream`, and this machine can run the chosen one | Moondream 3 Preview (BSL 1.1, about 19 GB, an NVIDIA GPU or a Mac with 32 GB) or Moondream 2 (Apache-2.0, about 3.7 GB), `llm.moondream_model`, at a pinned revision, with its tokenizer, PyTorch (the CPU build without an NVIDIA GPU) and transformers, to [describe pictures](configuration.md#descriptions) on this machine | | Prompt-injection classifier | `ai.injection_check` is `classifier` | ProtectAI's deberta-v3-base-prompt-injection-v2 (Apache-2.0, about 740 MB), the tokenizers library and ONNX Runtime ([third-party content](assistant.md#third-party-content-and-prompt-injection)) | | Punctuation model | `analysis.cleanup` is on and `analysis.cleanup_punctuation` isn't `off` ([Cleaning up transcripts](processing.md#cleaning-up-transcripts)) | kredor/punctuate-all at a pinned revision (MIT, about 1.1 GB), with Transformers and PyTorch's CPU build | | spaCy | `analysis.entities` is `spacy` ([Entities](processing.md#entities-and-the-graph-explorer)) | spaCy (MIT) and the model in `analysis.spacy_model`: `en_core_web_sm`, `de_core_news_sm` or `xx_ent_wiki_sm` (MIT, 11 to 15 MB each), about 60 MB in all | diff --git a/docs/configuration.md b/docs/configuration.md index d964a5ddb..5dfa0214f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -234,16 +234,31 @@ them. Settings → LLM provider → Describing pictures chooses the model (`llm. 280 MB; a few seconds a picture on a laptop's CPU, longer on a Raspberry Pi) or large (0.77B, about 830 MB, more detail and about three times slower). It says what it sees in its own words: it reads no instructions, so it isn't told what the picture is, and it describes, rather than transcribes, the text in it (the OCR step reads that). +- **Moondream on this server** (`moondream`). M87 Labs' Moondream captions each picture on the worker, through + Hugging Face transformers and PyTorch: no picture leaves the machine and no provider is needed. Which one is + `llm.moondream_model`: + - `moondream3-preview`, Moondream 3 (Preview): 9B parameters (2B active), about 19 GB. It needs an NVIDIA GPU with + 20 GB or more, or a Mac with 32 GB of memory: not a Raspberry Pi. Its weights and code are under the Business + Source License 1.1 (production use is allowed, except offering it to others in competition with M87 Labs' own), + which isn't an open-source licence, so Lens fetches it only when you pick it. + - `moondream2`, Moondream 2 (Apache-2.0): 1.9B parameters, about 3.7 GB. It runs on a CPU with about 10 GB of + memory, slowly: minutes a picture on four cores, against seconds for Florence-2. + + PyTorch (its CPU build without an NVIDIA GPU), transformers and the weights are fetched the first time a job waits + to describe something ([Components](components.md)), the weights and the model's own code pinned to a revision; + where the chosen one can't run, nothing is fetched and the step says why. Moondream 3.1 runs only on M87 Labs' + Photon engine, whose kernels are licensed only under a separate agreement with them, so Lens doesn't offer it. Each page or shot is a call to the model, so a long document costs as many: `llm.describe_max` caps how many of a resource are described. Each call is in the [activity ledger](activity.md) (`model.chat` for the provider's model, -`model.describe` for Florence-2, with the tokens read and written and the time it took). +`model.describe` for Florence-2 and Moondream, with the time it took and, for Florence-2, the tokens read and written). | Setting | Default | | |---|---|---| -| `llm.describe_engine` | `llm` | what describes pages and shots: `llm` (the provider's model below) or `florence-2` (on this server) | +| `llm.describe_engine` | `llm` | what describes pages and shots: `llm` (the provider's model below), or on this server `florence-2` or `moondream` | | `llm.vision_model` | none | the provider's model that describes pages and shots; none (with `llm`): the describe step is skipped | | `llm.florence_model` | `base` | Florence-2's size: `base` or `large` | +| `llm.moondream_model` | `moondream3-preview` | which Moondream: `moondream3-preview` (a GPU) or `moondream2` (a CPU will do) | | `llm.describe_max` | 50 | pages or shots of a resource described at most, 1–1000 (in order: the rest aren't) | Agents read and change these with the MCP tools `picture_descriptions` and `set_picture_descriptions` (admins). diff --git a/docs/mcp.md b/docs/mcp.md index 9c527ac91..a7ac0307c 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -118,8 +118,8 @@ For an app that speaks OpenAI's API rather than MCP, the namespace can also answ | `preview_template` | what a template in a drafted automation gives before it's saved, against an activity or the trigger kind's sample; a prompt shows outside text marked as the model sees it; an error names the missing variable and its nearest name | | `file_storage` | where Lens keeps its own files (notes' images and attachments) and the connections it could use; `check` tests it. Admins only | | `set_file_storage` | keep new files on this machine or a storage connection (folder, rclone crypt); checked before it's saved. Admins only, with a write-scope token | -| `picture_descriptions` | what describes the pages and shots of resources ([Descriptions](configuration.md#descriptions)): the LLM provider's model that can see images or Florence-2 on this server, its model, whether it's ready and why not, and the engines there are. Admins only | -| `set_picture_descriptions` | choose what describes pictures (`engine` `llm` or `florence-2`), the provider's `vision_model`, Florence-2's size (`florence_model`) and `describe_max`. Admins only, with a write-scope token | +| `picture_descriptions` | what describes the pages and shots of resources ([Descriptions](configuration.md#descriptions)): the LLM provider's model that can see images, or Florence-2 or Moondream on this server, its model, whether it's ready and why not, and the engines there are. Admins only | +| `set_picture_descriptions` | choose what describes pictures (`engine` `llm`, `florence-2` or `moondream`), the provider's `vision_model`, Florence-2's size (`florence_model`), which Moondream (`moondream_model`) and `describe_max`. Admins only, with a write-scope token | | `install_settings` | whether other devices on the network reach Lens and its edition (full or lean), whether the machine's upgrade agent can change them, and the install line to run when it can't. Admins only | | `set_install_settings` | open Lens to the network or keep it to this machine, or change the edition: the machine sets itself up again and Lens restarts ([Updates](updates.md#setting-the-machine-up-again)). Admins only, with a write-scope token | | `propose_graph_change` | ask for two entities to be merged or linked; it waits in Proposed changes unless `apply` makes it at once (it can be undone). Needs a write-scope token and editor access | diff --git a/docs/requirements.md b/docs/requirements.md index 37741ba08..ee95385ba 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -143,6 +143,7 @@ request, to another check that proves the same thing; loosening a requirement or | What is most like a passage, an asset, a topic or an entity | [test_similar.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_similar.py) `test_like_a_text_by_its_key_words`, `test_like_a_resource_by_words_topics_and_entities`, `test_like_a_topic_and_an_entity`; [similar.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/similar.test.tsx) | | With anytopdf, photos found by what they show | [test_photos.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_photos.py) `test_photos_found_by_what_they_show`, `test_off_by_default` | | Pages and shots described by the provider's vision model or, off by default, by Florence-2 on this server, fetched pinned and checked when picked | [test_florence.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_florence.py) `test_florence_describes_pictures_without_a_model_provider`, `test_florence_is_fetched_pinned_and_checked`, `test_admins_and_agents_choose_what_describes_pictures`, `test_a_picture_is_read_as_the_model_was_trained` | +| Or by Moondream on this server, off by default, fetched at a pinned revision only where the chosen model can run, and saying why where it can't | [test_moondream.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_moondream.py) `test_moondream_describes_pictures_without_a_model_provider`, `test_where_each_moondream_can_run`, `test_on_a_cpu_moondream_3_is_never_fetched_and_the_step_says_why`, `test_moondream_is_fetched_pinned`, `test_admins_and_agents_choose_moondream` | ## Learn from it diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index 44b5ebde2..83bbe398f 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -53,6 +53,7 @@ integrations, library, monitoring, + moondream, notebook, nsfw, rdf, @@ -2056,7 +2057,8 @@ def read_file(ctx: Context, path: str, offset: int) -> dict[str, Any]: "picture_descriptions", "Describing pictures", "What describes the pages of documents and images and the shots of videos (the describe step; search finds what " - "it says): the LLM provider's model that can see images, or Florence-2 on this server. Its model, whether it's " + "it says): the LLM provider's model that can see images, or Florence-2 or Moondream on this server. Its model, " + "whether it's " "ready and why not, how many pages or shots of a resource are described at most, and the engines there are. " "Admins only.", admin=True, @@ -2072,12 +2074,14 @@ def picture_descriptions(ctx: Context) -> dict[str, Any]: "Change how pictures are described", "Choose what describes the pages and shots of new and reprocessed resources: engine llm, the LLM provider's model " "that can see images (vision_model), or florence-2, Florence-2 (MIT) on this server's CPU, so no picture leaves it " - "(florence_model base, about 280 MB, or large, about 830 MB, fetched the first time a job needs it); and " - "describe_max, how many pages or shots of a resource are described at most. Admins only, with a token that has " - "the write scope.", + "(florence_model base, about 280 MB, or large, about 830 MB, fetched the first time a job needs it), or moondream, " + "Moondream on this server (moondream_model moondream3-preview, Moondream 3 under the BSL 1.1, not open source, " + "about 19 GB and a GPU; or moondream2, Apache-2.0, about 3.7 GB, on a CPU); and describe_max, how many pages or " + "shots of a resource are described at most. Admins only, with a token that has the write scope.", Arg("engine", "string", "what describes pictures", enum=tuple(descriptions.ENGINES)), Arg("vision_model", "string", "the LLM provider's model that can see images, for engine llm; empty for none", max_length=200), Arg("florence_model", "string", "Florence-2's size, for engine florence-2", enum=florence.SIZES), + Arg("moondream_model", "string", "which Moondream, for engine moondream", enum=tuple(moondream.MODELS)), Arg("describe_max", "integer", "pages or shots of a resource described at most, 1 to 1000"), writes=True, admin=True, @@ -2087,6 +2091,7 @@ def set_picture_descriptions( engine: str | None = None, vision_model: str | None = None, florence_model: str | None = None, + moondream_model: str | None = None, describe_max: int | None = None, ) -> dict[str, Any]: if not ctx.user.admin: @@ -2097,11 +2102,12 @@ def set_picture_descriptions( ("describe_engine", engine), ("vision_model", vision_model), ("florence_model", florence_model), + ("moondream_model", moondream_model), ("describe_max", describe_max), ) changes = {k: v for k, v in given if v is not None} if not changes: - raise ToolError("say what to change: engine, vision_model, florence_model or describe_max") + raise ToolError("say what to change: engine, vision_model, florence_model, moondream_model or describe_max") settings.save(ctx.db, ctx.request.app.state.archive.base, "llm", changes, ctx.user.email) auth.audit(ctx.db, ctx.user.as_audit(), "settings.save", "llm", sorted(changes) + ["mcp"]) return {"saved": changes, **descriptions.status(ctx.request.app.state.settings.current())} diff --git a/fastapi_backend/app/domain/components.py b/fastapi_backend/app/domain/components.py index 763f2b8ce..4e7239211 100644 --- a/fastapi_backend/app/domain/components.py +++ b/fastapi_backend/app/domain/components.py @@ -542,6 +542,30 @@ def fetch(self, cfg, m, say): fetch_file(cfg, n, say) +class Moondream(Component): + """PyTorch, transformers and the chosen Moondream (moondream.py), when Settings → LLM provider has it describe + pictures and this machine can run it (Moondream 3 needs a GPU: 19 GB isn't fetched where it can't run).""" + + kind = "model" + + def needed(self, cfg, m): + from . import moondream + + return (cfg.get("llm") or {}).get("describe_engine") == "moondream" and not moondream.cannot_run(cfg, m) + + def present(self, cfg): + from . import moondream + + return moondream.ready(cfg) + + def fetch(self, cfg, m, say): + from . import moondream + + if not importable("torch", "transformers"): + pip_install(cfg, "moondream", say) + moondream.fetch(cfg, say) + + class InjectionModel(Component): """The prompt-injection classifier (injection_model.py), fetched when ai.injection_check asks for it.""" @@ -920,6 +944,15 @@ def detail(self, cfg): size_mb=280, license="MIT", ), + Moondream( + "moondream", + "Moondream", + "describes pages and shots on this machine (llm.describe_engine moondream): Moondream 3 Preview, about 19 GB and " + "a GPU, or Moondream 2, about 3.7 GB, with PyTorch and transformers", + steps={"describe"}, + size_mb=19000, + license="BSL-1.1 (Moondream 3 Preview) or Apache-2.0 (Moondream 2)", + ), LAYA, Spacy( "spacy", diff --git a/fastapi_backend/app/domain/descriptions.py b/fastapi_backend/app/domain/descriptions.py index 94adb78fa..a8b74e5f7 100644 --- a/fastapi_backend/app/domain/descriptions.py +++ b/fastapi_backend/app/domain/descriptions.py @@ -1,8 +1,9 @@ """Descriptions of what's seen (docs/api.md#descriptions): a model that can see images says in a few sentences what each of a document's or an image's pages shows, and each of a video's shots (from its keyframe), so that they can be searched and read by people who can't see them. Which model is llm.describe_engine: "llm", the LLM provider's model an -admin chose as llm.vision_model (the default), or "florence-2", Florence-2 on this machine (florence.py), which needs -no provider and sends no picture anywhere. Without one the describe step is skipped, saying why.""" +admin chose as llm.vision_model (the default); or, on this machine, needing no provider and sending no picture +anywhere, "florence-2", Florence-2 (florence.py), or "moondream", Moondream (moondream.py). Without one the describe +step is skipped, saying why.""" from __future__ import annotations @@ -28,6 +29,9 @@ "llm": "the LLM provider's model that can see images (llm.vision_model): each picture is a request to it", "florence-2": "Florence-2 (MIT) on this server's CPU: no picture leaves it and no provider is called; base is about " "280 MB, large about 830 MB, fetched the first time a job needs it", + "moondream": "Moondream on this server, through PyTorch: no picture leaves it; moondream3-preview is Moondream 3 " + "(BSL 1.1, not open source; about 19 GB, needs an NVIDIA GPU or a Mac's GPU), moondream2 is Moondream 2 " + "(Apache-2.0, about 3.7 GB, runs on a CPU, slowly); fetched the first time a job needs it", } @@ -80,11 +84,12 @@ def describe(self, path, about): def engine(cfg): """(what describes pictures, None), or (None, why there's none): its `name` and `describe(path, about)`.""" - if (cfg["llm"].get("describe_engine") or "llm") == "florence-2": - from . import florence + which = cfg["llm"].get("describe_engine") or "llm" + if which in ("florence-2", "moondream"): + from . import florence, moondream try: - return florence.Florence(cfg), None + return (florence.Florence(cfg) if which == "florence-2" else moondream.Moondream(cfg)), None except RuntimeError as e: return None, str(e) model, why = model_why(cfg) @@ -94,13 +99,17 @@ def engine(cfg): def status(cfg): """What describes pictures: the engine and its model, whether it's ready (and why not), the settings, and the engines there are. Nothing is loaded to say so.""" - from . import florence + from . import florence, moondream l = cfg["llm"] which = l.get("describe_engine") or "llm" + later = "it isn't here yet: it's fetched the first time a job waits to describe something" if which == "florence-2": model = florence.name(florence.size(cfg)) - why = None if florence.ready(cfg) else "it isn't here yet: it's fetched the first time a job waits to describe something" + why = None if florence.ready(cfg) else later + elif which == "moondream": + model = moondream.name(moondream.choice(cfg)) + why = moondream.cannot_run(cfg) or (None if moondream.ready(cfg) else later) else: model, why = model_why(cfg) return { @@ -110,6 +119,7 @@ def status(cfg): "why_not": why, "vision_model": l.get("vision_model"), "florence_model": florence.size(cfg), + "moondream_model": moondream.choice(cfg), "describe_max": int(l.get("describe_max") or 50), "engines": ENGINES, } diff --git a/fastapi_backend/app/domain/moondream.py b/fastapi_backend/app/domain/moondream.py new file mode 100644 index 000000000..30989186e --- /dev/null +++ b/fastapi_backend/app/domain/moondream.py @@ -0,0 +1,177 @@ +"""Moondream describes pictures on this machine (docs/configuration.md#descriptions): with llm.describe_engine +"moondream", M87 Labs' Moondream captions each page and shot, through Hugging Face transformers on PyTorch (the +optional `moondream` extra), on the worker's NVIDIA GPU, a Mac's GPU, or its CPU. No picture leaves the machine and no +model provider is called. + +Which Moondream is llm.moondream_model, each pinned to a revision of its Hugging Face repository: + +* `moondream3-preview`, Moondream 3 (Preview): 9B parameters, 2B of them active for each token (a mixture of experts), + about 19 GB of weights in bfloat16. It wants an NVIDIA GPU with 24 GB or a Mac with 32 GB or more: not a Raspberry + Pi. Its weights and code are under the Business Source License 1.1 with an additional use grant (production use is + allowed, except offering it to others in competition with M87 Labs' paid versions), and become Apache-2.0 two years + after its release. That isn't an open-source licence, so nothing fetches it unless an admin picks it. +* `moondream2`, Moondream 2 (Apache-2.0): 1.9B parameters, about 3.7 GB; it runs on a CPU, slowly. + +Moondream 3.1 runs only on M87 Labs' Photon engine, whose kernels are proprietary and licensed only under a separate +agreement with them, so Lens doesn't install it. + +The model's code comes with its weights (transformers' trust_remote_code), at the pinned revision, so what runs can't +change under Lens; it loads Moondream's tokenizer (moondream/starmie-v1) from Hugging Face itself. components.py +fetches PyTorch (its CPU build when there's no NVIDIA GPU), transformers and the weights the first time a job waits at +the describe step with Moondream chosen. Each picture is a row in the activity ledger (`model.describe`), with the time +it took. +""" + +from __future__ import annotations + +import os +import threading + +from . import activity, components, machine +from .descriptions import Failed + +MODELS = { + "moondream3-preview": { + "repo": "moondream/moondream3-preview", + "revision": "5112966d1a723413b1c9a1e8bea272b72e647b35", + "name": "Moondream-3-Preview", + "license": "BSL-1.1 (Moondream 3 Preview)", + "gb": 19, + "cpu": False, # 19 GB of bfloat16 weights on a CPU: too slow to be any use + "gpu_gb": 20, # an NVIDIA GPU's memory to hold them (24 GB cards do) + "mac_gb": 32, # a Mac's memory, which its GPU shares + }, + "moondream2": { + "repo": "vikhyatk/moondream2", + "revision": "5d6c926f44e26b07957b0dd315bbedcb4c17a5fe", + "name": "Moondream-2", + "license": "Apache-2.0", + "gb": 3.7, + "cpu": True, + "memory_gb": 10, # on a CPU it runs in float32: about 7.5 GB of weights, and room to work + }, +} +DEFAULT = "moondream3-preview" +TOKENIZER = "moondream/starmie-v1" # both models' tokenizer, which their code loads from Hugging Face +LENGTH = "normal" # caption length: short, normal or long + +_lock = threading.Lock() +_loaded: dict = {} + + +def choice(cfg): + m = (cfg.get("llm") or {}).get("moondream_model") or DEFAULT + return m if m in MODELS else DEFAULT + + +def name(key): + return MODELS[key]["name"] + + +def cannot_run(cfg, m=None): + """Why this machine can't run the chosen Moondream, or None when it can. `m` is machine.probe().""" + m = m or machine.probe() + key = choice(cfg) + need, nm = MODELS[key], name(key) + gpus = [g.get("memory_gb") for g in m.get("gpus") or []] + if gpus and any(g is None or g >= need.get("gpu_gb", 0) for g in gpus): + return None # a GPU's memory nvidia-smi didn't say is given the benefit of the doubt + if not need["cpu"]: + if m.get("apple_silicon") and (m.get("memory_gb") or 0) >= need["mac_gb"]: + return None + return ( + f"{nm} needs an NVIDIA GPU with {need['gpu_gb']} GB, or a Mac with {need['mac_gb']} GB of memory, and this " + "server has neither: pick moondream2 (llm.moondream_model) or Florence-2" + ) + if m.get("memory_gb") is not None and m["memory_gb"] < need["memory_gb"]: + return f"{nm} needs about {need['memory_gb']} GB of memory on a CPU and this server has {m['memory_gb']} GB: pick Florence-2" + return None + + +def device(): + """Where the model runs: an NVIDIA GPU, a Mac's GPU, else the CPU.""" + import torch + + if torch.cuda.is_available(): + return "cuda" + if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available(): + return "mps" + return "cpu" + + +def _mark(key): + return f"moondream-{key}" + + +def ready(cfg): + """Whether PyTorch, transformers and the chosen model's weights are here.""" + components.activate(cfg) + return components.importable("torch", "transformers") and components._marked(cfg, _mark(choice(cfg))) + + +def fetch(cfg, say=None): + """Download the chosen model's weights and code, at its revision, into the Hugging Face cache.""" + from huggingface_hub import snapshot_download + + key = choice(cfg) + m = MODELS[key] + if say: + say(f"downloading {m['name']} (about {m['gb']} GB)") + snapshot_download(m["repo"], revision=m["revision"], token=components.hf_token(cfg)) + snapshot_download(TOKENIZER, token=components.hf_token(cfg)) # its code loads it by name, from the same cache + components._mark(cfg, _mark(key)) + + +def _load(cfg, key): + with _lock: # one per process: loading gigabytes for every picture would take far longer than describing it + if _loaded.get("key") != key: + import torch + from transformers import AutoModelForCausalLM + + m, where = MODELS[key], device() + limit = os.environ.get("OMP_THREAD_LIMIT", "") + if limit.isdigit() and 0 < int(limit) < torch.get_num_threads(): + # more threads than OpenMP will give makes PyTorch's CPU kernels write nonsense: "and, (, (, (…" + torch.set_num_threads(int(limit)) + _loaded.clear() + model = AutoModelForCausalLM.from_pretrained( + m["repo"], + revision=m["revision"], + trust_remote_code=True, + dtype=torch.float32 if where == "cpu" else torch.bfloat16, + token=components.hf_token(cfg), + ).to(where) + model.eval() + _loaded.update(key=key, model=model, device=where) + return _loaded + + +class Moondream: + """Moondream 3 (Preview) or 2 (llm.moondream_model), through transformers.""" + + def __init__(self, cfg): + self.cfg, self.key = cfg, choice(cfg) + self.name = name(self.key) + if why := cannot_run(cfg): + raise RuntimeError(why) + components.activate(cfg) + if not components.importable("torch", "transformers"): + raise RuntimeError( + "PyTorch and transformers aren't installed (they're fetched on first use, or pip install \"lens[moondream]\")" + ) + if not components._marked(cfg, _mark(self.key)): + raise RuntimeError(f"the {self.name} model isn't here yet (it's fetched the first time a job needs it)") + self.m = _load(cfg, self.key) + + def describe(self, path, about=None): + """What the model says the picture at `path` shows (`about` isn't read: its caption skill takes no prompt).""" + from PIL import Image + + with Image.open(path) as im: + picture = im.convert("RGB") + with activity.call("model.describe", self.cfg, self.name, detail={"engine": "moondream"}, price=activity.model_cost(self.cfg)): + try: + with __import__("torch").inference_mode(): + return self.m["model"].caption(picture, length=LENGTH)["caption"] + except Exception as e: # noqa: BLE001 - the model's own code raises what it raises + raise Failed(f"{self.name} couldn't read it: {e}") from e diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index ac23542dd..ee41520de 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -162,8 +162,9 @@ ("search", "stemming"): {"english", "none"}, ("search", "engine"): set(store.SEARCH_ENGINES), ("reports", "audio"): {"link", "embed", "none"}, - ("llm", "describe_engine"): {"llm", "florence-2"}, + ("llm", "describe_engine"): {"llm", "florence-2", "moondream"}, ("llm", "florence_model"): {"base", "large"}, + ("llm", "moondream_model"): {"moondream3-preview", "moondream2"}, ("video", "ocr_engine"): {"auto", "tesseract", "apple-vision", "rapidocr", "doctr", "none"}, ("video", "face_engine"): {"opencv", "insightface", "anytopdf", "none"}, ("video", "object_engine"): {"yolox", "ultralytics", "anytopdf", "off"}, diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 250f8f3f0..4d847f8dd 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -176,9 +176,11 @@ def labels(): # a model the admin knows can see images: it describes pages and shots (the describe step); none, none are "vision_model": None, "describe_max": 50, # pages or shots of a resource described at most - # what describes them: "llm" (vision_model, above) or "florence-2" (on this machine, florence.py) + # what describes them: "llm" (vision_model, above), or on this machine "florence-2" (florence.py) or + # "moondream" (moondream.py) "describe_engine": "llm", "florence_model": "base", # Florence-2 base (about 280 MB) or large (about 830 MB) + "moondream_model": "moondream3-preview", # Moondream 3 Preview (BSL 1.1, a GPU) or moondream2 (Apache-2.0) }, "graph": {"max_nodes": 150, "min_edge_weight": 2}, "search": { diff --git a/fastapi_backend/pyproject.toml b/fastapi_backend/pyproject.toml index 11f3ae338..70ad51f3a 100644 --- a/fastapi_backend/pyproject.toml +++ b/fastapi_backend/pyproject.toml @@ -53,6 +53,9 @@ spacy = ["spacy>=3.8,<3.9"] injection = ["onnxruntime", "tokenizers"] # Florence-2 (MIT) describing pictures on this machine (llm.describe_engine "florence-2"): its ONNX export on ONNX Runtime florence = ["onnxruntime", "tokenizers"] +# Moondream describing pictures on this machine (llm.describe_engine "moondream"): its own code on transformers and +# PyTorch. Moondream 3 Preview's weights are BSL 1.1 (not open source), Moondream 2's Apache-2.0; fetched only when picked +moondream = ["torch", "transformers"] # Checking IIIF manifests against the official schema validate = ["jsonschema"] # the SMB share of the file tree (smb_share.py): impacket's SMB server (a modified Apache 1.1 license); fetched when diff --git a/fastapi_backend/tests/api/test_florence.py b/fastapi_backend/tests/api/test_florence.py index cf04443f1..f828673d8 100644 --- a/fastapi_backend/tests/api/test_florence.py +++ b/fastapi_backend/tests/api/test_florence.py @@ -225,7 +225,7 @@ def test_admins_and_agents_choose_what_describes_pictures(client, people, db): assert "only admins" in tool_error(client, people["he"], "picture_descriptions") got = tool(client, ha, "picture_descriptions") assert (got["engine"], got["ready"], got["why_not"]) == ("llm", False, "no LLM is configured") - assert set(got["engines"]) == {"llm", "florence-2"} and got["settings"].endswith("/settings/llm") + assert {"llm", "florence-2"} <= set(got["engines"]) and got["settings"].endswith("/settings/llm") assert "say what to change" in tool_error(client, ha, "set_picture_descriptions") assert tool_error(client, ha, "set_picture_descriptions", describe_max=0) == "llm.describe_max is a whole number from 1 to 1000" out = tool(client, ha, "set_picture_descriptions", engine="florence-2", florence_model="large") diff --git a/fastapi_backend/tests/api/test_moondream.py b/fastapi_backend/tests/api/test_moondream.py new file mode 100644 index 000000000..f817fd6a8 --- /dev/null +++ b/fastapi_backend/tests/api/test_moondream.py @@ -0,0 +1,220 @@ +"""Moondream describing pictures on this machine (docs/configuration.md#descriptions): with llm.describe_engine +"moondream" the describe step needs no LLM provider; Moondream 3 Preview is fetched only where it can run (a GPU, or a +Mac with the memory), Moondream 2 on a CPU with the memory; both are pinned to a revision; each picture is a ledger +row; and admins and agents choose it. PyTorch isn't installed for these tests: the model is a fake.""" + +from __future__ import annotations + +import contextlib +import os +import sys +import types + +import huggingface_hub +import pytest +from PIL import Image + +from app.domain import components, descriptions, machine, moondream, store +from tests.api.test_documents import _png, _upload +from tests.api.test_mcp import tool, tool_error +from tests.helpers import drain, login, make_user + +R = store.R +CPU = {**machine.probe(), "os": "linux", "cuda": False, "gpus": [], "memory_gb": 16.0, "apple_silicon": False} +GPU = {**CPU, "cuda": True, "gpus": [{"name": "NVIDIA RTX 4090", "memory_gb": 24.0}]} + + +class Model: + """Moondream's caption skill: says whether the picture is redder than it is blue.""" + + def __init__(self): + self.lengths = [] + + def caption(self, picture, length): + assert picture.mode == "RGB" + self.lengths.append(length) + r, _, b = picture.getpixel((0, 0)) + return {"caption": "A red square." if r > b else "A blue square."} + + +@pytest.fixture +def server(monkeypatch): + """This machine, as the tests say it is.""" + box = {"m": CPU} + monkeypatch.setattr(machine, "probe", lambda data_dir=None: dict(box["m"])) + return box + + +@pytest.fixture +def model(cfg, monkeypatch, server): + """Moondream 2 "fetched": PyTorch and transformers "importable", its weights marked here, the model a fake.""" + torch = types.ModuleType("torch") + torch.inference_mode = contextlib.nullcontext + monkeypatch.setitem(sys.modules, "torch", torch) + monkeypatch.setitem(sys.modules, "transformers", types.ModuleType("transformers")) + components._mark(cfg, moondream._mark("moondream2")) + fake, loads = Model(), [] + monkeypatch.setattr(moondream, "_load", lambda cfg, key: loads.append(key) or {"key": key, "model": fake, "device": "cpu"}) + cfg["llm"].update(base_url=None, model=None, vision_model=None, describe_engine="moondream", moondream_model="moondream2") + return fake, loads + + +@pytest.fixture +def people(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + return {"ha": login(client, "root@x.io", "root password 1"), "he": login(client, "ed@x.io", "editor password 1")} + + +def test_moondream_describes_pictures_without_a_model_provider(client, people, db, cfg, model): + fake, loads = model + red = _upload(client, people["he"], _png(Image.new("RGB", (64, 48), (220, 20, 20))), "red.png")["recording"] + blue = _upload(client, people["he"], _png(Image.new("RGBA", (64, 48), (20, 20, 220, 255))), "blue.png")["recording"] + drain(db, cfg) + got = {rid: descriptions.for_recording(db, rid) for rid in (red, blue)} + assert [(d["text"], d["model"], d["paged"]) for d in got[red]] == [("A red square.", "Moondream-2", True)] + assert got[blue][0]["text"] == "A blue square." + assert set(loads) == {"moondream2"} and fake.lengths == ["normal", "normal"] + log = " ".join(db.one("SELECT log FROM job WHERE recording = $r", r=red)["log"]) + assert "1 page(s) described (Moondream-2)" in log + # each picture is a row in the activity ledger, for its resource + rows = db.rows("SELECT model, resources, detail FROM activity WHERE action = 'model.describe'") + assert len(rows) == 2 and {r["model"] for r in rows} == {"Moondream-2"} + assert all(r["detail"] == {"engine": "moondream"} for r in rows) + assert any(f"recording:{red}" in r["resources"] for r in rows) + + +def test_a_model_that_fails_fails_the_picture_not_the_job(cfg, folder, model): + fake, _ = model + fake.caption = lambda picture, length: 1 / 0 + Image.new("RGB", (8, 8), "white").save(folder / "w.png") + eye, why = descriptions.engine(cfg) + assert why is None + with pytest.raises(descriptions.Failed, match="Moondream-2 couldn't read it: division by zero"): + eye.describe(folder / "w.png") + + +def test_where_each_moondream_can_run(cfg, server): + cfg["llm"]["describe_engine"] = "moondream" + # Moondream 3 Preview (the default): an NVIDIA GPU with the memory, or a Mac with 32 GB; never a CPU + assert moondream.choice(cfg) == "moondream3-preview" + assert "needs an NVIDIA GPU with 20 GB, or a Mac with 32 GB" in moondream.cannot_run(cfg, CPU) + assert moondream.cannot_run(cfg, GPU) is None + assert moondream.cannot_run(cfg, {**GPU, "gpus": [{"name": "small", "memory_gb": 8.0}]}) + assert moondream.cannot_run(cfg, {**GPU, "gpus": [{"name": "NVIDIA GPU", "memory_gb": None}]}) is None + mac = {**CPU, "os": "darwin", "apple_silicon": True} + assert moondream.cannot_run(cfg, {**mac, "memory_gb": 64.0}) is None + assert moondream.cannot_run(cfg, {**mac, "memory_gb": 16.0}) + # Moondream 2: anywhere with the memory + cfg["llm"]["moondream_model"] = "moondream2" + assert moondream.cannot_run(cfg, CPU) is None + assert moondream.cannot_run(cfg, {**CPU, "memory_gb": 4.0}) == ( + "Moondream-2 needs about 10 GB of memory on a CPU and this server has 4.0 GB: pick Florence-2" + ) + # an unknown model is the default + cfg["llm"]["moondream_model"] = "moondream9" + assert moondream.choice(cfg) == "moondream3-preview" + + +def test_on_a_cpu_moondream_3_is_never_fetched_and_the_step_says_why(client, people, db, cfg, server): + cfg["llm"]["describe_engine"] = "moondream" + c = components.BY_ID["moondream"] + assert not c.needed(cfg, CPU) and c.needed(cfg, GPU) # 19 GB isn't fetched where it can't run + rid = _upload(client, people["he"], _png(Image.new("RGB", (64, 48), "white")), "white.png")["recording"] + drain(db, cfg) + log = " ".join(db.one("SELECT log FROM job WHERE recording = $r", r=rid)["log"]) + assert "describe skipped: Moondream-3-Preview needs an NVIDIA GPU" in log + st = descriptions.status(cfg) + assert (st["engine"], st["model"], st["ready"]) == ("moondream", "Moondream-3-Preview", False) + assert "pick moondream2" in st["why_not"] + + +def test_moondream_is_fetched_pinned(cfg, monkeypatch, server): + c = components.BY_ID["moondream"] + assert not c.needed(cfg, CPU) and c.steps == {"describe"} and "BSL-1.1" in c.license and "Apache-2.0" in c.license + cfg["llm"].update(describe_engine="moondream", moondream_model="moondream2") + assert c.needed(cfg, CPU) and not c.present(cfg) + got = [] + monkeypatch.setattr(components, "importable", lambda *mods: False) + monkeypatch.setattr(components, "pip_install", lambda cfg, extra, say=None: got.append(extra)) + monkeypatch.setattr(huggingface_hub, "snapshot_download", lambda repo, revision=None, token=None: got.append((repo, revision))) + c.fetch(cfg, CPU, None) + assert got == [ + "moondream", + ("vikhyatk/moondream2", "5d6c926f44e26b07957b0dd315bbedcb4c17a5fe"), + ("moondream/starmie-v1", None), + ] + monkeypatch.setattr(components, "importable", lambda *mods: True) + assert c.present(cfg) # marked: the next check doesn't fetch it again + cfg["llm"]["moondream_model"] = "moondream3-preview" + assert not c.present(cfg) + assert moondream.MODELS["moondream3-preview"]["revision"] == "5112966d1a723413b1c9a1e8bea272b72e647b35" + + +def test_admins_and_agents_choose_moondream(client, people, db, server): + ha = people["ha"] + assert client.get("/api/v1/settings", headers=ha).json()["llm"]["values"]["moondream_model"] == "moondream3-preview" + assert client.put("/api/v1/settings/llm", json={"moondream_model": "moondream9"}, headers=ha).status_code == 400 + got = tool(client, ha, "picture_descriptions") + assert set(got["engines"]) == {"llm", "florence-2", "moondream"} and "BSL 1.1" in got["engines"]["moondream"] + out = tool(client, ha, "set_picture_descriptions", engine="moondream", moondream_model="moondream2") + assert out["saved"] == {"describe_engine": "moondream", "moondream_model": "moondream2"} + assert (out["engine"], out["model"], out["moondream_model"], out["ready"]) == ("moondream", "Moondream-2", "moondream2", False) + assert "fetched the first time a job waits" in out["why_not"] + assert tool_error(client, ha, "set_picture_descriptions", moondream_model="moondream9") + audit = db.rows("SELECT target, detail FROM audit_log WHERE action = 'settings.save'") + assert {"target": "llm", "detail": ["describe_engine", "moondream_model", "mcp"]} in audit + + +def test_pytorch_never_asks_for_more_threads_than_openmp_gives(cfg, monkeypatch, server): + """With OMP_THREAD_LIMIT below PyTorch's threads, its CPU kernels write nonsense ("and, (, (, (…"): seen with the + real Moondream 2 under this suite's OMP_THREAD_LIMIT=1.""" + calls = [] + torch = types.ModuleType("torch") + torch.float32, torch.bfloat16 = "float32", "bfloat16" + torch.get_num_threads = lambda: 4 + torch.set_num_threads = calls.append + torch.cuda = types.SimpleNamespace(is_available=lambda: False) + torch.backends = types.SimpleNamespace() + + class Loaded: + def to(self, where): + calls.append(where) + return self + + def eval(self): + pass + + transformers = types.ModuleType("transformers") + transformers.AutoModelForCausalLM = types.SimpleNamespace( + from_pretrained=lambda repo, **kw: calls.append((repo, kw["revision"], kw["dtype"], kw["trust_remote_code"])) or Loaded() + ) + monkeypatch.setitem(sys.modules, "torch", torch) + monkeypatch.setitem(sys.modules, "transformers", transformers) + monkeypatch.setattr(moondream, "_loaded", {}) + monkeypatch.setenv("OMP_THREAD_LIMIT", "1") + got = moondream._load(cfg, "moondream2") + assert calls == [1, ("vikhyatk/moondream2", moondream.MODELS["moondream2"]["revision"], "float32", True), "cpu"] + assert moondream._load(cfg, "moondream2") is got and len(calls) == 3 # loaded once a process + monkeypatch.setattr(moondream, "_loaded", {}) + monkeypatch.delenv("OMP_THREAD_LIMIT") + calls.clear() + moondream._load(cfg, "moondream2") + assert 1 not in calls + + +@pytest.mark.skipif(not os.environ.get("LENS_TEST_MOONDREAM"), reason="LENS_TEST_MOONDREAM: fetch and run the real Moondream 2") +def test_the_real_model(cfg, folder): + from PIL import ImageDraw + + cfg["llm"].update(describe_engine="moondream", moondream_model="moondream2") + c = components.BY_ID["moondream"] + m = machine.probe(cfg["data_dir"]) + c.fetch(cfg, m, print) + assert c.present(cfg) + img = Image.new("RGB", (640, 480), "white") + ImageDraw.Draw(img).ellipse((220, 140, 420, 340), fill=(220, 20, 20)) + img.save(folder / "dot.png") + eye, why = descriptions.engine(cfg) + assert why is None and eye.name == "Moondream-2" + assert "red" in eye.describe(folder / "dot.png").lower() diff --git a/fastapi_backend/uv.lock b/fastapi_backend/uv.lock index 5df6b2925..6265b8e43 100644 --- a/fastapi_backend/uv.lock +++ b/fastapi_backend/uv.lock @@ -1953,6 +1953,10 @@ mac-ocr = [ mlx = [ { name = "mlx-whisper" }, ] +moondream = [ + { name = "torch" }, + { name = "transformers" }, +] msg = [ { name = "extract-msg" }, ] @@ -2069,15 +2073,17 @@ requires-dist = [ { name = "surrealdb", specifier = ">=2.0,<3" }, { name = "tokenizers", marker = "extra == 'florence'" }, { name = "tokenizers", marker = "extra == 'injection'" }, + { name = "torch", marker = "extra == 'moondream'" }, { name = "torch", marker = "extra == 'punctuation'" }, { name = "torch", marker = "extra == 'sensevoice'" }, { name = "torch", marker = "extra == 'voices'" }, { name = "torchaudio", marker = "extra == 'sensevoice'" }, { name = "torchaudio", marker = "extra == 'voices'" }, + { name = "transformers", marker = "extra == 'moondream'" }, { name = "transformers", marker = "extra == 'punctuation'" }, { name = "webauthn", specifier = ">=3.0.1,<4" }, ] -provides-extras = ["sensevoice", "whisper", "mlx", "laya", "voices", "pyannote", "faces", "mac-ocr", "rapidocr", "doctr", "insightface", "objects", "punctuation", "spacy", "injection", "florence", "validate", "smb", "sandbox", "web", "msg"] +provides-extras = ["sensevoice", "whisper", "mlx", "laya", "voices", "pyannote", "faces", "mac-ocr", "rapidocr", "doctr", "insightface", "objects", "punctuation", "spacy", "injection", "florence", "moondream", "validate", "smb", "sandbox", "web", "msg"] [package.metadata.requires-dev] dev = [ diff --git a/nextjs-frontend/__tests__/settings-llm-presets.test.tsx b/nextjs-frontend/__tests__/settings-llm-presets.test.tsx index da1654a0b..7126cb5a5 100644 --- a/nextjs-frontend/__tests__/settings-llm-presets.test.tsx +++ b/nextjs-frontend/__tests__/settings-llm-presets.test.tsx @@ -31,6 +31,7 @@ const view = (keySet: boolean, locked: string[] = []) => describe_max: 20, describe_engine: "llm", florence_model: "base", + moondream_model: "moondream3-preview", api_key: { secret: true, set: keySet }, api_key_env: null, max_chars: 24000, @@ -117,6 +118,21 @@ describe("Settings › LLM provider presets", () => { }); }); + it("picks which Moondream describes pictures on this server", async () => { + show(view(true)); + expect(screen.queryByLabelText("Moondream model")).not.toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("Describe pictures with"), { target: { value: "moondream" } }); + expect(screen.getByLabelText("Moondream model")).toHaveValue("moondream3-preview"); + expect(screen.queryByLabelText("Florence-2 size")).not.toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("Moondream model"), { target: { value: "moondream2" } }); + fireEvent.click(screen.getByRole("button", { name: "Save" })); + await waitFor(() => expect(Admin.updateSettings).toHaveBeenCalledTimes(1)); + expect(m(Admin.updateSettings).mock.calls[0][0]).toMatchObject({ + path: { section: "llm" }, + body: { describe_engine: "moondream", moondream_model: "moondream2" }, + }); + }); + it("offers no presets when .env sets the address", async () => { show(view(false, ["base_url"])); expect(screen.getByLabelText("Base URL (OpenAI-compatible)")).toHaveAttribute("readonly"); diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index c99953a5b..ef864a991 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -692,13 +692,15 @@ function LlmBody({ ctx }: { ctx: BodyCtx }) {
{ctx.state("llm.describe_engine").value === "florence-2" ? ( + ) : ctx.state("llm.describe_engine").value === "moondream" ? ( + ) : ( )} diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 999c4c408..c77f1070b 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -818,8 +818,9 @@ export const FIELDS: FieldSpec[] = [ options: [ { value: "llm", label: "The provider’s model that can see images" }, { value: "florence-2", label: "Florence-2 on this server (MIT, runs on the CPU)" }, + { value: "moondream", label: "Moondream on this server (PyTorch)" }, ], - hint: "The Describe step says what each page and shot shows. Florence-2 sends no picture anywhere and is fetched the first time it’s needed", + hint: "The Describe step says what each page and shot shows. Florence-2 and Moondream send no picture anywhere and are fetched the first time they’re needed", }, { section: "llm", @@ -831,6 +832,16 @@ export const FIELDS: FieldSpec[] = [ { value: "large", label: "Large (about 830 MB, more detail, slower)" }, ], }, + { + section: "llm", + key: "moondream_model", + label: "Moondream model", + kind: "select", + options: [ + { value: "moondream3-preview", label: "Moondream 3 Preview (about 19 GB, needs a GPU; BSL 1.1, not open source)" }, + { value: "moondream2", label: "Moondream 2 (about 3.7 GB, runs on the CPU, slowly; Apache-2.0)" }, + ], + }, { section: "llm", key: "vision_model", diff --git a/threat-model.json b/threat-model.json index 5e49187cd..04e904247 100644 --- a/threat-model.json +++ b/threat-model.json @@ -6,7 +6,7 @@ "scope_subpath": null, "description": "Lens is a self-hosted archive for documents, recordings, photos and messages: it reads, transcribes, indexes and publishes them, and people search and chat over them. One operator (the instance admin) runs an instance for their own people: a FastAPI API, a Next.js web app that proxies the API and holds the sign-in session, and background workers, around one SurrealDB database, with content split into namespaces and per-namespace roles. Exposing an instance to the internet behind HTTPS (reverse proxy or Cloudflare tunnel) is supported. Only the instance admin (equivalent to host code execution) and the host operator are trusted; everyone else is trusted only as far as their role.", "confidence": { - "documented": 250, + "documented": 251, "inferred": 99 }, "components": [ diff --git a/threat-model.md b/threat-model.md index 3e1ad5581..cdb39d239 100644 --- a/threat-model.md +++ b/threat-model.md @@ -467,6 +467,7 @@ third-party source; dependencies come from `uv.lock`, the pnpm lockfile and cont | Florence-2 (Florence-2-base-ft or -large-ft, their 8-bit ONNX exports, `florence` extra, opt-in: `llm.describe_engine` `florence-2`) | describing the archive's pictures on the worker's CPU, so none is sent to a provider; the model is fetched at a pinned revision and checked against its SHA-256 | upstream for what the model says: text a picture shows that ends up in its description reaches the assistant as archive text (N-PROMPT); a model file that passes unchecked is in model | *(documented, docs/configuration.md "pinned to a revision and checked against its SHA-256")* | | Punctuation model (kredor/punctuate-all, `punctuation` extra, opt-in: `analysis.cleanup_punctuation`) | full stops, commas and question marks for a transcript's lines; the model is fetched from Hugging Face at a pinned revision, and Transformers loads it there with no remote code | upstream for the model's judgement (a missed or wrong mark, which the edit history can revert); a model file from another revision is in model | *(documented, docs/components.md "at a pinned revision")* | | spaCy and its small models (`spacy` extra, opt-in: `analysis.entities` `spacy`) | finding names in text; the MIT model wheels (`en_core_web_sm`, `de_core_news_sm`, `xx_ent_wiki_sm` 3.8.0) come from spaCy's GitHub releases, are checked against their SHA-256 and installed without dependencies; a model is Python code that runs when it loads, so the checksum is what keeps a swapped file out | upstream for the model's judgement (a missed or wrong name); a model file that installs unchecked is in model | *(documented, docs/components.md "checked against their SHA-256")* | +| Moondream (Moondream 3 Preview, BSL 1.1, or Moondream 2, Apache-2.0; `moondream` extra, opt-in: `llm.describe_engine` `moondream`) | describing the archive's pictures on the worker, so none is sent to a provider; the weights and the model's own Python code (transformers' `trust_remote_code`) are fetched at a pinned revision, so the code that runs in the worker can't change under Lens; its tokenizer (`moondream/starmie-v1`, data, not code) is fetched by name | upstream for what the model says (text a picture shows reaches the assistant as archive text, N-PROMPT) and for the files' integrity beyond HTTPS and the Hub; the pinned code runs with the worker's access, an admin's choice like any extra | *(documented, docs/configuration.md "the weights and the model's own code pinned to a revision")* | | Optional ML extras (whisper, pyannote, speechbrain, insightface, doctr) | model inference; some fetch weights on their own | upstream | *(inferred, Q19)* | | StyleTTS2 service (`styletts2/`, opt-in Compose profile) and macOS `say` | text to speech; StyleTTS2 fetches its weights from Hugging Face and GitHub on first use and listens only on the Compose network; `say` gets the text on stdin, never as arguments | upstream for the model and synthesis; the argument handling in model | *(maintainer, 2026-10)* | | MCP runner (`mcp-runner/`, opt-in Compose profile) and the stdio MCP servers admins add (usually `npx` or `uvx` packages) | running those servers away from Lens: a container with none of Lens's data, keys or database, a non-root user, every capability dropped, `no-new-privileges`, a read-only image, memory and process limits, a network shared only with the backend (plus the internet to fetch packages), an empty home and only its extension's environment per server, and a bearer token on a volume only the two share | upstream for the servers and the packages they fetch (an admin's choice, trusted input, §1.3); the runner's isolation and token check in model | *(documented, docs/assistant.md "The MCP runner")* | diff --git a/threat-model.yaml b/threat-model.yaml index 72edbd42f..26391986c 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -2,7 +2,7 @@ # Authority order: prose > yaml > json. Regenerate whenever the prose changes. schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:b41fc97af52afeb4402f75c826637ee297dee746ce119b8d5e8e009632f55775" +prose_version: "threat-model.md@sha256:7facd52fd5c65fa45df1a131f8efad9d6c367e6804c299522a9dddca790ce3a6" model_status: under-review triage_policy: strict confidence: {documented: 204, maintainer: 46, inferred: 91, assumption: 8} From f6c733d86295f859fcc905211d026ce7248e8d8e Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 06:52:44 +0000 Subject: [PATCH 2/3] style(settings): format the Moondream settings with Prettier Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_013kCtgLNMjPQYdZpV5gbwUW --- nextjs-frontend/components/settings/bodies.tsx | 8 +++++++- nextjs-frontend/components/settings/model.ts | 5 ++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index ef864a991..c514ac907 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -692,7 +692,13 @@ function LlmBody({ ctx }: { ctx: BodyCtx }) { diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index c77f1070b..ab7f94172 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -838,7 +838,10 @@ export const FIELDS: FieldSpec[] = [ label: "Moondream model", kind: "select", options: [ - { value: "moondream3-preview", label: "Moondream 3 Preview (about 19 GB, needs a GPU; BSL 1.1, not open source)" }, + { + value: "moondream3-preview", + label: "Moondream 3 Preview (about 19 GB, needs a GPU; BSL 1.1, not open source)", + }, { value: "moondream2", label: "Moondream 2 (about 3.7 GB, runs on the CPU, slowly; Apache-2.0)" }, ], }, From 5852a9954c20aeec1fecd4bb9873e816fdec5949 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 09:23:27 +0000 Subject: [PATCH 3/3] fix(threat-model): count the Moondream row after main's own Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_013kCtgLNMjPQYdZpV5gbwUW --- threat-model.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threat-model.json b/threat-model.json index c9e859ae0..15df0245e 100644 --- a/threat-model.json +++ b/threat-model.json @@ -6,7 +6,7 @@ "scope_subpath": null, "description": "Lens is a self-hosted archive for documents, recordings, photos and messages: it reads, transcribes, indexes and publishes them, and people search and chat over them. One operator (the instance admin) runs an instance for their own people: a FastAPI API, a Next.js web app that proxies the API and holds the sign-in session, and background workers, around one SurrealDB database, with content split into namespaces and per-namespace roles. Exposing an instance to the internet behind HTTPS (reverse proxy or Cloudflare tunnel) is supported. Only the instance admin (equivalent to host code execution) and the host operator are trusted; everyone else is trusted only as far as their role.", "confidence": { - "documented": 251, + "documented": 252, "inferred": 99 }, "components": [