From 9d6e151abd0c5bbeddb921a42404881d74293d25 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:15:16 +0000 Subject: [PATCH 1/4] feat(documents): read documents with an Apache Tika server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An opt-in Tika server (`--profile tika` in the compose files, or any Tika server set in Settings → Documents) reads, as text, the documents nothing here can make into a PDF, and what every document says about itself as Dublin Core terms (`described` on the resource; its authors and language feed the resource's metadata). Off until documents.tika_url is set. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01X6p3skuCAfa2nTxSmDKvxq --- CHANGELOG.md | 5 + docker-compose.prod.yml | 15 ++ docker-compose.yml | 14 ++ docs/api.md | 5 +- docs/configuration.md | 34 +++ docs/deployment.md | 8 +- fastapi_backend/app/api/v1/routes/admin.py | 13 + fastapi_backend/app/domain/convert.py | 41 +++- fastapi_backend/app/domain/documents.py | 66 +++-- fastapi_backend/app/domain/metadata.py | 7 +- fastapi_backend/app/domain/settings.py | 17 +- fastapi_backend/app/domain/store.py | 3 + fastapi_backend/app/domain/tika.py | 231 ++++++++++++++++++ fastapi_backend/app/schemas/admin.py | 7 + fastapi_backend/app/schemas/recordings.py | 17 ++ fastapi_backend/app/schemas/uploads.py | 1 + fastapi_backend/tests/api/test_anytopdf.py | 3 +- fastapi_backend/tests/api/test_conversion.py | 5 +- fastapi_backend/tests/api/test_tika.py | 207 ++++++++++++++++ fastapi_backend/tests/domain/test_convert.py | 6 +- fastapi_backend/tests/fake_tika.py | 81 ++++++ .../__tests__/document-model.test.ts | 23 +- .../__tests__/import-documents.test.tsx | 2 +- nextjs-frontend/app/openapi-client/index.ts | 5 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 16 ++ .../app/openapi-client/types.gen.ts | 126 ++++++++++ nextjs-frontend/components/import/files.ts | 6 +- .../components/recording/details-tab.tsx | 16 +- .../components/recording/document/model.ts | 36 ++- .../components/recording/labels.ts | 1 + .../components/settings/bodies.tsx | 52 ++++ nextjs-frontend/components/settings/model.ts | 24 ++ nextjs-frontend/openapi.json | 223 +++++++++++++++++ threat-model.json | 8 + threat-model.md | 4 +- threat-model.yaml | 10 +- 36 files changed, 1285 insertions(+), 53 deletions(-) create mode 100644 fastapi_backend/app/domain/tika.py create mode 100644 fastapi_backend/tests/api/test_tika.py create mode 100644 fastapi_backend/tests/fake_tika.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5112764c6..33840472e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Apache Tika.** Lens can use an Apache Tika server (`--profile tika` in the compose files, or one of your own, set + in Settings → Documents). Documents nothing here can make into a PDF, such as Word files on a lean server or a + Raspberry Pi, are read by Tika as text, and every document's own title, authors, language and dates are read as + Dublin Core terms. Off until a Tika server is set ([Configuration](docs/configuration.md#apache-tika)). + - **Install on Windows, or by double-clicking.** `install.ps1` installs Lens on Windows: it gets Python 3 if needed and opens the web installer, which installs Docker Desktop (and WSL) and runs Lens in it. Each release now carries launchers that need no terminal at all: **Install Lens.app** for macOS, **Install Lens.cmd** for Windows and diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index c5b41f574..53ffe6c3a 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -139,6 +139,21 @@ services: restart: unless-stopped networks: [lens] + # Apache Tika (docs/configuration.md#apache-tika): reads the text and description of about a thousand kinds of file where Lens can't make + # them into PDFs itself, and what every document says about itself. Opt-in, about 250 MB of memory: + # `docker compose --profile tika up -d` (or COMPOSE_PROFILES=tika in .env), then Settings → Documents → Tika server: + # http://tika:9998. Reachable only inside Lens's network; it has no sign-in of its own. + tika: + image: apache/tika:${TIKA_VERSION:-4.0.0-1} + profiles: [tika] + security_opt: ["no-new-privileges:true"] + read_only: true + tmpfs: [/tmp] + environment: + JAVA_TOOL_OPTIONS: -Xmx${TIKA_HEAP:-512m} + restart: unless-stopped + networks: [lens] + # Fedora 6 (docs/fedora.md), the repository Lens can keep a copy of the archive in. Opt-in, and not needed on small # machines: `docker compose --profile fedora up`, with LENS_FEDORA_URL=http://fedora:8080/fcrepo/rest (and # LENS_FEDORA_USER / LENS_FEDORA_PASSWORD) in .env, or set it in Settings → Fedora. diff --git a/docker-compose.yml b/docker-compose.yml index 179d67cda..e92fd124e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -131,6 +131,20 @@ services: - matterbridge-data:/matterbridge networks: [lens] + # Apache Tika (docs/configuration.md#apache-tika): reads the text and description of about a thousand kinds of file where Lens can't make + # them into PDFs itself, and what every document says about itself. Opt-in, about 250 MB of memory: + # `docker compose --profile tika up -d` (or COMPOSE_PROFILES=tika in .env), then Settings → Documents → Tika server: + # http://tika:9998. Reachable only inside Lens's network; it has no sign-in of its own. + tika: + image: apache/tika:${TIKA_VERSION:-4.0.0-1} + profiles: [tika] + security_opt: ["no-new-privileges:true"] + read_only: true + tmpfs: [/tmp] + environment: + JAVA_TOOL_OPTIONS: -Xmx${TIKA_HEAP:-512m} + networks: [lens] + # Fedora 6 (docs/fedora.md), the repository Lens can keep a copy of the archive in. Opt-in, and not needed on small # machines: `docker compose --profile fedora up`, with LENS_FEDORA_URL=http://fedora:8080/fcrepo/rest (and # LENS_FEDORA_USER / LENS_FEDORA_PASSWORD) in .env, or set it in Settings → Fedora. diff --git a/docs/api.md b/docs/api.md index 8d862bc80..48ccdf5c5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -507,7 +507,10 @@ made, unless someone named or dated it. Its attachments are kept as its files (r read become resources of their own beside it, each with `attached_to: {resource, file, title}` (the email, and its title when the caller may see it); an attachment file's `resource` is the resource it became, or the one it was already in the namespace (`documents.attachment_resources`). Transcribing an email again keeps and makes nothing twice. -`GET /uploads/limits` says what the server can convert (`convert: {office, pages, msg}`); an upload it can't read is +`GET /uploads/limits` says what the server can convert (`convert: {office, pages, msg, tika}`); with an +[Apache Tika](configuration.md#apache-tika) server set, what nothing else can convert is read by Tika as text without +pages (`engine` `text:tika`), and every document has `described`, what it says about itself as Dublin Core terms +(`POST /settings/documents/tika/test` checks the server). An upload it can't read is refused, saying what it needs, and the files that can be transcripts too (PDFs, Word, text and Markdown) can be imported as transcripts instead (`POST /import`). Its pipeline's transcribe step draws its pages and reads their text: a PDF's own text, block by block with where each block is on its page (poppler's diff --git a/docs/configuration.md b/docs/configuration.md index 550663fb8..3f2ca08a0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -210,6 +210,9 @@ makes of them, or by LibreOffice where there's no Chromium. Outlook `.msg` email | `documents.converter` | `auto` | who makes the PDFs: `auto` uses [anytopdf](#anytopdf) only for what LibreOffice and Chromium can't do here, `anytopdf` uses it for every document and image (and fetches it), `lens` never uses it | | `documents.anytopdf_url` | none | a conversion node: another machine running `anytopdf queue serve` and `anytopdf queue work`, such as `https://convert.home:8640`. Set, it converts instead of the program here | | `documents.anytopdf_token` | none | the node's bearer token (its `ANYTOPDF_QUEUE_TOKEN`), kept encrypted | +| `documents.tika_url` | none (off) | an [Apache Tika](#apache-tika) server, such as `http://tika:9998` for the compose files' `tika` service | +| `documents.tika_text` | true | with a Tika server: documents nothing here can make into a PDF are read by Tika, as text without pages | +| `documents.tika_metadata` | true | with a Tika server: what every document says about itself (title, authors, language, dates) is read by Tika | Set at startup only (the config file; the web app can't choose what the server runs): @@ -220,6 +223,37 @@ Set at startup only (the config file; the web app can't choose what the server r | `documents.anytopdf` | the one Lens downloaded, else `anytopdf` on PATH | the anytopdf program | | `documents.web_networks` | `[]` | networks (CIDR, such as `10.20.0.0/16`) that web pages and calendar feeds may be fetched from besides the public internet: for an intranet, or a calendar server at home; loopback and cloud metadata addresses stay out unless listed. `LENS_WEB_NETWORKS` in `.env` (comma-separated) adds to it, for Docker and the packages | +### Apache Tika + +[Apache Tika](https://tika.apache.org) (Apache-2.0) reads the text and the description of about a thousand kinds of +file. Lens can use a Tika server; it's off until `documents.tika_url` is set. The compose files have one ready, as an +opt-in profile (about 250 MB of memory, so leave it off on a small Raspberry Pi): + +```sh +docker compose -f docker-compose.prod.yml --profile tika up -d +``` + +(or add `COMPOSE_PROFILES=tika` to Lens's `.env` and run the install command again). Then, in Settings → Documents, +set the Tika server to `http://tika:9998` and press Test. A Tika server elsewhere works the same way. + +- **Reading what nothing else can** (`documents.tika_text`). When neither LibreOffice, Chromium nor + [anytopdf](#anytopdf) can make a document into a PDF here (a Word file on a lean server, say), or one of them fails + on it, Tika reads it instead. Uploads and folder scans then take every document type, Office files included. Such + a document has its text, which is searched, analyzed and asked about like any other's, but no pages to look at; + its `engine` is `text:tika`. An email or an archive read this way gives its text, including its attachments' and + files', but they aren't kept as files or resources of their own. +- **What a document says about itself** (`documents.tika_metadata`). Every document is also described by Tika, as + Dublin Core terms (`GET /resources/{id}` → `described`: title, creators, subject, description, publisher, + language, created, modified, pages, words, format). Its title and creation date become the resource's while it's + still named and dated after its file (an email's own subject and date come first), and its authors and language + are its [metadata](rdf.md)'s until someone saves others. If Tika can't describe it, the document is read anyway. + +The document goes only to the address set: http(s) with no user name or password in it, no redirect followed, and +sent as `document.`, never by its own name. Tika has no sign-in of its own, so the compose service is reachable +only inside Lens's network; put one elsewhere behind TLS on a network you trust, since every document is sent to it. +Its answers are bounded (64 MB) before Lens reads them. The compose service runs read-only, as a user without +privileges, with `TIKA_HEAP` (default `512m`) as its Java heap and `TIKA_VERSION` (default `4.0.0-1`) as its image. + ### anytopdf [anytopdf](https://github.com/adeelahmad/anytopdf-rs) (MIT or Apache-2.0) is Lens's sister project: one static program diff --git a/docs/deployment.md b/docs/deployment.md index 66489da98..5dcd4813d 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,10 +1,12 @@ # Deployment Lens runs as containers: SurrealDB, the API, one or more workers, and the Next.js app. `docker-compose.prod.yml` -is a production-shaped starting point. Two more services are opt-in profiles: `--profile fedora` adds a Fedora 6 -repository to keep a copy of the archive in ([Fedora](fedora.md)), and `--profile opensearch` adds OpenSearch for word +is a production-shaped starting point. More services are opt-in profiles: `--profile fedora` adds a Fedora 6 +repository to keep a copy of the archive in ([Fedora](fedora.md)), `--profile opensearch` adds OpenSearch for word search on a big archive (then choose it in Settings → Search and reindex; the default is a SQLite index in -`data_dir/search`). +`data_dir/search`), and `--profile tika` adds an Apache Tika server that reads documents nothing else here can, and +what every document says about itself (then set it in Settings → Documents; +[Configuration](configuration.md#apache-tika)). ```bash export ACCESS_SECRET_KEY=$(openssl rand -hex 32) ARCHIVE_SECRET_KEY=$(openssl rand -hex 32) \ diff --git a/fastapi_backend/app/api/v1/routes/admin.py b/fastapi_backend/app/api/v1/routes/admin.py index 5249bfb4a..22c92ef1c 100644 --- a/fastapi_backend/app/api/v1/routes/admin.py +++ b/fastapi_backend/app/api/v1/routes/admin.py @@ -30,6 +30,7 @@ speech, store, telemetry, + tika, tunnel, ) from app.schemas.admin import ( @@ -54,6 +55,7 @@ Started, TelemetryStatus, TelemetryTestResult, + TikaTestResult, TunnelStatus, ) from app.schemas.auth import AccountToken @@ -193,6 +195,17 @@ def test_conversion_node(user: AdminWriter, cfg: Cfg) -> ConversionNodeTestResul return ConversionNodeTestResult(ok=error is None, error=error, ms=None if error else int((time.time() - t0) * 1000)) +@router.post("/settings/documents/tika/test") +def test_tika(user: AdminWriter, cfg: Cfg) -> TikaTestResult: + """Check that a Tika server answers at documents.tika_url, and say which version it is.""" + t0 = time.time() + try: + said = tika.version(cfg) + except ValueError as e: + return TikaTestResult(ok=False, error=str(e)) + return TikaTestResult(ok=True, version=said, ms=int((time.time() - t0) * 1000)) + + @router.get("/settings/decisions/status") def decision_status(user: AdminReader, cfg: Cfg) -> DecisionStatus: """Who takes routine decisions, and whether a local Laya model can here: on this machine (Apple Silicon, once diff --git a/fastapi_backend/app/domain/convert.py b/fastapi_backend/app/domain/convert.py index 2cda12425..eca22f829 100644 --- a/fastapi_backend/app/domain/convert.py +++ b/fastapi_backend/app/domain/convert.py @@ -32,7 +32,7 @@ from email.utils import getaddresses, parsedate_to_datetime from html.parser import HTMLParser -from . import anytopdf, netguard, store +from . import anytopdf, netguard, store, tika R = store.R OFFICE = { @@ -140,18 +140,35 @@ def _has_msg(): return True +def _can(cfg): + """What this server can make into PDFs itself, or with anytopdf: {office, pages, msg}.""" + by = None if anytopdf.mode(cfg) == "lens" else anytopdf.available(cfg) + office, pages = bool(soffice(cfg) or by == "node"), bool(chromium(cfg) or soffice(cfg) or by) + return {"office": office, "pages": pages, "msg": pages and _has_msg()} + + def capabilities(cfg): """What this server can make into PDFs: Office files (LibreOffice), text, Markdown, saved web pages and emails (Chromium or LibreOffice), Outlook .msg emails (those, and the extract-msg package), and web pages captured from their address (Chromium). anytopdf does the same where Lens can't (anytopdf.py): emails, text and pages here, and - Office files too on a conversion node.""" - by = None if anytopdf.mode(cfg) == "lens" else anytopdf.available(cfg) - office, pages = bool(soffice(cfg) or by == "node"), bool(chromium(cfg) or soffice(cfg) or by) - return {"office": office, "pages": pages, "msg": pages and _has_msg(), "web": bool(chromium(cfg))} + Office files too on a conversion node. A Tika server reads all of those as text where nothing here can (tika.py).""" + got = {**_can(cfg), "web": bool(chromium(cfg)), "tika": tika.reads_text(cfg)} + if got["tika"]: + got.update(office=True, pages=True, msg=True) + return got def unavailable(cfg, name): - """Why this server can't read this document (None when it can): what it would need.""" + """Why this server can't read this document (None when it can): what it would need. A Tika server reads, as text, + any document Lens can't make into a PDF here.""" + why = unconvertible(cfg, name) + if why and tika.reads_text(cfg) and ext_of(name) in store.DOCUMENT_EXT: + return None + return why + + +def unconvertible(cfg, name): + """Why this server can't make this document into a PDF (None when it can): what it would need.""" ext = ext_of(name) if ext in store.CAMERA_EXT: if anytopdf.mode(cfg) == "lens" or not anytopdf.available(cfg): @@ -160,11 +177,13 @@ def unavailable(cfg, name): return None if ext not in store.DOCUMENT_EXT or ext == ".pdf": return None - can = capabilities(cfg) + can = _can(cfg) if ext in OFFICE and not can["office"]: - return f"converting {word(name)}s needs LibreOffice on the server (the lens:full image) or an anytopdf conversion node" + return ( + f"converting {word(name)}s needs LibreOffice on the server (the lens:full image), an anytopdf conversion node or a Tika server" + ) if ext not in OFFICE and not can["pages"]: - return f"converting {word(name)}s needs Chromium or LibreOffice on the server (the lens:full image)" + return f"converting {word(name)}s needs Chromium or LibreOffice on the server (the lens:full image), or a Tika server" if ext == ".msg" and not can["msg"]: return "reading Outlook .msg emails needs the extract-msg package on the server (pip install -e '.[msg]')" return None @@ -222,7 +241,7 @@ def office_pdf(cfg, src, out): """An Office, OpenDocument or RTF file made into a PDF by LibreOffice, offline and with a profile of its own.""" exe = soffice(cfg) if not exe: - raise Unavailable(unavailable(cfg, src) or "LibreOffice isn't installed") + raise Unavailable(unconvertible(cfg, src) or "LibreOffice isn't installed") with tempfile.TemporaryDirectory(prefix="lens-office-") as tmp: doc = pathlib.Path(tmp) / f"document{ext_of(src)}" shutil.copyfile(src, doc) @@ -765,7 +784,7 @@ def page_pdf(cfg, page, out, via_anytopdf): def to_pdf(cfg, src, out): """Make `out`, the PDF of the document at `src`. Returns what was learnt on the way: {by, title?, email?, attachments?}. Unavailable when the server can't; ValueError when the file can't be read.""" - why = unavailable(cfg, src) + why = unconvertible(cfg, src) if why: raise Unavailable(why) ext, path, via = ext_of(src), pathlib.Path(src), by_anytopdf(cfg, src) diff --git a/fastapi_backend/app/domain/documents.py b/fastapi_backend/app/domain/documents.py index d8267fc45..dc58dd845 100644 --- a/fastapi_backend/app/domain/documents.py +++ b/fastapi_backend/app/domain/documents.py @@ -19,7 +19,7 @@ import tempfile import xml.etree.ElementTree as ET -from . import anytopdf, convert, ingest, keyring, store, video, webcapture +from . import anytopdf, convert, ingest, keyring, store, tika, video, webcapture R = store.R TYPES = { @@ -417,7 +417,7 @@ def read_image(path, d, opts, engine, say, why="no OCR engine is available"): # ---------- the transcribe step for documents and images ---------- -BY = {"libreoffice": "LibreOffice", "chromium": "Chromium", "anytopdf": "anytopdf"} # who made its PDF, for people +BY = {"libreoffice": "LibreOffice", "chromium": "Chromium", "anytopdf": "anytopdf", "tika": "Tika"} # who made its PDF, for people def segments_of(blocks): @@ -433,16 +433,37 @@ def segments_of(blocks): def _named(rec, path, learnt): """An email's subject as its title and its date as when it was made, unless someone named or dated it otherwise: - an upload is titled after its file and dated by it until then.""" + an upload is titled after its file and dated by it until then. A document's own title and date (Tika's) the same, + where it says them.""" out = {} stem = pathlib.PurePosixPath(str(path)).stem - if learnt.get("title") and (not rec.get("title") or rec["title"] == stem): - out["title"] = learnt["title"][:200] - if learnt["email"].get("date"): - out["recorded_at"] = learnt["email"]["date"] + told, email = learnt.get("described") or {}, learnt.get("email") + title = (learnt.get("title") if email else None) or told.get("title") + if title and (not rec.get("title") or rec["title"] == stem): + out["title"] = title[:200] + when = (email or {}).get("date") or told.get("created") + if when: + out["recorded_at"] = when return out +def _tika_text(cfg, path, say): + """A document read by Tika as text alone, with no pages: (pages, blocks by page, notes, OCRed, learnt).""" + got = tika.read(cfg, path) + blocks = plain_blocks(got["text"]) + say(f"read by Tika as text ({len(blocks)} block(s), no pages to look at)") + return [], {0: blocks} if blocks else {}, [], 0, {"by": "tika", "described": got["described"]} + + +def _tika_described(cfg, path, say): + """What a document says about itself, read by Tika (documents.tika_metadata), or {} when it can't.""" + try: + return tika.read(cfg, path)["described"] + except ValueError as e: + say(f"Tika couldn't say what it is: {e}") + return {} + + def _clear(d): for pattern in ("page-*.jpg", "thumb-*.jpg"): for f in d.glob(pattern): @@ -464,14 +485,19 @@ def transcribe(db, cfg, rid, say): if not path or not os.path.exists(path): raise FileNotFoundError(f"its file isn't there ({rec.get('path')})") opts, (engine, why) = cfg["documents"], video.ocr_engine_why(cfg) - learnt, pdf, source = {}, path, rec["source"] + learnt, pdf, source, read = {}, path, rec["source"], None if source == "document" and convert.needs(path): pdf = convert.rendition_path(cfg, rid) try: learnt = convert.to_pdf(cfg, path, pdf) - except convert.Unavailable as e: - raise ValueError(f"this {convert.word(path)} can't be read here: {e}") from None - say(f"made into a PDF by {BY[learnt['by']]}") + say(f"made into a PDF by {BY[learnt['by']]}") + except (convert.Unavailable, ValueError) as e: + if not tika.reads_text(cfg): # nothing else can read it + if isinstance(e, convert.Unavailable): + raise ValueError(f"this {convert.word(path)} can't be read here: {e}") from None + raise + say(f"not made into a PDF ({e}): Tika reads it instead") + *read, learnt = _tika_text(cfg, path, say) elif source == "image" and convert.ext_of(path) in store.CAMERA_EXT: # a camera RAW or HEIC photo: Lens can't open it, anytopdf can; its PDF is the photo's rendition why = convert.unavailable(cfg, path) @@ -495,13 +521,17 @@ def transcribe(db, cfg, rid, say): known = anytopdf.page_words(pdf) if learnt.get("by") == "anytopdf" else {} try: with keyring.sealing(db, cfg, rec["space"], d): # the pages drawn: encrypted, even if this fails part way - if source == "image" and not learnt: + if read: # Tika's text: there are no pages + pages, blocks, notes, ocred = read + elif source == "image" and not learnt: pages, blocks, notes, ocred = read_image(pdf, d, opts, engine, say, why) else: pages, blocks, notes, ocred = read_pdf(pdf, d, opts, engine, say, why, known) finally: - if learnt: # the rendition, read: kept encrypted from here on, like the file it was made from + if learnt and not read: # the rendition, read: kept encrypted from here on, like the file it was made from keyring.protect(db, cfg, rec["space"], pdf) + if source == "document" and not read and tika.describes(cfg): + learnt["described"] = _tika_described(cfg, path, say) segs = segments_of(blocks) rows = [{**p, "id": R("page", f"{rid}-{p['idx']}"), "recording": rid, "space": rec["space"]} for p in pages] db.run( @@ -514,15 +544,17 @@ def transcribe(db, cfg, rid, say): media = store.clean({"kind": rec["source"], "pages": len(pages), "width": first.get("width"), "height": first.get("height")}) theirs = sum(1 for p in pages if p["idx"] in known and known[p["idx"]]["ocr"]) # pages anytopdf's OCR read ocr_by = engine.name if ocred and engine else "anytopdf" if theirs else None - how = ("pdf" if rec["source"] == "document" else "image") + (f"+ocr:{ocr_by}" if ocr_by else "") + how = ("text:tika" if read else "pdf" if rec["source"] == "document" else "image") + (f"+ocr:{ocr_by}" if ocr_by else "") patch = {"status": "transcribed", "engine": how, "media": media, "transcribed_at": store.now()} - if learnt: - patch["rendition"] = {"from": convert.ext_of(path), "by": learnt["by"]} + if learnt.get("by"): + patch["rendition"] = None if read else {"from": convert.ext_of(path), "by": learnt["by"]} + if source == "document": + patch["described"] = learnt.get("described") or None # what it says about itself (Tika), else nothing kept + patch.update(_named(rec, path, learnt)) if learnt.get("archive"): patch["archive"] = learnt["archive"] # {files, left_out} if learnt.get("email"): patch["email"] = learnt["email"] - patch.update(_named(rec, path, learnt)) ingest.write_transcript(db, rid, rec["space"], segs, patch) for note in notes: say(note) diff --git a/fastapi_backend/app/domain/metadata.py b/fastapi_backend/app/domain/metadata.py index 95e32de05..159ce9b17 100644 --- a/fastapi_backend/app/domain/metadata.py +++ b/fastapi_backend/app/domain/metadata.py @@ -403,7 +403,7 @@ def _write(db, rid, meta): def defaults(db, cfg, rid): """What a recording says about itself before anyone edits its metadata.""" - rec = db.one("SELECT title, recorded_at, language, summary, space FROM $r", r=R("recording", rid)) or {} + rec = db.one("SELECT title, recorded_at, language, summary, space, described FROM $r", r=R("recording", rid)) or {} ns = namespace(db, rec.get("space")) lang = cfg["iiif"].get("default_language") or "none" level, open_ = acc.namespace_defaults(db, [rec["space"]]).get(rec.get("space"), ("private", list(acc.PARTS))) @@ -415,6 +415,11 @@ def defaults(db, cfg, rid): pass if rec.get("language") and LANG_RX.match(rec["language"]) and rec["language"] not in ("none", "nospeech"): d["language"] = [rec["language"]] + told = rec.get("described") or {} # what a document says about itself (tika.py) + if "language" not in d and told.get("language") and LANG_RX.match(told["language"]): + d["language"] = [told["language"]] + if told.get("creators"): + d["creators"] = [{"name": n, "role": "author"} for n in told["creators"]] apps = db.rows("SELECT speaker FROM appearance WHERE recording = $r", r=rid) names = render.speaker_names(db, [a["speaker"] for a in apps]) if names: diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index 4941151ea..0e6cc6987 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -104,6 +104,9 @@ "converter", "anytopdf_url", "anytopdf_token", + "tika_url", + "tika_text", + "tika_metadata", ), } SECRETS = { @@ -432,7 +435,12 @@ def _check(section, key, value, default): return _local_llm_setting(key, value) if section == "decisions" and key != "engine": return _decision_setting(key, value) - if (section, key) in (("documents", "attachment_resources"), ("documents", "video_urls")): + if (section, key) in ( + ("documents", "attachment_resources"), + ("documents", "video_urls"), + ("documents", "tika_text"), + ("documents", "tika_metadata"), + ): if not isinstance(value, bool): raise ValueError(f"documents.{key} is true or false") return value @@ -444,6 +452,13 @@ def _check(section, key, value, default): if not (isinstance(value, str) and VIEWER_URL.match(value.strip())): raise ValueError("documents.anytopdf_url is a conversion node's http(s) address, such as https://convert.home:8640") return value.strip().rstrip("/") + if (section, key) == ("documents", "tika_url"): + if value in (None, ""): + return None + u = urllib.parse.urlsplit(value.strip()) if isinstance(value, str) else None + if not (u and VIEWER_URL.match(value.strip()) and u.hostname and u.username is None and u.password is None): + raise ValueError("documents.tika_url is a Tika server's http(s) address with no password in it, such as http://tika:9998") + return value.strip().rstrip("/") if (section, key) == ("documents", "anytopdf_token"): if value in (None, ""): return None diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index cbae2c2df..9bbc5b40f 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -245,6 +245,9 @@ def labels(): "anytopdf": None, # the anytopdf program; None: the one downloaded on first use, else one on PATH "anytopdf_url": None, # a conversion node: `anytopdf queue serve` on another machine "anytopdf_token": None, + "tika_url": None, # tika.py: a Tika server (the compose files' `tika` profile: http://tika:9998); None: off + "tika_text": True, # with a Tika server: it reads, as text, documents Lens can't make into PDFs + "tika_metadata": True, # with a Tika server: it reads what every document says about itself }, "workers": { "inline": 1, diff --git a/fastapi_backend/app/domain/tika.py b/fastapi_backend/app/domain/tika.py new file mode 100644 index 000000000..5a29517f6 --- /dev/null +++ b/fastapi_backend/app/domain/tika.py @@ -0,0 +1,231 @@ +"""Apache Tika (tika.apache.org) as a document reader: the text and the description of about a thousand kinds of file, +read by a Tika server (docs/api.md#documents-and-images). + +Off unless documents.tika_url names a Tika server: the `tika` service Lens's compose files can start next to it +(`--profile tika`, http://tika:9998), or one elsewhere. It has two uses: +- documents.tika_text (on): a document that neither Lens's converters nor anytopdf can make into a PDF here (an Office + file on a server without LibreOffice, say) is read by Tika instead, as text alone: it has no pages to look at, but + its words are searched, analyzed and asked about like any other's. +- documents.tika_metadata (on): what every document says about itself (its title, authors, language, dates, pages) is + read by Tika too, as Dublin Core terms. Its title and date become the resource's while it's still named and dated + after its file, and its authors and language its metadata's (metadata.defaults). + +The document goes to the address the admin set and nowhere else: only http(s), with no user name or password in the +address, and no redirect is followed. It is sent as document., never by its own name. What Tika answers is +bounded (MAX_ANSWER) before Lens reads it, and only the Dublin Core terms Lens keeps (TERMS) are taken from it. +""" + +from __future__ import annotations + +import datetime as dt +import json +import pathlib +import re +import urllib.error +import urllib.parse +import urllib.request + +MAX_ANSWER = 64 << 20 # the most of Tika's answer that is read +MAX_TEXT = 20_000_000 # the most text kept of one document (and what's in it) +MAX_PARTS = 500 # what's in a document (attachments, embedded files) read with it, at most +CONTENT = ("tk:content", "X-TIKA:content") # where Tika puts a part's text: 4.x, then 2.x and 3.x +FAILED = ("tk:exception:container", "x-tika:exception:runtime", "x-tika:exception:container") # why it couldn't +# Dublin Core terms Lens keeps, and the Tika keys they are read from (the first one there) +TERMS = { + "title": ("dc:title", "pdf:docinfo:title"), + "creators": ("dc:creator", "meta:author", "Author"), + "subject": ("dc:subject", "cp:subject"), + "description": ("dc:description",), + "publisher": ("dc:publisher",), + "language": ("dc:language",), + "created": ("dcterms:created", "meta:creation-date"), + "modified": ("dcterms:modified",), + "pages": ("xmpTPg:NPages", "meta:page-count"), + "words": ("meta:word-count",), + "format": ("Content-Type",), +} +LANG = re.compile(r"^[a-zA-Z]{2,3}(-[A-Za-z0-9]{2,8})*$") + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """A Tika server that answers with a redirect is refused: the document goes to the address set, never on.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise urllib.error.HTTPError(req.full_url, code, "it redirected elsewhere, and Lens doesn't follow", headers, fp) + + +_OPENER = urllib.request.build_opener(_NoRedirect) + + +def _opts(cfg): + return cfg.get("documents") or {} + + +def url(cfg): + """The Tika server's address, or None when Tika is off.""" + return (_opts(cfg).get("tika_url") or "").strip().rstrip("/") or None + + +def reads_text(cfg): + """Whether Tika reads documents Lens can't make into PDFs here.""" + return bool(url(cfg)) and _opts(cfg).get("tika_text", True) is not False + + +def describes(cfg): + """Whether Tika reads what every document says about itself.""" + return bool(url(cfg)) and _opts(cfg).get("tika_metadata", True) is not False + + +def _url(cfg): + """The server's address, refused unless it is http(s) with a host and no user name or password.""" + got = url(cfg) + u = urllib.parse.urlsplit(got or "") + if u.scheme not in ("http", "https") or not u.hostname or u.username is not None or u.password is not None: + raise ValueError("documents.tika_url is a Tika server's http(s) address, such as http://tika:9998") + return got + + +def _call(cfg, method, path, body=None, headers=None, seconds=60): + req = urllib.request.Request(_url(cfg) + path, data=body, method=method, headers=headers or {}) + return _OPENER.open(req, timeout=seconds) + + +def _bounded(r): + raw = r.read(MAX_ANSWER + 1) + if len(raw) > MAX_ANSWER: + raise ValueError(f"Tika's answer ran past {MAX_ANSWER >> 20} MB") + return raw + + +def _said(e): + try: + return (e.read(2000) or b"").decode("utf-8", "replace").strip().splitlines()[0][:300] or e.reason + except (AttributeError, IndexError, OSError): + return getattr(e, "reason", str(e)) + + +def parts(cfg, path): + """What Tika read in the file at `path`: [{key: value}], the document first, then what's in it (its /rmeta).""" + path = pathlib.Path(path) + seconds = _opts(cfg).get("convert_seconds") or 300 + headers = { + "Accept": "application/json", + "Content-Disposition": f"attachment; filename=document{path.suffix.lower()}", # its kind, not its name + "Content-Type": "application/octet-stream", + } + try: + with _call(cfg, "PUT", "/rmeta/text", path.read_bytes(), headers, seconds) as r: + raw = _bounded(r) + except urllib.error.HTTPError as e: + if e.code in (415, 422): + raise ValueError(f"Tika can't read it ({_said(e)})") from None + raise ValueError(f"the Tika server said {e.code}: {_said(e)}") from None + except (urllib.error.URLError, OSError) as e: + raise ValueError(f"the Tika server can't be reached ({getattr(e, 'reason', e)})") from None + try: + got = json.loads(raw) + except ValueError: + raise ValueError("the Tika server didn't answer like Tika") from None + if not isinstance(got, list) or not got or not all(isinstance(p, dict) for p in got): + raise ValueError("the Tika server didn't answer like Tika") + return got[:MAX_PARTS] + + +def _first(part, keys): + for k in keys: + v = part.get(k) + if isinstance(v, list): + v = [str(x).strip() for x in v if str(x).strip()] + if v: + return v + elif v is not None and str(v).strip(): + return str(v).strip() + return None + + +def _when(v): + try: + d = dt.datetime.fromisoformat(str(v).replace("Z", "+00:00")) + except ValueError: + return None + return (d if d.tzinfo else d.replace(tzinfo=dt.timezone.utc)).astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _count(v): + try: + n = int(str(v).strip()) + except ValueError: + return None + return n if n >= 0 else None + + +def described(part): + """What a document says about itself, as the Dublin Core terms Lens keeps: {title, creators, subject, description, + publisher, language, created, modified, pages, words, format}, those it says.""" + out = {} + for term, keys in TERMS.items(): + v = _first(part, keys) + if v is None: + continue + if term == "creators": + names = v if isinstance(v, list) else re.split(r"\s*;\s*", v) + v = list(dict.fromkeys(n[:200] for n in names if n))[:20] + elif isinstance(v, list): + v = v[0] + if term in ("created", "modified"): + v = _when(v) + elif term in ("pages", "words"): + v = _count(v) + elif term == "language": + v = v if LANG.match(v) else None + elif term == "format": + v = v.split(";")[0].strip().lower()[:100] + elif isinstance(v, str): + v = v[: 2000 if term == "description" else 500] + if v not in (None, "", []): + out[term] = v + return out + + +def text_of(found): + """All the text Tika read: the document's, then each thing in it, at most MAX_TEXT characters.""" + out, size = [], 0 + for part in found: + t = _first(part, CONTENT) + t = "\n".join(t) if isinstance(t, list) else t + if not t: + continue + t = t.strip()[: MAX_TEXT - size] + out.append(t) + size += len(t) + if size >= MAX_TEXT: + break + return "\n\n".join(out) + + +def read(cfg, path): + """The text Tika reads in the file at `path`, and what it says about itself: {text, described}. ValueError when + the server can't be reached or can't read it.""" + found = parts(cfg, path) + text = text_of(found) + broke = next((v for k, v in found[0].items() if k.lower().startswith(FAILED)), None) + if broke and not text: + said = str(broke[0] if isinstance(broke, list) else broke).strip().splitlines()[0] + raise ValueError(f"Tika couldn't read it ({said.split(': ', 1)[-1][:300]})") + return {"text": text, "described": described(found[0])} + + +def version(cfg): + """The Tika server's version (`Apache Tika 4.0.0`); ValueError saying what's wrong when it doesn't answer as one.""" + if not url(cfg): + raise ValueError("set the Tika server's address first (documents.tika_url)") + try: + with _call(cfg, "GET", "/version", headers={"Accept": "text/plain"}, seconds=10) as r: + said = r.read(200).decode("utf-8", "replace").strip() + except urllib.error.HTTPError as e: + raise ValueError(f"it said {e.code}: {_said(e)}") from None + except (urllib.error.URLError, OSError) as e: + raise ValueError(f"it can't be reached ({getattr(e, 'reason', e)})") from None + if not said.startswith("Apache Tika"): + raise ValueError("it answered, but not like a Tika server") + return said diff --git a/fastapi_backend/app/schemas/admin.py b/fastapi_backend/app/schemas/admin.py index 6a843ad4b..581cfbdd2 100644 --- a/fastapi_backend/app/schemas/admin.py +++ b/fastapi_backend/app/schemas/admin.py @@ -112,6 +112,13 @@ class ConversionNodeTestResult(ResponseModel): ms: int | None = None +class TikaTestResult(ResponseModel): + ok: bool + error: str | None = None + version: str | None = Field(None, description="what the Tika server says it is: Apache Tika 4.0.0") + ms: int | None = None + + class EncryptionProgress(ResponseModel): """Converting the files already kept after encryption.files changed: `to` "encrypted" or "plain" (vaults stay encrypted), how many changed and how many were skipped (a locked vault, a damaged file), and why a run stopped diff --git a/fastapi_backend/app/schemas/recordings.py b/fastapi_backend/app/schemas/recordings.py index 12daee883..91d220439 100644 --- a/fastapi_backend/app/schemas/recordings.py +++ b/fastapi_backend/app/schemas/recordings.py @@ -130,6 +130,22 @@ class Rendition(ResponseModel): by: Literal["libreoffice", "chromium", "anytopdf"] +class Described(ResponseModel): + """What a document says about itself, read by a Tika server (documents.tika_metadata), as Dublin Core terms.""" + + title: str | None = Field(None, description="dc:title") + creators: list[str] = Field(default_factory=list, description="dc:creator: its authors") + subject: str | None = Field(None, description="dc:subject") + description: str | None = Field(None, description="dc:description") + publisher: str | None = Field(None, description="dc:publisher") + language: str | None = Field(None, description="dc:language, a code such as en") + created: str | None = Field(None, description="dcterms:created (ISO 8601)") + modified: str | None = Field(None, description="dcterms:modified (ISO 8601)") + pages: int | None = Field(None, description="how many pages it says it has") + words: int | None = Field(None, description="how many words it says it has") + format: str | None = Field(None, description="dc:format: its media type, as Tika found it") + + class WebPage(ResponseModel): """A web page captured as a document: where it was, and when.""" @@ -205,6 +221,7 @@ class Recording(ResponseModel): tags: list[str] = Field(default_factory=list) email: EmailInfo | None = Field(None, description="an email: its subject, sender, recipients and date") rendition: Rendition | None = Field(None, description="a document that isn't a PDF: how the PDF it's read from was made") + described: Described | None = Field(None, description="a document: what it says about itself (title, authors, dates), read by Tika") attached_to: AttachedTo | None = Field(None, description="an email's attachment made a resource of its own: that email") web: WebPage | None = Field(None, description="a web page captured as a document") enrichment: Enrichment | None = Field(None, description="the places and dates anytopdf found (analysis.anytopdf)") diff --git a/fastapi_backend/app/schemas/uploads.py b/fastapi_backend/app/schemas/uploads.py index 99cb003f7..050394a93 100644 --- a/fastapi_backend/app/schemas/uploads.py +++ b/fastapi_backend/app/schemas/uploads.py @@ -14,6 +14,7 @@ class Converters(ResponseModel): pages: bool = Field(description="text, Markdown, saved web pages and .eml emails (Chromium or LibreOffice)") msg: bool = Field(description="Outlook .msg emails (those, and the extract-msg package)") web: bool = Field(False, description="web pages captured from their address (Chromium; POST /import/web)") + tika: bool = Field(False, description="a Tika server reads, as text, the documents nothing else here can (documents.tika_url)") class UploadLimits(ResponseModel): diff --git a/fastapi_backend/tests/api/test_anytopdf.py b/fastapi_backend/tests/api/test_anytopdf.py index 0a17e5096..0f4b0bfb1 100644 --- a/fastapi_backend/tests/api/test_anytopdf.py +++ b/fastapi_backend/tests/api/test_anytopdf.py @@ -62,6 +62,7 @@ def test_where_lens_cannot_convert_anytopdf_does(lean, here, client, env, db, cf "pages": True, "msg": False, "web": False, + "tika": False, } txt = _upload(client, he, b"Tide notes\n\nHigh water at noon.\n", "notes.txt") mail = _upload(client, he, _email("Harbour report", 9), "mail.eml") @@ -88,7 +89,7 @@ def test_where_lens_cannot_convert_anytopdf_does(lean, here, client, env, db, cf # Office files still need LibreOffice here r = _start(client, he, b"x", "letter.docx") - assert r.status_code == 400 and "or an anytopdf conversion node" in r.json()["detail"] + assert r.status_code == 400 and "an anytopdf conversion node or a Tika server" in r.json()["detail"] def test_with_lens_chosen_anytopdf_is_never_used(lean, here, client, env, db, cfg): diff --git a/fastapi_backend/tests/api/test_conversion.py b/fastapi_backend/tests/api/test_conversion.py index 9b983638c..8b0afb748 100644 --- a/fastapi_backend/tests/api/test_conversion.py +++ b/fastapi_backend/tests/api/test_conversion.py @@ -214,15 +214,16 @@ def test_documents_the_server_cannot_convert(client, env, db, cfg, monkeypatch): "pages": False, "msg": False, "web": False, + "tika": False, } r = _start(client, he, b"x", "letter.docx") assert (r.status_code, r.json()["detail"]) == ( 400, "this Word document can't be read here: converting Word documents needs LibreOffice on the server " - "(the lens:full image) or an anytopdf conversion node; import it as a transcript instead", + "(the lens:full image), an anytopdf conversion node or a Tika server; import it as a transcript instead", ) r = _start(client, he, b"x", "deck.pptx") - assert r.status_code == 400 and r.json()["detail"].endswith("(the lens:full image) or an anytopdf conversion node") + assert r.status_code == 400 and r.json()["detail"].endswith("(the lens:full image), an anytopdf conversion node or a Tika server") assert _start(client, he, b"%PDF", "a.pdf").status_code == 201 # a source's files: read as transcripts where they can be (an email's text, without its attachments), else left assert [sources.file_kind(cfg, n) for n in ("a.docx", "a.txt", "a.pptx", "a.eml", "a.pdf")] == [ diff --git a/fastapi_backend/tests/api/test_tika.py b/fastapi_backend/tests/api/test_tika.py new file mode 100644 index 000000000..f2b399292 --- /dev/null +++ b/fastapi_backend/tests/api/test_tika.py @@ -0,0 +1,207 @@ +"""Apache Tika as a document reader (tika.py): with a Tika server set, documents nothing here can make into a PDF are +read as text, and every document's own description is read as Dublin Core terms; without one, nothing changes.""" + +from __future__ import annotations + +import os + +import pytest + +from app.domain import anytopdf, convert, settings, store, tika +from tests import fake_tika +from tests.api.test_conversion import _text +from tests.api.test_documents import _start, _upload +from tests.helpers import drain, login, make_user, text_pdf, write_docx + +R = store.R +REAL = os.environ.get("LENS_TEST_TIKA") # a real Tika server's address, such as http://127.0.0.1:9998 + + +@pytest.fixture +def env(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + return {"ha": login(client, "root@x.io", "root password 1"), "he": login(client, "ed@x.io", "editor password 1")} + + +@pytest.fixture +def lean(monkeypatch): + """A server with neither LibreOffice, Chromium nor anytopdf (the lean image on a Raspberry Pi).""" + monkeypatch.setattr(convert, "soffice", lambda cfg: None) + monkeypatch.setattr(convert, "chromium", lambda cfg: None) + monkeypatch.setattr(anytopdf, "available", lambda cfg: None) + + +@pytest.fixture +def server(): + srv, url = fake_tika.server() + yield url + srv.shutdown() + srv.server_close() + + +def _set(client, ha, **changes): + r = client.put("/api/v1/settings/documents", headers=ha, json=changes) + assert r.status_code == 200, r.text + + +def _rec(db, rid): + return db.one("SELECT status, engine, rendition, described, title, recorded_at, media, error FROM $r", r=R("recording", rid)) + + +def test_off_by_default_nothing_changes(lean, client, env): + he = env["he"] + assert client.get("/api/v1/uploads/limits", headers=env["he"]).json()["convert"] == { + "office": False, + "pages": False, + "msg": False, + "web": False, + "tika": False, + } + r = _start(client, he, b"x", "letter.docx") + assert r.status_code == 400 and "or a Tika server" in r.json()["detail"] + got = client.get("/api/v1/settings", headers=env["ha"]).json()["documents"]["values"] + assert (got["tika_url"], got["tika_text"], got["tika_metadata"]) == (None, True, True) + + +def test_what_nothing_here_can_convert_tika_reads_as_text(lean, server, client, env, db, cfg, folder): + he, ha = env["he"], env["ha"] + _set(client, ha, tika_url=server + "/") + assert client.get("/api/v1/settings", headers=ha).json()["documents"]["values"]["tika_url"] == server + assert client.get("/api/v1/uploads/limits", headers=he).json()["convert"] == { + "office": True, + "pages": True, + "msg": True, + "web": False, + "tika": True, + } + write_docx(folder / "letter.docx", ["The harbour report", "Ships arrived at dawn."]) + doc = _upload(client, he, (folder / "letter.docx").read_bytes(), "letter.docx") + drain(db, settings.effective(db, cfg)) + + rec = _rec(db, doc["recording"]) + assert (rec["status"], rec["engine"], rec.get("rendition")) == ("analyzed", "text:tika", None), rec.get("error") + assert rec["media"]["pages"] == 0 + text = _text(db, doc["recording"]) + assert "Ships arrived at dawn." in text and "A note tucked inside." in text # what's in it too + log = client.get(f"/api/v1/jobs/{doc['job']}/log", headers=he).json()["lines"] + assert any("Tika reads it instead" in x for x in log) and any("read by Tika as text" in x for x in log) + + # what it says about itself: kept as Dublin Core terms, its title and date the resource's, its authors its metadata's + assert rec["described"] == { + "title": "Tide tables", + "creators": ["Ada Harbour", "Ben Quay"], + "language": "en", + "created": "2024-03-01T12:00:00Z", + "pages": 3, + "format": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + } + assert rec["title"] == "Tide tables" and str(rec["recorded_at"]).startswith("2024-03-01") + got = client.get(f"/api/v1/resources/{doc['recording']}", headers=he).json() + assert got["described"]["creators"] == ["Ada Harbour", "Ben Quay"] + meta = client.get(f"/api/v1/recordings/{doc['recording']}/metadata", headers=he).json()["meta"] + assert [c["name"] for c in meta["creators"]] == ["Ada Harbour", "Ben Quay"] and meta["language"] == ["en"] + + # sent as its kind, never by its own name, with nothing but the document + put = [s for s in fake_tika.Tika.seen if s["method"] == "PUT"] + assert len(put) == 1 and put[0]["headers"]["Content-Disposition"] == "attachment; filename=document.docx" + + +def test_a_document_tika_cannot_read_fails_saying_so(lean, server, client, env, db, cfg): + he, ha = env["he"], env["ha"] + _set(client, ha, tika_url=server) + bad = _upload(client, he, b"unreadable\n", "bad.txt") + drain(db, settings.effective(db, cfg)) + job = client.get(f"/api/v1/jobs/{bad['job']}", headers=he).json() + assert job["status"] == "failed" and "Tika couldn't read it (Unexpected RuntimeException)" in job["error"] + + +def test_tika_describes_what_lens_reads_itself(server, client, env, db, cfg): + he, ha = env["he"], env["ha"] + _set(client, ha, tika_url=server) + up = _upload(client, he, text_pdf([["Harbour report"]]), "harbour.pdf", title="Harbour report") + plain = _upload(client, he, text_pdf([["nothing about itself"]]), "plain.pdf") + drain(db, settings.effective(db, cfg)) + rec = _rec(db, up["recording"]) + assert (rec["status"], rec["engine"]) == ("analyzed", "pdf"), rec.get("error") + assert rec["described"]["title"] == "Tide tables" and rec["title"] == "Harbour report" # named by someone: kept + assert "Harbour report" in _text(db, up["recording"]) # its text is still Lens's own reading + rec = _rec(db, plain["recording"]) + assert rec["described"] == {"format": "application/octet-stream"} + assert rec["title"] == "plain" + + # off: nothing is sent, and nothing kept + _set(client, ha, tika_metadata=False) + sent = len(fake_tika.Tika.seen) + again = _upload(client, he, text_pdf([["Tides"]]), "again.pdf") + drain(db, settings.effective(db, cfg)) + assert _rec(db, again["recording"]).get("described") is None and len(fake_tika.Tika.seen) == sent + + +def test_a_tika_server_that_is_gone_does_not_stop_a_document(server, client, env, db, cfg): + he, ha = env["he"], env["ha"] + _set(client, ha, tika_url="http://127.0.0.1:9") + up = _upload(client, he, text_pdf([["Harbour report"]]), "harbour.pdf") + drain(db, settings.effective(db, cfg)) + rec = _rec(db, up["recording"]) + assert rec["status"] == "analyzed" and rec.get("described") is None, rec.get("error") + log = client.get(f"/api/v1/jobs/{up['job']}/log", headers=he).json()["lines"] + assert any("Tika couldn't say what it is: the Tika server can't be reached" in x for x in log) + + +def test_tika_settings_and_check(server, client, env, cfg): + ha = env["ha"] + put = lambda changes: client.put("/api/v1/settings/documents", headers=ha, json=changes) # noqa: E731 + assert put({"tika_url": "ftp://tika"}).status_code == 400 + assert put({"tika_url": "http://user:secret@tika:9998"}).status_code == 400 + assert put({"tika_text": "yes"}).status_code == 400 + test = lambda: client.post("/api/v1/settings/documents/tika/test", headers=ha).json() # noqa: E731 + assert test() == {"ok": False, "error": "set the Tika server's address first (documents.tika_url)", "version": None, "ms": None} + assert put({"tika_url": server}).status_code == 200 + got = test() + assert got["ok"] and got["version"] == "Apache Tika 4.0.0" + assert client.post("/api/v1/settings/documents/tika/test", headers=env["he"]).status_code == 403 + assert put({"tika_url": "http://127.0.0.1:9"}).status_code == 200 + got = test() + assert not got["ok"] and "it can't be reached" in got["error"] + assert put({"tika_url": ""}).status_code == 200 + + +def test_a_redirect_is_never_followed(server, cfg, tmp_path): + cfg["documents"]["tika_url"] = server + fake_tika.Tika.redirect = True + f = tmp_path / "a.txt" + f.write_text("Tides") + with pytest.raises(ValueError, match="the Tika server said 302"): + tika.read(cfg, f) + with pytest.raises(ValueError, match="it said 302"): + tika.version(cfg) + assert all("elsewhere" not in s["path"] for s in fake_tika.Tika.seen) + + +def test_what_tika_says_is_bounded(cfg, tmp_path, monkeypatch): + part = { + "dc:title": "x" * 900, + "dc:creator": "Ada Harbour; Ben Quay; ", + "dcterms:created": "not a date", + "dc:language": "English, mostly", + "meta:page-count": "-2", + "tk:content": "a" * 50, + } + d = tika.described(part) + assert d == {"title": "x" * 500, "creators": ["Ada Harbour", "Ben Quay"]} + monkeypatch.setattr(tika, "MAX_TEXT", 70) + assert len(tika.text_of([part, {"X-TIKA:content": "b" * 50}])) == 70 + 2 # Tika 2.x and 3.x keys read too + + +@pytest.mark.skipif(not REAL, reason="LENS_TEST_TIKA names a real Tika server") +def test_a_real_tika_server(lean, client, env, db, cfg, folder): + he, ha = env["he"], env["ha"] + _set(client, ha, tika_url=REAL) + assert client.post("/api/v1/settings/documents/tika/test", headers=ha).json()["ok"] + write_docx(folder / "letter.docx", ["The harbour report", "Ships arrived at dawn."]) + doc = _upload(client, he, (folder / "letter.docx").read_bytes(), "letter.docx") + drain(db, settings.effective(db, cfg)) + rec = _rec(db, doc["recording"]) + assert (rec["status"], rec["engine"]) == ("analyzed", "text:tika"), rec.get("error") + assert "Ships arrived at dawn." in _text(db, doc["recording"]) diff --git a/fastapi_backend/tests/domain/test_convert.py b/fastapi_backend/tests/domain/test_convert.py index 2d13f1408..9d8bc49fe 100644 --- a/fastapi_backend/tests/domain/test_convert.py +++ b/fastapi_backend/tests/domain/test_convert.py @@ -117,16 +117,16 @@ def ask(method, target): def test_what_the_server_can_convert(cfg, monkeypatch): monkeypatch.setattr(convert, "soffice", lambda cfg: None) monkeypatch.setattr(convert, "chromium", lambda cfg: None) - assert convert.capabilities(cfg) == {"office": False, "pages": False, "msg": False, "web": False} + assert convert.capabilities(cfg) == {"office": False, "pages": False, "msg": False, "web": False, "tika": False} assert convert.unavailable(cfg, "a.pdf") is None and convert.unavailable(cfg, "a.mp3") is None assert ( convert.unavailable(cfg, "a.docx") - == "converting Word documents needs LibreOffice on the server (the lens:full image) or an anytopdf conversion node" + == "converting Word documents needs LibreOffice on the server (the lens:full image), an anytopdf conversion node or a Tika server" ) assert "Chromium or LibreOffice" in convert.unavailable(cfg, "a.eml") monkeypatch.setattr(convert, "chromium", lambda cfg: "/usr/bin/chromium") monkeypatch.setattr(convert, "_has_msg", lambda: False) - assert convert.capabilities(cfg) == {"office": False, "pages": True, "msg": False, "web": True} + assert convert.capabilities(cfg) == {"office": False, "pages": True, "msg": False, "web": True, "tika": False} assert convert.unavailable(cfg, "a.md") is None and "extract-msg" in convert.unavailable(cfg, "a.msg") assert convert.bootstrap(cfg)["chromium"] == "/usr/bin/chromium" diff --git a/fastapi_backend/tests/fake_tika.py b/fastapi_backend/tests/fake_tika.py new file mode 100644 index 000000000..0fe7e2279 --- /dev/null +++ b/fastapi_backend/tests/fake_tika.py @@ -0,0 +1,81 @@ +"""An Apache Tika server for tests (tika.py), on loopback: `GET /version`, and `PUT /rmeta/text` answering like Tika +4.x. A Word file's text is its paragraphs; anything else is read as UTF-8. A file saying `unreadable` is one Tika can't +parse; one saying `nothing about itself` has no title, authors or dates. Every request is kept in Tika.seen.""" + +from __future__ import annotations + +import http.server +import io +import json +import re +import threading +import zipfile + + +def text_of(raw): + if raw[:2] == b"PK": # a Word file: the text of its paragraphs + xml = zipfile.ZipFile(io.BytesIO(raw)).read("word/document.xml").decode() + return "\n".join(re.sub(r"<[^>]+>", "", p) for p in re.findall(r"].*?", xml)) + return raw.decode("utf-8", "replace") + + +class Tika(http.server.BaseHTTPRequestHandler): + seen: list = [] + redirect = False + + def log_message(self, *args): + pass + + def _send(self, code, body, kind="application/json", more=None): + raw = body if isinstance(body, bytes) else json.dumps(body).encode() + self.send_response(code) + self.send_header("Content-Type", kind) + self.send_header("Content-Length", str(len(raw))) + for k, v in (more or {}).items(): + self.send_header(k, v) + self.end_headers() + self.wfile.write(raw) + + def do_GET(self): # noqa: N802 + Tika.seen.append({"method": "GET", "path": self.path}) + if Tika.redirect: + return self._send(302, b"", "text/plain", {"Location": "http://example.org/elsewhere"}) + if self.path == "/version": + return self._send(200, b"Apache Tika 4.0.0", "text/plain") + return self._send(404, b"not found", "text/plain") + + def do_PUT(self): # noqa: N802 + raw = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + Tika.seen.append({"method": "PUT", "path": self.path, "headers": dict(self.headers), "size": len(raw)}) + if Tika.redirect: + return self._send(302, b"", "text/plain", {"Location": "http://example.org/elsewhere"}) + if self.path != "/rmeta/text": + return self._send(404, b"not found", "text/plain") + name = (self.headers.get("Content-Disposition") or "").rpartition("filename=")[2] + part = {"tk:resource-name": name, "Content-Type": "application/octet-stream", "tk:embedded-depth": "0"} + text = text_of(raw) + if "unreadable" in text: + part["tk:exception:container-exception"] = "org.apache.tika.exception.TikaException: Unexpected RuntimeException\n\tat x" + return self._send(200, [part]) + if "nothing about itself" not in text: + part.update( + { + "dc:title": "Tide tables", + "dc:creator": ["Ada Harbour", "Ben Quay"], + "dcterms:created": "2024-03-01T12:00:00Z", + "dc:language": "en", + "xmpTPg:NPages": "3", + "Content-Type": "application/vnd.openxmlformats-officedocument.wordprocessingml.document; charset=x", + } + ) + part["tk:content"] = "\n\n\n" + text + "\n" + inside = {"tk:embedded-depth": "1", "tk:content": "A note tucked inside.", "dc:title": "not the document's"} + return self._send(200, [part, inside]) + + +def server(): + """A Tika server on loopback: (server, its address).""" + Tika.seen, Tika.redirect = [], False + srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Tika) + threading.Thread(target=srv.serve_forever, daemon=True).start() + return srv, f"http://127.0.0.1:{srv.server_address[1]}" diff --git a/nextjs-frontend/__tests__/document-model.test.ts b/nextjs-frontend/__tests__/document-model.test.ts index 897f8c9b4..48e02d334 100644 --- a/nextjs-frontend/__tests__/document-model.test.ts +++ b/nextjs-frontend/__tests__/document-model.test.ts @@ -26,6 +26,7 @@ import { pagesSummary, pageStart, parsePage, + describedRows, renditionNote, textNote, zoomStep, @@ -175,11 +176,21 @@ describe("a document's pages", () => { ["Sent", "on 2026-09-29"], ]); expect(emailRows(null, String)).toEqual([]); - expect([renditionNote({ by: "libreoffice" }), renditionNote({ by: "chromium" }), renditionNote(null)]).toEqual([ - "made into a PDF by LibreOffice", - "made into a PDF by Chromium", - null, + expect([ + renditionNote({ by: "libreoffice" }), + renditionNote({ by: "chromium" }), + renditionNote({ by: "anytopdf" }), + renditionNote(null), + ]).toEqual(["made into a PDF by LibreOffice", "made into a PDF by Chromium", "made into a PDF by anytopdf", null]); + const told = { title: "Tide tables", creators: ["Ada", "Ben"], created: "2024-03-01T12:00:00Z", pages: 3 }; + expect(describedRows(told, (iso) => `on ${iso.slice(0, 10)}`)).toEqual([ + ["Title", "Tide tables"], + ["Authors", "Ada; Ben"], + ["Created", "on 2024-03-01"], + ["Pages", "3"], ]); + expect(describedRows({ creators: ["Ada"] }, String)).toEqual([["Author", "Ada"]]); + expect(describedRows(null, String)).toEqual([]); }); it("puts a document's notes, rows and search matches on pages", () => { @@ -247,13 +258,13 @@ describe("importing documents and images", () => { ]).toEqual(["transcript", "transcript", "document", "document"]); expect(conversionProblem("a.pdf", none)).toBeNull(); expect(conversionProblem("a.eml", none)).toBe( - "Reading it needs Chromium or LibreOffice on the server (the lens:full image).", + "Reading it needs Chromium or LibreOffice on the server (the lens:full image), or a Tika server.", ); expect(conversionProblem("a.msg")).toBe("Reading Outlook emails needs the extract-msg package on the server."); expect(localProblem({ name: "deck.pptx", size: 10 }, none)).toMatchObject({ code: "unsupported", title: "This server can’t read powerpoint presentations", - body: "Reading it needs LibreOffice on the server (the lens:full image).", + body: "Reading it needs LibreOffice on the server (the lens:full image) or a Tika server.", }); expect(localProblem({ name: "notes.docx", size: 10 }, none, "document")?.body).toMatch( /Import it as a transcript instead\.$/, diff --git a/nextjs-frontend/__tests__/import-documents.test.tsx b/nextjs-frontend/__tests__/import-documents.test.tsx index dd24b3026..a89114c91 100644 --- a/nextjs-frontend/__tests__/import-documents.test.tsx +++ b/nextjs-frontend/__tests__/import-documents.test.tsx @@ -43,7 +43,7 @@ describe("importing a PDF", () => { rerender( {}} limits={noOffice} />); expect(screen.queryByRole("radio")).toBeNull(); expect(container).toHaveTextContent( - "Imported as a transcript: this server can’t keep word documents as documents. Reading it needs LibreOffice on the server (the lens:full image).", + "Imported as a transcript: this server can’t keep word documents as documents. Reading it needs LibreOffice on the server (the lens:full image) or a Tika server.", ); rerender( {}} />); expect(container).toBeEmptyDOMElement(); // a presentation is a document only diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 32ab2f424..e83c04347 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -673,6 +673,7 @@ export type { DeleteWatchErrors, DeleteWatchResponse, DeleteWatchResponses, + Described, Description, DetectLlmData, DetectLlmResponse, @@ -2581,7 +2582,11 @@ export type { TestTelemetryData, TestTelemetryResponse, TestTelemetryResponses, + TestTikaData, + TestTikaResponse, + TestTikaResponses, TicketRequest, + TikaTestResult, TokenData, TokenError, TokenErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 6e5801f34..95212a152 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -1250,6 +1250,8 @@ import type { TestSpeechResponses, TestTelemetryData, TestTelemetryResponses, + TestTikaData, + TestTikaResponses, TokenData, TokenErrors, TokenLimitsData, @@ -2507,6 +2509,20 @@ export class Admin { }); } + /** + * Test Tika + * + * Check that a Tika server answers at documents.tika_url, and say which version it is. + */ + public static testTika( + options?: Options, + ): RequestResult { + return (options?.client ?? client).post({ + url: "/api/v1/settings/documents/tika/test", + ...options, + }); + } + /** * Decision Status * diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index fd83bbb05..0a4a6af46 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -2830,6 +2830,12 @@ export type Converters = { * web pages captured from their address (Chromium; POST /import/web) */ web?: boolean; + /** + * Tika + * + * a Tika server reads, as text, the documents nothing else here can (documents.tika_url) + */ + tika?: boolean; [key: string]: unknown; }; @@ -3095,6 +3101,81 @@ export type DecisionTestResult = { [key: string]: unknown; }; +/** + * Described + * + * What a document says about itself, read by a Tika server (documents.tika_metadata), as Dublin Core terms. + */ +export type Described = { + /** + * Title + * + * dc:title + */ + title?: string | null; + /** + * Creators + * + * dc:creator: its authors + */ + creators?: Array; + /** + * Subject + * + * dc:subject + */ + subject?: string | null; + /** + * Description + * + * dc:description + */ + description?: string | null; + /** + * Publisher + * + * dc:publisher + */ + publisher?: string | null; + /** + * Language + * + * dc:language, a code such as en + */ + language?: string | null; + /** + * Created + * + * dcterms:created (ISO 8601) + */ + created?: string | null; + /** + * Modified + * + * dcterms:modified (ISO 8601) + */ + modified?: string | null; + /** + * Pages + * + * how many pages it says it has + */ + pages?: number | null; + /** + * Words + * + * how many words it says it has + */ + words?: number | null; + /** + * Format + * + * dc:format: its media type, as Tika found it + */ + format?: string | null; + [key: string]: unknown; +}; + /** * Description * @@ -11499,6 +11580,10 @@ export type Recording = { * a document that isn't a PDF: how the PDF it's read from was made */ rendition?: Rendition | null; + /** + * a document: what it says about itself (title, authors, dates), read by Tika + */ + described?: Described | null; /** * an email's attachment made a resource of its own: that email */ @@ -15491,6 +15576,31 @@ export type TicketRequest = { ticket: string; }; +/** + * TikaTestResult + */ +export type TikaTestResult = { + /** + * Ok + */ + ok: boolean; + /** + * Error + */ + error?: string | null; + /** + * Version + * + * what the Tika server says it is: Apache Tika 4.0.0 + */ + version?: string | null; + /** + * Ms + */ + ms?: number | null; + [key: string]: unknown; +}; + /** * TokenLimits * @@ -19037,6 +19147,22 @@ export type TestConversionNodeResponses = { export type TestConversionNodeResponse = TestConversionNodeResponses[keyof TestConversionNodeResponses]; +export type TestTikaData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/settings/documents/tika/test"; +}; + +export type TestTikaResponses = { + /** + * Successful Response + */ + 200: TikaTestResult; +}; + +export type TestTikaResponse = TestTikaResponses[keyof TestTikaResponses]; + export type DecisionStatusData = { body?: never; path?: never; diff --git a/nextjs-frontend/components/import/files.ts b/nextjs-frontend/components/import/files.ts index 9024be8b3..f8c2cb90f 100644 --- a/nextjs-frontend/components/import/files.ts +++ b/nextjs-frontend/components/import/files.ts @@ -146,9 +146,11 @@ export function conversionProblem(name: string, limits: UploadLimits = DEFAULT_L const can = limits.convert; if (!DOCUMENT_EXT.includes(ext) || ext === ".pdf" || !can) return null; if (OFFICE_EXT.includes(ext)) - return can.office ? null : "Reading it needs LibreOffice on the server (the lens:full image)."; + return can.office ? null : "Reading it needs LibreOffice on the server (the lens:full image) or a Tika server."; if (ext === ".msg") return can.msg ? null : "Reading Outlook emails needs the extract-msg package on the server."; - return can.pages ? null : "Reading it needs Chromium or LibreOffice on the server (the lens:full image)."; + return can.pages + ? null + : "Reading it needs Chromium or LibreOffice on the server (the lens:full image), or a Tika server."; } /** What a file becomes unless someone chooses otherwise: a document where this server can read it as one, else the diff --git a/nextjs-frontend/components/recording/details-tab.tsx b/nextjs-frontend/components/recording/details-tab.tsx index b7bff897d..5bce63deb 100644 --- a/nextjs-frontend/components/recording/details-tab.tsx +++ b/nextjs-frontend/components/recording/details-tab.tsx @@ -20,7 +20,14 @@ import type { Enrichment } from "@/app/openapi-client"; import { uploadKindName } from "@/components/import/files"; import { ContentTypePicker } from "@/components/recording/content-type"; import { useRec } from "@/components/recording/context"; -import { emailRows, renditionNote, webRows, type EmailInfo } from "@/components/recording/document/model"; +import { + describedRows, + emailRows, + renditionNote, + webRows, + type DescribedInfo, + type EmailInfo, +} from "@/components/recording/document/model"; import { sourceLabel, transcriptOrigin } from "@/components/recording/labels"; import { Button } from "@/components/ui/button"; import { absolute, bytes, count, plural, tc } from "@/lib/format"; @@ -150,6 +157,13 @@ export function DetailsTab() { {(video || rec.channels) && } {rec.email && } + {rec.described && describedRows(rec.described as DescribedInfo, absolute).length > 0 && ( + + )} {rec.web && ( = { libreoffice: "LibreOffice", chromium: "Chromium", anytopdf: "anytopdf" }; + return `made into a PDF by ${by[r.by] ?? r.by}`; +} + +export type DescribedInfo = { + title?: string | null; + creators?: string[]; + subject?: string | null; + description?: string | null; + publisher?: string | null; + language?: string | null; + created?: string | null; + modified?: string | null; + pages?: number | null; + words?: number | null; + format?: string | null; +}; + +/** What a document says about itself (read by Tika) as rows for the Details tab (only the ones it says). */ +export function describedRows(d: DescribedInfo | null | undefined, when: (iso: string) => string): [string, string][] { + if (!d) return []; + const rows: [string, string | null | undefined][] = [ + ["Title", d.title], + [d.creators && d.creators.length > 1 ? "Authors" : "Author", d.creators?.join("; ")], + ["Subject", d.subject], + ["Description", d.description], + ["Publisher", d.publisher], + ["Language", d.language], + ["Created", d.created ? when(d.created) : null], + ["Modified", d.modified ? when(d.modified) : null], + ["Pages", d.pages != null ? String(d.pages) : null], + ["Words", d.words != null ? String(d.words) : null], + ["Format", d.format], + ]; + return rows.filter((r): r is [string, string] => Boolean(r[1])); } export type WebPageInfo = { url: string; final?: string | null; captured_at?: string | null; how?: string | null }; diff --git a/nextjs-frontend/components/recording/labels.ts b/nextjs-frontend/components/recording/labels.ts index d19439534..9f54f9373 100644 --- a/nextjs-frontend/components/recording/labels.ts +++ b/nextjs-frontend/components/recording/labels.ts @@ -31,6 +31,7 @@ export function transcriptOrigin(engine: string | null | undefined): string | nu const f = engine.slice(7); return `from ${IMPORT_FORMATS[f] ?? f.toUpperCase()}`; } + if (engine === "text:tika") return "read as text by Tika"; return engine; } diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index d7d16bf49..1305ab206 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -415,6 +415,7 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) { + ); @@ -513,6 +514,57 @@ function ConversionNodeCheck({ ctx }: { ctx: BodyCtx }) { ); } +/** The compose files' `tika` service, as Lens's containers reach it. */ +const TIKA_SERVICE = "http://tika:9998"; + +function TikaSettings({ ctx }: { ctx: BodyCtx }) { + const client = useApiClient(); + const url = ctx.state("documents.tika_url"); + const test = useMutation({ mutationFn: () => data(Admin.testTika({ client })) }); + const saved = Boolean(ctx.view.documents?.values?.tika_url); + return ( + <> + + {!url.value && ( +
+ +
+ )} + {(saved || url.value) && ( + <> + + + + )} + {saved && ( +
+ + + {ctx.dirty + ? "Uses the saved settings, not your unsaved changes" + : "Asks the Tika server which version it is"} + +
+ )} + {test.data && + (test.data.ok ? ( + + {test.data.version} ({test.data.ms} ms) + + ) : ( + + {test.data.error} + + ))} + {test.isError && {test.error.message}} + + ); +} + function exportLine(e: { at: number; ok: boolean; error?: string | null } | null | undefined, what: string) { if (!e) return null; const when = new Date(e.at * 1000).toLocaleTimeString(); diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 6c59496a1..256c9959d 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -2112,6 +2112,30 @@ export const FIELDS: FieldSpec[] = [ hint: "Another machine running anytopdf queue serve: it converts instead, Office files too", }, { section: "documents", key: "anytopdf_token", label: "Conversion node token", kind: "secret" }, + { + section: "documents", + key: "tika_url", + label: "Tika server", + kind: "text", + mono: true, + nullable: true, + placeholder: "http://tika:9998", + hint: "Apache Tika, such as the compose files’ tika profile: off while empty. Every document is sent to it", + }, + { + section: "documents", + key: "tika_text", + label: "Read with Tika what nothing else here can", + kind: "switch", + hint: "Office files on a lean server, say: their text, without pages to look at", + }, + { + section: "documents", + key: "tika_metadata", + label: "Read what each document says about itself with Tika", + kind: "switch", + hint: "Its title, authors, language and dates, as Dublin Core", + }, // IIIF & metadata { section: "iiif", diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index cfe128052..2a6bf5778 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -2510,6 +2510,28 @@ } } }, + "/api/v1/settings/documents/tika/test": { + "post": { + "tags": [ + "admin" + ], + "summary": "Test Tika", + "description": "Check that a Tika server answers at documents.tika_url, and say which version it is.", + "operationId": "test_tika", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TikaTestResult" + } + } + } + } + } + } + }, "/api/v1/settings/decisions/status": { "get": { "tags": [ @@ -29245,6 +29267,12 @@ "title": "Web", "description": "web pages captured from their address (Chromium; POST /import/web)", "default": false + }, + "tika": { + "type": "boolean", + "title": "Tika", + "description": "a Tika server reads, as text, the documents nothing else here can (documents.tika_url)", + "default": false } }, "additionalProperties": true, @@ -29757,6 +29785,142 @@ ], "title": "DecisionTestResult" }, + "Described": { + "properties": { + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title", + "description": "dc:title" + }, + "creators": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Creators", + "description": "dc:creator: its authors" + }, + "subject": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Subject", + "description": "dc:subject" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description", + "description": "dc:description" + }, + "publisher": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Publisher", + "description": "dc:publisher" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language", + "description": "dc:language, a code such as en" + }, + "created": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created", + "description": "dcterms:created (ISO 8601)" + }, + "modified": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Modified", + "description": "dcterms:modified (ISO 8601)" + }, + "pages": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Pages", + "description": "how many pages it says it has" + }, + "words": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Words", + "description": "how many words it says it has" + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Format", + "description": "dc:format: its media type, as Tika found it" + } + }, + "additionalProperties": true, + "type": "object", + "title": "Described", + "description": "What a document says about itself, read by a Tika server (documents.tika_metadata), as Dublin Core terms." + }, "Description": { "properties": { "idx": { @@ -44205,6 +44369,17 @@ ], "description": "a document that isn't a PDF: how the PDF it's read from was made" }, + "described": { + "anyOf": [ + { + "$ref": "#/components/schemas/Described" + }, + { + "type": "null" + } + ], + "description": "a document: what it says about itself (title, authors, dates), read by Tika" + }, "attached_to": { "anyOf": [ { @@ -51142,6 +51317,54 @@ ], "title": "TicketRequest" }, + "TikaTestResult": { + "properties": { + "ok": { + "type": "boolean", + "title": "Ok" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version", + "description": "what the Tika server says it is: Apache Tika 4.0.0" + }, + "ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Ms" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "ok" + ], + "title": "TikaTestResult" + }, "TokenLimits": { "properties": { "default_days": { diff --git a/threat-model.json b/threat-model.json index bbc99cbd5..3b3a0a6bc 100644 --- a/threat-model.json +++ b/threat-model.json @@ -958,6 +958,14 @@ "provenance": "documented", "source": "docs/configuration.md \"the token never goes on to another server\"" }, + { + "property": "P-TIKA-SERVER", + "violation_symptom": "info-leak", + "severity_tier": "security", + "conditions": "a Tika server (documents.tika_url, off by default) gets the document only at the address the admin set: http(s) with no user name or password, no redirect is followed, and the document goes as document., never by its own name; its answer is read to 64 MB, only the Dublin Core terms Lens keeps are taken from it (bounded in length), and the text kept is capped at 20 million characters; a Tika server that can't be reached fails only a document nothing else can read; the compose service runs read-only, without privileges, reachable only inside Lens's network", + "provenance": "documented", + "source": "docs/configuration.md \"The document goes only to the address set\"" + }, { "property": "P-TEMPLATE", "violation_symptom": "integrity-bypass", diff --git a/threat-model.md b/threat-model.md index 01f0272e9..e4e375f2d 100644 --- a/threat-model.md +++ b/threat-model.md @@ -169,7 +169,8 @@ flowchart LR - **B2, request to role.** The API turns a bearer token, signed link, share link, IIIF cookie or vouched-for address into a role per namespace. This is the main boundary, and namespace isolation lives here, in application code. - **B3, content to parser.** Uploaded and fetched bytes cross into ffmpeg, LibreOffice, Chromium, poppler, Tesseract, - anytopdf (here, or on an admin-set conversion node, P-CONVERSION-NODE) and the LLM prompt. + anytopdf (here, or on an admin-set conversion node, P-CONVERSION-NODE), an admin-set Apache Tika server + (P-TIKA-SERVER) and the LLM prompt. - **B4, Lens to the outside.** Fetches whose URL a non-admin sets, and iCal feeds, go through `netguard`. Other admin-set URLs do not; the telemetry endpoint is only shape-checked (P-ADMIN-GUARD). - **B5, peers.** Workers, the API and the web app trust the database and each other. Workers hold the DB credential. @@ -493,6 +494,7 @@ Paths in the Voided by column are relative to `fastapi_backend/app/`. Searches c | P-CONVERT-OFFLINE | Chromium and LibreOffice reach nothing while converting. HTML is cleaned of anything that could fetch or run. | info-leak | security-critical | *(documented, docs/configuration.md "Neither may reach anything")* | Voided by: Chromium without sandbox — `domain/convert.py:273`; operator-set `documents.chromium` or `soffice` binaries. | | P-CONVERT-TIMEOUT | Every document conversion and capture ends within `documents.convert_seconds` and leaves no process of its group running; ffmpeg decoding (6 h), antiword, catdoc and pdftotext reads (300 s) and ffmpeg encoding for a speech service (600 s) end at their timeouts. | hang | correctness-only | *(maintainer, 2026-10-06, Q20)* | Voided by: children that ffmpeg or a reader starts itself (only converters run in their own process group) — `domain/ingest.py:794`, `domain/speech.py:183`. | | P-CONVERSION-NODE | A conversion node (`documents.anytopdf_url`) gets the document and its token only at the address the admin set: http(s) with no user name or password, and no redirect is followed. Its job answers are read to 1 MB, its PDF to 2 GB, and a PDF that doesn't start like one is refused before Lens reads it. anytopdf converting here has no network, no plugins and no config file. | info-leak, bad-data-accepted | security-critical | *(documented, docs/configuration.md "the token never goes on to another server")* | Voided by: The node's own address is trusted input (N-ADMIN); a node on plain `http://` off loopback sends the token in clear (§1.13) — `domain/anytopdf.py:237-244`. The node converts what it is sent: what it reads from the document is the node's, not Lens's. | +| P-TIKA-SERVER | A Tika server (`documents.tika_url`, off by default) gets the document only at the address the admin set: http(s) with no user name or password, no redirect is followed, and the document goes as `document.`, never by its own name. Its answer is read to 64 MB, only the Dublin Core terms Lens keeps are taken from it (bounded in length), and the text kept is capped at 20 million characters. A Tika server that can't be reached fails only a document nothing else can read. The compose service runs read-only, without privileges, reachable only inside Lens's network. | info-leak, bad-data-accepted | security-critical | *(documented, docs/configuration.md "The document goes only to the address set")* | Voided by: The server's address is trusted input (N-ADMIN); a Tika server off this machine on plain `http://` sees every document in clear, and Tika itself has no sign-in, so whoever reaches it can use it — `domain/tika.py`. What Tika reads from the document is Tika's, not Lens's. | | P-TEMPLATE | Templates render in a sandboxed Jinja environment. They cannot reach Python internals, output is capped, and reports escape HTML. | integrity-bypass | security-critical | *(documented, docs/processing.md "sandboxed Jinja environment")* | Voided by: Shipped report templates use an unsandboxed environment over server-built values — `domain/render.py:22`. `grep -rn 'Environment(\\|from_string' fastapi_backend/app` — hits in `domain/render.py` and `domain/templates.py`. | | P-ENCRYPTION | With `encryption.files` on, files are encrypted per namespace. A file cut short, edited or reordered fails to open. A vault has no recovery code, password or admin override, and separate workers never get its key. | info-leak, integrity-bypass | security-critical | *(documented, docs/encryption.md "There is no recovery code, password or admin override")* | Voided by: `encryption.files` off — `domain/keyring.py:545`; shipped default off (`domain/store.py:402`) until a new archive's first start or `lens encrypt`. Unlocked vault keys stay in the API for `encryption.vault_minutes`. `secret.key` placement (Q14). | | P-TELEMETRY | Telemetry is off unless turned on, has no built-in destination, and never sends content, names, addresses or error messages. | info-leak | security-critical | *(documented, docs/telemetry.md "Never sent")* | Voided by: none identified (not required for this symptom; no exhaustive search) | diff --git a/threat-model.yaml b/threat-model.yaml index 872a1a204..ffd40d2f9 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -2,7 +2,7 @@ # Authority order: prose > yaml > json. Regenerate whenever the prose changes. schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:c7c945e760490e73dbb947e89beff868a218b89b839b4172f9d928d4853e992a" +prose_version: "threat-model.md@sha256:d0b8917737e49cfc11da9252837c3fc1dab8aa13052987c42bff9efa0bd64adc" model_status: under-review triage_policy: strict confidence: {documented: 201, maintainer: 46, inferred: 91, assumption: 8} @@ -1344,6 +1344,14 @@ properties_claimed: violation_symptoms: [info-leak, bad-data-accepted] x-voided-by: "the node's address is trusted input (N-ADMIN); a node on plain http off loopback sends the token in clear (domain/anytopdf.py:237-244); what the node reads from the document is the node's" provenance: {kind: documented, source: 'docs/configuration.md "the token never goes on to another server"'} + - id: P-TIKA-SERVER + kind: confidentiality + components: [outbound, ingest] + tier: security-critical + conditions: "a Tika server (documents.tika_url, off by default) gets the document only at the address the admin set: http(s) with no user name or password, no redirect is followed, and the document goes as document., never by its own name; its answer is read to 64 MB, only the Dublin Core terms Lens keeps are taken from it (bounded in length), and the text kept is capped at 20 million characters; a Tika server that can't be reached fails only a document nothing else can read; the compose service runs read-only, without privileges, reachable only inside Lens's network" + violation_symptoms: [info-leak, bad-data-accepted] + x-voided-by: "the server's address is trusted input (N-ADMIN); a Tika server off this machine on plain http sees every document in clear, and Tika has no sign-in of its own (domain/tika.py); what Tika reads from the document is Tika's" + provenance: {kind: documented, source: 'docs/configuration.md "The document goes only to the address set"'} - id: P-TEMPLATE kind: integrity components: [ingest, assistant] From 7e33d965009e23a3f47d3cfe5ee3662d145eae41 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:23:10 +0000 Subject: [PATCH 2/4] test(documents): allow OCR on a short PDF page where tesseract is installed Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01X6p3skuCAfa2nTxSmDKvxq --- fastapi_backend/tests/api/test_tika.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastapi_backend/tests/api/test_tika.py b/fastapi_backend/tests/api/test_tika.py index f2b399292..7fc4351ec 100644 --- a/fastapi_backend/tests/api/test_tika.py +++ b/fastapi_backend/tests/api/test_tika.py @@ -123,7 +123,7 @@ def test_tika_describes_what_lens_reads_itself(server, client, env, db, cfg): plain = _upload(client, he, text_pdf([["nothing about itself"]]), "plain.pdf") drain(db, settings.effective(db, cfg)) rec = _rec(db, up["recording"]) - assert (rec["status"], rec["engine"]) == ("analyzed", "pdf"), rec.get("error") + assert rec["status"] == "analyzed" and rec["engine"].startswith("pdf"), rec.get("error") # +ocr where tesseract is assert rec["described"]["title"] == "Tide tables" and rec["title"] == "Harbour report" # named by someone: kept assert "Harbour report" in _text(db, up["recording"]) # its text is still Lens's own reading rec = _rec(db, plain["recording"]) From 944ace7bae3d56d58756dd22b1966392e00c0f63 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 22:24:44 +0000 Subject: [PATCH 3/4] ci(clicks): wait for the servers to exit before removing their data folder The survey passed, but rm -rf failed with 'Directory not empty' while the just-killed API was still writing, failing the job. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01X6p3skuCAfa2nTxSmDKvxq --- ci/clicks/run.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/clicks/run.sh b/ci/clicks/run.sh index 89b3f454e..ba479d542 100755 --- a/ci/clicks/run.sh +++ b/ci/clicks/run.sh @@ -11,7 +11,9 @@ tmp=$(mktemp -d) pids="" stop() { for p in $pids; do kill "$p" 2>/dev/null || true; done - rm -rf "$tmp" + # wait for them to exit: a server still writing its data folder makes rm fail with "Directory not empty" + for p in $pids; do wait "$p" 2>/dev/null || true; done + rm -rf "$tmp" 2>/dev/null || { sleep 2; rm -rf "$tmp" || true; } } trap stop EXIT INT TERM From 540361ec5205bdedd65fcdf49ef2b9adfb1ab007 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Wed, 14 Oct 2026 10:01:46 +0000 Subject: [PATCH 4/4] test(convert): expect the tika key in capabilities after main's fetchable converters Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01X6p3skuCAfa2nTxSmDKvxq --- fastapi_backend/tests/domain/test_convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastapi_backend/tests/domain/test_convert.py b/fastapi_backend/tests/domain/test_convert.py index 97390e262..1582e8a64 100644 --- a/fastapi_backend/tests/domain/test_convert.py +++ b/fastapi_backend/tests/domain/test_convert.py @@ -137,7 +137,7 @@ def test_libreoffice_and_chromium_are_installed_the_first_time_theyre_needed(cfg root, bin_dir = fake_programs.sandbox(tmp_path, monkeypatch) monkeypatch.setattr(convert, "_has_msg", lambda: False) assert convert.fetchable("libreoffice") and convert.fetchable("chromium") and not convert.fetchable("ffmpeg") - assert convert.capabilities(cfg) == {"office": True, "pages": True, "msg": False, "web": True} + assert convert.capabilities(cfg) == {"office": True, "pages": True, "msg": False, "web": True, "tika": False} assert convert.unavailable(cfg, "a.docx") is None and convert.unavailable(cfg, "a.eml") is None rows = {r["id"]: r for r in components.catalog(cfg)} assert (rows["libreoffice"]["here"], rows["libreoffice"]["hint"]) == (False, "installed the first time it's needed")