From 97ac455a890b07579540de7b57f5e5be875be2ae Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 06:47:30 +0000 Subject: [PATCH] feat(imports): link a file or address already here instead of importing it again Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QFUNzy1QZy5HhKAyJxTAYJ --- README.md | 3 ++- changelog.d/duplicates.md | 4 ++++ docs/api.md | 9 ++++++--- docs/requirements.md | 1 + fastapi_backend/app/api/v1/routes/imports.py | 4 ++++ fastapi_backend/app/domain/import_hooks.py | 3 +++ fastapi_backend/app/domain/sources.py | 7 ++++++- fastapi_backend/app/domain/webcapture.py | 12 +++++++++++ fastapi_backend/app/schemas/imports.py | 3 ++- fastapi_backend/tests/api/test_uploads.py | 20 +++++++++++++++++++ fastapi_backend/tests/api/test_video_urls.py | 12 +++++++++++ .../app/openapi-client/types.gen.ts | 8 +++++++- .../components/import/use-import.ts | 10 ++++++++-- nextjs-frontend/components/import/web-tab.tsx | 8 +++++--- nextjs-frontend/openapi.json | 18 ++++++++++++++--- 15 files changed, 107 insertions(+), 15 deletions(-) create mode 100644 changelog.d/duplicates.md diff --git a/README.md b/README.md index 43e7f92a4..9832593cb 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,8 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p nothing else here can read; capture web pages one at a time, from a link list or a bookmarks file, by API or by asking the assistant, from private networks you allow in Settings too; and import videos and podcasts by their address (off by default). Say what a file or a page is as it comes in (a podcast, an interview, a scan), in Import, by API or - to the assistant, and its content type's pipeline runs from the start. + to the assistant, and its content type's pipeline runs from the start. A file, page or video the namespace already + has links to the one there instead of being imported again. ([Sensors](docs/sensors.md), [processing](docs/processing.md), [Apache Tika](docs/configuration.md#apache-tika), [import webhooks](docs/api.md#import-webhooks); [checks](docs/requirements.md#bring-in-everything)) - **Clip from the browser.** The Lens web clipper for Chrome, Edge, Firefox and Safari turns a page, a selection or a diff --git a/changelog.d/duplicates.md b/changelog.d/duplicates.md new file mode 100644 index 000000000..6d8da6f8c --- /dev/null +++ b/changelog.d/duplicates.md @@ -0,0 +1,4 @@ +- **A copy you have already isn't read again.** The same file in two folders of a source on this machine (or uploaded, + or found by a scan) is one resource, matched by what's in it, so it's transcribed once; a web page or a video + imported by an address the namespace has already gives that resource back, with `duplicate` true, instead of + fetching and reading it twice. Copies on cloud storage are still matched by their path ([Imports](docs/api.md#web-pages)). diff --git a/docs/api.md b/docs/api.md index ce17ab953..16653fcd1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1000,7 +1000,8 @@ a storage source now, rather than watching their folder (admins, like sources; u `GET /sources/{sid}/browse` lists them). Audio, video, documents and images stay on the source and run the namespace's pipeline, or `pipeline`; transcripts are imported. A PDF, Word, text or Markdown file is a document unless `documents_as` is `transcript` (its text only), or the server can't make a PDF of it. Each path gets a result: `queued` (with `recording` and `job`), `already` (the namespace has it from this -source), `skipped` (a folder, or not audio, video, a document, an image or a transcript) or `error`. Choosing a file +source, or, for a folder on this machine, has the same file from anywhere: another folder, an upload or a scan, by +what's in it, so it isn't read twice; with that `recording`), `skipped` (a folder, or not audio, video, a document, an image or a transcript) or `error`. Choosing a file whose asset was deleted brings it back. Audited as `import.source`. ### Web pages @@ -1010,7 +1011,8 @@ of the namespace; admins may name a new one). Its pipeline keeps the page as it any document ([Documents and images](#documents-and-images)): a link to a PDF is kept as it is, any other page is printed by headless Chromium with its scripts run. Its `web` is `{url, final, captured_at, how}`: the address given, the one it ended at after redirects, when it was captured, and `printed` or `pdf`. Its title is the page's own once it's -captured, unless `title` gave one. Capturing happens once: reading it again keeps the page it captured. +captured, unless `title` gave one. Capturing happens once: reading it again keeps the page it captured. An address the +namespace has a page from already isn't captured again: the answer is that resource, with `duplicate` true and no `job`. Only public addresses are reached, on ports 80 and 443: the address is checked when it's given (400, saying why: not http(s), a user name or password in it, another port, or a host with an address that isn't public), and every request @@ -1036,7 +1038,8 @@ as the checks above would). Audited as `import.web` once, with how many were que `POST /api/v1/import/video {url, namespace, title?, collection?, pipeline?}` imports a video or a podcast episode by its address (editors of the namespace; admins may name a new one): a page on a video site, a podcast's episode page, or a -media file. It's off by default: `documents.video_urls` (Settings › Documents) turns it on, and until then it answers +media file. An address the namespace has the video from already isn't downloaded again: the answer is that resource, +with `duplicate` true and no `job`. It's off by default: `documents.video_urls` (Settings › Documents) turns it on, and until then it answers 400. Its transcribe step downloads the media with [yt-dlp](https://github.com/yt-dlp/yt-dlp) (Unlicense), which Lens fetches on first use (release 2026.08.19, checked against its SHA-256, into `data_dir/models/`) unless one is on PATH, then transcribes it like an upload. One video is downloaded, never a playlist: the best single file with both sound and diff --git a/docs/requirements.md b/docs/requirements.md index aeb1c0b3e..1f6b4b1a7 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -158,6 +158,7 @@ request, to another check that proves the same thing; loosening a requirement or | Apache Tika (opt-in) reads documents nothing else here can | [test_tika.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_tika.py) `test_off_by_default_nothing_changes`, `test_what_nothing_here_can_convert_tika_reads_as_text` | | Web pages captured one at a time, from a link list or a bookmarks file, by API or by asking; private networks only when allowed | [test_webcapture.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_webcapture.py) `test_a_web_page_becomes_a_document`, `test_an_admin_allows_a_private_network_in_settings`, `test_private_addresses_are_refused`; [test_web_links.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_web_links.py) `test_one_or_several_addresses_as_a_list`, `test_a_bookmarks_file_becomes_pages_to_capture`; [test_setup_assistant.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_setup_assistant.py) `test_the_assistant_captures_web_pages` | | A content type chosen as a file or a page comes in, in Import, by API or to the assistant, runs that content type's pipeline from the start | [import-content-type.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/import-content-type.test.tsx) `offers the content types of the file's base type, and none chosen means recognised from the file`; [import-web.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/import-web.test.tsx) `captures pages as the content type chosen for them`; [test_uploads.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_uploads.py) `test_a_content_type_chosen_at_upload`, `test_a_held_file_is_placed_as_a_content_type`; [test_imports.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_imports.py) `test_choosing_the_content_type_at_import`; [test_web_links.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_web_links.py) `test_pages_captured_as_a_content_type`; [test_setup_assistant.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_setup_assistant.py) `test_the_assistant_imports_files_as_the_content_type_said` | +| A file, page or video the namespace already has links to the one there instead of being imported again: a copy in a watched folder on this machine, a page or video address imported twice | [test_uploads.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_uploads.py) `test_a_copy_in_a_folder_on_this_machine_links_to_the_one_there`; [test_video_urls.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_video_urls.py) `test_an_address_imported_again_links_to_what_the_namespace_has` | | Videos and podcasts imported by their address, off by default | [test_video_urls.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_video_urls.py) `test_off_by_default`, `test_a_video_by_its_address_is_downloaded_then_transcribed`; [import-web.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/import-web.test.tsx) `imports a video or podcast by its address, when asked to` | ## Clip from the browser diff --git a/fastapi_backend/app/api/v1/routes/imports.py b/fastapi_backend/app/api/v1/routes/imports.py index 258bc2808..88173dcfd 100644 --- a/fastapi_backend/app/api/v1/routes/imports.py +++ b/fastapi_backend/app/api/v1/routes/imports.py @@ -167,6 +167,8 @@ def import_web_page(body: WebImportRequest, acl: Acl, user: Writer, db: Db, cfg: except ValueError as e: raise HTTPException(400, str(e)) from None sid = sid if sid is not None else store.ns_id(db, ns) + if (had := webcapture.existing(db, sid, url)) is not None: # captured already: link to it + return ImportResult(id=had, duplicate=True) rid = webcapture.create(db, sid, url, body.title, body.collection, by=user.email) if body.content_type: content_types.choose(db, rid, body.content_type) @@ -202,6 +204,8 @@ def import_video(body: VideoImportRequest, acl: Acl, user: Writer, db: Db, cfg: except ValueError as e: raise HTTPException(400, str(e)) from None sid = sid if sid is not None else store.ns_id(db, ns) + if (had := webcapture.existing(db, sid, url, media=True)) is not None: # downloaded already: link to it + return ImportResult(id=had, duplicate=True) rid = mediaurl.create(db, sid, url, body.title, body.collection, by=user.email) job = jobs.enqueue(db, rid, None, by=user.email, pipeline=body.pipeline) auth.audit(db, user.as_audit(), "import.video", f"recording:{rid}", {"url": url, "namespace": ns}) diff --git a/fastapi_backend/app/domain/import_hooks.py b/fastapi_backend/app/domain/import_hooks.py index 4354b131c..6828c9d67 100644 --- a/fastapi_backend/app/domain/import_hooks.py +++ b/fastapi_backend/app/domain/import_hooks.py @@ -235,6 +235,9 @@ def accept_url(db, cfg, hook, url, title=None): if not convert.web_capture(cfg) and not urllib.parse.urlsplit(url).path.lower().endswith(".pdf"): raise ValueError("capturing web pages needs Chromium on the server (the lens:full image); a link to a PDF works without it") by = actor(hook) + if (had := webcapture.existing(db, hook["space"], url)) is not None: # the namespace has it: link to it + _used(db, hook, 1) + return {"kind": "url", "name": url, "recording": had, "duplicate": True} rid = webcapture.create(db, hook["space"], url, title, _collection(db, hook), by=by["email"]) job = jobs.enqueue(db, rid, None, by=by["email"], pipeline=hook.get("pipeline")) _used(db, hook, 1) diff --git a/fastapi_backend/app/domain/sources.py b/fastapi_backend/app/domain/sources.py index 2ae2104ce..796c86631 100644 --- a/fastapi_backend/app/domain/sources.py +++ b/fastapi_backend/app/domain/sources.py @@ -632,6 +632,11 @@ def _ingest(db, cfg, src, f, kind, space, by, steps=None, pipeline=None, collect known = db.one("SELECT record::id(id) AS id FROM recording WHERE fp_key = $k", k=f"{space}:{fp}") if known: return known["id"], None + if src["type"] == "local" and not f.get("key"): # a file on this machine: by what's in it, as uploads are + fp = ingest.fingerprint(check_path(cfg, src, f["path"])) + copy = db.one("SELECT record::id(id) AS id FROM recording WHERE space = $s AND fingerprint = $f LIMIT 1", s=space, f=fp) + if copy: # the same file in another folder, uploaded, or found by a scan: link to it, not read it twice + return copy["id"], None rid, mail = db.next_id("recording"), _mail_of(f) db.q( "CREATE $r CONTENT $d", @@ -868,7 +873,7 @@ def import_files(db, cfg, sid, paths, space, by, pipeline=None, collection=None, try: deletion.forget(db, space, path=f"{src['name']}:{p}") # chosen on purpose: it may come back rid, job = _ingest(db, cfg, src, f, kind, space, by, None, pipeline, collection) - results.append(store.clean({"path": p, "status": "queued", "recording": rid, "job": job})) + results.append(store.clean({"path": p, "status": "queued" if job else "already", "recording": rid, "job": job})) except Exception as e: # noqa: BLE001 - one bad file must not stop the rest results.append({"path": p, "status": "error", "detail": f"{type(e).__name__}: {e}"[:300]}) return results diff --git a/fastapi_backend/app/domain/webcapture.py b/fastapi_backend/app/domain/webcapture.py index 5e6444ce5..2d1fe34c7 100644 --- a/fastapi_backend/app/domain/webcapture.py +++ b/fastapi_backend/app/domain/webcapture.py @@ -57,6 +57,18 @@ def placeholder(url): return (u.netloc + rest)[:200] +def existing(db, sid, url, media=False): + """The resource namespace `sid` already has for the page (or, with `media`, the video or episode) at `url`, so + importing it again links to it rather than fetching and reading it twice: its id, or None.""" + row = db.one( + "SELECT record::id(id) AS id FROM recording WHERE space = $s AND web.url = $u AND (web.media ?? false) = $m LIMIT 1", + s=int(sid), + u=url, + m=bool(media), + ) + return row["id"] if row else None + + def create(db, sid, url, title=None, collection=None, by=None): """A resource for the page at `url` in namespace `sid` (in `collection`, else the namespace's default), to be captured when its pipeline runs. Returns its id.""" diff --git a/fastapi_backend/app/schemas/imports.py b/fastapi_backend/app/schemas/imports.py index 27afc76a3..69710dadf 100644 --- a/fastapi_backend/app/schemas/imports.py +++ b/fastapi_backend/app/schemas/imports.py @@ -32,7 +32,8 @@ class ImportRequest(ImportPreviewRequest): class ImportResult(ResponseModel): ok: bool = True id: int - job: int + job: int | None = None # None when nothing was queued: the namespace had it already + duplicate: bool = Field(False, description="the namespace already had this page or video: `id` is that resource") class PreviewLine(ResponseModel): diff --git a/fastapi_backend/tests/api/test_uploads.py b/fastapi_backend/tests/api/test_uploads.py index 92e229aa5..793caa864 100644 --- a/fastapi_backend/tests/api/test_uploads.py +++ b/fastapi_backend/tests/api/test_uploads.py @@ -390,3 +390,23 @@ def test_a_held_file_is_placed_as_a_content_type(db, cfg, folder): assert uploads.get(db, up["id"])["state"] == "held" # nothing changed done = uploads.place(db, cfg, up["id"], "pods", 1, content_type="interview") assert content_types.of_recording(db, done["recording"]) == (content_types.get(db, "interview"), True) + + +def test_a_copy_in_a_folder_on_this_machine_links_to_the_one_there(client, env, db, cfg, folder, monkeypatch): + """The same file in two folders of a source on this machine, or uploaded after, is one resource, read once.""" + from tests.api.test_sources import _lsjson_without_rclone + + _lsjson_without_rclone(monkeypatch) + data, ha, he = env["data"], env["ha"], env["he"] + for sub in ("phone", "laptop"): + (folder / "inbox" / sub).mkdir(parents=True) + (folder / "inbox" / sub / f"{sub}.wav").write_bytes(data) + (folder / "inbox" / "laptop" / "other.wav").write_bytes(data[:100] + b"\1\1" + data[102:]) + sid = client.post("/api/v1/sources", json={"name": "inbox", "type": "local"}, headers=ha).json()["id"] + paths = [str(folder / "inbox" / p) for p in ("phone/phone.wav", "laptop/laptop.wav", "laptop/other.wav")] + got = client.post("/api/v1/import/source", json={"source": sid, "paths": paths, "namespace": "pods"}, headers=ha).json()["results"] + assert [x["status"] for x in got] == ["queued", "already", "queued"] + assert got[1]["recording"] == got[0]["recording"] != got[2]["recording"] and got[1]["job"] is None + assert db.one("SELECT count() AS n FROM recording GROUP ALL")["n"] == 2 + done = _upload(client, he, data) # and an upload of it finds it there + assert (done["recording"], done["duplicate"]) == (got[0]["recording"], True) diff --git a/fastapi_backend/tests/api/test_video_urls.py b/fastapi_backend/tests/api/test_video_urls.py index e18ed5c09..615599867 100644 --- a/fastapi_backend/tests/api/test_video_urls.py +++ b/fastapi_backend/tests/api/test_video_urls.py @@ -134,3 +134,15 @@ def test_yt_dlp_is_fetched_only_when_it_matches_its_checksum(cfg, monkeypatch): monkeypatch.setattr(mediaurl, "build", lambda *a: ("yt-dlp_linux", hashlib.sha256(body).hexdigest())) got = mediaurl.fetch(cfg) assert pathlib.Path(got).read_bytes() == body and mediaurl.program(cfg) == [got] + + +def test_an_address_imported_again_links_to_what_the_namespace_has(client, env, cfg, db): + """A video (or a page) at an address the namespace has already is not fetched again: the import gives that one.""" + assert client.put("/api/v1/settings/documents", headers=env["ha"], json={"video_urls": True}).status_code == 200 + body = {"namespace": "pods", "url": "https://8.8.8.8/watch?v=7"} + first = client.post("/api/v1/import/video", headers=env["he"], json=body).json() + again = client.post("/api/v1/import/video", headers=env["he"], json=body).json() + assert (first["duplicate"], again) == (False, {"ok": True, "id": first["id"], "job": None, "duplicate": True}) + other = client.post("/api/v1/import/video", headers=env["ha"], json={**body, "namespace": "calls"}).json() + assert other["id"] != first["id"] and not other["duplicate"] # namespaces keep their own + assert len(db.values("SELECT VALUE id FROM job")) == 2 diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index d28812274..0c2e71d89 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -10825,7 +10825,13 @@ export type ImportResult = { /** * Job */ - job: number; + job?: number | null; + /** + * Duplicate + * + * the namespace already had this page or video: `id` is that resource + */ + duplicate?: boolean; [key: string]: unknown; }; diff --git a/nextjs-frontend/components/import/use-import.ts b/nextjs-frontend/components/import/use-import.ts index 11779e8ca..01f263b02 100644 --- a/nextjs-frontend/components/import/use-import.ts +++ b/nextjs-frontend/components/import/use-import.ts @@ -368,13 +368,19 @@ export function useImportQueue() { const res = await data(Imports.importTranscript({ client, body: await j.body() })); if (j.audio) { // then its audio, attached to the transcript that just landed - update(j.key, { recording: res.id, job: res.job, audio: j.audio.name, sent: 0, size: j.audio.size }); + update(j.key, { + recording: res.id, + job: res.job ?? undefined, + audio: j.audio.name, + sent: 0, + size: j.audio.size, + }); media.current.set(j.key, { file: j.audio, namespace: j.namespace, title: j.title, attach: res.id }); landed(); await upload(j.key); continue; } - update(j.key, { state: "sent", recording: res.id, job: res.job }); + update(j.key, { state: "sent", recording: res.id, job: res.job ?? undefined }); landed(); } catch (e) { update(j.key, { diff --git a/nextjs-frontend/components/import/web-tab.tsx b/nextjs-frontend/components/import/web-tab.tsx index 8fe8e4f68..0da4e2d2f 100644 --- a/nextjs-frontend/components/import/web-tab.tsx +++ b/nextjs-frontend/components/import/web-tab.tsx @@ -66,9 +66,11 @@ export function WebTab({ onSuccess: (r) => { setCaptured((c) => [{ url: webAddress(url), id: r.id }, ...c].slice(0, 10)); toast( - media - ? { title: "Importing the video", body: "Its media is downloaded, then transcribed.", tone: "green" } - : { title: "Capturing the page", body: "It’s kept as a PDF and read like any document.", tone: "green" }, + r.duplicate + ? { title: "It’s here already", body: "This namespace has it from that address, so it isn’t fetched again." } + : media + ? { title: "Importing the video", body: "Its media is downloaded, then transcribed.", tone: "green" } + : { title: "Capturing the page", body: "It’s kept as a PDF and read like any document.", tone: "green" }, ); setUrl(""); setTitle(""); diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 0ad9c8e8f..21032f741 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -54322,15 +54322,27 @@ "title": "Id" }, "job": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Job" + }, + "duplicate": { + "type": "boolean", + "title": "Duplicate", + "description": "the namespace already had this page or video: `id` is that resource", + "default": false } }, "additionalProperties": true, "type": "object", "required": [ - "id", - "job" + "id" ], "title": "ImportResult" },