From 904385e9d3af1c5cfa2dd2394bba1de5c917aacc Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 19:58:59 +0000 Subject: [PATCH] feat(import): capture one URL, several or a list from the API and the assistant POST /import/web/links takes urls as a JSON list besides pasted text and a file; the queueing moves to links.queue so the assistant's new import_urls tool (addresses, pasted list, or an attached list/bookmark file) shares it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01WQdDgCNPBWVWxydBXSenZD --- docs/api.md | 6 +- docs/assistant.md | 3 + fastapi_backend/app/api/v1/routes/imports.py | 39 +++------ fastapi_backend/app/domain/ai_tools.py | 4 +- fastapi_backend/app/domain/chat.py | 5 +- fastapi_backend/app/domain/links.py | 35 +++++++- fastapi_backend/app/domain/ops_tools.py | 82 ++++++++++++++++++- fastapi_backend/app/domain/uploads.py | 7 ++ fastapi_backend/app/schemas/imports.py | 3 +- .../tests/api/test_setup_assistant.py | 41 ++++++++++ fastapi_backend/tests/api/test_web_links.py | 14 ++++ nextjs-frontend/app/openapi-client/sdk.gen.ts | 4 +- .../app/openapi-client/types.gen.ts | 8 +- nextjs-frontend/components/settings/model.ts | 1 + nextjs-frontend/openapi.json | 20 ++++- 15 files changed, 227 insertions(+), 45 deletions(-) diff --git a/docs/api.md b/docs/api.md index 8d862bc80..b55d660cf 100644 --- a/docs/api.md +++ b/docs/api.md @@ -719,9 +719,9 @@ intranet. Without Chromium on the server it answers 400 (`GET /uploads/limits` `import.web`. The PDF is the resource's file (`GET /resources/{rid}/media`), named after the link (`annual-report.pdf`) or the page's title (`harbour-news.pdf`). -`POST /api/v1/import/web/links {namespace, text | filename + data, folders_as_tags?, collection?, pipeline?}` captures -many pages at once: pasted addresses (`text`, one per line; a line's other words become its title, `#` lines are -skipped), or a file (base64 `data`): a list of addresses, a browser's bookmark export (the HTML file every browser +`POST /api/v1/import/web/links {namespace, text | urls | filename + data, folders_as_tags?, collection?, pipeline?}` +captures one or many pages at once: pasted addresses (`text`, one per line; a line's other words become its title, `#` +lines are skipped), a JSON list of addresses (`urls`), or a file (base64 `data`): a list of addresses, a browser's bookmark export (the HTML file every browser writes) or Chrome's own Bookmarks file (JSON). Each link becomes a page captured as above, titled with its bookmark's name until it's captured, and tagged with the bookmark folders it was in (the innermost five, without the browser's own: Bookmarks bar, Other bookmarks), unless `folders_as_tags` is false. At most 500 links at once (400 beyond); links diff --git a/docs/assistant.md b/docs/assistant.md index 1ca8ebef2..7b28d259c 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -99,6 +99,9 @@ When the configured model supports function calling, chat becomes an agent. - **Files:** attach files to a message with the clip, or by dropping or pasting them on the box. They upload as you type (`POST /api/v1/uploads` with `hold`), stay out of the archive, and the assistant puts them in a namespace with `import_files` (`attachments` on `POST /api/v1/chats//messages`). +- **Web pages:** give it one address, several, or a list of links (pasted, or attached as a text file or a browser's + bookmark export) and `import_urls` captures each page as a document, as `POST /api/v1/import/web/links` does. Like + `import_files`, it picks the namespace when you don't name one, and waits for your approval. - **Keep typing:** what you send while an answer is being written waits its turn and goes next. ## A namespace's own assistant diff --git a/fastapi_backend/app/api/v1/routes/imports.py b/fastapi_backend/app/api/v1/routes/imports.py index 95d8832a9..46210ebb9 100644 --- a/fastapi_backend/app/api/v1/routes/imports.py +++ b/fastapi_backend/app/api/v1/routes/imports.py @@ -14,7 +14,7 @@ from fastapi import APIRouter, HTTPException, Request from app.api.deps import Acl, AdminWriter, Cfg, Db, Writer, domain_errors -from app.domain import auth, convert, ingest, jobs, library, links, mediaurl, pipelines, sources, store, webcapture +from app.domain import auth, convert, ingest, jobs, links, mediaurl, pipelines, sources, store, webcapture from app.domain.store import DB from app.schemas.imports import ( ImportPreview, @@ -192,8 +192,8 @@ def import_video(body: VideoImportRequest, acl: Acl, user: Writer, db: Db, cfg: @router.post("/web/links") def import_web_links(body: LinksImportRequest, acl: Acl, user: Writer, db: Db, cfg: Cfg) -> LinksImport: - """Capture many web pages at once (editors; admins may name a new namespace): pasted addresses, one per line, or a - file: a list of addresses, a browser's bookmark export (HTML) or Chrome's Bookmarks file (JSON). Each link becomes + """Capture one or many web pages at once (editors; admins may name a new namespace): pasted addresses, one per + line, a JSON list of them (`urls`), or a file: a list of addresses, a browser's bookmark export (HTML) or Chrome's Bookmarks file (JSON). Each link becomes a page captured as a document, as `POST /import/web` does, tagged with the bookmark folders it was in (`folders_as_tags`). At most 500 links; a link already captured in the namespace isn't captured again, and one that can't be is skipped, saying why. Audited as `import.web` once, with how many were queued.""" @@ -202,8 +202,9 @@ def import_web_links(body: LinksImportRequest, acl: Acl, user: Writer, db: Db, c raise HTTPException(400, "choose a namespace: lowercase letters, digits, - and _") if not convert.chromium(cfg): raise HTTPException(400, "capturing web pages needs Chromium on the server (the lens:full image)") - if bool(body.data) == bool((body.text or "").strip()): - raise HTTPException(400, "paste the links or send a file, not both") + given = [bool(body.data), bool((body.text or "").strip()), bool(body.urls)] + if sum(given) != 1: + raise HTTPException(400, "paste the links, list them in urls, or send a file: one of them") check_pipeline(db, body.pipeline) sid: int | None try: @@ -214,35 +215,15 @@ def import_web_links(body: LinksImportRequest, acl: Acl, user: Writer, db: Db, c raise HTTPException(403, "only admins can create namespaces") from None sid = None check_collection(db, sid, body.collection) - found = links.read(_decode(body.data, cfg) if body.data else body.text or "") + found = links.read(_decode(body.data, cfg) if body.data else "\n".join(body.urls or []) or body.text or "") if not found: raise HTTPException(400, "there are no http:// or https:// links in it") if len(found) > links.MAX: raise HTTPException(400, f"import {links.MAX} links at most at once; this has {len(found)}") sid = sid if sid is not None else store.ns_id(db, ns) - have = { - (r.get("web") or {}).get("url"): r["id"] - for r in db.rows("SELECT record::id(id) AS id, web FROM recording WHERE space = $s AND web != NONE", s=sid) - } - out = [] - for link in found: - try: - url = webcapture.check_url(cfg, link["url"]) - except ValueError as e: - out.append(LinkImportResult(url=link["url"], status="skipped", detail=str(e))) - continue - if url in have: - out.append(LinkImportResult(url=url, status="already", recording=have[url])) - continue - rid = webcapture.create(db, sid, url, link["title"], body.collection, by=user.email) - have[url] = rid - if body.folders_as_tags and link["folders"]: - try: - library.set_tags(db, rid, links.tags(link["folders"])) - except ValueError: - pass # tags are a nicety: the page is captured without them - job = jobs.enqueue(db, rid, None, by=user.email, pipeline=body.pipeline) - out.append(LinkImportResult(url=url, status="queued", recording=rid, job=job)) + out = [ + LinkImportResult(**r) for r in links.queue(db, cfg, sid, found, user.email, body.collection, body.pipeline, body.folders_as_tags) + ] queued = sum(r.status == "queued" for r in out) auth.audit(db, user.as_audit(), "import.web", f"space:{sid}", {"links": queued, "namespace": ns}) return LinksImport(results=out) diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index c41b2f824..ca0ae9076 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -1060,7 +1060,7 @@ def cited(self, text): def approve(db, cfg, aid, user, editable, decision="approve", base=None, admin=False, readable=None): """Carry out an approved action: a batch run (or a sample of it), an entity change, or a change to the server - (ops_tools.py: settings, a namespace, importing attached files).""" + (ops_tools.py: settings, a namespace, importing attached files or web pages).""" a = db.one("SELECT record::id(id) AS id, chat, account, tool, args, status FROM $r", r=R("approval", int(aid))) if not a or a["status"] != "pending": raise ValueError("nothing to approve") @@ -1088,7 +1088,7 @@ def _carry_out(db, cfg, a, user, editable, decision, base, admin, readable): result = run_extension(db, cfg, args, user, admin, box.readable, box) elif a["tool"] in ("save_extension", "switch_extension"): result = author_extension(db, a["tool"], args, user, admin, readable if readable is not None else editable, editable) - elif a["tool"] in ("change_settings", "create_namespace", "import_files"): + elif a["tool"] in ("change_settings", "create_namespace", "import_files", "import_urls"): result = ops_tools.apply(db, cfg, base or cfg, a["tool"], args, user, editable, admin) elif a["tool"] == "forget_memory": sid = {v: k for k, v in store.space_names(db).items()}.get(args["namespace"]) diff --git a/fastapi_backend/app/domain/chat.py b/fastapi_backend/app/domain/chat.py index 35245b38f..7d65e3981 100644 --- a/fastapi_backend/app/domain/chat.py +++ b/fastapi_backend/app/domain/chat.py @@ -494,8 +494,9 @@ def attached_note(files): if not files: return "" lines = [f"- {f['filename']} ({round(f['size'] / 1e6, 1)} MB, attachment id {f['id']})" for f in files] - return "\n\nAttached files, not in the archive yet (import_files puts them in a namespace, choosing one when not named):\n" + "\n".join( - lines + return ( + "\n\nAttached files, not in the archive yet (import_files puts them in a namespace, choosing one when not named; a list of links is captured as web pages with import_urls upload_id):\n" + + "\n".join(lines) ) diff --git a/fastapi_backend/app/domain/links.py b/fastapi_backend/app/domain/links.py index e73864684..b79609cb0 100644 --- a/fastapi_backend/app/domain/links.py +++ b/fastapi_backend/app/domain/links.py @@ -2,7 +2,8 @@ export (the HTML file every browser writes), or Chrome's own Bookmarks file (JSON). Each link becomes a web page captured as a document (webcapture.py); the bookmark folders it was in can become its tags. -Only what a link says is read here: nothing is fetched. Addresses are checked when each is imported. +Only what a link says is read here: nothing is fetched. Addresses are checked when each is imported (`queue`), by the +API, the web app and the assistant's import_urls alike. """ from __future__ import annotations @@ -12,6 +13,8 @@ import re from html.parser import HTMLParser +from . import jobs, library, webcapture + MAX = 500 # links in one import, at most URL = re.compile(r"https?://[^\s<>\"']+", re.I) SKIP = {"bookmarks bar", "bookmarks menu", "bookmarks toolbar", "other bookmarks", "mobile bookmarks", "favorites bar"} @@ -101,3 +104,33 @@ def read(raw): def tags(folders, most=5, width=40): """A link's bookmark folders as tags: the innermost few, each cut to a tag's length.""" return [" ".join(f.split())[:width].strip() for f in folders[-most:] if f.strip()] + + +def queue(db, cfg, sid, found, by, collection=None, pipeline=None, folders_as_tags=True): + """Capture each of `found` (from `read`) in namespace `sid` as a web page document: [{url, status, recording, job, + detail}], status queued, already (captured in the namespace before) or skipped (detail says why). The caller + checks the person may add to the namespace and the collection is one of its own.""" + have = { + (r.get("web") or {}).get("url"): r["id"] + for r in db.rows("SELECT record::id(id) AS id, web FROM recording WHERE space = $s AND web != NONE", s=sid) + } + out = [] + for link in found: + try: + url = webcapture.check_url(cfg, link["url"]) + except ValueError as e: + out.append({"url": link["url"], "status": "skipped", "detail": str(e)}) + continue + if url in have: + out.append({"url": url, "status": "already", "recording": have[url]}) + continue + rid = webcapture.create(db, sid, url, link.get("title"), collection, by=by) + have[url] = rid + if folders_as_tags and link.get("folders"): + try: + library.set_tags(db, rid, tags(link["folders"])) + except ValueError: + pass # tags are a nicety: the page is captured without them + job = jobs.enqueue(db, rid, None, by=by, pipeline=pipeline) + out.append({"url": url, "status": "queued", "recording": rid, "job": job}) + return out diff --git a/fastapi_backend/app/domain/ops_tools.py b/fastapi_backend/app/domain/ops_tools.py index 8b494c627..9976e55d6 100644 --- a/fastapi_backend/app/domain/ops_tools.py +++ b/fastapi_backend/app/domain/ops_tools.py @@ -1,7 +1,8 @@ """The assistant's tools for running the server: what's set up and what isn't, the settings, namespaces, and files dropped into the conversation. -Admins get the server tools; anyone gets `import_files` for their own attachments, into namespaces they edit. A change +Admins get the server tools; anyone gets `import_files` for their own attachments and `import_urls` for web pages, +into namespaces they edit. A change waits for the person's approval, unless the conversation lets the assistant act (a setup conversation does: the admin asked it to set things up): then it's made at once, audited, and said in the answer. Telemetry, which sends data elsewhere, always waits for approval. @@ -9,7 +10,7 @@ from __future__ import annotations -from . import auth, blobs, decide, feeds, jobs, llm, metadata, settings, setup, sources, store, uploads +from . import auth, blobs, convert, decide, feeds, jobs, links, llm, metadata, settings, setup, sources, store, uploads R = store.R _S = {"type": "string"} @@ -102,7 +103,24 @@ ["upload_ids"], True, ), + ( + "import_urls", + "Capture web pages as documents in a namespace, each kept as a PDF of the page as it is now and read like any " + "document: one address, several, or a whole list. Give urls, or text holding them (a pasted list, one per line), " + "or upload_id: an attached file listing them (a text list, a browser's bookmark export, Chrome's Bookmarks). " + "Leave namespace out unless the person named one: it's chosen for them, and you're told when it's unclear and to ask.", + { + "urls": {"type": "array", "items": _S}, + "text": _S, + "upload_id": _S, + "namespace": _S, + "collection_id": {"type": "integer"}, + }, + [], + True, + ), ] +LIST_BYTES = 5 * 2**20 # an attached list of links is read up to this def status(db, cfg, base): @@ -176,6 +194,22 @@ def apply(db, cfg, base, tool, args, user, editable, admin): auth.audit(db, user, "upload", f"recording:{up['recording']}", {"file": up["filename"], "namespace": ns, "assistant": True}) placed.append({"file": up["filename"], "recording": up["recording"], "duplicate": bool(up.get("duplicate"))}) return {"namespace": ns, "imported": placed} + if tool == "import_urls": + ns = args["namespace"].strip().lower() + try: + sid = store.ns_id(db, ns, create=False) + except KeyError: + if not admin: + raise PermissionError(f"there's no namespace {ns}") from None + sid = store.ns_id(db, ns) + if not admin and sid not in editable: + raise PermissionError(f"you can't add to {ns}") + if args.get("collection_id") is not None: + store.home(db, sid, args["collection_id"]) + out = links.queue(db, cfg, sid, args["links"], user["email"], args.get("collection_id")) + queued = sum(r["status"] == "queued" for r in out) + auth.audit(db, user, "import.web", f"space:{sid}", {"links": queued, "namespace": ns, "assistant": True}) + return {"namespace": ns, "pages": out} raise ValueError(f"no tool {tool}") @@ -296,3 +330,47 @@ def t_import_files(self, upload_ids, namespace=None, collection_id=None): if chosen: out = {**out, "namespace_chosen": {"by": chosen["by"], "confidence": chosen["confidence"]}} return out, said + + def t_import_urls(self, urls=None, text=None, upload_id=None, namespace=None, collection_id=None): + if isinstance(urls, str): + urls = [urls] + given = [bool(urls), bool((text or "").strip()), bool(upload_id)] + if sum(given) != 1: + raise ValueError("give the addresses as urls, as text, or as an attached file's upload_id: one of them") + if not convert.chromium(self.cfg): + raise ValueError("capturing web pages needs Chromium on the server (the lens:full image)") + if upload_id: + try: + up = uploads.get(self.db, str(upload_id)) + except KeyError: + raise ValueError(f"no attachment {upload_id}") from None + if up.get("account") != self.user["id"] or up.get("state") != "held": + raise ValueError(f"{upload_id} isn't a file attached here") + raw = uploads.held_bytes(self.cfg, up, LIST_BYTES) + else: + raw = "\n".join(str(u) for u in urls) if urls else text + found = links.read(raw) + if not found: + raise ValueError("there are no http:// or https:// links in it") + if len(found) > links.MAX: + raise ValueError(f"import {links.MAX} links at most at once; this has {len(found)}") + shown = found[0]["url"] if len(found) == 1 else f"{len(found)} web pages" + chosen = None + if not (namespace or "").strip(): + pages = [{"url": f["url"], "title": f["title"] or ""} for f in found[:20]] + namespace, chosen = self._where_to(pages) + if namespace is None: + ranked = chosen.get("ranked") or [{"option": o} for o in chosen["options"]] + return ( + {"status": "ask", "note": "It isn't clear where these go: ask the person, suggesting the first.", "namespaces": ranked}, + f"Not sure where to put {shown}", + ) + namespace = namespace.strip().lower() + args = {"links": found, "namespace": namespace, "collection_id": collection_id} + summary = f"Capture {shown} into {namespace}" + if chosen and chosen.get("by") != "only option": + summary += f" (chosen, {round(100 * chosen['confidence'])}% sure)" + out, said = self._change("import_urls", store.clean(args), summary) + if chosen: + out = {**out, "namespace_chosen": {"by": chosen["by"], "confidence": chosen["confidence"]}} + return out, said diff --git a/fastapi_backend/app/domain/uploads.py b/fastapi_backend/app/domain/uploads.py index b579af418..b135036cd 100644 --- a/fastapi_backend/app/domain/uploads.py +++ b/fastapi_backend/app/domain/uploads.py @@ -454,6 +454,13 @@ def incoming(cfg): return folder +def held_bytes(cfg, row, most): + """The first `most` bytes of a held upload (all of it here), read where it waits: e.g. a list of links attached + to a conversation. It stays held.""" + with open(_part(cfg, row["id"]), "rb") as f: + return f.read(most) + + def cancel(db, cfg, row): """Stop an upload; its partial file goes. A finished one is only forgotten: it's a recording now.""" if row.get("state") != "done": diff --git a/fastapi_backend/app/schemas/imports.py b/fastapi_backend/app/schemas/imports.py index 7205faa0a..834e2c2d8 100644 --- a/fastapi_backend/app/schemas/imports.py +++ b/fastapi_backend/app/schemas/imports.py @@ -76,11 +76,12 @@ class VideoImportRequest(RequestModel): class LinksImportRequest(RequestModel): - """Links pasted as ``text`` (one per line), or a file (``filename`` + base64 ``data``): a list of addresses, a + """Links pasted as ``text`` (one per line), listed in ``urls``, or a file (``filename`` + base64 ``data``): a list of addresses, a browser's bookmark export (HTML) or Chrome's Bookmarks file (JSON).""" namespace: str = Field(description="a namespace you edit; admins may name a new one") text: str | None = Field(None, max_length=1_000_000, description="addresses, one per line") + urls: list[str] | None = Field(None, max_length=500, description="addresses, as a list") filename: str | None = None data: str | None = Field(None, description="the file, base64 encoded") folders_as_tags: bool = Field(True, description="tag each page with the bookmark folders it was in") diff --git a/fastapi_backend/tests/api/test_setup_assistant.py b/fastapi_backend/tests/api/test_setup_assistant.py index 2a100c8ee..774e93db8 100644 --- a/fastapi_backend/tests/api/test_setup_assistant.py +++ b/fastapi_backend/tests/api/test_setup_assistant.py @@ -147,3 +147,44 @@ def test_a_held_upload_stays_out_until_placed(app, db, cfg, folder, new_client, uploads.place( db, cfg, up["id"], "pods", db.values("SELECT VALUE record::id(id) FROM account WHERE email = 'ed@x.io'")[0], collection=12345 ) + + +def test_the_assistant_captures_web_pages(app, db, cfg, folder, new_client, llm, monkeypatch): + from app.domain import convert + + monkeypatch.setattr(convert, "chromium", lambda cfg: "/usr/bin/chromium") # nothing is captured: no job runs here + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + listed = held(c, h, b"# reading\nhttps://8.8.8.8/tides Tide tables\nhttps://1.1.1.1/lamps\n", "links.txt") + llm.tool_script = [ + {"content": "", "tool_calls": [call(1, "import_urls", {"urls": ["https://8.8.8.8/harbour"], "namespace": "pods"})]}, + {"content": "", "tool_calls": [call(2, "import_urls", {"upload_id": listed["id"], "namespace": "pods"})]}, + {"content": "I asked to capture them."}, + ] + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "keep these", "attachments": [listed["id"]]}).text) + offered = {t["function"]["name"] for t in llm.seen[-3]["tools"]} + assert "import_urls" in offered + appr = ev["approval"] + assert [a["summary"] for a in appr] == ["Capture https://8.8.8.8/harbour into pods", "Capture 2 web pages into pods"] + assert not db.values("SELECT VALUE id FROM recording WHERE web != NONE") # nothing until approved + for a in appr: + r = c.post(f"/api/v1/approvals/{a['id']}", headers=h, json={"decision": "approve"}) + assert r.json()["status"] == "done", r.text + got = db.rows("SELECT id, title, web, space FROM recording WHERE web != NONE ORDER BY id") + assert [(x["web"]["url"], x["space"]) for x in got] == [ + ("https://8.8.8.8/harbour", s.pods), + ("https://8.8.8.8/tides", s.pods), + ("https://1.1.1.1/lamps", s.pods), + ] + assert got[1]["title"] == "Tide tables" + assert c.get(f"/api/v1/uploads/{listed['id']}", headers=h).json()["state"] == "held" # the list itself isn't imported + # a viewer can't add pages to the namespace + cv, hv = s.cl["viewer"] + llm.tool_script = [ + {"content": "", "tool_calls": [call(1, "import_urls", {"urls": ["https://8.8.8.8/x"], "namespace": "pods"})]}, + {"content": "No."}, + ] + vid = cv.post("/api/v1/chats", headers=hv, json={}).json()["id"] + ev = sse(cv.post(f"/api/v1/chats/{vid}/messages", headers=hv, json={"content": "keep https://8.8.8.8/x"}).text) + assert "approval" not in ev diff --git a/fastapi_backend/tests/api/test_web_links.py b/fastapi_backend/tests/api/test_web_links.py index 9269a9fde..94e7900a0 100644 --- a/fastapi_backend/tests/api/test_web_links.py +++ b/fastapi_backend/tests/api/test_web_links.py @@ -143,3 +143,17 @@ def test_what_is_refused(client, env, monkeypatch): monkeypatch.setattr(convert, "chromium", lambda cfg: None) r = client.post(url, headers=he, json={"namespace": "pods", "text": "https://8.8.8.8/1"}) assert r.status_code == 400 and "Chromium" in r.json()["detail"] + + +def test_one_or_several_addresses_as_a_list(client, env, db): + he = env["he"] + url = "/api/v1/import/web/links" + r = client.post(url, headers=he, json={"namespace": "pods", "urls": ["https://8.8.8.8/one"]}) + assert [(x["url"], x["status"]) for x in r.json()["results"]] == [("https://8.8.8.8/one", "queued")] + r = client.post(url, headers=he, json={"namespace": "pods", "urls": ["https://8.8.8.8/one", "https://8.8.4.4/two", "ftp://x/3"]}) + assert [(x["url"], x["status"]) for x in r.json()["results"]] == [ + ("https://8.8.8.8/one", "already"), + ("https://8.8.4.4/two", "queued"), + ] + r = client.post(url, headers=he, json={"namespace": "pods", "urls": ["https://8.8.8.8/1"], "text": "https://8.8.8.8/2"}) + assert r.status_code == 400 and "one of them" in r.json()["detail"] diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 6e5801f34..d2afecfac 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -4928,8 +4928,8 @@ export class Imports { /** * Import Web Links * - * Capture many web pages at once (editors; admins may name a new namespace): pasted addresses, one per line, or a - * file: a list of addresses, a browser's bookmark export (HTML) or Chrome's Bookmarks file (JSON). Each link becomes + * Capture one or many web pages at once (editors; admins may name a new namespace): pasted addresses, one per + * line, a JSON list of them (`urls`), or a file: a list of addresses, a browser's bookmark export (HTML) or Chrome's Bookmarks file (JSON). Each link becomes * a page captured as a document, as `POST /import/web` does, tagged with the bookmark folders it was in * (`folders_as_tags`). At most 500 links; a link already captured in the namespace isn't captured again, and one that * can't be is skipped, saying why. Audited as `import.web` once, with how many were queued. diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index fd83bbb05..7589eeb8a 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -6720,7 +6720,7 @@ export type LinksImport = { /** * LinksImportRequest * - * Links pasted as ``text`` (one per line), or a file (``filename`` + base64 ``data``): a list of addresses, a + * Links pasted as ``text`` (one per line), listed in ``urls``, or a file (``filename`` + base64 ``data``): a list of addresses, a * browser's bookmark export (HTML) or Chrome's Bookmarks file (JSON). */ export type LinksImportRequest = { @@ -6736,6 +6736,12 @@ export type LinksImportRequest = { * addresses, one per line */ text?: string | null; + /** + * Urls + * + * addresses, as a list + */ + urls?: Array | null; /** * Filename */ diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 6c59496a1..9be4d7262 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -386,6 +386,7 @@ export const AI_TOOLS: { name: string; label: string; acts: boolean }[] = [ acts: true, }, { name: "import_files", label: "Import files sent in a conversation", acts: true }, + { name: "import_urls", label: "Capture web pages from addresses or a list of links", acts: true }, { name: "server_status", label: "Check what the server has set up (admins)", acts: false }, { name: "find_model_servers", label: "Find model servers nearby (admins)", acts: false }, { name: "read_settings", label: "Read settings (admins)", acts: false }, diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index cfe128052..bf825488c 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -10179,7 +10179,7 @@ "imports" ], "summary": "Import Web Links", - "description": "Capture many web pages at once (editors; admins may name a new namespace): pasted addresses, one per line, or a\nfile: a list of addresses, a browser's bookmark export (HTML) or Chrome's Bookmarks file (JSON). Each link becomes\na page captured as a document, as `POST /import/web` does, tagged with the bookmark folders it was in\n(`folders_as_tags`). At most 500 links; a link already captured in the namespace isn't captured again, and one that\ncan't be is skipped, saying why. Audited as `import.web` once, with how many were queued.", + "description": "Capture one or many web pages at once (editors; admins may name a new namespace): pasted addresses, one per\nline, a JSON list of them (`urls`), or a file: a list of addresses, a browser's bookmark export (HTML) or Chrome's Bookmarks file (JSON). Each link becomes\na page captured as a document, as `POST /import/web` does, tagged with the bookmark folders it was in\n(`folders_as_tags`). At most 500 links; a link already captured in the namespace isn't captured again, and one that\ncan't be is skipped, saying why. Audited as `import.web` once, with how many were queued.", "operationId": "import_web_links", "requestBody": { "content": { @@ -35944,6 +35944,22 @@ "title": "Text", "description": "addresses, one per line" }, + "urls": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array", + "maxItems": 500 + }, + { + "type": "null" + } + ], + "title": "Urls", + "description": "addresses, as a list" + }, "filename": { "anyOf": [ { @@ -36004,7 +36020,7 @@ "namespace" ], "title": "LinksImportRequest", - "description": "Links pasted as ``text`` (one per line), or a file (``filename`` + base64 ``data``): a list of addresses, a\nbrowser's bookmark export (HTML) or Chrome's Bookmarks file (JSON)." + "description": "Links pasted as ``text`` (one per line), listed in ``urls``, or a file (``filename`` + base64 ``data``): a list of addresses, a\nbrowser's bookmark export (HTML) or Chrome's Bookmarks file (JSON)." }, "LlmEstimate": { "properties": {