From c159244c509cbb807ed26d001d291a7a33579ce1 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 08:45:47 +0000 Subject: [PATCH] feat(imports): make each bulk import one run you can undo whole Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QFUNzy1QZy5HhKAyJxTAYJ --- README.md | 6 +- changelog.d/import-runs.md | 8 + docs/api.md | 34 +- docs/assistant.md | 3 + docs/database.md | 3 +- docs/processing.md | 6 +- docs/requirements.md | 1 + fastapi_backend/app/api/v1/routes/imports.py | 93 ++++- fastapi_backend/app/api/v1/routes/jobs.py | 2 +- fastapi_backend/app/domain/ai_tools.py | 1 + fastapi_backend/app/domain/convert.py | 9 +- fastapi_backend/app/domain/deletion.py | 9 +- fastapi_backend/app/domain/import_runs.py | 180 ++++++++++ fastapi_backend/app/domain/ingest.py | 8 +- fastapi_backend/app/domain/links.py | 16 +- fastapi_backend/app/domain/moving.py | 2 +- fastapi_backend/app/domain/ops_tools.py | 73 +++- fastapi_backend/app/domain/sources.py | 12 +- fastapi_backend/app/domain/store.py | 4 + fastapi_backend/app/schemas/imports.py | 31 ++ fastapi_backend/app/schemas/recordings.py | 1 + fastapi_backend/tests/api/test_import_runs.py | 184 ++++++++++ .../tests/api/test_mail_calendar_sources.py | 4 +- .../tests/api/test_setup_assistant.py | 3 +- .../__tests__/import-runs.test.tsx | 112 ++++++ nextjs-frontend/app/openapi-client/index.ts | 17 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 56 +++ .../app/openapi-client/types.gen.ts | 207 +++++++++++ .../components/import/import-runs.ts | 39 ++ .../components/import/import-screen.tsx | 2 + .../components/import/recent-imports.tsx | 108 ++++++ .../components/import/source-tab.tsx | 10 +- .../components/import/web-links.tsx | 15 +- nextjs-frontend/openapi.json | 338 ++++++++++++++++++ threat-model.json | 9 + threat-model.md | 1 + threat-model.yaml | 9 + 37 files changed, 1564 insertions(+), 52 deletions(-) create mode 100644 changelog.d/import-runs.md create mode 100644 fastapi_backend/app/domain/import_runs.py create mode 100644 fastapi_backend/tests/api/test_import_runs.py create mode 100644 nextjs-frontend/__tests__/import-runs.test.tsx create mode 100644 nextjs-frontend/components/import/import-runs.ts create mode 100644 nextjs-frontend/components/import/recent-imports.tsx diff --git a/README.md b/README.md index 27c6dbb1d..fd6ca8369 100644 --- a/README.md +++ b/README.md @@ -151,9 +151,11 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p file or a page is as it comes in (a podcast, an interview, a scan), in Import, by API or to the assistant, and its content type's pipeline runs from the start. A file, page or video the namespace already has links to the one there instead of being imported again. Media browsers can't play, such as an AVI or HEVC from a phone, can get a copy they - can (off by default). + can (off by default). A bulk import (a folder scan, an archive's files, a link list, a source's files) is one run: + the toast after it, and the Import screen's list of recent runs, undo it whole. ([Sensors](docs/sensors.md), [processing](docs/processing.md), [Apache Tika](docs/configuration.md#apache-tika), - [import webhooks](docs/api.md#import-webhooks), [playable copies](docs/video.md#playable-copies); + [import webhooks](docs/api.md#import-webhooks), [playable copies](docs/video.md#playable-copies), + [import runs](docs/api.md#import-runs); [checks](docs/requirements.md#bring-in-everything)) - **Clip from the browser.** The Lens web clipper for Chrome, Edge, Firefox and Safari turns a page, a selection or a part you pick into Markdown, lets you look it over, and sends it to a namespace. It signs in to Lens with nothing to diff --git a/changelog.d/import-runs.md b/changelog.d/import-runs.md new file mode 100644 index 000000000..5359e5dc1 --- /dev/null +++ b/changelog.d/import-runs.md @@ -0,0 +1,8 @@ +- **A bulk import is one run you can undo whole.** A folder scan, a zip or tar archive's files, a list of links or a + bookmarks file, and chosen files of a source each make one import run, and every asset it makes carries the run's + id (`import_run`). `GET /api/v1/import/runs` lists recent runs with how many assets each made and how many are left; + `POST /api/v1/import/runs/{id}/undo` deletes them all through the normal delete, needing owner rights on each + (nothing goes when any isn't yours to delete), audited per asset and for the run. The toast after a link or source + import has Undo, the Import screen lists recent runs with Undo on each, and the assistant has `import_runs` and + `undo_import` (always after your yes). Assets imported before keep no run, and nothing changes for them. + ([Import runs](docs/api.md#import-runs)) diff --git a/docs/api.md b/docs/api.md index 446aec32f..41c8457a9 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1046,6 +1046,9 @@ POST /api/v1/import POST /api/v1/import/preview POST /api/v1/import/web POST /api/v1/import/web/links +GET /api/v1/import/runs +GET /api/v1/import/runs/{id} +POST /api/v1/import/runs/{id}/undo ``` `POST /import` queues the namespace's pipeline after the import, or the saved pipeline named by `pipeline` (any of @@ -1068,7 +1071,33 @@ pipeline, or `pipeline`; transcripts are imported. A PDF, Word, text or Markdown `documents_as` is `transcript` (its text only), or the server can't make a PDF of it. Each path gets a result: `queued` (with `recording` and `job`), `already` (the namespace has it from this source, or, for a folder on this machine, has the same file from anywhere: another folder, an upload or a scan, by what's in it, so it isn't read twice; with that `recording`), `skipped` (a folder, or not audio, video, a document, an image or a transcript) or `error`. Choosing a file -whose asset was deleted brings it back. Audited as `import.source`. +whose asset was deleted brings it back. The files queued are one [import run](#import-runs) (`import_run`). Audited +as `import.source`. + +### Import runs + +A bulk import is one import run, so a mistaken one is undone whole rather than asset by asset: a folder scan that +finds new files (`scan`), a zip or tar archive whose files become resources of their own (`archive`: the archive and +those resources), a list of links or a bookmarks file (`links`, `POST /import/web/links`) and chosen files of a source +(`source`, `POST /import/source`). Every asset a run made carries its id as `import_run` (`GET /recordings/{id}`), and +the import's answer names it (`import_run`). Any other upload, a pasted transcript or one web page makes no run, +and assets imported before runs existed have none. + +`GET /api/v1/import/runs?limit=` lists recent runs, newest first: the ones you started and those in namespaces where +you may delete assets (everyone's, for admins), each with `kind`, `namespace`, `source` (what it read: the folders, +the archive, the links' file, the source and how many files), `created_by`, `created_at`, `assets` (how many it made), +`left` (how many are still here) and `status` (`imported`, `partly undone` or `undone`). `GET /import/runs/{id}` reads +one (404 when it isn't one you'd see listed). + +`POST /api/v1/import/runs/{id}/undo` deletes every asset the run made that is still here, each exactly as +`DELETE /recordings/{id}` deletes one (the media files stay where they are, and scans and watched folders don't import +them again). It needs the rights a delete needs on each one (the remove permission in its namespace, owners, or admin +of its collection): when any is one you may not delete, it answers 403 and nothing is deleted. An asset a job is +working on is left (`busy`) and its job asked to stop; undoing the run again takes the rest. It answers `{run, +deleted, busy}`. Lens keeps no recently deleted list of assets, so an undo can't itself be undone: importing the files +again brings them back. Audited as `recording.delete` for each asset (with the run) and `import.undo` for the run. The +Import screen lists recent runs with Undo on each, and the toast after a bulk import has Undo for the whole run; the +assistant has `import_runs` and `undo_import`, which waits for your approval. ### Web pages @@ -1098,7 +1127,8 @@ name until it's captured, and tagged with the bookmark folders it was in (the in own: Bookmarks bar, Other bookmarks), unless `folders_as_tags` is false. At most 500 links at once (400 beyond); links are taken in order, without repeats. It answers with each link's `status`: `queued` (with its `recording` and `job`), `already` (a page captured from that address is in the namespace: its `recording`) or `skipped` (`detail` says why, -as the checks above would). Audited as `import.web` once, with how many were queued. +as the checks above would). The pages queued are one [import run](#import-runs) (`import_run`). Audited as +`import.web` once, with how many were queued. ### Videos and podcasts diff --git a/docs/assistant.md b/docs/assistant.md index 99c3ab09d..b49966494 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -303,6 +303,9 @@ When the configured model supports function calling, chat becomes an agent. bookmark export) and `import_urls` captures each page as a document, as `POST /api/v1/import/web/links` does. Like `import_files`, it picks the namespace when you don't name one, takes a content type when you say what the pages are, and waits for your approval. +- **Undo an import:** `import_runs` lists your recent bulk imports (a folder scan, an archive's files, a list of links, + a source's files) and `undo_import` deletes everything one of them made, as the Import screen's Undo does + ([Import runs](api.md#import-runs)). It needs owner rights on each asset and always waits for your approval. - **Keep typing:** what you send while an answer is being written waits its turn and goes next. - **Your browser** (Chrome and Edge): asked from the Lens extension's side panel, the assistant can open, read, click and type in that browser and download files into the conversation, on sites you allow, asking before anything that diff --git a/docs/database.md b/docs/database.md index 2ec70dc89..455985acd 100644 --- a/docs/database.md +++ b/docs/database.md @@ -93,7 +93,8 @@ as missing on older rows: a chat message's record (`meta`, [What each answer records](assistant.md#what-each-answer-records)) is one, and messages from before it simply have none. [Deep research](assistant.md#deep-research) is another: its tables (`research`, and `research_run` for the claim of the process running one) are new, a message's `kind` is missing on older ones, and notes accept `chat` links -without rewriting any. Anything else, such as rewriting stored values, moving data between +without rewriting any. [Import runs](api.md#import-runs) are a third: the `import_run` table and the index on a +recording's `import_run` are new, and assets imported before have no run. Anything else, such as rewriting stored values, moving data between tables, or changing or removing an index that already exists, is a named step in `app/domain/migrations.py`: * Every process that opens the database (the API, workers, `lens watch`, `lens` commands) runs the steps the database diff --git a/docs/processing.md b/docs/processing.md index f8c1cb486..c633c7c7e 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -6,7 +6,7 @@ How assets move through Lens: where they come from, what each step does, and how - `scan` finds audio under each namespace's paths and fingerprints it: moved files keep their history, duplicates are skipped, and so are files whose asset someone deleted or moved to another namespace (at the same path, or a copy of the same - file). + file). What one namespace's scan brings in is one [import run](api.md#import-runs), undone whole. - `clean_audio` (only in a pipeline that has it) cleans up the sound before it's transcribed: noise, volume, rumble, echo, and silence left out of what's transcribed ([Cleaning up audio](#cleaning-up-audio)). - `transcribe` uses SenseVoice, faster-whisper or mlx-whisper. When the configured engine isn't installed on a worker (or @@ -158,7 +158,9 @@ it, and its files are kept as its files, as an email's attachments are: those Le their own that say which archive they came from, a chat export in it a chat and JSON in it records (so a WhatsApp export with media is the chat and its photos). Only plain files are taken, an encrypted entry or a link is left out, an archive inside one is kept but not opened, and reading stops at 100 files or 512 MB unpacked, so a zip bomb -can't fill the disk. What was left out, and why, is on its page. Archives get the Archive content type. +can't fill the disk. What was left out, and why, is on its page. Archives get the Archive content type. The archive +and the resources made of its files are one [import run](api.md#import-runs), so Undo on the Import screen takes +them all away again. **Records.** JSON and JSON Lines that aren't a transcript (their rows have fields a transcript's lines don't, like an order's `total`) are read as records, one line each: a JSON Lines file a record per line, a JSON file its list (or the diff --git a/docs/requirements.md b/docs/requirements.md index 7dd1d06d6..7523d1c42 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -178,6 +178,7 @@ request, to another check that proves the same thing; loosening a requirement or | A file, page or video the namespace already has links to the one there instead of being imported again: a copy in a watched folder on this machine, a page or video address imported twice | [test_uploads.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_uploads.py) `test_a_copy_in_a_folder_on_this_machine_links_to_the_one_there`; [test_video_urls.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_video_urls.py) `test_an_address_imported_again_links_to_what_the_namespace_has` | | A video's own captions, written by people, become its transcript instead of transcribing it, off by default | [test_video_urls.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_video_urls.py) `test_a_videos_own_captions_become_its_transcript` | | Videos and podcasts imported by their address, off by default | [test_video_urls.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_video_urls.py) `test_off_by_default`, `test_a_video_by_its_address_is_downloaded_then_transcribed`; [import-web.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/import-web.test.tsx) `imports a video or podcast by its address, when asked to` | +| A bulk import (a folder scan, an archive's files, a link list, a source's files) is one run, listed and undone whole by its owners, by API, the Import screen or the assistant | [test_import_runs.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_import_runs.py) `test_a_link_import_is_one_run_undone_whole`, `test_an_uploaded_archive_and_its_files_are_one_run`, `test_a_folder_scan_is_one_run`, `test_chosen_files_of_a_source_are_one_run`, `test_the_assistant_lists_and_undoes_an_import_with_approval`; [import-runs.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/import-runs.test.tsx) | ## Clip from the browser diff --git a/fastapi_backend/app/api/v1/routes/imports.py b/fastapi_backend/app/api/v1/routes/imports.py index 88173dcfd..87db009b7 100644 --- a/fastapi_backend/app/api/v1/routes/imports.py +++ b/fastapi_backend/app/api/v1/routes/imports.py @@ -1,5 +1,5 @@ """Importing transcripts: an uploaded file (base64 in JSON) or pasted text, and a preview that saves nothing; and -chosen files of a storage source.""" +chosen files of a storage source. A bulk import is one import run, listed and undone whole (import_runs.py).""" from __future__ import annotations @@ -11,16 +11,32 @@ from contextlib import contextmanager from typing import Any -from fastapi import APIRouter, HTTPException, Request +from fastapi import APIRouter, BackgroundTasks, HTTPException, Query, Request -from app.api.deps import Acl, AdminWriter, Cfg, Db, Writer, domain_errors -from app.domain import auth, content_types, convert, ingest, jobs, links, mediaurl, pipelines, sources, store, webcapture +from app.api.deps import Access, Acl, AdminWriter, Cfg, CurrentUser, Db, Principal, Writer, domain_errors +from app.domain import ( + auth, + content_types, + convert, + import_runs, + ingest, + jobs, + links, + mediaurl, + pipelines, + render, + sources, + store, + webcapture, +) from app.domain.store import DB from app.schemas.imports import ( ImportPreview, ImportPreviewRequest, ImportRequest, ImportResult, + ImportRun, + ImportRunUndone, LinkImportResult, LinksImport, LinksImportRequest, @@ -245,11 +261,13 @@ def import_web_links(body: LinksImportRequest, acl: Acl, user: Writer, db: Db, c if len(found) > links.MAX: raise HTTPException(400, f"import {links.MAX} links at most at once; this has {len(found)}") sid = sid if sid is not None else store.ns_id(db, ns) - queued = links.queue(db, cfg, sid, found, user.email, body.collection, body.pipeline, body.folders_as_tags, body.content_type) - out = [LinkImportResult(**r) for r in queued] + said = body.filename or ("pasted links" if (body.text or "").strip() else f"{len(found)} link(s)") + made = links.queue(db, cfg, sid, found, user.email, body.collection, body.pipeline, body.folders_as_tags, body.content_type, said) + out = [LinkImportResult(**r) for r in made] queued = sum(r.status == "queued" for r in out) - auth.audit(db, user.as_audit(), "import.web", f"space:{sid}", {"links": queued, "namespace": ns}) - return LinksImport(results=out) + run = next((r.import_run for r in out if r.import_run), None) + auth.audit(db, user.as_audit(), "import.web", f"space:{sid}", store.clean({"links": queued, "namespace": ns, "import_run": run})) + return LinksImport(results=out, import_run=run) @router.post("/preview") @@ -304,8 +322,63 @@ def import_from_source(body: SourceImportRequest, request: Request, user: AdminW with domain_errors(): results = sources.import_files(db, cfg, body.source, body.paths, sid, user.email, body.pipeline, body.collection, body.documents_as) queued = [r for r in results if r["status"] == "queued"] + run = queued[0].get("import_run") if queued else None if queued: - detail = {"namespace": ns, "files": len(queued), "recordings": [r["recording"] for r in queued]} + detail = {"namespace": ns, "files": len(queued), "recordings": [r["recording"] for r in queued], "import_run": run} auth.audit(db, user.as_audit(), "import.source", f"source:{body.source}", detail) request.app.state.graph_cache.clear() - return SourceImport(results=[SourceImportResult(**r) for r in results]) + return SourceImport(results=[SourceImportResult(**r) for r in results], import_run=run) + + +# ---------- import runs: one bulk import, listed and undone whole ---------- +def _import_run(db: DB, acl: Access, user: Principal, run_id: int) -> dict[str, Any]: + """A run this person may see: theirs, one in a namespace where they may delete assets, or (admins) any; else 404.""" + try: + run = import_runs.get(db, run_id) + except KeyError: + raise HTTPException(404, "not found") from None + if not (user.admin or run.get("created_by") == user.email or acl.can(run.get("space"), "remove")): + raise HTTPException(404, "not found") + return run + + +@router.get("/runs") +def list_import_runs(acl: Acl, user: CurrentUser, db: Db, limit: int = Query(50, ge=1, le=200)) -> list[ImportRun]: + """Recent bulk imports, newest first, each with how many assets it made and how many are still here: the ones you + started and those in namespaces where you may delete assets (everyone's, for admins). A folder scan, an archive's + files, a list of links or bookmarks, and chosen files of a source are each one run.""" + mine = [s for s in acl.roles if acl.can(s, "remove")] + return [ImportRun.model_validate(r) for r in import_runs.listing(db, None if user.admin else user.email, mine, limit)] + + +@router.get("/runs/{run_id}") +def get_import_run(run_id: int, acl: Acl, user: CurrentUser, db: Db) -> ImportRun: + return ImportRun.model_validate(_import_run(db, acl, user, run_id)) + + +@router.post("/runs/{run_id}/undo") +def undo_import_run(run_id: int, acl: Acl, user: Writer, db: Db, cfg: Cfg, request: Request, tasks: BackgroundTasks) -> ImportRunUndone: + """Undo a bulk import: every asset it made that is still here is deleted, as `DELETE /recordings/{id}` deletes + one (the media files stay where they are, and scans don't import them again). It needs the rights a delete needs on + each of them (owners); when any is one you may not delete, nothing is deleted (403). An asset a job is working on + is left (`busy`) and its job asked to stop: undo the run again for the rest. Lens keeps no recently deleted list of + assets, so this can't itself be undone; importing the files again brings them back. Audited as `recording.delete` + for each asset and `import.undo` for the run.""" + run = _import_run(db, acl, user, run_id) + + def can(space: int, collection: int | None) -> bool: + try: + acl.need_in(space, collection, "remove") + except HTTPException: + return False + return True + + refused = import_runs.refused(db, run_id, can) + if refused: + raise HTTPException(403, f"you may not delete {len(refused)} of its {run['left']} assets; undoing it needs an owner") + with domain_errors(): + out = import_runs.undo(db, cfg, run_id, user.as_audit()) + request.app.state.graph_cache.clear() + for ns in out["namespaces"]: + tasks.add_task(render.refresh_overview, db, cfg, ns) + return ImportRunUndone(run=ImportRun.model_validate(import_runs.get(db, run_id)), deleted=out["deleted"], busy=out["busy"]) diff --git a/fastapi_backend/app/api/v1/routes/jobs.py b/fastapi_backend/app/api/v1/routes/jobs.py index f510f3a1a..462cbd8ec 100644 --- a/fastapi_backend/app/api/v1/routes/jobs.py +++ b/fastapi_backend/app/api/v1/routes/jobs.py @@ -131,7 +131,7 @@ def queue_step(step: str, acl: Acl, user: Writer, db: Db, cfg: Cfg) -> StepQueue def scan() -> None: try: - ingest.scan(db, cfg, log=log.info) + ingest.scan(db, cfg, log=log.info, by=user.email) jobs.enqueue_pending(db, by=user.email) except Exception: # noqa: BLE001 - background thread; report and carry on log.exception("scan failed") diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index 2c77e8aad..05f2be8a4 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -2685,6 +2685,7 @@ def _carry_out(db, cfg, a, user, editable, decision, base, admin, readable, via= "create_namespace", "import_files", "import_urls", + "undo_import", "invite_person", "change_person", "subscribe_feed", diff --git a/fastapi_backend/app/domain/convert.py b/fastapi_backend/app/domain/convert.py index 3eaafaedb..cfe9b3786 100644 --- a/fastapi_backend/app/domain/convert.py +++ b/fastapi_backend/app/domain/convert.py @@ -890,7 +890,8 @@ def keep_attachments(db, cfg, rid, attachments, say, archive=False): """An email's attachments (or an archive's files) kept as its files (once: a file it has already isn't added again), and those Lens can read made resources of their own beside it (documents.attachment_resources), queued for the namespace's pipeline. A resource made from an attachment says which email and file it came from (attached_to). - A chat export or JSON among them becomes a chat, records or a transcript; an archive in an archive stays a file.""" + A chat export or JSON among them becomes a chat, records or a transcript; an archive in an archive stays a file. + An archive and the resources made of its files are one import run (import_runs.archive).""" from . import archives, documents, files rec = db.one("SELECT space, collection, recorded_at FROM $r", r=R("recording", rid)) or {} @@ -928,6 +929,12 @@ def keep_attachments(db, cfg, rid, attachments, say, archive=False): say(f"only its first {MAX_ATTACHMENTS} {what}s were kept") if attachments: say(f"{len(attachments)} {what}(s): {kept} kept now, {made} made resource(s) of their own") + if archive: # the archive and the resources made of its files are one import run, undone whole + from . import import_runs + + run = import_runs.archive(db, rid) + if run and made: + say(f"import run {run}: undoing it deletes the archive and the resources made of its files") def _held(name, data): diff --git a/fastapi_backend/app/domain/deletion.py b/fastapi_backend/app/domain/deletion.py index 58fbd0a53..fde473991 100644 --- a/fastapi_backend/app/domain/deletion.py +++ b/fastapi_backend/app/domain/deletion.py @@ -185,10 +185,11 @@ def delete(db, cfg, rid, by=None): + [ "CREATE $g CONTENT $note", "DELETE meta_edit WHERE target = $target", - "UPDATE ip_group SET recordings = array::complement(recordings, [$r]) WHERE recordings CONTAINS $r", - "UPDATE saved_collection SET recordings = array::complement(recordings, [$r]) WHERE recordings CONTAINS $r", - "UPDATE chat SET scope.recordings = array::complement(scope.recordings, [$r]) WHERE scope.recordings CONTAINS $r", - "UPDATE batch SET recordings = array::complement(recordings, [$r]) WHERE recordings CONTAINS $r AND started CONTAINSNOT $r", + "UPDATE ip_group SET recordings = array::complement(recordings ?? [], [$r]) WHERE recordings CONTAINS $r", + "UPDATE saved_collection SET recordings = array::complement(recordings ?? [], [$r]) WHERE recordings CONTAINS $r", + "UPDATE chat SET scope.recordings = array::complement(scope.recordings ?? [], [$r]) WHERE scope.recordings CONTAINS $r", + "UPDATE batch SET recordings = array::complement(recordings ?? [], [$r])" + " WHERE recordings CONTAINS $r AND started CONTAINSNOT $r", "UPDATE resource_file SET resource = NONE WHERE resource = $r", # an email's attachment that this was "DELETE $rec", ], diff --git a/fastapi_backend/app/domain/import_runs.py b/fastapi_backend/app/domain/import_runs.py new file mode 100644 index 000000000..cd38f8043 --- /dev/null +++ b/fastapi_backend/app/domain/import_runs.py @@ -0,0 +1,180 @@ +"""Import runs: one bulk import is one run, so a mistaken one is undone whole (docs/api.md#import-runs). + +A folder scan that finds new files (`scan`), the files of a zip or tar archive made resources (`archive`), a list of +links or a bookmarks file captured as web pages (`links`) and chosen files of a storage source (`source`) each make +one `import_run`: what started it (`kind`), who (`created_by`), when, the namespace (`space`), a line saying what it +read (`source`) and how many assets it made (`assets`). Every asset it made carries the run's id (`import_run` on the +recording). Single imports (one upload, one pasted transcript, one web page) make no run, and nor did imports from +before runs existed: those assets have no `import_run` and nothing changes for them. + +Undoing a run deletes every asset of it still here through the normal deletion path (deletion.py), so each goes +exactly as a delete takes it, needing the same rights: the remove permission in its namespace (owners), or admin of +its collection. When any asset is one the person may not delete, nothing is deleted. An asset a job is working on is +left and its job asked to stop; undoing the run again takes the rest. Each deletion is audited as `recording.delete` +and the undo as `import.undo`. Lens keeps no recently deleted list of assets, so an undo can't itself be undone: the +media files stay where they were, and importing them again on purpose brings them back. + +A batch run (batches.py) is something else: steps run over assets that are already here, so runs are kept apart. +""" + +from __future__ import annotations + +from . import auth, deletion, hierarchy, roles, store + +R = store.R +KINDS = ("scan", "archive", "links", "source") +FIELDS = "record::id(id) AS id, kind, space, source, created_by, created_at, assets, undone_at, undone_by, undone, busy" + + +def mark(db): + """The last recording id handed out so far: what an import makes from here on has a higher one (`record`'s `after`).""" + got = db.values("SELECT VALUE n FROM $r", r=R("seq", "recording")) + return int(got[0] or 0) if got else 0 + + +def record(db, kind, space, ids, by=None, source=None, after=None): + """A run of `kind` in namespace `space` that made the recordings `ids`: its id, or None when it made nothing. With + `after` (from `mark`, taken before the import), only recordings made since count: an import that answered with an + asset that was already here (the same file) never puts that asset in its run.""" + ids = list(dict.fromkeys(int(i) for i in ids if i is not None and (after is None or int(i) > after))) + if not ids: + return None + run = db.next_id("import_run") + db.q( + "CREATE $r CONTENT $d", + r=R("import_run", run), + d=store.clean( + { + "kind": kind, + "space": space, + "source": str(source)[:300] if source else None, + "created_by": by, + "created_at": store.now(), + "assets": 0, # counted as they're tagged + } + ), + ) + tag(db, run, ids) + return run + + +def tag(db, run, ids): + """Mark recordings as made by run `run` (the ones it has already are left as they are).""" + ids = [R("recording", int(i)) for i in dict.fromkeys(ids)] + if not ids: + return 0 + new = db.values("SELECT VALUE record::id(id) FROM recording WHERE id IN $ids AND (import_run ?? 0) != $run", ids=ids, run=int(run)) + if new: + db.q("UPDATE recording SET import_run = $run WHERE id IN $ids", ids=[R("recording", i) for i in new], run=int(run)) + db.q("UPDATE $r SET assets = (assets ?? 0) + $n", r=R("import_run", int(run)), n=len(new)) + return len(new) + + +def archive(db, rid, by=None, title=None): + """An archive's files made resources of their own (convert.keep_attachments): they and the archive are one run, + the archive's own when it came in with one (a source's files), else a new `archive` run. None when it made none.""" + made = db.values("SELECT VALUE record::id(id) FROM recording WHERE attached_to.resource = $r", r=int(rid)) + if not made: + return None + rec = db.one("SELECT space, title, import_run FROM $r", r=R("recording", int(rid))) or {} + if rec.get("import_run"): + tag(db, rec["import_run"], made) + return rec["import_run"] + if by is None: # whoever brought the archive in: the person whose upload's job it is + first = db.one("SELECT created_by, created_at FROM job WHERE recording = $r ORDER BY created_at LIMIT 1", r=int(rid)) or {} + by = first.get("created_by") if "@" in str(first.get("created_by") or "") else None + return record(db, "archive", rec.get("space"), [int(rid), *made], by, title or rec.get("title")) + + +def _left(db, ids): + """{run: how many of its assets are still here}.""" + if not ids: + return {} + rows = db.rows("SELECT import_run, count() AS n FROM recording WHERE import_run IN $ids GROUP BY import_run", ids=list(ids)) + return {r["import_run"]: r["n"] for r in rows} + + +def _view(db, row, left, names): + n = left.get(row["id"], 0) + status = ("undone" if not n else "partly undone") if row.get("undone_at") else "imported" + return {**row, "namespace": names.get(row.get("space")), "left": n, "status": status} + + +def get(db, run): + """One run, with how many of its assets are still here (`left`) and its status. KeyError when there is none.""" + row = db.one(f"SELECT {FIELDS} FROM $r", r=R("import_run", int(run))) + if not row: + raise KeyError(run) + return _view(db, row, _left(db, [row["id"]]), store.space_names(db)) + + +def listing(db, by=None, spaces=None, limit=50): + """Recent runs, newest first: everyone's (`by` None), else those `by` started or made in `spaces`.""" + where = "" if by is None else " WHERE created_by = $e OR space IN $sp" + rows = db.rows( + f"SELECT {FIELDS} FROM import_run{where} ORDER BY created_at DESC, id DESC LIMIT $n", + e=by, + sp=sorted(spaces or ()), + n=int(limit), + ) + left, names = _left(db, [r["id"] for r in rows]), store.space_names(db) + return [_view(db, r, left, names) for r in rows] + + +def members(db, run): + """The assets of the run still here: [{id, space, collection}], an archive's files before the archive.""" + rows = db.rows("SELECT record::id(id) AS id, space, collection, attached_to FROM recording WHERE import_run = $run", run=int(run)) + rows.sort(key=lambda r: (r.get("attached_to") is None, r["id"])) + return [{k: r.get(k) for k in ("id", "space", "collection")} for r in rows] + + +def may_remove(db, user): + """can(space, collection): whether this person may delete an asset there, as a delete checks it + (deps.Access.need_in): the remove permission of their namespace role, or a collection role that ranks as one.""" + perms = auth.permissions(db, user) + colls = {} if user.get("admin") else hierarchy.roles_of(db, user.get("id")) + need = auth.ROLES[roles.minimum("remove")] + + def can(space, collection): + return "remove" in perms.get(space, ()) or hierarchy.ROLES.get((colls.get(space) or {}).get(collection) or "", 0) >= need + + return can + + +def refused(db, run, can): + """The assets of the run this person may not delete (can(space, collection), as may_remove gives it).""" + return [m["id"] for m in members(db, run) if not can(m["space"], m.get("collection"))] + + +def undo(db, cfg, run, who): + """Delete every asset of the run still here (`who`: the person, {id, email}, as the audit log keeps them), once the + caller has checked they may (`refused`). Returns {run, deleted, busy, namespaces}: busy are the assets a job is + working on, left for now with their jobs asked to stop.""" + deleted, busy, spaces = [], [], set() + for m in members(db, run): + try: + gone = deletion.delete(db, cfg, m["id"], who.get("email")) + except KeyError: + continue # deleted meanwhile + except deletion.Running: + db.q( + "UPDATE job SET cancel_requested = true, updated_at = $t WHERE recording = $r AND status = 'running'", + r=m["id"], + t=store.now(), + ) + busy.append(m["id"]) + continue + auth.audit(db, who, "recording.delete", f"recording:{m['id']}", {**gone, "import_run": int(run)}) + deleted.append(m["id"]) + if gone.get("namespace"): + spaces.add(gone["namespace"]) + db.q( + "UPDATE $r SET undone_at = $t, undone_by = $by, undone = (undone ?? 0) + $n, busy = $busy", + r=R("import_run", int(run)), + t=store.now(), + by=who.get("email"), + n=len(deleted), + busy=len(busy), + ) + auth.audit(db, who, "import.undo", f"import_run:{int(run)}", {"deleted": len(deleted), "busy": len(busy), "recordings": deleted}) + return {"run": int(run), "deleted": deleted, "busy": busy, "namespaces": sorted(spaces)} diff --git a/fastapi_backend/app/domain/ingest.py b/fastapi_backend/app/domain/ingest.py index c1e43c6ec..991238263 100644 --- a/fastapi_backend/app/domain/ingest.py +++ b/fastapi_backend/app/domain/ingest.py @@ -236,8 +236,9 @@ def envelope(x, bins=1600): return bytes((np.clip(np.sqrt(rms / peak), 0, 1) * 255).astype(np.uint8)) -def scan(db, cfg, only=None, log=print): - from . import deletion +def scan(db, cfg, only=None, log=print, by=None): + """Import the new audio in each namespace's folders; what one namespace's scan finds is one import run (import_runs.py).""" + from . import deletion, import_runs exts = {e.lower() for e in cfg["audio"]["extensions"]} stats = {"new": 0, "known": 0, "changed": 0, "duplicate": 0, "deleted": 0} @@ -247,6 +248,7 @@ def scan(db, cfg, only=None, log=print): nid = store.ns_id(db, name) home = store.default_collection(db, nid) # where new files go gone_paths, gone_fps = deletion.gone(db, nid) # recordings someone deleted stay deleted + made = [] for root in spec["paths"]: rootp = pathlib.Path(root) if not rootp.exists(): @@ -318,6 +320,8 @@ def scan(db, cfg, only=None, log=print): ), ) stats["new"] += 1 + made.append(rid) + import_runs.record(db, "scan", nid, made, by, "scan of " + ", ".join(spec["paths"])) return stats diff --git a/fastapi_backend/app/domain/links.py b/fastapi_backend/app/domain/links.py index 0747a9bef..8a937715d 100644 --- a/fastapi_backend/app/domain/links.py +++ b/fastapi_backend/app/domain/links.py @@ -13,7 +13,7 @@ import re from html.parser import HTMLParser -from . import content_types, jobs, library, webcapture +from . import content_types, import_runs, jobs, library, webcapture MAX = 500 # links in one import, at most URL = re.compile(r"https?://[^\s<>\"']+", re.I) @@ -106,11 +106,13 @@ def tags(folders, most=5, width=40): return [" ".join(f.split())[:width].strip() for f in folders[-most:] if f.strip()] -def queue(db, cfg, sid, found, by, collection=None, pipeline=None, folders_as_tags=True, content_type=None): +def queue(db, cfg, sid, found, by, collection=None, pipeline=None, folders_as_tags=True, content_type=None, source=None): """Capture each of `found` (from `read`) in namespace `sid` as a web page document (of `content_type`, a text one, - when it's chosen): [{url, status, recording, job, detail}], status queued, already (captured in the namespace - before) or skipped (detail says why). The caller checks the person may add to the namespace and the collection is - one of its own.""" + when it's chosen): [{url, status, recording, job, import_run, detail}], status queued, already (captured in the + namespace before) or skipped (detail says why). The pages queued are one import run (import_runs.py; `source` says + what the links came from), which undoes them whole. The caller checks the person may add to the namespace and the + collection is one of its own.""" + since = import_runs.mark(db) have = { (r.get("web") or {}).get("url"): r["id"] for r in db.rows("SELECT record::id(id) AS id, web FROM recording WHERE space = $s AND web != NONE", s=sid) @@ -136,4 +138,6 @@ def queue(db, cfg, sid, found, by, collection=None, pipeline=None, folders_as_ta content_types.choose(db, rid, content_type) job = jobs.enqueue(db, rid, None, by=by, pipeline=pipeline) out.append({"url": url, "status": "queued", "recording": rid, "job": job}) - return out + made = [r["recording"] for r in out if r["status"] == "queued"] + run = import_runs.record(db, "links", sid, made, by, source or f"{len(found)} link(s)", since) + return [{**r, "import_run": run} if r["status"] == "queued" else r for r in out] diff --git a/fastapi_backend/app/domain/moving.py b/fastapi_backend/app/domain/moving.py index 15e691e3b..7a428cbfc 100644 --- a/fastapi_backend/app/domain/moving.py +++ b/fastapi_backend/app/domain/moving.py @@ -259,7 +259,7 @@ def _move(db, cfg, rid, dst, rec, names, home, rediarize, revoke_shares, by): "DELETE entity_override WHERE recording = $r", # and so do topics: the recording is about the new namespace's topics, once someone says which "DELETE topic_about WHERE recording = $r", - "UPDATE ip_group SET recordings = array::complement(recordings, [$r]) WHERE space = $s AND recordings CONTAINS $r", + "UPDATE ip_group SET recordings = array::complement(recordings ?? [], [$r]) WHERE space = $s AND recordings CONTAINS $r", "CREATE $g CONTENT $note", "UPDATE $rec SET space = $d, collection = $home" + (", fp_key = $k" if fp else ""), ], diff --git a/fastapi_backend/app/domain/ops_tools.py b/fastapi_backend/app/domain/ops_tools.py index bf5b81b10..33ab2dfb5 100644 --- a/fastapi_backend/app/domain/ops_tools.py +++ b/fastapi_backend/app/domain/ops_tools.py @@ -2,7 +2,7 @@ dropped into the conversation. Admins get the server tools; anyone gets `import_files` for their own attachments and `import_urls` for web pages, -into namespaces they edit. A change +into namespaces they edit, and `import_runs` and `undo_import` for their bulk imports (import_runs.py). A change waits for the person's approval, unless the conversation lets the assistant act (a setup conversation does: the admin asked it to set things up): then it's made at once, audited, and said in the answer, as long as the answer hasn't read text from outside Lens (Toolbox.wary), which could be what asked for it. Telemetry, which sends data @@ -18,6 +18,7 @@ convert, decide, feeds, + import_runs, jobs, links, llm, @@ -77,8 +78,8 @@ # machines ALWAYS_ASK = {"telemetry", "analytics", "sensors", "shares", "files", "web", "secrets", "scaling"} # who gets in always waits for the admin's approval: a person, or a machine joining the mesh; and so does taking a -# machine off it, which stops what runs there -ASK_TOOLS = {"invite_person", "change_person", "add_mesh_node", "remove_mesh_node", "let_go_scaling_stray"} +# machine off it, which stops what runs there; and deleting a whole import +ASK_TOOLS = {"invite_person", "change_person", "add_mesh_node", "remove_mesh_node", "let_go_scaling_stray", "undo_import"} # within a scaling group's approved terms (its min, max and budget), or a change to the mesh that's undone as easily # (renaming a node, the namespaces it serves, bringing a removed one back, withdrawing a token), so made at once # unless the answer read outside text @@ -387,6 +388,24 @@ [], True, ), + ( + "import_runs", + "Recent bulk imports (a folder scan, an archive's files, a list of links or bookmarks, a source's chosen " + "files): each run's id, kind, namespace, what it read, who started it and when, how many assets it made and how " + "many are still here. undo_import undoes one.", + {"limit": {"type": "integer", "description": "how many, newest first (default 20)"}}, + [], + False, + ), + ( + "undo_import", + "Undo a bulk import whole: every asset it made that is still here is deleted, as deleting each one would. " + "It needs owner rights on each, always waits for the person's yes, and can't itself be undone (importing the " + "files again brings them back).", + {"run": {"type": "integer", "description": "the run's id, from import_runs"}}, + ["run"], + True, + ), ] LIST_BYTES = 5 * 2**20 # an attached list of links is read up to this @@ -499,6 +518,12 @@ def apply(db, cfg, base, tool, args, user, editable, admin, via="agent"): if not admin: raise PermissionError("only admins manage people") return _people(db, cfg, tool, args, user) + if tool == "undo_import": + run = _import_run(db, user, admin, args["run"]) + if import_runs.refused(db, run["id"], import_runs.may_remove(db, {"id": user.get("id"), "admin": admin})): + raise PermissionError("undoing it needs an owner: some of its assets aren't yours to delete") + out = import_runs.undo(db, cfg, run["id"], {"id": user.get("id"), "email": user.get("email")}) + return {"run": run["id"], "deleted": len(out["deleted"]), "busy": len(out["busy"])} if tool == "import_urls": ns = args["namespace"].strip().lower() try: @@ -511,7 +536,16 @@ def apply(db, cfg, base, tool, args, user, editable, admin, via="agent"): raise PermissionError(f"you can't add to {ns}") if args.get("collection_id") is not None: store.home(db, sid, args["collection_id"]) - out = links.queue(db, cfg, sid, args["links"], user["email"], args.get("collection_id"), content_type=args.get("content_type")) + out = links.queue( + db, + cfg, + sid, + args["links"], + user["email"], + args.get("collection_id"), + content_type=args.get("content_type"), + source="the assistant's list", + ) queued = sum(r["status"] == "queued" for r in out) auth.audit(db, user, "import.web", f"space:{sid}", {"links": queued, "namespace": ns, "assistant": True}) return {"namespace": ns, "pages": out} @@ -582,6 +616,22 @@ def people(db): ] +def _owned(db, user, admin): + """The namespaces where this person may delete assets.""" + return [s for s, p in auth.permissions(db, {"id": user.get("id"), "admin": admin}).items() if "remove" in p] + + +def _import_run(db, user, admin, run): + """A run this person may see (theirs, one in a namespace where they may delete, or any for admins).""" + try: + r = import_runs.get(db, int(run)) + except (KeyError, TypeError, ValueError): + raise ValueError("there's no import run with that id: import_runs lists them") from None + if not (admin or r.get("created_by") == user.get("email") or r.get("space") in _owned(db, user, admin)): + raise ValueError("there's no import run with that id: import_runs lists them") + return r + + def _space(db, ns): try: return store.ns_id(db, (ns or "").strip().lower(), create=False) @@ -992,6 +1042,21 @@ def t_import_files(self, upload_ids, namespace=None, collection_id=None, content out = {**out, "namespace_chosen": {"by": chosen["by"], "confidence": chosen["confidence"]}} return out, said + def t_import_runs(self, limit=20): + mine = _owned(self.db, self.user, self.admin) + runs = import_runs.listing(self.db, None if self.admin else self.user.get("email"), mine, min(max(int(limit or 20), 1), 100)) + keep = ("id", "kind", "namespace", "source", "created_by", "created_at", "assets", "left", "status") + return {"runs": [{k: r.get(k) for k in keep} for r in runs]}, f"Listed {len(runs)} import run(s)" + + def t_undo_import(self, run): + r = _import_run(self.db, self.user, self.admin, run) + if not r["left"]: + return {"run": r["id"], "status": r["status"], "left": 0}, f"Import run {r['id']} has nothing left to undo" + if import_runs.refused(self.db, r["id"], import_runs.may_remove(self.db, {"id": self.user.get("id"), "admin": self.admin})): + raise PermissionError("undoing it needs an owner: some of its assets aren't yours to delete") + what = f" ({r['source']})" if r.get("source") else "" + return self._change("undo_import", {"run": r["id"]}, f"Undo import run {r['id']}{what}: delete its {r['left']} asset(s)") + def t_import_urls(self, urls=None, text=None, upload_id=None, namespace=None, collection_id=None, content_type=None): if isinstance(urls, str): urls = [urls] diff --git a/fastapi_backend/app/domain/sources.py b/fastapi_backend/app/domain/sources.py index 4ce89a946..31d1a7860 100644 --- a/fastapi_backend/app/domain/sources.py +++ b/fastapi_backend/app/domain/sources.py @@ -864,9 +864,13 @@ def import_files(db, cfg, sid, paths, space, by, pipeline=None, collection=None, watched: audio, video, documents and images stay on the source and run the pipeline (the namespace's, or `pipeline`); transcripts are imported, and PDFs, Word and text files too when `documents_as` is 'transcript'. One result per path: - queued (with the recording and job), already (it's a recording of the namespace from this source), skipped (a - kind of file Lens doesn't import; a folder) or error. Choosing a file on purpose brings back one deleted before.""" + queued (with the recording, job and import run), already (it's a recording of the namespace from this source), + skipped (a kind of file Lens doesn't import; a folder) or error. Choosing a file on purpose brings back one deleted + before. The files queued are one import run (import_runs.py), which undoes them whole.""" + from . import import_runs + src = get(db, sid) + since = import_runs.mark(db) folders = {} for path in paths: p = check_path(cfg, src, path) @@ -897,7 +901,9 @@ def import_files(db, cfg, sid, paths, space, by, pipeline=None, collection=None, results.append(store.clean({"path": p, "status": "queued" if job else "already", "recording": rid, "job": job})) except Exception as e: # noqa: BLE001 - one bad file must not stop the rest results.append({"path": p, "status": "error", "detail": f"{type(e).__name__}: {e}"[:300]}) - return results + made = [r["recording"] for r in results if r["status"] == "queued" and r["recording"] > since] + run = import_runs.record(db, "source", space, made, by, f"{src['name']}: {len(made)} chosen file(s)", since) + return [{**r, "import_run": run} if r["status"] == "queued" and r["recording"] in made else r for r in results] def poll_due(db, cfg, log=print): diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index dca1c44d5..bbd383112 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -1810,6 +1810,10 @@ def close(self): "DEFINE INDEX IF NOT EXISTS saved_collection_owner ON saved_collection FIELDS account", "DEFINE TABLE IF NOT EXISTS batch SCHEMALESS", "DEFINE INDEX IF NOT EXISTS job_batch ON job FIELDS batch", + # one bulk import (a folder scan, an archive's files, a list of links, a source's files), undone whole (import_runs.py) + "DEFINE TABLE IF NOT EXISTS import_run SCHEMALESS", + "DEFINE INDEX IF NOT EXISTS import_run_created ON import_run FIELDS created_at", + "DEFINE INDEX IF NOT EXISTS recording_import_run ON recording FIELDS import_run", "DEFINE TABLE IF NOT EXISTS approval SCHEMALESS", "DEFINE INDEX IF NOT EXISTS approval_chat ON approval FIELDS chat", # the inbox (approvals.py): a person's waiting approvals, an automation's (capped), and the expiry sweep diff --git a/fastapi_backend/app/schemas/imports.py b/fastapi_backend/app/schemas/imports.py index 69710dadf..512663b00 100644 --- a/fastapi_backend/app/schemas/imports.py +++ b/fastapi_backend/app/schemas/imports.py @@ -106,11 +106,13 @@ class LinkImportResult(ResponseModel): ) recording: int | None = None job: int | None = None + import_run: int | None = Field(None, description="the import run it is part of, which undoes the whole import") detail: str | None = None class LinksImport(ResponseModel): results: list[LinkImportResult] + import_run: int | None = Field(None, description="the import run of the pages queued (POST /import/runs/{id}/undo undoes it)") class SourceImportResult(ResponseModel): @@ -118,8 +120,37 @@ class SourceImportResult(ResponseModel): status: Literal["queued", "already", "skipped", "error"] recording: int | None = None job: int | None = None + import_run: int | None = Field(None, description="the import run it is part of, which undoes the whole import") detail: str | None = None class SourceImport(ResponseModel): results: list[SourceImportResult] + import_run: int | None = Field(None, description="the import run of the files queued (POST /import/runs/{id}/undo undoes it)") + + +ImportRunKind = Literal["scan", "archive", "links", "source"] + + +class ImportRun(ResponseModel): + """One bulk import: a folder scan, an archive's files, a list of links or bookmarks, or chosen files of a source.""" + + id: int + kind: ImportRunKind = Field(description="what made it: scan, archive, links or source") + namespace: str | None = None + source: str | None = Field(None, description="what it read: the folders, the archive, the links' file, the source") + created_by: str | None = Field(None, description="who started it (none: a scan from the command line)") + created_at: str + assets: int = Field(0, description="how many assets it made") + left: int = Field(0, description="how many of them are still here") + status: Literal["imported", "partly undone", "undone"] = Field( + description="imported; partly undone (some assets were busy: undo it again for the rest); undone" + ) + undone_at: str | None = None + undone_by: str | None = None + + +class ImportRunUndone(ResponseModel): + run: ImportRun + deleted: list[int] = Field(description="the assets deleted") + busy: list[int] = Field(description="assets a job is working on, left for now: their jobs were asked to stop") diff --git a/fastapi_backend/app/schemas/recordings.py b/fastapi_backend/app/schemas/recordings.py index ad8ccaa1b..abf3a059d 100644 --- a/fastapi_backend/app/schemas/recordings.py +++ b/fastapi_backend/app/schemas/recordings.py @@ -375,6 +375,7 @@ class Recording(ResponseModel): attached_to: AttachedTo | None = Field(None, description="an email's attachment made a resource of its own: that email") converted_from: ConvertedFrom | None = Field(None, description="made by a convert step: the asset it was made from") web: WebPage | None = Field(None, description="a web page captured as a document") + import_run: int | None = Field(None, description="the bulk import that made it (GET /import/runs/{id}), which undoes it whole") enrichment: Enrichment | None = Field(None, description="the places and dates anytopdf found (analysis.anytopdf)") recorded_at: str | None = Field( None, diff --git a/fastapi_backend/tests/api/test_import_runs.py b/fastapi_backend/tests/api/test_import_runs.py new file mode 100644 index 000000000..a144e58a9 --- /dev/null +++ b/fastapi_backend/tests/api/test_import_runs.py @@ -0,0 +1,184 @@ +"""Import runs (import_runs.py, /import/runs): a bulk import is one run, listed with its counts and undone whole, +through the normal deletion path and its rights, leaving every other asset alone.""" + +from __future__ import annotations + +import json + +import pytest + +from app.domain import convert, import_runs, ingest, settings, store +from tests.api._assist import Assist, sse, start_llm +from tests.api.test_anytopdf import here, lean # noqa: F401 - fixtures: the fake anytopdf makes an archive's page +from tests.api.test_documents import _upload +from tests.helpers import drain, login, make_user, quiet, write_wav + +R = store.R + + +@pytest.fixture +def web(monkeypatch): + monkeypatch.setattr(convert, "chromium", lambda cfg: "/usr/bin/chromium") # web pages are queued, never captured here + + +@pytest.fixture +def env(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + make_user(db, "own@x.io", "owner password 1", roles={"pods": "owner"}) + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + make_user(db, "other@x.io", "other password 1", roles={"calls": "owner"}) + return { + "ha": login(client, "root@x.io", "root password 1"), + "ho": login(client, "own@x.io", "owner password 1"), + "he": login(client, "ed@x.io", "editor password 1"), + "hx": login(client, "other@x.io", "other password 1"), + } + + +def _links(client, h, *urls): + r = client.post("/api/v1/import/web/links", headers=h, json={"namespace": "pods", "text": "\n".join(urls)}) + assert r.status_code == 200, r.text + return r.json() + + +def _ids(db): + return set(db.values("SELECT VALUE record::id(id) FROM recording")) + + +def test_a_link_import_is_one_run_undone_whole(web, client, env, db): + ho, he, hx = env["ho"], env["he"], env["hx"] + first = _links(client, ho, "https://8.8.8.8/a", "https://8.8.4.4/b") + second = _links(client, he, "https://1.1.1.1/c") + run, other = first["import_run"], second["import_run"] + assert run and other and run != other + mine = {r["recording"] for r in first["results"]} + assert all(r["import_run"] == run for r in first["results"]) + assert set(db.values("SELECT VALUE record::id(id) FROM recording WHERE import_run = $r", r=run)) == mine + assert client.get(f"/api/v1/recordings/{min(mine)}", headers=ho).json()["import_run"] == run + + listed = client.get("/api/v1/import/runs", headers=ho).json() # an owner of pods sees both, newest first + assert [(x["id"], x["kind"], x["namespace"], x["assets"], x["left"], x["status"]) for x in listed] == [ + (other, "links", "pods", 1, 1, "imported"), + (run, "links", "pods", 2, 2, "imported"), + ] + assert listed[1]["created_by"] == "own@x.io" and listed[1]["source"] == "pasted links" + assert [x["id"] for x in client.get("/api/v1/import/runs", headers=he).json()] == [other] # an editor: their own + assert client.get("/api/v1/import/runs", headers=hx).json() == [] + assert client.get(f"/api/v1/import/runs/{run}", headers=hx).status_code == 404 + + # without the rights a delete needs, nothing goes: an editor who started the run, someone of another namespace + r = client.post(f"/api/v1/import/runs/{other}/undo", headers=he) + assert r.status_code == 403 and "owner" in r.json()["detail"], r.text + assert client.post(f"/api/v1/import/runs/{run}/undo", headers=hx).status_code == 404 + assert len(_ids(db)) == 3 + + before = _ids(db) + r = client.post(f"/api/v1/import/runs/{run}/undo", headers=ho) + assert r.status_code == 200, r.text + out = r.json() + assert set(out["deleted"]) == mine and out["busy"] == [] + assert (out["run"]["left"], out["run"]["status"], out["run"]["undone_by"]) == (0, "undone", "own@x.io") + assert _ids(db) == before - mine # exactly the run's assets: the other run's page stays + assert not db.values("SELECT VALUE id FROM job WHERE recording IN $r AND status = 'queued'", r=sorted(mine)) + actions = db.rows("SELECT action, target, detail FROM audit_log WHERE action IN ['recording.delete', 'import.undo']") + assert sorted(a["target"] for a in actions if a["action"] == "recording.delete") == sorted(f"recording:{i}" for i in mine) + assert [a["detail"]["deleted"] for a in actions if a["action"] == "import.undo"] == [2] + # undoing it again finds nothing left; the other run is still there to undo + assert client.post(f"/api/v1/import/runs/{run}/undo", headers=ho).json()["deleted"] == [] + assert client.post(f"/api/v1/import/runs/{other}/undo", headers=ho).json()["deleted"] == [second["results"][0]["recording"]] + + +def test_an_uploaded_archive_and_its_files_are_one_run(lean, here, client, env, db, cfg): # noqa: F811 - the fixtures + from tests.api.test_archives import zip_of + + ho, he = env["ho"], env["he"] + alone = _upload(client, ho, b"Harbour log. Nothing to undo here.\n", "log.txt") + up = _upload(client, ho, zip_of({"notes.txt": "Tide notes. High water at noon.", "plan.md": "# Plan\n\nLamps."}), "export.zip") + drain(db, settings.effective(db, cfg)) + rid = up["recording"] + rec = db.one("SELECT status, error FROM $r", r=R("recording", rid)) + assert rec["status"] == "analyzed", rec + made = set(db.values("SELECT VALUE record::id(id) FROM recording WHERE attached_to.resource = $r", r=rid)) + assert len(made) == 2 + run = db.one("SELECT import_run FROM $r", r=R("recording", rid))["import_run"] + assert run and set(db.values("SELECT VALUE record::id(id) FROM recording WHERE import_run = $r", r=run)) == made | {rid} + assert not db.one("SELECT import_run FROM $r", r=R("recording", alone["recording"])).get("import_run") # one upload: no run + got = client.get(f"/api/v1/import/runs/{run}", headers=ho).json() + assert (got["kind"], got["source"], got["assets"], got["created_by"]) == ("archive", "export", 3, "own@x.io") + + assert client.post(f"/api/v1/import/runs/{run}/undo", headers=he).status_code == 404 # not theirs, and not an owner + out = client.post(f"/api/v1/import/runs/{run}/undo", headers=ho).json() + assert set(out["deleted"]) == made | {rid} and out["deleted"][-1] == rid # the files, then the archive + assert _ids(db) == {alone["recording"]} + + +def test_a_folder_scan_is_one_run(db, cfg, folder): + audio = folder / "audio" + audio.mkdir() + for name, seconds in (("a.wav", 1.0), ("b.wav", 1.5)): + write_wav(audio / name, seconds=seconds) + cfg["namespaces"]["pods"]["paths"] = [str(audio)] + assert ingest.scan(db, cfg, only="pods", log=quiet, by="root@x.io")["new"] == 2 + (run,) = import_runs.listing(db) + assert (run["kind"], run["assets"], run["left"], run["created_by"]) == ("scan", 2, 2, "root@x.io") + assert run["source"] == f"scan of {audio}" + assert ingest.scan(db, cfg, only="pods", log=quiet)["new"] == 0 + assert len(import_runs.listing(db)) == 1 # a scan that finds nothing new makes no run + + +def test_chosen_files_of_a_source_are_one_run(client, env, db, folder, monkeypatch): + from tests.api.test_sources import _lsjson_without_rclone + + _lsjson_without_rclone(monkeypatch) + ha, ho = env["ha"], env["ho"] + inbox = folder / "inbox" / "talks" + inbox.mkdir(parents=True) + for name, seconds in (("ep1.wav", 1.0), ("ep2.wav", 1.5)): + write_wav(inbox / name, seconds=seconds) + sid = client.post("/api/v1/sources", json={"name": "inbox", "type": "local"}, headers=ha).json()["id"] + paths = [str(inbox / "ep1.wav"), str(inbox / "ep2.wav")] + r = client.post("/api/v1/import/source", json={"source": sid, "paths": paths, "namespace": "pods"}, headers=ha).json() + run = r["import_run"] + assert run and [x["import_run"] for x in r["results"]] == [run, run] + got = client.get(f"/api/v1/import/runs/{run}", headers=ho).json() # an owner of pods sees an admin's run there + assert (got["kind"], got["assets"], got["source"], got["created_by"]) == ("source", 2, "inbox: 2 chosen file(s)", "root@x.io") + # chosen again: already there, so no new run, and nothing already here joins one + again = client.post("/api/v1/import/source", json={"source": sid, "paths": paths, "namespace": "pods"}, headers=ha).json() + assert again["import_run"] is None and [x["status"] for x in again["results"]] == ["already", "already"] + since = import_runs.mark(db) + old = r["results"][0]["recording"] + assert import_runs.record(db, "source", store.ns_id(db, "pods"), [old], after=since) is None + + +@pytest.fixture +def llm(cfg): + from tests import fake_llm + + srv = start_llm(cfg) + yield fake_llm.Handler + srv.shutdown() + + +def test_the_assistant_lists_and_undoes_an_import_with_approval(web, llm, app, db, cfg, folder, new_client): + from tests.api.test_setup_assistant import call + + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["admin"] + r = c.post("/api/v1/import/web/links", headers=h, json={"namespace": "pods", "text": "https://8.8.8.8/a\nhttps://8.8.4.4/b"}) + run = r.json()["import_run"] + llm.tool_script = [ + {"content": "", "tool_calls": [call(1, "import_runs", {})]}, + {"content": "", "tool_calls": [call(2, "undo_import", {"run": run})]}, + {"content": "I asked to undo it."}, + ] + cid = c.post("/api/v1/chats", headers=h, json={"kind": "setup"}).json()["id"] # even in setup, it waits for a yes + ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "undo that import"}).text) + said = llm.seen[-2]["messages"][-1]["content"] + listed = json.loads(said[said.index("{") : said.rindex("}") + 1])["runs"] + assert [(x["id"], x["left"]) for x in listed] == [(run, 2)] + appr = ev["approval"][0] + assert appr["summary"] == f"Undo import run {run} (pasted links): delete its 2 asset(s)" + assert import_runs.get(db, run)["left"] == 2 # nothing until approved + done = c.post(f"/api/v1/approvals/{appr['id']}", headers=h, json={"decision": "approve"}).json() + assert (done["status"], done["deleted"]) == ("done", 2), done + assert import_runs.get(db, run)["status"] == "undone" diff --git a/fastapi_backend/tests/api/test_mail_calendar_sources.py b/fastapi_backend/tests/api/test_mail_calendar_sources.py index 22064af60..146dd1635 100644 --- a/fastapi_backend/tests/api/test_mail_calendar_sources.py +++ b/fastapi_backend/tests/api/test_mail_calendar_sources.py @@ -269,7 +269,9 @@ def test_a_calendar_feed_as_a_source(feed, client, db, cfg, admin): # feed firs # chosen on purpose, from the listing: already there r = client.post("/api/v1/import/source", headers=h, json={"source": sid, "paths": [standup["path"]], "namespace": "pods"}) - assert r.json()["results"] == [{"path": standup["path"], "status": "already", "recording": one["id"], "job": None, "detail": None}] + assert r.json()["results"] == [ + {"path": standup["path"], "status": "already", "recording": one["id"], "job": None, "import_run": None, "detail": None} + ] # the event's own .ics can be fetched (what a remote resource's file is) assert b"SUMMARY:Harbour stand-up" in b"".join(sources.stream(db, cfg, sid, standup["path"])) diff --git a/fastapi_backend/tests/api/test_setup_assistant.py b/fastapi_backend/tests/api/test_setup_assistant.py index 9d75fb030..a4e015b5b 100644 --- a/fastapi_backend/tests/api/test_setup_assistant.py +++ b/fastapi_backend/tests/api/test_setup_assistant.py @@ -155,9 +155,10 @@ def test_a_held_upload_stays_out_until_placed(app, db, cfg, folder, new_client, ) -def test_the_assistant_manages_people_with_approval(app, db, cfg, folder, new_client, llm): +def test_the_assistant_manages_people_with_approval(app, db, cfg, folder, new_client, llm, monkeypatch): from app.domain import auth + monkeypatch.setattr(chat, "CONTEXT_TOKENS", 128000) # room for the people list beside the admin's tools (chat.fit) s = Assist(app, db, cfg, folder, new_client) c, h = s.cl["admin"] llm.tool_script = [ diff --git a/nextjs-frontend/__tests__/import-runs.test.tsx b/nextjs-frontend/__tests__/import-runs.test.tsx new file mode 100644 index 000000000..b7d994e9b --- /dev/null +++ b/nextjs-frontend/__tests__/import-runs.test.tsx @@ -0,0 +1,112 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import "@testing-library/jest-dom"; +import type { ReactNode } from "react"; + +import { Imports } from "@/app/openapi-client"; +import { canUndoRun, runDetail, runTitle, undoneToast } from "@/components/import/import-runs"; +import { RecentImports } from "@/components/import/recent-imports"; +import { WebLinks } from "@/components/import/web-links"; +import { TooltipProvider } from "@/components/ui/tooltip"; + +jest.mock("@/app/openapi-client", () => ({ + Imports: { listImportRuns: jest.fn(), undoImportRun: jest.fn(), importWebLinks: jest.fn() }, +})); +jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); +const toast = jest.fn(); +jest.mock("@/components/ui/toast", () => ({ useToast: () => toast })); +const m = (f: unknown) => f as jest.Mock; +const ok = (data: unknown) => ({ data, response: { ok: true, status: 200 } }); + +const RUN = { + id: 7, + kind: "archive" as const, + namespace: "pods", + source: "export.zip", + created_by: "own@x.io", + created_at: new Date().toISOString(), + assets: 240, + left: 240, + status: "imported" as const, +}; + +function wrap(children: ReactNode) { + const qc = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }); + return render( + + {children} + , + ); +} + +beforeAll(() => { + globalThis.ResizeObserver ??= class { + observe() {} + unobserve() {} + disconnect() {} + } as unknown as typeof ResizeObserver; +}); +beforeEach(() => jest.clearAllMocks()); + +describe("import runs", () => { + it("says what a run made, what's left and what an undo did", () => { + expect(runTitle(RUN)).toBe("Imported 240 assets"); + expect(runTitle({ ...RUN, left: 3, status: "partly undone" })).toBe("Imported 240 assets, 3 left"); + expect(runTitle({ ...RUN, left: 0, status: "undone" })).toBe("Undone: 240 assets removed"); + expect(runDetail(RUN)).toBe("Archive · export.zip · into pods"); + expect(runDetail({ kind: "scan", source: null, namespace: null })).toBe("Folder scan"); + expect(canUndoRun(RUN)).toBe(true); + expect(canUndoRun({ left: 0 })).toBe(false); + expect(undoneToast({ deleted: [1, 2], busy: [] })).toEqual({ title: "Removed 2 assets" }); + expect(undoneToast({ deleted: [1], busy: [5] })).toEqual({ + title: "Removed 1 asset", + body: "1 asset is being processed: undo the import again once it stops.", + }); + }); + + it("lists recent runs and undoes one whole, asking once in the row", async () => { + m(Imports.listImportRuns).mockResolvedValue(ok([RUN, { ...RUN, id: 6, left: 0, status: "undone" }])); + m(Imports.undoImportRun).mockResolvedValue( + ok({ run: { ...RUN, left: 0, status: "undone" }, deleted: Array.from({ length: 240 }, (_, i) => i), busy: [] }), + ); + wrap(); + expect(await screen.findByText("Imported 240 assets")).toBeInTheDocument(); + expect(screen.getAllByRole("button", { name: "Undo" })).toHaveLength(1); // the undone run has nothing left + fireEvent.click(screen.getByRole("button", { name: "Undo" })); + fireEvent.click(screen.getByRole("button", { name: "Keep" })); + expect(Imports.undoImportRun).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole("button", { name: "Undo" })); + fireEvent.click(screen.getByRole("button", { name: "Delete 240 assets" })); + await waitFor(() => expect(Imports.undoImportRun).toHaveBeenCalled()); + expect(m(Imports.undoImportRun).mock.calls[0][0].path).toEqual({ run_id: 7 }); + await waitFor(() => + expect(toast).toHaveBeenCalledWith(expect.objectContaining({ title: "Removed 240 assets", tone: "green" })), + ); + }); + + it("offers Undo on the toast after a bulk import, which undoes the whole run", async () => { + m(Imports.importWebLinks).mockResolvedValue( + ok({ + results: [ + { url: "https://example.org/a", status: "queued", recording: 1, job: 1, import_run: 9 }, + { url: "https://example.org/b", status: "queued", recording: 2, job: 2, import_run: 9 }, + ], + import_run: 9, + }), + ); + m(Imports.undoImportRun).mockResolvedValue(ok({ run: { ...RUN, id: 9 }, deleted: [1, 2], busy: [] })); + wrap(); + fireEvent.change(screen.getByLabelText(/Links/), { + target: { value: "https://example.org/a\nhttps://example.org/b" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Capture them all" })); + await waitFor(() => expect(toast).toHaveBeenCalled()); + const shown = toast.mock.calls[0][0]; + expect(shown).toEqual(expect.objectContaining({ title: "Capturing 2 pages", action: expect.anything() })); + expect(shown.action.label).toBe("Undo"); + shown.action.onClick(); + await waitFor(() => expect(Imports.undoImportRun).toHaveBeenCalled()); + expect(m(Imports.undoImportRun).mock.calls[0][0].path).toEqual({ run_id: 9 }); + await waitFor(() => expect(toast).toHaveBeenCalledWith(expect.objectContaining({ title: "Removed 2 assets" }))); + }); +}); diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index e6380a5ce..929ec752b 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -1455,6 +1455,11 @@ export type { GetHealthData, GetHealthResponse, GetHealthResponses, + GetImportRunData, + GetImportRunError, + GetImportRunErrors, + GetImportRunResponse, + GetImportRunResponses, GetInstallData, GetInstallResponse, GetInstallResponses, @@ -1881,6 +1886,8 @@ export type { ImportPreviewRequest, ImportRequest, ImportResult, + ImportRun, + ImportRunUndone, ImportTarget, ImportTargetData, ImportTargetResponse, @@ -2171,6 +2178,11 @@ export type { ListImportHooksErrors, ListImportHooksResponse, ListImportHooksResponses, + ListImportRunsData, + ListImportRunsError, + ListImportRunsErrors, + ListImportRunsResponse, + ListImportRunsResponses, ListIpGroupsData, ListIpGroupsError, ListIpGroupsErrors, @@ -4121,6 +4133,11 @@ export type { UndoGraphChangeErrors, UndoGraphChangeResponse, UndoGraphChangeResponses, + UndoImportRunData, + UndoImportRunError, + UndoImportRunErrors, + UndoImportRunResponse, + UndoImportRunResponses, Undone, UndoRunData, UndoRunError, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index f8a9306d0..d394ef7c0 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -685,6 +685,9 @@ import type { GetGroupResponses, GetHealthData, GetHealthResponses, + GetImportRunData, + GetImportRunErrors, + GetImportRunResponses, GetInstallData, GetInstallResponses, GetJobData, @@ -1072,6 +1075,9 @@ import type { ListImportHooksData, ListImportHooksErrors, ListImportHooksResponses, + ListImportRunsData, + ListImportRunsErrors, + ListImportRunsResponses, ListIpGroupsData, ListIpGroupsErrors, ListIpGroupsResponses, @@ -1963,6 +1969,9 @@ import type { UndoGraphChangeData, UndoGraphChangeErrors, UndoGraphChangeResponses, + UndoImportRunData, + UndoImportRunErrors, + UndoImportRunResponses, UndoRunData, UndoRunErrors, UndoRunResponses, @@ -7695,6 +7704,53 @@ export class Imports { }, }); } + + /** + * List Import Runs + * + * Recent bulk imports, newest first, each with how many assets it made and how many are still here: the ones you + * started and those in namespaces where you may delete assets (everyone's, for admins). A folder scan, an archive's + * files, a list of links or bookmarks, and chosen files of a source are each one run. + */ + public static listImportRuns( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/import/runs", + ...options, + }); + } + + /** + * Get Import Run + */ + public static getImportRun( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/import/runs/{run_id}", + ...options, + }); + } + + /** + * Undo Import Run + * + * Undo a bulk import: every asset it made that is still here is deleted, as `DELETE /recordings/{id}` deletes + * one (the media files stay where they are, and scans don't import them again). It needs the rights a delete needs on + * each of them (owners); when any is one you may not delete, nothing is deleted (403). An asset a job is working on + * is left (`busy`) and its job asked to stop: undo the run again for the rest. Lens keeps no recently deleted list of + * assets, so this can't itself be undone; importing the files again brings them back. Audited as `recording.delete` + * for each asset and `import.undo` for the run. + */ + public static undoImportRun( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/import/runs/{run_id}/undo", + ...options, + }); + } } export class Uploads { diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 7d1b2e21e..d651b1fca 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -11515,6 +11515,91 @@ export type ImportResult = { [key: string]: unknown; }; +/** + * ImportRun + * + * One bulk import: a folder scan, an archive's files, a list of links or bookmarks, or chosen files of a source. + */ +export type ImportRun = { + /** + * Id + */ + id: number; + /** + * Kind + * + * what made it: scan, archive, links or source + */ + kind: "scan" | "archive" | "links" | "source"; + /** + * Namespace + */ + namespace?: string | null; + /** + * Source + * + * what it read: the folders, the archive, the links' file, the source + */ + source?: string | null; + /** + * Created By + * + * who started it (none: a scan from the command line) + */ + created_by?: string | null; + /** + * Created At + */ + created_at: string; + /** + * Assets + * + * how many assets it made + */ + assets?: number; + /** + * Left + * + * how many of them are still here + */ + left?: number; + /** + * Status + * + * imported; partly undone (some assets were busy: undo it again for the rest); undone + */ + status: "imported" | "partly undone" | "undone"; + /** + * Undone At + */ + undone_at?: string | null; + /** + * Undone By + */ + undone_by?: string | null; + [key: string]: unknown; +}; + +/** + * ImportRunUndone + */ +export type ImportRunUndone = { + run: ImportRun; + /** + * Deleted + * + * the assets deleted + */ + deleted: Array; + /** + * Busy + * + * assets a job is working on, left for now: their jobs were asked to stop + */ + busy: Array; + [key: string]: unknown; +}; + /** * ImportTarget * @@ -12367,6 +12452,12 @@ export type LinkImportResult = { * Job */ job?: number | null; + /** + * Import Run + * + * the import run it is part of, which undoes the whole import + */ + import_run?: number | null; /** * Detail */ @@ -12382,6 +12473,12 @@ export type LinksImport = { * Results */ results: Array; + /** + * Import Run + * + * the import run of the pages queued (POST /import/runs/{id}/undo undoes it) + */ + import_run?: number | null; [key: string]: unknown; }; @@ -20801,6 +20898,12 @@ export type Recording = { * a web page captured as a document */ web?: WebPage | null; + /** + * Import Run + * + * the bulk import that made it (GET /import/runs/{id}), which undoes it whole + */ + import_run?: number | null; /** * the places and dates anytopdf found (analysis.anytopdf) */ @@ -26235,6 +26338,12 @@ export type SourceImport = { * Results */ results: Array; + /** + * Import Run + * + * the import run of the files queued (POST /import/runs/{id}/undo undoes it) + */ + import_run?: number | null; [key: string]: unknown; }; @@ -26296,6 +26405,12 @@ export type SourceImportResult = { * Job */ job?: number | null; + /** + * Import Run + * + * the import run it is part of, which undoes the whole import + */ + import_run?: number | null; /** * Detail */ @@ -40601,6 +40716,98 @@ export type ImportFromSourceResponses = { export type ImportFromSourceResponse = ImportFromSourceResponses[keyof ImportFromSourceResponses]; +export type ListImportRunsData = { + body?: never; + path?: never; + query?: { + /** + * Limit + */ + limit?: number; + }; + url: "/api/v1/import/runs"; +}; + +export type ListImportRunsErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ListImportRunsError = ListImportRunsErrors[keyof ListImportRunsErrors]; + +export type ListImportRunsResponses = { + /** + * Response Imports-List Import Runs + * + * Successful Response + */ + 200: Array; +}; + +export type ListImportRunsResponse = ListImportRunsResponses[keyof ListImportRunsResponses]; + +export type GetImportRunData = { + body?: never; + path: { + /** + * Run Id + */ + run_id: number; + }; + query?: never; + url: "/api/v1/import/runs/{run_id}"; +}; + +export type GetImportRunErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GetImportRunError = GetImportRunErrors[keyof GetImportRunErrors]; + +export type GetImportRunResponses = { + /** + * Successful Response + */ + 200: ImportRun; +}; + +export type GetImportRunResponse = GetImportRunResponses[keyof GetImportRunResponses]; + +export type UndoImportRunData = { + body?: never; + path: { + /** + * Run Id + */ + run_id: number; + }; + query?: never; + url: "/api/v1/import/runs/{run_id}/undo"; +}; + +export type UndoImportRunErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type UndoImportRunError = UndoImportRunErrors[keyof UndoImportRunErrors]; + +export type UndoImportRunResponses = { + /** + * Successful Response + */ + 200: ImportRunUndone; +}; + +export type UndoImportRunResponse = UndoImportRunResponses[keyof UndoImportRunResponses]; + export type UploadLimitsData = { body?: never; path?: never; diff --git a/nextjs-frontend/components/import/import-runs.ts b/nextjs-frontend/components/import/import-runs.ts new file mode 100644 index 000000000..3cc88b6fd --- /dev/null +++ b/nextjs-frontend/components/import/import-runs.ts @@ -0,0 +1,39 @@ +import type { ImportRun, ImportRunUndone } from "@/app/openapi-client/types.gen"; +import { plural } from "@/lib/format"; + +/** What started a run, as the list names it. */ +export const RUN_KIND: Record = { + scan: "Folder scan", + archive: "Archive", + links: "Links", + source: "From a source", +}; + +/** "Imported 240 assets", or how many are left once some went. */ +export function runTitle(run: Pick): string { + const made = run.assets ?? 0; + const left = run.left ?? 0; + if (run.status === "undone") return `Undone: ${plural(made, "asset")} removed`; + if (left < made) return `Imported ${plural(made, "asset")}, ${left} left`; + return `Imported ${plural(made, "asset")}`; +} + +/** The line under it: what made it, what it read and where it went. */ +export function runDetail(run: Pick): string { + return [RUN_KIND[run.kind], run.source, run.namespace ? `into ${run.namespace}` : null].filter(Boolean).join(" · "); +} + +/** Whether the run has anything left to undo. */ +export function canUndoRun(run: Pick): boolean { + return (run.left ?? 0) > 0; +} + +/** The toast after an undo: how many went, and what was left because a job is working on it. */ +export function undoneToast(r: Pick): { title: string; body?: string } { + const title = r.deleted.length ? `Removed ${plural(r.deleted.length, "asset")}` : "Nothing was left to remove"; + if (!r.busy.length) return { title }; + return { + title, + body: `${plural(r.busy.length, "asset")} ${r.busy.length === 1 ? "is" : "are"} being processed: undo the import again once ${r.busy.length === 1 ? "it stops" : "they stop"}.`, + }; +} diff --git a/nextjs-frontend/components/import/import-screen.tsx b/nextjs-frontend/components/import/import-screen.tsx index 1e7312b68..b61025472 100644 --- a/nextjs-frontend/components/import/import-screen.tsx +++ b/nextjs-frontend/components/import/import-screen.tsx @@ -13,6 +13,7 @@ import { } from "@/components/import/files"; import { ImportQueue } from "@/components/import/import-queue"; import { PasteTab } from "@/components/import/paste-tab"; +import { RecentImports } from "@/components/import/recent-imports"; import { chooseFiles, defaultImportNamespace, isFileDrag, takeFiles } from "@/components/import/pending"; import { SourceTab } from "@/components/import/source-tab"; import { WebTab } from "@/components/import/web-tab"; @@ -498,6 +499,7 @@ export function ImportScreen() { ))} + {tab !== "upload" || items.length === 0 ? : null} )} diff --git a/nextjs-frontend/components/import/recent-imports.tsx b/nextjs-frontend/components/import/recent-imports.tsx new file mode 100644 index 000000000..2c25fd91e --- /dev/null +++ b/nextjs-frontend/components/import/recent-imports.tsx @@ -0,0 +1,108 @@ +"use client"; + +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useCallback, useState } from "react"; + +import { Imports } from "@/app/openapi-client"; +import { canUndoRun, runDetail, runTitle, undoneToast } from "@/components/import/import-runs"; +import { Button } from "@/components/ui/button"; +import { useToast } from "@/components/ui/toast"; +import { data, useApiClient } from "@/lib/api/browser"; +import { plural, relative } from "@/lib/format"; + +const RUNS = ["import-runs"]; + +/** Undo a whole import run: every asset it made is deleted, then a toast says how many went. */ +export function useUndoImport() { + const client = useApiClient(); + const qc = useQueryClient(); + const toast = useToast(); + return useCallback( + async (run: number) => { + try { + const r = await data(Imports.undoImportRun({ client, path: { run_id: run } })); + toast({ ...undoneToast(r), tone: "green" }); + } catch (e) { + toast({ title: "Couldn’t undo the import", body: (e as Error).message, tone: "red" }); + } finally { + void qc.invalidateQueries({ queryKey: RUNS }); + void qc.invalidateQueries({ queryKey: ["recordings"] }); + void qc.invalidateQueries({ queryKey: ["jobs"] }); + } + }, + [client, qc, toast], + ); +} + +/** The toast right after a bulk import: how many came in, with Undo for the whole run. */ +export function useImportedToast() { + const toast = useToast(); + const qc = useQueryClient(); + const undo = useUndoImport(); + return useCallback( + (run: number | null | undefined, title: string, body?: string) => { + void qc.invalidateQueries({ queryKey: RUNS }); + toast({ + title, + body, + tone: "green", + duration: run ? 10000 : undefined, + action: run ? { label: "Undo", onClick: () => void undo(run) } : undefined, + }); + }, + [qc, toast, undo], + ); +} + +/** Recent bulk imports (folder scans, archives, link lists, a source's files), each with Undo for the whole run. */ +export function RecentImports() { + const client = useApiClient(); + const undo = useUndoImport(); + const [asking, setAsking] = useState(null); + const runs = useQuery({ + queryKey: RUNS, + queryFn: () => data(Imports.listImportRuns({ client, query: { limit: 10 } })), + staleTime: 10_000, + }); + if (!runs.data?.length) return null; + return ( +
+

Recent imports

+
    + {runs.data.map((r) => ( +
  • +
    +
    {runTitle(r)}
    +
    + {runDetail(r)} · {relative(r.created_at)} +
    +
    + {canUndoRun(r) && + (asking === r.id ? ( + + + + + ) : ( + + ))} +
  • + ))} +
+
+ ); +} diff --git a/nextjs-frontend/components/import/source-tab.tsx b/nextjs-frontend/components/import/source-tab.tsx index 77228cbb5..0d58aa1b1 100644 --- a/nextjs-frontend/components/import/source-tab.tsx +++ b/nextjs-frontend/components/import/source-tab.tsx @@ -8,6 +8,7 @@ import { useEffect, useState } from "react"; import { Imports, Sources } from "@/app/openapi-client"; import type { BrowseEntry } from "@/app/openapi-client/types.gen"; import { canBeTranscript, importable, kindOf, pipelineOptions, sourceImportSummary } from "@/components/import/files"; +import { useImportedToast } from "@/components/import/recent-imports"; import { fileIcon } from "@/components/import/upload-tab"; import { useNamespacePipeline } from "@/components/import/use-import"; import { CollectionField } from "@/components/library/collections-ui"; @@ -48,6 +49,7 @@ export function SourceTab({ namespace, namespaces }: { namespace: string | null; const client = useApiClient(); const qc = useQueryClient(); const toast = useToast(); + const imported = useImportedToast(); const sources = useQuery({ queryKey: ["sources"], queryFn: () => data(Sources.listSources({ client })), @@ -277,12 +279,12 @@ export function SourceTab({ namespace, namespaces }: { namespace: string | null; paths={[...chosen]} defaultNs={namespace ?? namespaces[0] ?? ""} namespaces={namespaces} - onDone={(summary) => { + onDone={(summary, run) => { setChosen(new Set()); void qc.invalidateQueries({ queryKey: ["browse", sid, path] }); void qc.invalidateQueries({ queryKey: ["recordings"] }); void qc.invalidateQueries({ queryKey: ["jobs"] }); - toast({ title: "Imported from the source", body: summary, tone: "green" }); + imported(run, "Imported from the source", summary); // Undo takes the whole import away again }} /> )} @@ -325,7 +327,7 @@ function ImportChosenDialog({ paths: string[]; defaultNs: string; namespaces: string[]; - onDone: (summary: string) => void; + onDone: (summary: string, run?: number | null) => void; }) { const client = useApiClient(); const [ns, setNs] = useState(defaultNs); @@ -376,7 +378,7 @@ function ImportChosenDialog({ }), ); onOpenChange(false); - onDone(sourceImportSummary(r.results)); + onDone(sourceImportSummary(r.results), r.import_run); } catch (e) { setError((e as Error).message); } finally { diff --git a/nextjs-frontend/components/import/web-links.tsx b/nextjs-frontend/components/import/web-links.tsx index edd44ed80..0cbb07b70 100644 --- a/nextjs-frontend/components/import/web-links.tsx +++ b/nextjs-frontend/components/import/web-links.tsx @@ -6,9 +6,9 @@ import { useId, useRef, useState, type FormEvent } from "react"; import { Imports } from "@/app/openapi-client"; import type { LinkImportResult } from "@/app/openapi-client/types.gen"; import { fileToBase64 } from "@/components/import/files"; +import { useImportedToast } from "@/components/import/recent-imports"; import { Button } from "@/components/ui/button"; import { Checkbox, Field, Textarea } from "@/components/ui/field"; -import { useToast } from "@/components/ui/toast"; import { data, useApiClient } from "@/lib/api/browser"; const SAID = { queued: "Capturing", already: "Already here", skipped: "Skipped" } as const; @@ -32,7 +32,7 @@ export function WebLinks({ contentType?: string | null; }) { const client = useApiClient(); - const toast = useToast(); + const imported = useImportedToast(); const id = useId(); const picker = useRef(null); const [text, setText] = useState(""); @@ -57,11 +57,12 @@ export function WebLinks({ onSuccess: (r) => { setResults(r.results); const queued = r.results.filter((x) => x.status === "queued").length; - toast({ - title: `Capturing ${queued} ${queued === 1 ? "page" : "pages"}`, - body: "Each is kept as a PDF and read like any document.", - tone: "green", - }); + // one import run: Undo takes every page it queued away again + imported( + r.import_run, + `Capturing ${queued} ${queued === 1 ? "page" : "pages"}`, + "Each is kept as a PDF and read like any document.", + ); setText(""); setFile(null); if (picker.current) picker.current.value = ""; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index c487cbca9..0b3e7e749 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -15931,6 +15931,141 @@ } } }, + "/api/v1/import/runs": { + "get": { + "tags": [ + "imports" + ], + "summary": "List Import Runs", + "description": "Recent bulk imports, newest first, each with how many assets it made and how many are still here: the ones you\nstarted and those in namespaces where you may delete assets (everyone's, for admins). A folder scan, an archive's\nfiles, a list of links or bookmarks, and chosen files of a source are each one run.", + "operationId": "list_import_runs", + "parameters": [ + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 200, + "minimum": 1, + "default": 50, + "title": "Limit" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ImportRun" + }, + "title": "Response Imports-List Import Runs" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/import/runs/{run_id}": { + "get": { + "tags": [ + "imports" + ], + "summary": "Get Import Run", + "operationId": "get_import_run", + "parameters": [ + { + "name": "run_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Run Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ImportRun" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/import/runs/{run_id}/undo": { + "post": { + "tags": [ + "imports" + ], + "summary": "Undo Import Run", + "description": "Undo a bulk import: every asset it made that is still here is deleted, as `DELETE /recordings/{id}` deletes\none (the media files stay where they are, and scans don't import them again). It needs the rights a delete needs on\neach of them (owners); when any is one you may not delete, nothing is deleted (403). An asset a job is working on\nis left (`busy`) and its job asked to stop: undo the run again for the rest. Lens keeps no recently deleted list of\nassets, so this can't itself be undone; importing the files again brings them back. Audited as `recording.delete`\nfor each asset and `import.undo` for the run.", + "operationId": "undo_import_run", + "parameters": [ + { + "name": "run_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Run Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ImportRunUndone" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/uploads/limits": { "get": { "tags": [ @@ -57126,6 +57261,149 @@ ], "title": "ImportResult" }, + "ImportRun": { + "properties": { + "id": { + "type": "integer", + "title": "Id" + }, + "kind": { + "type": "string", + "enum": [ + "scan", + "archive", + "links", + "source" + ], + "title": "Kind", + "description": "what made it: scan, archive, links or source" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Namespace" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source", + "description": "what it read: the folders, the archive, the links' file, the source" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By", + "description": "who started it (none: a scan from the command line)" + }, + "created_at": { + "type": "string", + "title": "Created At" + }, + "assets": { + "type": "integer", + "title": "Assets", + "description": "how many assets it made", + "default": 0 + }, + "left": { + "type": "integer", + "title": "Left", + "description": "how many of them are still here", + "default": 0 + }, + "status": { + "type": "string", + "enum": [ + "imported", + "partly undone", + "undone" + ], + "title": "Status", + "description": "imported; partly undone (some assets were busy: undo it again for the rest); undone" + }, + "undone_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Undone At" + }, + "undone_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Undone By" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "kind", + "created_at", + "status" + ], + "title": "ImportRun", + "description": "One bulk import: a folder scan, an archive's files, a list of links or bookmarks, or chosen files of a source." + }, + "ImportRunUndone": { + "properties": { + "run": { + "$ref": "#/components/schemas/ImportRun" + }, + "deleted": { + "items": { + "type": "integer" + }, + "type": "array", + "title": "Deleted", + "description": "the assets deleted" + }, + "busy": { + "items": { + "type": "integer" + }, + "type": "array", + "title": "Busy", + "description": "assets a job is working on, left for now: their jobs were asked to stop" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "run", + "deleted", + "busy" + ], + "title": "ImportRunUndone" + }, "ImportTarget": { "properties": { "kind": { @@ -58576,6 +58854,18 @@ ], "title": "Job" }, + "import_run": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Import Run", + "description": "the import run it is part of, which undoes the whole import" + }, "detail": { "anyOf": [ { @@ -58604,6 +58894,18 @@ }, "type": "array", "title": "Results" + }, + "import_run": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Import Run", + "description": "the import run of the pages queued (POST /import/runs/{id}/undo undoes it)" } }, "additionalProperties": true, @@ -72814,6 +73116,18 @@ ], "description": "a web page captured as a document" }, + "import_run": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Import Run", + "description": "the bulk import that made it (GET /import/runs/{id}), which undoes it whole" + }, "enrichment": { "anyOf": [ { @@ -82257,6 +82571,18 @@ }, "type": "array", "title": "Results" + }, + "import_run": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Import Run", + "description": "the import run of the files queued (POST /import/runs/{id}/undo undoes it)" } }, "additionalProperties": true, @@ -82368,6 +82694,18 @@ ], "title": "Job" }, + "import_run": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Import Run", + "description": "the import run it is part of, which undoes the whole import" + }, "detail": { "anyOf": [ { diff --git a/threat-model.json b/threat-model.json index 96853f74d..9be6faac9 100644 --- a/threat-model.json +++ b/threat-model.json @@ -771,6 +771,15 @@ "provenance": "documented", "source": "docs/assistant.md \"The gateway\"" }, + { + "entry_point": "import runs", + "parameter": "GET /api/v1/import/runs, POST /api/v1/import/runs/{id}/undo; assistant import_runs, undo_import: a run id", + "attacker_controllable": "yes", + "component": "authz", + "caller_must_enforce": "none — a run is seen only by those (404 otherwise); undo deletes each asset through deletion.delete and needs the remove permission (or a collection admin role) on every one, else nothing is deleted (403); the assistant's undo always waits for the person's approval; audited per asset (recording.delete) and for the run (import.undo)", + "provenance": "documented", + "source": "docs/api.md \"Import runs\"" + }, { "entry_point": "code sandbox", "parameter": "assistant run_python, run_bash, sandbox_files, sandbox_nodes, share_sandbox_file and import_sandbox_file: code the model writes, attachment ids, a path in the sandbox, a compute node's id or name; sandbox.* settings (admins)", diff --git a/threat-model.md b/threat-model.md index 902bb79b8..f6a2e8fd6 100644 --- a/threat-model.md +++ b/threat-model.md @@ -331,6 +331,7 @@ classified by gate only *(inferred, Q8)*; unread modules are listed in Q34. | `PUT /api/v1/admin/services/{profile}`, MCP `set_service` | `profile` (`mcp`, `browser`, `monitoring`), `on` | admin only (MCP: with the write scope) | x-env-config | admin is trusted; audited (`lens.services`, `lens.monitoring`); an unknown profile is a 404; one change at a time; it reaches the machine only as an inbox request, which the agent checks again | *(documented, docs/assistant.md "Optional containers")* | | `/api/v1/scaling` (providers, groups, units), `PUT /api/v1/settings/scaling`, MCP and assistant scaling tools | provider credentials, a group's terms and rules, a unit id | admin only (MCP: with the write scope) | x-credential, x-env-config | admin is trusted; off until turned on; audited (`scaling.*`) and in the activity ledger; a provider's credentials are sealed per provider and field and shown only as set; a group launches nothing until a person signed in to the web app approves its terms, and agents act only within them (P-SCALING) | *(documented, docs/scaling.md "The person approves")* | | `PUT /api/v1/admin/sandbox`, MCP `set_sandbox_gateway` | `on` | admin only (MCP: with the write scope) | x-env-config | admin is trusted; audited (`lens.sandbox`); refused when the machine can't run the gateway (preflight); reaches the machine only as an inbox request (`action=services`, `profile=sandbox`) the agent checks again; off deletes every sandbox first | *(documented, docs/assistant.md "The gateway")* | +| `GET /api/v1/import/runs`, `POST /api/v1/import/runs/{id}/undo`; assistant `import_runs`, `undo_import` | a run id | yes: whoever started the run, owners of its namespace, admins; indirectly through content the model reads | data, x-prompt-injection | a run is seen only by those (404 otherwise); undo deletes each asset through `deletion.delete` and needs the remove permission (or a collection admin role) on every one, else nothing is deleted (403); the assistant's undo always waits for the person's approval; audited per asset (`recording.delete`) and for the run (`import.undo`) | *(documented, docs/api.md "Import runs")* | | MCP `edit_entity`, `edit_topic`, `merge_topics`, `delete_topic`, `undo_topic_change`, `undo_graph_change`; assistant `propose_topic_change` | entity and topic ids, names, labels, definitions | yes, an agent with an editor's write-scope token; indirectly through content the model reads | data, x-prompt-injection | the curate permission in each namespace touched and a write-capable principal (the same route functions as the web app); the assistant's changes wait for the person's approval; merges and deletes are kept to undo, and audited | *(documented, docs/mcp.md "curate entities and topics as an editor does")* | | `/api/v1/automations` (save, try, run now, decide, draft from a sentence), the MCP and assistant automation tools; the activities an automation runs on | triggers, `where` and If match trees, the event workflow graph (action params are Jinja templates), `policy`, `limits`, an activity id; the text of the activities themselves | yes: any editor, for automations in namespaces they edit; indirectly through activity text (mail, titles, chat) that reaches action params | callback/code (declarative), data, x-prompt-injection | none; it runs as its maker with their permissions at run time (P-AUTOMATION-ACTOR), actions reaching outside Lens ask first, outside text marks intents untrusted and breaks up links (P-INJECTION-EGRESS), chains and bursts are bounded (P-AUTOMATION-LOOP), and `matches` patterns refuse nested repeats and repeated alternatives that start alike, read at most 20,000 characters and give up after 50 ms | *(documented, docs/automations.md "Running automations")*. Tested by `fastapi_backend/tests/domain/test_match_ops.py` `test_checks_refuse_values_a_test_cant_use`, `test_a_pattern_reads_a_long_value_only_so_far_and_gives_up_when_slow`; the rest under the properties it names | | Assistant code sandbox tools (`run_python`, `run_bash`, `sandbox_files`, `sandbox_nodes`, `share_sandbox_file`, `import_sandbox_file`) and the `sandbox.*` settings | code and scripts the model writes, attachment ids, a path in the sandbox, a compute node's id or name; the gateway address, image and hosts it may reach (admins) | yes: the people `sandbox.who` names, and indirectly through content the model reads | callback/code, data, x-prompt-injection | none; off by default; the code runs in the conversation's own OpenShell sandbox, which holds none of Lens's data or keys and reaches no network unless `sandbox.network` allows it (P-SANDBOX); with the network on, a run after outside text asks first (P-INJECTION-EGRESS); files go in only from the person's own attachments to that conversation, and come out only as an import the person approves or as a download for that person (`share_sandbox_file`); with compute nodes registered by the mesh, a run may go to a ready compute node's own gateway, reached only over the mesh, with the same sandbox settings, and only where the model passed `node` or the conversation's sandbox already is | *(documented, docs/assistant.md "Code sandbox")* | diff --git a/threat-model.yaml b/threat-model.yaml index 0e116d28a..82f8a76b5 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -740,6 +740,15 @@ entry_points: obligation_id: admin-is-trusted caller_must_enforce: "admin is trusted; audited (lens.sandbox); refused when preflight finds the machine can't run the gateway; reaches the machine only as an inbox request (action=services, profile=sandbox) the agent checks again; off deletes every sandbox first" provenance: {kind: documented, source: 'docs/assistant.md "The gateway"'} + - id: "import runs" + component: authz + parameters: + - name: "GET /api/v1/import/runs, POST /api/v1/import/runs/{id}/undo; assistant import_runs, undo_import: a run id" + attacker_controllable: true + x-condition: "whoever started the run, owners of its namespace, admins; indirectly through content the model reads" + control_kinds: [data, x-prompt-injection] + caller_must_enforce: "none — a run is seen only by those (404 otherwise); undo deletes each asset through deletion.delete and needs the remove permission (or a collection admin role) on every one, else nothing is deleted (403); the assistant's undo always waits for the person's approval; audited per asset (recording.delete) and for the run (import.undo)" + provenance: {kind: documented, source: 'docs/api.md "Import runs"'} - id: "entity and topic curation tools" component: authz parameters: