From c78d988199d730f56e8159e2a5af2c5c9d6420d3 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Mon, 19 Oct 2026 15:51:22 +0000 Subject: [PATCH] feat(video): blur faces in a copy of a video for sharing A person can ask for a copy of a video with every face, the unnamed ones or chosen ones blurred. ffmpeg draws a blur over each face track's boxes over time, as a heavy background step (blur_faces) that respects quiet hours; the copy is kept as an attachment beside the asset, and an owner can have visitors play it instead of the original (members keep the original; a missing copy is never replaced by the faces). API, MCP tools and a one-click button in the People tab with the job's progress. Off until asked. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QFUNzy1QZy5HhKAyJxTAYJ --- README.md | 8 +- changelog.d/blur-faces.md | 8 + docs/api.md | 13 + docs/architecture.md | 4 +- docs/mcp.md | 7 +- docs/requirements.md | 1 + docs/video.md | 35 ++ fastapi_backend/app/api/iiif.py | 2 +- fastapi_backend/app/api/mcp_tools.py | 77 ++++ fastapi_backend/app/api/v1/routes/feeds.py | 2 +- .../app/api/v1/routes/recordings.py | 21 +- fastapi_backend/app/api/v1/routes/video.py | 58 ++- fastapi_backend/app/domain/face_blur.py | 322 ++++++++++++++++ fastapi_backend/app/domain/files.py | 6 +- fastapi_backend/app/domain/jobs.py | 11 + fastapi_backend/app/domain/quiet.py | 4 +- fastapi_backend/app/domain/store.py | 2 +- fastapi_backend/app/schemas/video.py | 44 ++- fastapi_backend/tests/api/test_face_blur.py | 167 +++++++++ fastapi_backend/tests/api/test_jobs.py | 2 +- fastapi_backend/tests/api/test_mcp.py | 2 + nextjs-frontend/__tests__/face-blur.test.ts | 50 +++ nextjs-frontend/app/openapi-client/index.ts | 20 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 74 +++- .../app/openapi-client/types.gen.ts | 227 +++++++++++ nextjs-frontend/components/recording/hooks.ts | 51 ++- .../recording/video/face-blur-model.ts | 50 +++ .../components/recording/video/people-tab.tsx | 160 +++++++- nextjs-frontend/components/settings/model.ts | 22 +- nextjs-frontend/components/ui/loop.tsx | 2 + nextjs-frontend/openapi.json | 353 +++++++++++++++++- threat-model.md | 1 + threat-model.yaml | 7 + 33 files changed, 1776 insertions(+), 37 deletions(-) create mode 100644 changelog.d/blur-faces.md create mode 100644 fastapi_backend/app/domain/face_blur.py create mode 100644 fastapi_backend/tests/api/test_face_blur.py create mode 100644 nextjs-frontend/__tests__/face-blur.test.ts create mode 100644 nextjs-frontend/components/recording/video/face-blur-model.ts diff --git a/README.md b/README.md index 83b71fdb5..1fe97edf8 100644 --- a/README.md +++ b/README.md @@ -321,8 +321,9 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p Open signed in), encryption at rest with a key per namespace, roles per namespace (viewer, editor, owner, or custom roles made of permissions), who sees each asset, set for many at once from the Library, access that ends on a date you pick (a role in a namespace or on a collection, or permission on one asset), a downloads switch per asset (off, - it still plays and reads, but only its owners save its files), email invitations, removing someone for good, and an - audit log. Prompt injection meets + it still plays and reads, but only its owners save its files), a video's copy with faces blurred for sharing (made + when you ask, and played to visitors instead of the original if you pick that), email invitations, removing someone + for good, and an audit log. Prompt injection meets several layers: what the assistant reads (recordings, documents, web pages, tool results) reaches the model marked as data, not instructions, and text that reads like instructions to an AI is flagged (by free patterns, or an optional classifier model that runs on your machine and is fetched only when you pick it). Once an answer has read such text, @@ -336,7 +337,8 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p field in Settings can refer to an item there, and Lens reads it without anyone, agents included, seeing it. ([Encryption](docs/encryption.md), [secrets](docs/secrets.md), [access](docs/access.md), [prompt injection](docs/assistant.md#third-party-content-and-prompt-injection), [HTTPS](docs/https.md), - [remote access](docs/remote-access.md); [checks](docs/requirements.md#stay-private-and-safe)) + [remote access](docs/remote-access.md), [blurring faces](docs/video.md#blurring-faces-for-sharing); + [checks](docs/requirements.md#stay-private-and-safe)) - **Use open standards.** RDF and Dublin Core for every record, OAuth for apps and OpenID Connect sign-in, MCP, MQTT, IIIF for publishing (with your team's shared notes and comments as W3C Web Annotations, for people with a role on the asset only), OpenTelemetry and Prometheus for monitoring, an optional Fedora mirror, and a CSV catalogue of the assets diff --git a/changelog.d/blur-faces.md b/changelog.d/blur-faces.md new file mode 100644 index 000000000..8115e2af7 --- /dev/null +++ b/changelog.d/blur-faces.md @@ -0,0 +1,8 @@ +- **Blur faces for sharing.** A video's People tab has **Blur faces for sharing**: every face found is ticked, and + Enter makes a copy of the video with those faces blurred, by ffmpeg over the faces' tracks, in the background as a + heavy step (quiet processing hours apply), with its progress shown. The copy is kept with the video's files, the + original stays as it is, and an owner can have visitors (public pages, embeds, share links, IIIF, feeds) play the + copy instead. Nothing changes for a video until someone asks. Agents use `GET` and + `POST /api/v1/recordings/{id}/blur-faces` (all, unnamed or chosen faces), `PUT .../blur-faces/visitors`, and the + MCP tools `blur_faces`, `get_face_blur` and `set_face_blur_for_visitors` + ([Blurring faces for sharing](docs/video.md#blurring-faces-for-sharing)). diff --git a/docs/api.md b/docs/api.md index 1d5ccac87..e1597a020 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1574,6 +1574,9 @@ DELETE /api/v1/faces/{fid} POST /api/v1/faces/{fid}/merge POST /api/v1/faces/{fid}/speaker POST /api/v1/faces/{fid}/dismiss +GET /api/v1/resources/{rid}/blur-faces +POST /api/v1/resources/{rid}/blur-faces +PUT /api/v1/resources/{rid}/blur-faces/visitors ``` `PUT /namespaces/{name}/faces/mode {mode?, purpose?, pixelate?, reprocess?}` (owners) sets the namespace's face mode @@ -1591,6 +1594,16 @@ suggested for it. `POST /faces/{fid}/name {name}` (editors) names one: a name an case) joins it to that face, a merge whose id comes back as `merge` to undo with `POST /faces/merges/{mid}/undo`; any other name becomes its own, undone with `POST /faces/{fid} {"name": ""}` ([Naming faces](video.md#naming-faces)). +`POST /resources/{rid}/blur-faces {who?, tracks?}` (the process permission) makes a copy of a video with faces +blurred for sharing, in the background: every face found (`who` `all`, the default), the ones nobody has named +(`unnamed`), or the face tracks in `tracks` (`chosen`); 202 with the job, 400 for anything but a video or with no faces +to blur. The copy is kept as one of the resource's files (an attachment labelled "Faces blurred"); asking again +replaces it. `GET /resources/{rid}/blur-faces` (members) gives `{blurred, job}`: the copy (`file`, `name`, `size`, a +signed `download`, `faces`, `who`, `tracks`, `made_at`, `visitors`) and the latest job making one (`status`, `progress` +in percent while ffmpeg runs, `waits` while quiet processing holds it, `error`). `PUT .../blur-faces/visitors {on}` +(owners) has visitors play the copy from `/audio` and `/media` (and IIIF and feeds) instead of the original +([Blurring faces for sharing](video.md#blurring-faces-for-sharing)). + ### Objects The `objects` step finds the people, vehicles, animals and everyday things (the 80 kinds of the COCO dataset) on a diff --git a/docs/architecture.md b/docs/architecture.md index c8013f87e..5d090e628 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -82,8 +82,8 @@ jobs), drain it (it also hands its job back after the step it's on) and resume i record, so it holds across restarts. `GET /api/v1/events` streams job progress as server-sent events. **Quiet processing** (Settings › Workers, both off by default) keeps the heavy steps (`workers.heavy_steps`: by -default transcribe, read, diarize, shots, ocr, faces, objects and nsfw, the ones that work through media on this -machine) to when it suits a small machine at home: +default transcribe, read, diarize, shots, ocr, faces, objects, nsfw and blur_faces, the ones that work through media +on this machine) to when it suits a small machine at home: - `workers.heavy_hours`, a window like `01:00-06:00` in `workers.timezone` (UTC by default; it may pass midnight). Outside it no job starts a heavy step: a job that reaches one goes back on the queue, its log saying why, and waits diff --git a/docs/mcp.md b/docs/mcp.md index f06c04d5d..ed0a05fea 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -131,6 +131,9 @@ For an app that speaks OpenAI's API rather than MCP, the namespace can also answ | `mark_nsfw` | mark a resource NSFW or not by hand, or hand it back to the model's score. Needs a write-scope token and editor access | | `set_language` | set or correct the [language an asset is in](api.md#an-assets-language) as a BCP 47 tag with a region if it matters (`en-GB`, `pt-BR`), written the usual way and kept over the one transcription detects; left out, it's forgotten. Needs a write-scope token and editor access | | `enhance` | [enhance an image](enhance.md) whatever the settings, keeping the original, and read it again from the enhanced copy. Needs a write-scope token and the process permission | +| `get_face_blur` | a video's faces (track ids and names, to pick from) and its [copy with faces blurred](video.md#blurring-faces-for-sharing): the file and where to download it, whose faces it blurs and whether visitors play it; and the latest job making one, with how far it got, why it waits or why it failed. Members of its namespace | +| `blur_faces` | make a [copy of a video with faces blurred](video.md#blurring-faces-for-sharing) for sharing: all faces, the unnamed ones or the tracks given; it runs in the background and is kept with the video's files, the original unchanged. Needs a write-scope token and the process permission | +| `set_face_blur_for_visitors` | have visitors (public pages, embeds, share links, IIIF, feeds) play a video's copy with faces blurred instead of the original, or not. Needs a write-scope token and owner access | | `namespace_integrations` | whether a namespace has [its own MCP server](#one-namespaces-mcp-server) and [chat completions](api.md#chat-completions-for-any-app) on, their addresses, and the tools offered there | | `set_namespace_integrations` | turn a namespace's own MCP server or chat completions on or off, and pick the read tools its chat completions may use. Owners, with a write-scope token | | `list_files` | a folder of [the graph as files](file-tree.md): your namespaces, then their collections, topics (nested by narrower topics) and entities, each holding its assets' files | @@ -215,7 +218,9 @@ app (merges and deletes can be undone), `write_note`, which adds a note, `create queues an episode, `create_feed` and `publish_to_feed`, which make and fill a [published feed](feeds.md), `save_circle`, `share_with_circle` and `undo_share`, which change your [circles](circles.md) and what you share with them, `write_spec`, which queues a spec, `translate_transcript`, which queues a translation, `draft_reply` and `edit_reply`, which draft and change [email replies](email-replies.md) (nothing is sent), `send_reply`, which asks to send one (it waits in Approvals), `set_nsfw` and `mark_nsfw`, which change [NSFW classification](nsfw.md) (owners or admins, and -editors), `set_language`, which sets an asset's language (editors), `enhance`, which enhances an image and reads it again (the process permission), +editors), `set_language`, which sets an asset's language (editors), `enhance`, which enhances an image and reads it again (the process permission), `blur_faces` and +`set_face_blur_for_visitors`, which make a video's copy with faces blurred and choose whether visitors play it (the +process permission, and owners), `set_namespace_integrations`, which turns a namespace's own MCP server or chat completions on or off (owners), `set_picture_descriptions`, which changes what describes pictures (admins), `set_file_storage`, which changes where new files are kept (admins), `set_secret_reference`, which points a secret field at a password manager's item (admins), `install_plugin`, which installs a plugin from a marketplace as your own extension, `add_mcp_server`, which adds an MCP server (admins), `subscribe_feed` and `unsubscribe_feed`, which follow and stop following news feeds (admins), `set_mail_routing`, which changes where an email account's messages go (admins), `add_mesh_node`, diff --git a/docs/requirements.md b/docs/requirements.md index 6e6c3a116..358cde46b 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -410,6 +410,7 @@ request, to another check that proves the same thing; loosening a requirement or | Roles per namespace: viewer, editor, owner, or custom roles made of permissions | [test_roles.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_roles.py) `test_builtin_roles_keep_their_meaning`, `test_custom_roles_allow_exactly_their_permissions`; [journeys.mjs](https://github.com/adeelahmad/lense/blob/main/ci/clicks/journeys.mjs) `person-add` | | Access that ends on a date: a role in a namespace or on a collection, or permission on an asset, opens nothing after its end and lists show it as expired | [test_grant_ends.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_grant_ends.py) `test_asset_permission_holds_until_it_ends`, `test_namespace_role_holds_until_it_ends`, `test_collection_role_holds_until_it_ends`; [test_setup_assistant.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_setup_assistant.py) `test_the_assistant_gives_a_role_until_a_date`; [access-until.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/access-until.test.ts) `sends the end of the day picked, and nothing for an empty field` | | A downloads switch per asset: off, only its owners save its files (its own file, its PDF, supplementary files) while everyone who sees it still plays it and reads its text | [test_downloads.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_downloads.py) `test_downloads_off_refuses_a_viewers_download_but_still_plays`, `test_downloads_off_keeps_iiif_playing`; [access-model.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/access-model.test.ts) `sends the downloads switch only when it changes, on by default`, `says why the files can't be saved, only when they can't` | +| A video's copy with faces blurred for sharing, made by ffmpeg over the faces' tracks only when asked (all, unnamed or chosen faces; API, MCP and the People tab), kept with its files, and played to visitors instead of the original only when an owner picks that | [test_face_blur.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_face_blur.py) `test_regions_cover_each_track_between_and_around_its_samples`, `test_the_filter_graph_blurs_each_region_for_its_time`, `test_blurring_faces_for_sharing`; [face-blur.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/face-blur.test.ts) `asks for every face when all are ticked, else for the ones ticked in order`, `says how far it got, why it waits and why it failed` | | Email invitations, and removing someone for good | [test_users.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_users.py) `test_invites_are_emailed_when_mail_is_set_up`, `test_admins_remove_people_for_good` | | An audit log | [test_settings.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_settings.py) `test_settings_are_editable_and_secret`; [test_files.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_files.py) `test_change_a_file_and_the_audit_log` | | Outside text reaches the model marked as data; text that reads like instructions is flagged, by patterns or an optional local classifier | [test_prompt_injection.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_prompt_injection.py) `test_what_tools_read_is_marked_as_data`, `test_a_result_that_reads_like_instructions_is_flagged`; [test_injection_model.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_injection_model.py) `test_until_the_model_is_here_the_patterns_stand_alone` | diff --git a/docs/video.md b/docs/video.md index 7bb482329..bf0fa4715 100644 --- a/docs/video.md +++ b/docs/video.md @@ -49,6 +49,41 @@ videos, documents and images. A published video recording becomes a IIIF Video on a Canvas with its width, height and duration. It gets a thumbnail, a "Text on screen" annotation layer targeting `#xywh=…&t=…`, and its shots as a Range. +## Blurring faces for sharing + +A person who may process a video can ask for a **copy with faces blurred**, to share it without showing who is in it. +Nothing changes for a video until someone asks; the original stays as it is. + +- **Asking:** the People tab's **Blur faces for sharing** button opens a list of the faces found, all ticked; Enter (or + the button) starts it. Agents ask with `POST /api/v1/recordings//blur-faces` (`who`: `all`, the default, + `unnamed` for the faces nobody has named, or `chosen` with `tracks`, the face tracks' ids) or the MCP tool + `blur_faces`. Asking while a copy is being made gives the same job. It needs faces found first (the faces step, with + detection or recognition on). +- **How it's made:** ffmpeg, which Lens already uses, draws a strong blur over each face track's boxes over time, as a + background job step (`blur_faces`). The faces step saw each face only on the frames it sampled, so between two + samples that saw a face the blur covers both of its boxes, it starts a sample before the first and ends a sample + after the last, and a margin goes round each box: the copy errs on blurring too much, never too little. Many boxes + are merged into fewer, larger ones (at most 300 at a time). The copy is H.264 video with AAC sound in MP4, and what + the file says about itself (its title, where a phone recorded it) is left out. Faces the detector missed aren't + blurred: check the copy before sharing it. +- **Heavy work:** `blur_faces` is one of the heavy steps (`workers.heavy_steps`), so Settings › Workers › Quiet + processing keeps it to its hours and to so many at a time. The People tab shows the job: waiting (and why), how far + ffmpeg got, or why it failed. +- **The copy:** it's kept with the video's files (the Files tab), an attachment labelled "Faces blurred", encrypted + like the rest when the namespace is, and deleted with the video. Asking again replaces it. `GET /api/v1/recordings//blur-faces` (members of its namespace) and the MCP tool + `get_face_blur` say what there is and how the job is doing, with a link to save the copy. +- **Visitors:** an owner can have visitors play the copy instead of the original: the People tab's **Visitors see + this copy** switch, `PUT /api/v1/recordings//blur-faces/visitors` with `{"on": true}`, or the MCP tool + `set_face_blur_for_visitors`. Then a request without a role in the namespace (public pages, embeds, share links, + IIIF and podcast feeds) gets the copy from `/audio` and `/media`; members still get the original. If the copy goes + missing while this is on, visitors get nothing rather than the faces; deleting it in the Files tab turns the choice + off with it. Pictures (frames, keyframes, pages) are pixelated for visitors by the namespace's own setting above. +- Asking for a copy is audited as `recording.blur_faces`, and the visitors' choice as `recording.blur_faces.visitors`. + +Refine later: a document's or an image's pages; following a face frame by frame between samples instead of covering +both boxes; IIIF manifests and podcast feeds giving the copy's own type and size; the in-app assistant asking for a +copy; a copy made on a mesh worker node reaching the primary. + ## Playable copies Phones and cameras write files many browsers can't play: HEVC video in a .mov, AVI, MPEG transport streams, AMR voice diff --git a/fastapi_backend/app/api/iiif.py b/fastapi_backend/app/api/iiif.py index fb112d989..81c4b5646 100644 --- a/fastapi_backend/app/api/iiif.py +++ b/fastapi_backend/app/api/iiif.py @@ -508,7 +508,7 @@ def iiif_audio(rid: int, request: Request, user: OptionalUser, db: Db, cfg: Cfg) if full.get("source") in documents.KINDS: # a document or an image: its file, to save _may_save(request, db, user, rid) return serve_document(db, cfg, full, request) - return serve_audio(db, cfg, full, rid, request) + return serve_audio(db, cfg, full, rid, request, visitor=not _member(request, db, user, full["space"])) @router.get("/iiif/{rid}/pdf", response_class=FileResponse, responses={200: {"content": {"application/pdf": {}}}}) diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index 24ece72c0..b94baded4 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -3099,6 +3099,83 @@ def enhance_image(ctx: Context, recording_id: int) -> dict[str, Any]: return {"recording_id": recording_id, "job": job, "url": ctx.link(recording_id)} +# ---------- faces blurred for sharing (docs/video.md) ---------- +def _face_blur_out(ctx: Context, rid: int) -> dict[str, Any]: + from app.domain import face_blur, faces + + out = face_blur.status(ctx.db, ctx.cfg, rid) + if out["blurred"]: + out["blurred"]["download"] = f"{store.API}/recordings/{rid}/files/{out['blurred']['file']}/download" + people = [ + {"track": t["id"], "name": t["name"], "named": bool(t.get("face")), "first_seen": store.tc(t.get("first_ms") or 0)} + for t in faces.tracks_for(ctx.db, rid) + ] + return store.clean({**out, "faces": people, "recording_url": ctx.link(rid)}) + + +@tool( + "get_face_blur", + "A video's copy with the faces blurred", + "The faces found in a video (track ids and names, to pick from) and its copy with faces blurred for sharing, once " + "one is made: the file's id and where to download it with your token, whose faces it blurs, and whether visitors " + "play it instead of the original; and the latest job to make one, with its status, how far it got, why it waits or " + "why it failed. Members of the recording's namespace.", + Arg("recording_id", "integer", "the video", required=True), +) +def get_face_blur(ctx: Context, recording_id: int) -> dict[str, Any]: + rec = ctx.acl.recording(recording_id) + if not ctx.acl.member(rec): + raise ToolError("not found") + return _face_blur_out(ctx, recording_id) + + +@tool( + "blur_faces", + "Blur faces in a video for sharing", + "Make a copy of a video with faces blurred, for sharing: every face found (who all, the default), the ones nobody " + "has named (unnamed), or the face tracks given (chosen, with tracks from get_face_blur). It runs in the background " + "as a heavy step and is kept as one of the recording's files, an attachment labelled Faces blurred; asking again " + "replaces it, and the original stays as it is. Check on it with get_face_blur. Needs a token with the write scope " + "and the right to process the recording; refused for anything but a video, or with no faces to blur.", + Arg("recording_id", "integer", "the video", required=True), + Arg("who", "string", "whose faces: all, unnamed or chosen", default="all", enum=("all", "unnamed", "chosen")), + Arg("tracks", "string[]", "with who=chosen: the face tracks to blur"), + writes=True, +) +def blur_faces(ctx: Context, recording_id: int, who: str = "all", tracks: list[str] | None = None) -> dict[str, Any]: + from app.api.v1.routes import video as video_routes + from app.schemas.video import FaceBlurCreate + + if not ctx.user.can_write: + raise ToolError("this token is read-only: ask for a token with the write scope to blur faces") + try: + body = FaceBlurCreate(who=who or "all", tracks=tracks or []) + except ValidationError: + raise ToolError("who is all, unnamed or chosen, and tracks is a list of face track ids") from None + out = video_routes.blur_faces(recording_id, body, ctx.acl, ctx.user, ctx.db) + return {"job": out.job, "status": "blurring", "recording_url": ctx.link(recording_id)} + + +@tool( + "set_face_blur_for_visitors", + "Show visitors the copy with faces blurred", + "Have visitors of a video (its public page, embeds, share links, IIIF and podcast feeds) play its copy with faces " + "blurred instead of the original, or not; members of its namespace play the original either way. Make the copy " + "with blur_faces first. Owners, with a token that has the write scope.", + Arg("recording_id", "integer", "the video", required=True), + Arg("on", "boolean", "true: visitors play the copy; false: the original", required=True), + writes=True, +) +def set_face_blur_for_visitors(ctx: Context, recording_id: int, on: bool) -> dict[str, Any]: + from app.api.v1.routes import video as video_routes + from app.schemas.video import FaceBlurVisitors + + if not ctx.user.can_write: + raise ToolError("this token is read-only: ask for a token with the write scope to change what visitors play") + video_routes.set_face_blur_visitors(recording_id, FaceBlurVisitors(on=on), ctx.acl, ctx.user, ctx.db, ctx.cfg) + return _face_blur_out(ctx, recording_id) + + # ---------- a namespace's own ways in for other apps (integrations.py) ---------- def _integrations(ctx: Context, namespace: str, sid: int) -> dict[str, Any]: return { diff --git a/fastapi_backend/app/api/v1/routes/feeds.py b/fastapi_backend/app/api/v1/routes/feeds.py index 7a09183b6..11e2fd172 100644 --- a/fastapi_backend/app/api/v1/routes/feeds.py +++ b/fastapi_backend/app/api/v1/routes/feeds.py @@ -72,7 +72,7 @@ def read_feed_audio(token: str, iid: int, request: Request, db: Db, cfg: Cfg) -> rid, rec = syndication.item_recording(db, feed, iid) except KeyError: raise HTTPException(404, "not found") from None - got = recording_routes.serve_audio(db, cfg, rec, rid, request) + got = recording_routes.serve_audio(db, cfg, rec, rid, request, visitor=True) # a feed's listeners are visitors got.headers.update(PRIVATE) if request.method == "HEAD": # the headers alone; nothing is read or decrypted return Response(status_code=got.status_code, headers=dict(got.headers)) diff --git a/fastapi_backend/app/api/v1/routes/recordings.py b/fastapi_backend/app/api/v1/routes/recordings.py index c825cd935..4f31915ff 100644 --- a/fastapi_backend/app/api/v1/routes/recordings.py +++ b/fastapi_backend/app/api/v1/routes/recordings.py @@ -25,6 +25,7 @@ documents, enhance, events_content, + face_blur, hierarchy, iiif, ipgroups, @@ -672,11 +673,22 @@ def _media_type(rec: dict[str, Any], path: Any) -> str | None: return None -def serve_audio(db: DB, cfg: dict[str, Any], rec: dict[str, Any], rid: int, request: Request, player: bool = False) -> Response: +def serve_audio( + db: DB, cfg: dict[str, Any], rec: dict[str, Any], rid: int, request: Request, player: bool = False, visitor: bool = False +) -> Response: """The recording's audio (or video) file with byte ranges: local, from a local source, from the cache, or streamed. - For the player, its playable copy when one was made (playable.py); the file itself otherwise.""" + For the player, its playable copy when one was made (playable.py); the file itself otherwise. For a `visitor` (no + role in its namespace), the copy with the faces blurred where an owner chose that (face_blur.py), and never the + file itself while that copy is missing.""" if rec.get("source") != "audio": raise HTTPException(404, "no audio for this recording") + if visitor: + try: + blurred = face_blur.for_visitors(db, cfg, rid) + except face_blur.Missing: + raise HTTPException(404, "the copy with the faces blurred is missing") from None + if blurred: + return file_response(blurred[0], request, blurred[1], db, cfg) copy = playable.copy_of(db, cfg, rid) if player else None if copy: return file_response(copy[0], request, copy[1], db, cfg) @@ -706,8 +718,9 @@ def serve_audio(db: DB, cfg: dict[str, Any], rec: dict[str, Any], rid: int, requ def get_audio(rid: int, request: Request, acl: Acl, db: Db, cfg: Cfg, s: str = "") -> Response: """The audio (or video) for the player, with byte ranges: a copy every browser plays when Settings › Video made one, else the file itself (``/media`` always gives the file). Accepts a bearer token, a share link (``?s=``) or a signed - link.""" - return serve_audio(db, cfg, acl.recording(rid, share=s), rid, request, player=True) + link. Visitors get the copy with the faces blurred where an owner chose that (docs/video.md).""" + rec = acl.recording(rid, share=s) + return serve_audio(db, cfg, rec, rid, request, player=True, visitor=not acl.member(rec)) @router.get("/{rid}/wordcloud.svg", response_class=Response, responses={200: {"content": {"image/svg+xml": {}}}}) diff --git a/fastapi_backend/app/api/v1/routes/video.py b/fastapi_backend/app/api/v1/routes/video.py index 9152bf2d7..91119b4ee 100644 --- a/fastapi_backend/app/api/v1/routes/video.py +++ b/fastapi_backend/app/api/v1/routes/video.py @@ -3,7 +3,8 @@ Face detection and recognition are off until a namespace owner turns them on and, for recognition, says what it is for. Descriptors (face prints) exist only while recognition is on; turning faces off removes every face and track. Media and frames are served to anyone the recording is visible to: a bearer token with a role in its namespace, a -share link (``?s=``) or a signed link from a JSON response. +share link (``?s=``) or a signed link from a JSON response. A copy of a video with its faces blurred is made only when +someone asks for it, and visitors play it instead of the original only where an owner chooses that. """ from __future__ import annotations @@ -19,11 +20,15 @@ from app.api.media import sign_urls from app.api.streaming import LOCKED, picture_response, stored_file from app.api.v1.routes.recordings import serve_audio -from app.domain import auth, convert, documents, enhance, files, ingest, jobs, keyring, store, video +from app.domain import auth, convert, documents, enhance, face_blur, files, ingest, jobs, keyring, store, video from app.domain import faces as facemod from app.domain.store import API, DB, R from app.schemas.common import Ok +from app.schemas.recordings import JobQueued from app.schemas.video import ( + FaceBlur, + FaceBlurCreate, + FaceBlurVisitors, FaceDismiss, FaceMerge, FaceMerged, @@ -55,12 +60,13 @@ def get_media(rid: int, request: Request, acl: Acl, db: Db, cfg: Cfg, s: str = "") -> Response: """The video or audio file, with byte ranges; a document's or an image's file, to save. Accepts a bearer token, a share link (``?s=``) or a signed link. With the asset's downloads off, 403 for anyone below its owners (players - stream /recordings/{rid}/audio, which stays open).""" + stream /recordings/{rid}/audio, which stays open). Visitors get the copy with the faces blurred where an owner chose + that.""" rec = acl.recording(rid, share=s) acl.may_save(rec) # this is the file to save; players stream /audio (docs/access.md#downloads) if rec.get("source") in documents.KINDS: return serve_document(db, cfg, rec, request) - return serve_audio(db, cfg, rec, rid, request) + return serve_audio(db, cfg, rec, rid, request, visitor=not acl.member(rec)) def serve_document(db: DB, cfg: dict[str, Any], rec: dict[str, Any], request: Request) -> Response: @@ -156,6 +162,50 @@ def delete_face_track(rid: int, track: str, user: Writer, acl: Acl, db: Db, cfg: return Ok() +# ---------- faces blurred for sharing (face_blur.py) ---------- +def _face_blur(db: DB, cfg: dict[str, Any], rid: int, save: bool) -> FaceBlur: + return FaceBlur.model_validate(sign_urls(face_blur.status(db, cfg, rid), full=True, save=save)) + + +@router.get("/recordings/{rid}/blur-faces") +def get_face_blur(rid: int, acl: Acl, db: Db, cfg: Cfg) -> FaceBlur: + """The video's copy with the faces blurred for sharing, once one is made (with a signed link to save it, and whether + visitors play it), and the latest job to make one: its status, how far ffmpeg got, why it waits or why it failed. + Members of its namespace.""" + rec = acl.recording(rid) + if not acl.member(rec): + raise HTTPException(404, "not found") + return _face_blur(db, cfg, rid, acl.owns(rec)) + + +@router.post("/recordings/{rid}/blur-faces", status_code=202) +def blur_faces(rid: int, body: FaceBlurCreate, acl: Acl, user: Writer, db: Db) -> JobQueued: + """Make a copy of the video with faces blurred, for sharing: every face found (`who` all, the default), the ones + nobody has named (unnamed), or the face tracks in `tracks` (chosen). It runs in the background as a heavy step + (quiet processing hours apply) and is kept as one of the asset's files, an attachment labelled "Faces blurred"; + asking again replaces it, and asking while it runs gives the same job. The original stays as it is. Needs the + right to process the asset; 400 for anything but a video, or no faces to blur. Audited as + `recording.blur_faces`.""" + acl.recording(rid, "process") + with domain_errors(): + jid = face_blur.queue(db, rid, body.who, body.tracks, by=user.email, by_id=user.id) + auth.audit(db, user.as_audit(), "recording.blur_faces", f"recording:{rid}", store.clean({"who": body.who, "job": jid})) + return JobQueued(job=jid) + + +@router.put("/recordings/{rid}/blur-faces/visitors") +def set_face_blur_visitors(rid: int, body: FaceBlurVisitors, acl: Acl, user: Writer, db: Db, cfg: Cfg) -> FaceBlur: + """Have visitors (public pages, embeds, share links, IIIF, podcast feeds) play the copy with the faces blurred + instead of the original, or not; members play the original either way. While it's on and the copy is missing, + visitors get nothing rather than the faces. Owners (the right to publish it); 400 before a copy is made. Audited + as `recording.blur_faces.visitors`.""" + rec = acl.recording(rid, "publish") + with domain_errors(): + face_blur.set_visitors(db, rid, body.on) + auth.audit(db, user.as_audit(), "recording.blur_faces.visitors", f"recording:{rid}", {"on": body.on}) + return _face_blur(db, cfg, rid, acl.owns(rec)) + + # ---------- people on screen, per namespace ---------- def _face_space(db: DB, acl: Access, fid: int, role: str = "viewer") -> int: row = db.one("SELECT space FROM $r", r=R("face", int(fid))) diff --git a/fastapi_backend/app/domain/face_blur.py b/fastapi_backend/app/domain/face_blur.py new file mode 100644 index 000000000..d178f9502 --- /dev/null +++ b/fastapi_backend/app/domain/face_blur.py @@ -0,0 +1,322 @@ +"""Faces blurred for sharing (docs/video.md, Blurring faces for sharing): a copy of a video with the faces found in it +blurred (all of them, the ones nobody has named, or the ones a person picks), to download or to show visitors. + +Nothing happens until someone asks (the API, MCP or the People tab): a video keeps playing as it is. Asking queues a +`blur_faces` step after what the asset's job still has to do; it is a heavy step (quiet.py), so Settings › Workers keeps +it to its hours and to so many at a time. ffmpeg draws a blur over each face track's boxes over time. The faces step +saw each face only on the frames it sampled (every sample_ms), so between two samples that saw it the blur covers both +boxes, it starts a sample before the first and ends a sample after the last, and a margin goes round each box: the +copy errs on blurring too much, never too little. A video with many faces has its boxes merged into fewer, larger ones, +so the filter graph stays small. The sound is kept; what the file says about itself (its title, where a phone recorded +it) is left out. + +The copy is one of the asset's supplementary files (files.py): an attachment labelled "Faces blurred", in the Files +tab with the rest. The asset keeps which file it is (`face_blur`); asking again replaces it. An owner can have +visitors (public pages, embeds, share links, IIIF, podcast feeds) play the copy instead of the original +(`face_blur.visitors`); members still play the original. Should the copy go missing while that's on, visitors get +nothing rather than the faces; deleting the file (Files tab) takes the choice with it. + +Refine later: blurring a document's or an image's pages; following a face between samples by tracking it frame by +frame instead of covering both boxes; a podcast feed's enclosure giving the copy's size; the in-app assistant asking +for a copy. +""" + +from __future__ import annotations + +import pathlib +import subprocess +import tempfile +import time + +from . import faces, files, ingest, quiet, store + +R = store.R +STEP = "blur_faces" +WHO = ("all", "unnamed", "chosen") # every face, the ones nobody has named, or the tracks picked +LABEL = "Faces blurred" +MARGIN = faces.MARGIN # of a box's size around it, blurred too +SMALLEST = 0.02 # of the picture: a box is never narrower or shorter (ffmpeg crops in whole pairs of pixels) +MOST = 300 # boxes in one filter graph; more are merged two by two (each covering more) until there are no more +SECONDS = 6 * 3600 # the longest ffmpeg may take over one copy +SAY_EVERY = 10 # percent: the job's log says how far it got this often + + +class Missing(LookupError): + """Visitors are to play the copy with the faces blurred, and it isn't there.""" + + +# ---------- what to blur ---------- +def chosen(db, rid, who="all", tracks=None): + """The video's face tracks to blur, in the order they first appear: every one, the ones whose face nobody has + named, or the ones picked (`tracks`, their ids). ValueError for any other `who`.""" + if who not in WHO: + raise ValueError(f"say whose faces to blur: {', '.join(WHO)}") + rows = [ + r + for r in db.rows( + "SELECT record::id(id) AS id, face, boxes, paged, first_ms FROM face_track WHERE recording = $r ORDER BY first_ms", r=int(rid) + ) + if not r.get("paged") + ] + if who == "unnamed": + ids = [R("face", t["face"]) for t in rows if t.get("face")] + named = set(db.values("SELECT VALUE record::id(id) FROM face WHERE id IN $ids AND name != NONE", ids=ids)) if ids else set() + rows = [t for t in rows if t.get("face") not in named] + elif who == "chosen": + want = {str(t) for t in tracks or []} + rows = [t for t in rows if str(t["id"]) in want] + return rows + + +def _run(run, step): + """One stretch of a track (samples no more than two steps apart, as faces.spans bridges them) as pieces + [t0, t1, x0, y0, x1, y1] in ms and fractions: each pair of samples covered by both boxes, a step either side.""" + boxes = [(b[0], b[1], b[2], b[1] + b[3], b[2] + b[4]) for b in run] + if len(boxes) == 1: + t, x0, y0, x1, y1 = boxes[0] + return [[t - step, t + step, x0, y0, x1, y1]] + out = [[a[0], b[0], min(a[1], b[1]), min(a[2], b[2]), max(a[3], b[3]), max(a[4], b[4])] for a, b in zip(boxes, boxes[1:])] + out[0][0] -= step + out[-1][1] += step + return out + + +def _union(pieces): + return [ + min(p[0] for p in pieces), + max(p[1] for p in pieces), + min(p[2] for p in pieces), + min(p[3] for p in pieces), + max(p[4] for p in pieces), + max(p[5] for p in pieces), + ] + + +def _pad(p, margin): + t0, t1, x0, y0, x1, y1 = p + w, h = x1 - x0, y1 - y0 + x0, y0 = max(0.0, x0 - w * margin), max(0.0, y0 - h * margin) + x1, y1 = min(1.0, x1 + w * margin), min(1.0, y1 + h * margin) + w, h = max(SMALLEST, x1 - x0), max(SMALLEST, y1 - y0) + x0, y0 = min(x0, 1.0 - w), min(y0, 1.0 - h) + return (round(max(0, t0) / 1000, 3), round(t1 / 1000, 3), round(x0, 4), round(y0, 4), round(w, 4), round(h, 4)) + + +def regions(tracks, step_ms, margin=MARGIN, most=MOST): + """Where and when to blur: [(from s, to s, x, y, w, h)], the box in fractions of the picture, for these tracks' + boxes ([t ms, x, y, w, h] each, as the faces step keeps them). At most `most` of them while each track has more + than one: past that, a track's neighbouring pieces are merged, each covering both.""" + per = [] + for tr in tracks: + pieces, run = [], [] + for b in sorted((b for b in tr.get("boxes") or [] if len(b) >= 5), key=lambda b: b[0]): + if run and b[0] - run[-1][0] > step_ms * 2: + pieces += _run(run, step_ms) + run = [] + run.append(b) + if run: + pieces += _run(run, step_ms) + per.append(pieces) + while sum(len(p) for p in per) > most and any(len(p) > 1 for p in per): + per = [[_union(p[i : i + 2]) for i in range(0, len(p), 2)] for p in per] + return [_pad(p, margin) for pieces in per for p in pieces] + + +def graph(regs): + """The ffmpeg filter graph blurring these regions of input 0's video, its output labelled [v]: one crop, blur and + overlay each, switched on only for its time.""" + n = len(regs) + parts = [f"[0:v]format=yuv420p,split={n + 1}[m0]" + "".join(f"[c{i}]" for i in range(1, n + 1))] + for i, (t0, t1, x, y, w, h) in enumerate(regs, 1): + on = f"enable='between(t,{t0:.3f},{t1:.3f})'" + parts.append( + f"[c{i}]crop=w=iw*{w:.4f}:h=ih*{h:.4f}:x=iw*{x:.4f}:y=ih*{y:.4f}," + f"boxblur=luma_radius='min(w,h)/3':luma_power=3:chroma_radius='min(cw,ch)/3':chroma_power=3:{on}[b{i}]" + ) + parts.append(f"[m{i - 1}][b{i}]overlay=x=main_w*{x:.4f}:y=main_h*{y:.4f}:{on}[{'v' if i == n else f'm{i}'}]") + return ";".join(parts) + + +# ---------- asking for a copy ---------- +def latest(db, rid): + """The latest job to blur the asset's faces: {id, status, error, step: whether it's on that step now}, or None.""" + for j in db.rows( + "SELECT record::id(id) AS id, status, steps, step_index, next_step, error FROM job WHERE recording = $r ORDER BY id DESC LIMIT 20", + r=int(rid), + ): + if any(isinstance(s, dict) and s.get("type") == STEP for s in j.get("steps") or []): + return {"id": j["id"], "status": j["status"], "error": j.get("error"), "on_step": j.get("next_step") == STEP} + return None + + +def queue(db, rid, who="all", tracks=None, by=None, by_id=None): + """Make a copy of the video with these faces blurred, in the background: a `blur_faces` step after what its job + still has to do (jobs.add_steps), or a new job; the job already blurring them if there is one. ValueError when it + isn't a video, or has no faces (of those asked for) to blur; KeyError for no such asset.""" + from . import jobs + + rec = db.one("SELECT source, media FROM $r", r=R("recording", int(rid))) + if rec is None: + raise KeyError(rid) + if rec.get("source") != "audio" or (rec.get("media") or {}).get("kind") != "video": + raise ValueError("Only a video's faces are blurred so far.") + picked = chosen(db, rid, who, tracks) + if not picked: + raise ValueError("There are no faces to blur: pick at least one." if who == "chosen" else "No faces were found to blur.") + spec = store.clean({"type": STEP, "who": who, "tracks": [t["id"] for t in picked] if who == "chosen" else None, "by": by_id}) + j = latest(db, rid) + if j and j["status"] in jobs.ACTIVE: + return j["id"] + return jobs.add_steps(db, int(rid), [spec], by=by) + + +# ---------- the job step ---------- +def _name(rec): + stem = pathlib.PurePosixPath(str((rec.get("remote") or {}).get("path") or rec.get("path") or "")).stem + return files.clean_name(f"{stem or rec.get('title') or 'video'}-faces-blurred.mp4") + + +def _ffmpeg(src, regs, dest, duration_ms, progress): + """Run ffmpeg over `src` into `dest` (MP4: H.264 with the faces blurred, AAC sound, no metadata), calling + progress(percent) as it goes. RuntimeError with ffmpeg's last words when it fails.""" + argv = ["ffmpeg", "-nostdin", "-y", "-v", "error", "-nostats", "-progress", "pipe:1"] + argv += ["-format_whitelist", ingest.MEDIA_DEMUXERS, "-i", str(src), "-filter_complex", graph(regs)] + argv += ["-map", "[v]", "-map", "0:a:0?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-pix_fmt", "yuv420p"] + argv += ["-c:a", "aac", "-b:a", "160k", "-map_metadata", "-1", "-map_chapters", "-1", "-movflags", "+faststart", "-f", "mp4", str(dest)] + with tempfile.TemporaryFile("w+") as err: + try: + proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=err, text=True) + except OSError as e: # no ffmpeg + raise RuntimeError(f"couldn't blur the faces: {e}") from None + start = time.monotonic() + assert proc.stdout is not None + for line in proc.stdout: + key, _, value = line.strip().partition("=") + if key in ("out_time_us", "out_time_ms") and value.isdigit() and duration_ms: # both are in microseconds + progress(min(99, int(int(value) / 10 / duration_ms))) + if time.monotonic() - start > SECONDS: + proc.kill() + break + code = proc.wait() + err.seek(0) + last = (err.read().strip().splitlines() or ["ffmpeg took too long" if code < 0 else "ffmpeg failed"])[-1] + if code != 0 or not pathlib.Path(dest).is_file(): + raise RuntimeError(f"couldn't blur the faces: {last}") + + +def step(db, cfg, rid, say, spec=None): + """The `blur_faces` step: the copy made, kept as the asset's file, and the asset told which file it is.""" + from . import jobs + + spec = spec or {} + rec = ( + db.one("SELECT space, source, path, remote, media, title, sample_ms, duration_ms, face_blur FROM $r", r=R("recording", int(rid))) + or {} + ) + if rec.get("source") != "audio" or (rec.get("media") or {}).get("kind") != "video": + raise jobs.Skip("only a video's faces are blurred") + who = spec.get("who") or "all" + tracks = chosen(db, rid, who, spec.get("tracks")) + if not tracks: + raise jobs.Skip("no faces to blur: the faces step found none, or none of those picked are left") + regs = regions(tracks, rec.get("sample_ms") or 5000) + say(f"blurring {len(tracks)} face(s) in {len(regs)} place(s)") + src, tmp = ingest.audio_path(db, cfg, rec), files.incoming(cfg) + said, kept = [0], [None] + + def progress(pct): + if pct == kept[0]: # ffmpeg says where it is twice a second + return + kept[0] = pct + db.q("UPDATE $r SET face_blur_progress = $p", r=R("recording", int(rid)), p=pct) + if pct >= said[0] + SAY_EVERY: + said[0] = pct - pct % SAY_EVERY + say(f" {said[0]}%") + + try: + _ffmpeg(src, regs, tmp, rec.get("duration_ms"), progress) + old = (rec.get("face_blur") or {}).get("file") + about = _described(db, rid, who, tracks) + f = files.add(db, cfg, rid, tmp, _name(rec), "attachment", label=LABEL, description=about, by=spec.get("by")) + finally: + tmp.unlink(missing_ok=True) + db.q("UPDATE $r SET face_blur_progress = NONE", r=R("recording", int(rid))) + blur = { + "file": f["id"], + "who": who, + "tracks": [t["id"] for t in tracks], + "faces": len(tracks), + "made_at": store.now(), + "by": spec.get("by"), + "visitors": bool((rec.get("face_blur") or {}).get("visitors")), # an owner's choice outlasts a new copy + } + db.q("UPDATE $r SET face_blur = $b", r=R("recording", int(rid)), b=store.clean(blur)) + if old and old != f["id"]: + try: + files.delete(db, cfg, rid, old) + except KeyError: # deleted already + pass + say(f"blurred {len(tracks)} face(s) in a copy kept as {f['name']} ({f['size'] // 1024} KB)") + + +def _described(db, rid, who, tracks): + """The copy's description in the Files tab: whose faces are blurred.""" + names = {t["id"]: t["name"] for t in faces.tracks_for(db, int(rid))} + whose = {"all": "every face found", "unnamed": "the faces nobody has named", "chosen": "the faces picked"}[who] + listed = ", ".join(names[t["id"]] for t in tracks if t["id"] in names) + return f"A copy with {whose} blurred ({len(tracks)})" + (f": {listed}." if listed else ".") + + +# ---------- what there is, and visitors ---------- +def status(db, cfg, rid): + """The copy with the faces blurred, if there is one ({file, name, size, download, faces, who, tracks, made_at, + visitors}), and the latest job to make one ({id, status, error, progress, waits}).""" + rec = db.one("SELECT face_blur, face_blur_progress FROM $r", r=R("recording", int(rid))) or {} + blur, copy = rec.get("face_blur") or {}, None + if blur.get("file"): + try: + f = files.get(db, rid, blur["file"]) + copy = { + **{k: blur.get(k) for k in ("who", "tracks", "faces", "made_at", "visitors")}, + "file": f["id"], + "name": f["name"], + "size": f.get("size"), + "download": f"{store.API}/recordings/{int(rid)}/files/{f['id']}/download", + } + except KeyError: + pass + job = latest(db, rid) + if job: + on_step = job.pop("on_step") + job["progress"] = rec.get("face_blur_progress") if job["status"] == "running" and on_step else None + job["waits"] = quiet.why(cfg) if job["status"] == "queued" and STEP in quiet.held(db, cfg) else None + return {"blurred": copy, "job": job} + + +def set_visitors(db, rid, on): + """Have visitors play the copy with the faces blurred instead of the original, or not. ValueError without a copy.""" + blur = (db.one("SELECT face_blur FROM $r", r=R("recording", int(rid))) or {}).get("face_blur") or {} + if on and not blur.get("file"): + raise ValueError("Make a copy with the faces blurred first.") + if blur: + db.q("UPDATE $r SET face_blur.visitors = $on", r=R("recording", int(rid)), on=bool(on)) + + +def for_visitors(db, cfg, rid): + """What a visitor plays instead of the original, when an owner chose that: (path, media type); None when they + didn't. Missing when they did and the copy isn't there: visitors then get nothing rather than the faces.""" + blur = (db.one("SELECT face_blur FROM $r", r=R("recording", int(rid))) or {}).get("face_blur") or {} + if not blur.get("visitors"): + return None + try: + path = files.path_of(cfg, files.get(db, rid, blur["file"])) + except KeyError: + raise Missing(rid) from None + if not path.is_file(): + raise Missing(rid) + return str(path), "video/mp4" + + +def forget_file(db, rid, fid): + """The copy's file was deleted: the asset no longer has one, and visitors no longer play it.""" + db.q("UPDATE $r SET face_blur = NONE WHERE face_blur.file = $f", r=R("recording", int(rid)), f=int(fid)) diff --git a/fastapi_backend/app/domain/files.py b/fastapi_backend/app/domain/files.py index 9816441d2..9339ff607 100644 --- a/fastapi_backend/app/domain/files.py +++ b/fastapi_backend/app/domain/files.py @@ -461,9 +461,13 @@ def update(db, cfg, rid, fid, changes): def delete(db, cfg, rid, fid): - """Delete one of the resource's files and its lines. Returns what it was.""" + """Delete one of the resource's files and its lines. Returns what it was. A video's copy with the faces blurred + (face_blur.py) goes with the choice to show it to visitors.""" + from . import face_blur + f = get(db, rid, fid) db.run(["DELETE file_line WHERE file = $f", "DELETE $r"], f=int(fid), r=R("resource_file", int(fid))) + face_blur.forget_file(db, rid, fid) shutil.rmtree(folder(cfg, rid) / str(int(fid)), ignore_errors=True) return f diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index 481bff1f2..1d7f686a2 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -39,6 +39,7 @@ "objects", "nsfw", "describe", + "blur_faces", } # they take longer the longer the recording FILED = ("audio", "document", "image") # sources with a file of their own for the steps to work on TIMINGS = 25 # recent timings kept per kind of step, for estimates @@ -281,6 +282,13 @@ def _nsfw(db, cfg, rid, say, spec=None): nsfw.step(db, cfg, rid, say) +def _blur_faces(db, cfg, rid, say, spec=None): + """A copy of a video with the faces found in it blurred, for sharing (face_blur.py); only when someone asks.""" + from . import face_blur + + face_blur.step(db, cfg, rid, say, spec) + + def _convert(db, cfg, rid, say, spec=None): """New assets made from this one: a video's soundtrack as audio, a document's pages as images (conversions.py).""" from . import conversions @@ -320,6 +328,7 @@ def _describe(db, cfg, rid, say, spec=None): "spec": _write_spec, "reply": _draft_reply, "translate": _translate, + "blur_faces": _blur_faces, } @@ -1175,6 +1184,8 @@ def __init__(self, db, cfg_fn, name=None, steps=None, log=None): self.can.add("clean") if not steps and self.can & {"transcribe", "read"}: # converting needs what they need; older lists run it too self.can.add("convert") + if not steps and "faces" in self.can: # blurring faces needs no more than finding them; older lists run it too + self.can.add("blur_faces") if not steps and "objects" in self.can: # NSFW classification runs on ONNX Runtime too; older lists run it as well self.can.add("nsfw") if not steps and self.can & {"transcribe", "read"}: # enhancing comes first; lists written before it run it too diff --git a/fastapi_backend/app/domain/quiet.py b/fastapi_backend/app/domain/quiet.py index 6d30105c9..eb19cce5f 100644 --- a/fastapi_backend/app/domain/quiet.py +++ b/fastapi_backend/app/domain/quiet.py @@ -8,7 +8,7 @@ already running finishes; two workers claiming in the same instant can briefly make it one more). The heavy steps are `workers.heavy_steps`: by default the ones that work through media on this machine (speech, -voices, shots, text on screen, faces, objects, NSFW, reading documents). Everything else runs as before. +voices, shots, text on screen, faces, objects, NSFW, reading documents, blurring faces). Everything else runs as before. """ from __future__ import annotations @@ -18,7 +18,7 @@ from . import schedule, store -HEAVY = ["transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw"] +HEAVY = ["transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "blur_faces"] WINDOW = re.compile(r"^([01]\d|2[0-3]):([0-5]\d)-([01]\d|2[0-3]):([0-5]\d)$") diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index c351f60e2..9a734ac13 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -309,7 +309,7 @@ def labels(): # quiet processing (quiet.py): a window like "01:00-06:00" for the heavy steps, and how many run at once; off "heavy_hours": None, "heavy_at_once": None, - "heavy_steps": ["transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw"], + "heavy_steps": ["transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "blur_faces"], "timezone": "UTC", "steps": [ "enhance", diff --git a/fastapi_backend/app/schemas/video.py b/fastapi_backend/app/schemas/video.py index 65dea3beb..5485eaa01 100644 --- a/fastapi_backend/app/schemas/video.py +++ b/fastapi_backend/app/schemas/video.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any +from typing import Any, Literal from pydantic import Field @@ -95,3 +95,45 @@ class FaceSpeaker(RequestModel): class FaceDismiss(RequestModel): kind: str = "face" id: int = 0 + + +class FaceBlurCreate(RequestModel): + who: Literal["all", "unnamed", "chosen"] = Field( + "all", description="whose faces to blur: every face found, the ones nobody has named, or the tracks in `tracks`" + ) + tracks: list[str] = Field( + default_factory=list, max_length=500, description="with who=chosen: the face tracks to blur (the player's faces[].id)" + ) + + +class FaceBlurVisitors(RequestModel): + on: bool = Field(description="visitors play the copy with the faces blurred instead of the original") + + +class FaceBlurCopy(ResponseModel): + """The copy with the faces blurred, kept as one of the asset's files (an attachment).""" + + file: int = Field(description="its id among the asset's files") + name: str + size: int | None = None + download: str = Field(description="a signed link to save it") + faces: int = Field(description="how many faces (tracks) are blurred") + who: str = Field(description="all, unnamed or chosen") + tracks: list[str] = Field(default_factory=list, description="the face tracks blurred") + made_at: str | None = None + visitors: bool = Field(False, description="visitors (public pages, embeds, share links, IIIF, feeds) play it instead of the original") + + +class FaceBlurJob(ResponseModel): + """The latest job to make the copy.""" + + id: int + status: str = Field(description="queued, running, held, succeeded, failed or cancelled") + error: str | None = None + progress: int | None = Field(None, description="how far ffmpeg is, in percent, while it runs") + waits: str | None = Field(None, description="why it waits to start (quiet processing hours, or heavy steps at once)") + + +class FaceBlur(ResponseModel): + blurred: FaceBlurCopy | None = Field(None, description="the copy with the faces blurred, once one is made") + job: FaceBlurJob | None = Field(None, description="the latest job to make one") diff --git a/fastapi_backend/tests/api/test_face_blur.py b/fastapi_backend/tests/api/test_face_blur.py new file mode 100644 index 000000000..5eee9f2e2 --- /dev/null +++ b/fastapi_backend/tests/api/test_face_blur.py @@ -0,0 +1,167 @@ +"""Faces blurred for sharing (face_blur.py, docs/video.md): only when someone asks, ffmpeg makes a copy of a video with +the faces found blurred over their tracks' boxes, kept as one of its files; owners can have visitors play it.""" + +from __future__ import annotations + +import datetime as dt +import subprocess + +import pytest + +from app.domain import face_blur, faces, files, ingest, jobs, metadata, quiet, store +from tests.api.test_mcp import tool, tool_error +from tests.helpers import drain +from tests.helpers import quiet as hush +from tests.video_helpers import needs_ffmpeg + +R = store.R + + +def test_regions_cover_each_track_between_and_around_its_samples(): + """Between two samples that saw a face, the blur covers both boxes; a sample before the first and after the last + too; samples more than two steps apart are separate stretches; and a margin goes round each box.""" + moving = {"boxes": [[2000, 0.1, 0.1, 0.2, 0.2], [3000, 0.3, 0.1, 0.2, 0.2], [4000, 0.5, 0.1, 0.2, 0.2], [9000, 0.7, 0.7, 0.1, 0.1]]} + assert face_blur.regions([moving], 1000, margin=0) == [ + (1.0, 3.0, 0.1, 0.1, 0.4, 0.2), # the first pair's boxes together, from a step before the first + (3.0, 5.0, 0.3, 0.1, 0.4, 0.2), # to a step after the last + (8.0, 10.0, 0.7, 0.7, 0.1, 0.1), # seen once, five steps later: a step either side + ] + edge = {"boxes": [[0, 0.0, 0.9, 0.1, 0.1]]} + assert face_blur.regions([edge], 1000) == [(0.0, 1.0, 0.0, 0.88, 0.12, 0.12)] # the margin stays on the picture + tiny = {"boxes": [[0, 0.5, 0.5, 0.001, 0.001]]} + assert face_blur.regions([tiny], 1000, margin=0)[0][4:] == (face_blur.SMALLEST, face_blur.SMALLEST) + assert face_blur.regions([{"boxes": []}, {}], 1000) == [] + + many = {"boxes": [[t * 1000, 0.1, 0.1, 0.1, 0.1] for t in range(40)]} # 39 pieces + merged = face_blur.regions([many], 1000, most=10) + assert len(merged) == 10 # 39, 20, then 10: halved until there are no more than `most` + assert (merged[0][0], merged[-1][1]) == (0.0, 40.0) # still all of the time it was seen, and a step after + + +def test_the_filter_graph_blurs_each_region_for_its_time(): + g = face_blur.graph([(1.0, 3.0, 0.1, 0.2, 0.3, 0.4), (8.0, 10.5, 0.5, 0.5, 0.2, 0.2)]) + chains = g.split(";") + assert chains[0] == "[0:v]format=yuv420p,split=3[m0][c1][c2]" + assert chains[1].startswith("[c1]crop=w=iw*0.3000:h=ih*0.4000:x=iw*0.1000:y=ih*0.2000,boxblur=") + assert chains[1].endswith(":enable='between(t,1.000,3.000)'[b1]") + assert chains[2] == "[m0][b1]overlay=x=main_w*0.1000:y=main_h*0.2000:enable='between(t,1.000,3.000)'[m1]" + assert chains[4] == "[m1][b2]overlay=x=main_w*0.5000:y=main_h*0.5000:enable='between(t,8.000,10.500)'[v]" # the output + + +@pytest.fixture +def env(request, cfg, db, folder, monkeypatch): + from tests.api.test_video import Env + + e = Env(request, cfg, db, folder, monkeypatch) + e.c.put("/api/v1/namespaces/pods/faces/mode", headers=e.h, json={"mode": "recognize", "purpose": "Our hosts", "reprocess": True}) + drain(db, cfg) + return e + + +def _probe(path): + out = subprocess.run( + ["ffprobe", "-v", "error", "-show_entries", "stream=codec_name:format_tags", "-of", "default=nw=1", str(path)], + capture_output=True, + text=True, + ) + return out.stdout + + +@needs_ffmpeg +def test_blurring_faces_for_sharing(env, new_client, monkeypatch): + c, h, he, rid, db, cfg = env.c, env.h, env.he, env.rid, env.db, env.cfg + anon = new_client() + tracks = faces.tracks_for(db, rid) + assert len(tracks) == 2 # FakeFaces: one person in scenes 1 and 3, another in scene 2 + assert c.get(f"/api/v1/recordings/{rid}/blur-faces", headers=he).json() == {"blurred": None, "job": None} # nothing asked + original = c.get(f"/api/v1/recordings/{rid}/audio", headers={**h, "Range": "bytes=0-4095"}).content + + # asking: the right to process it, a video, faces of those asked for + assert c.post(f"/api/v1/recordings/{rid}/blur-faces", json={"who": "chosen", "tracks": ["nope"]}, headers=he).status_code == 400 + assert c.post(f"/api/v1/recordings/{rid}/blur-faces", json={"who": "some"}, headers=he).status_code == 422 + (env.folder / "note.txt").write_text("[00:00] Mara: Just words.") + words = ingest.import_transcript(db, cfg, "pods", env.folder / "note.txt", log=hush) + r = c.post(f"/api/v1/recordings/{words}/blur-faces", json={}, headers=he) + assert (r.status_code, r.json()["detail"]) == (400, "Only a video's faces are blurred so far.") + + # heavy: it waits for the quiet processing hours like the other heavy steps + assert face_blur.STEP in quiet.HEAVY and face_blur.STEP in jobs.STEPS + jid = c.post(f"/api/v1/recordings/{rid}/blur-faces", json={}, headers=he).json()["job"] + assert c.post(f"/api/v1/recordings/{rid}/blur-faces", json={}, headers=he).json()["job"] == jid # the same job + hour = dt.datetime.now(dt.UTC).hour + away = {**cfg, "workers": {**cfg["workers"], "heavy_hours": f"{(hour + 2) % 24:02d}:00-{(hour + 3) % 24:02d}:00"}} + assert "heavy steps run" in face_blur.status(db, away, rid)["job"]["waits"] + + # ffmpeg failing fails the job, and says why + def broken(*a, **k): + raise OSError("no ffmpeg here") + + monkeypatch.setattr(face_blur.subprocess, "Popen", broken) + drain(db, cfg) + got = c.get(f"/api/v1/recordings/{rid}/blur-faces", headers=he).json() + assert got["blurred"] is None and got["job"]["status"] == "failed" and "no ffmpeg here" in got["job"]["error"] + monkeypatch.undo() + + # every face, then only the unnamed ones (one is named), the copy replaced each time + c.post(f"/api/v1/recordings/{rid}/blur-faces", json={}, headers=he) + drain(db, cfg) + first = c.get(f"/api/v1/recordings/{rid}/blur-faces", headers=he).json()["blurred"] + assert (first["faces"], first["who"], first["visitors"]) == (2, "all", False) + f = files.get(db, rid, first["file"]) + assert (f["role"], f["label"], f["name"]) == ("attachment", "Faces blurred", "review-faces-blurred.mp4") + assert "every face found" in f["description"] + saved = c.get(first["download"]) # a signed link + assert saved.status_code == 200 and saved.content[4:8] == b"ftyp" + probed = _probe(files.path_of(cfg, f)) + assert "codec_name=h264" in probed and "codec_name=aac" in probed + + named = next(t for t in tracks if t.get("face")) + faces.rename(db, named["face"], "Mara Quint") + c.post(f"/api/v1/recordings/{rid}/blur-faces", json={"who": "unnamed"}, headers=he) + drain(db, cfg) + second = c.get(f"/api/v1/recordings/{rid}/blur-faces", headers=he).json() + assert (second["blurred"]["faces"], second["blurred"]["who"], second["job"]["status"]) == (1, "unnamed", "succeeded") + assert named["id"] not in second["blurred"]["tracks"] + assert [x["id"] for x in files.of(db, rid)] == [second["blurred"]["file"]] # the first copy replaced + + # visitors: owners choose; members still play the original, visitors the copy + metadata.save(db, cfg, rid, {"access": "public"}) + url = anon.get(f"/api/v1/public/recordings/{rid}").json()["media"]["url"] + assert anon.get(url, headers={"Range": "bytes=0-4095"}).content == original # nothing changes until someone picks + on = {"on": True} + assert c.put(f"/api/v1/recordings/{rid}/blur-faces/visitors", json=on, headers=he).status_code == 403 + assert c.put(f"/api/v1/recordings/{rid}/blur-faces/visitors", json=on, headers=h).json()["blurred"]["visitors"] is True + copy = files.path_of(cfg, files.get(db, rid, second["blurred"]["file"])) + seen = anon.get(url, headers={"Range": "bytes=0-4095"}) + assert (seen.status_code, seen.headers["content-type"], seen.content) == (206, "video/mp4", copy.read_bytes()[:4096]) + assert anon.get(f"/iiif/{rid}/media", headers={"Range": "bytes=0-4095"}).content == seen.content + assert c.get(f"/api/v1/recordings/{rid}/audio", headers={**he, "Range": "bytes=0-4095"}).content == original + assert anon.get(f"/api/v1/recordings/{rid}/blur-faces").status_code in (401, 404) # members only + + # the copy gone missing: visitors get nothing rather than the faces + hidden = copy.with_name("elsewhere") + copy.rename(hidden) + assert anon.get(url).status_code == 404 + hidden.rename(copy) + # deleting the copy in the Files tab takes the choice with it + assert c.delete(f"/api/v1/recordings/{rid}/files/{second['blurred']['file']}", headers=he).status_code == 200 + assert anon.get(url, headers={"Range": "bytes=0-4095"}).content == original + assert c.get(f"/api/v1/recordings/{rid}/blur-faces", headers=he).json()["blurred"] is None + r = c.put(f"/api/v1/recordings/{rid}/blur-faces/visitors", json=on, headers=h) + assert (r.status_code, r.json()["detail"]) == (400, "Make a copy with the faces blurred first.") + + # agents: the same through MCP + out = tool(c, he, "get_face_blur", recording_id=rid) + assert [p["track"] for p in out["faces"]] == [t["id"] for t in tracks] and "blurred" not in out + other = next(t["id"] for t in tracks if t["id"] != named["id"]) + job = tool(c, he, "blur_faces", recording_id=rid, who="chosen", tracks=[other])["job"] + drain(db, cfg) + out = tool(c, he, "get_face_blur", recording_id=rid) + assert (out["job"]["id"], out["blurred"]["tracks"]) == (job, [other]) + assert out["blurred"]["download"].endswith(f"/files/{out['blurred']['file']}/download") + assert tool_error(c, he, "set_face_blur_for_visitors", recording_id=rid, on=True) # owners + assert tool(c, h, "set_face_blur_for_visitors", recording_id=rid, on=True)["blurred"]["visitors"] is True + assert tool_error(c, he, "blur_faces", recording_id=rid, who="everyone") == "who is one of all, unnamed, chosen" + + audited = [a["action"] for a in c.get("/api/v1/audit", headers=h).json()] + assert {"recording.blur_faces", "recording.blur_faces.visitors"} <= set(audited) diff --git a/fastapi_backend/tests/api/test_jobs.py b/fastapi_backend/tests/api/test_jobs.py index 528b5ef85..1485c1aac 100644 --- a/fastapi_backend/tests/api/test_jobs.py +++ b/fastapi_backend/tests/api/test_jobs.py @@ -375,7 +375,7 @@ def test_a_steps_list_from_before_the_video_steps_runs_them_too(db, cfg): w = jobs.Worker(db, lambda: {**cfg, "workers": {**cfg["workers"], "steps": old}}, name="old", log=said.append) assert w.can == set(jobs.STEPS) and not said w = jobs.Worker(db, lambda: {**cfg, "workers": {**cfg["workers"], "steps": [*old, "shots"]}}, name="chosen", log=said.append) - assert "ocr" not in w.can and "leaves out describe, faces, nsfw, objects, ocr" in said[0] + assert "ocr" not in w.can and "leaves out blur_faces, describe, faces, nsfw, objects, ocr" in said[0] assert jobs.Worker(db, lambda: cfg, name="mac", steps=["transcribe"], log=said.append).can == {"transcribe"} and len(said) == 1 diff --git a/fastapi_backend/tests/api/test_mcp.py b/fastapi_backend/tests/api/test_mcp.py index ac5b5800f..3c1d6e6bc 100644 --- a/fastapi_backend/tests/api/test_mcp.py +++ b/fastapi_backend/tests/api/test_mcp.py @@ -102,6 +102,8 @@ def test_clients_are_sent_to_sign_in(client, app, env): "mark_nsfw", "set_language", "enhance", + "blur_faces", + "set_face_blur_for_visitors", "set_namespace_integrations", "set_picture_descriptions", "set_file_storage", diff --git a/nextjs-frontend/__tests__/face-blur.test.ts b/nextjs-frontend/__tests__/face-blur.test.ts new file mode 100644 index 000000000..b37bfd985 --- /dev/null +++ b/nextjs-frontend/__tests__/face-blur.test.ts @@ -0,0 +1,50 @@ +import type { FaceBlur } from "@/app/openapi-client/types.gen"; +import { blurRequest, blurring, blurStatus, copyLine } from "@/components/recording/video/face-blur-model"; + +const job = (over: Partial>): FaceBlur => ({ job: { id: 7, status: "running", ...over } }); + +describe("blurring faces for sharing", () => { + it("asks for every face when all are ticked, else for the ones ticked in order", () => { + expect(blurRequest(new Set(["a", "b"]), ["a", "b"])).toEqual({ who: "all" }); + expect(blurRequest(new Set(["c", "a"]), ["a", "b", "c"])).toEqual({ who: "chosen", tracks: ["a", "c"] }); + }); + + it("keeps asking while the job is still to finish", () => { + expect(blurring(job({ status: "queued" }))).toBe(true); + expect(blurring(job({ status: "held" }))).toBe(true); + expect(blurring(job({ status: "succeeded" }))).toBe(false); + expect(blurring({ blurred: null, job: null })).toBe(false); + expect(blurring(undefined)).toBe(false); + }); + + it("says how far it got, why it waits and why it failed", () => { + expect(blurStatus(job({ progress: 40 }))).toEqual({ text: "Blurring faces… 40%", tone: "intent", progress: 0.4 }); + expect(blurStatus(job({}))).toEqual({ text: "Blurring faces…", tone: "intent", progress: null }); + expect(blurStatus(job({ status: "queued", waits: "heavy steps run 01:00-06:00 (UTC)" }))?.text).toBe( + "Waiting to start: heavy steps run 01:00-06:00 (UTC)", + ); + expect(blurStatus(job({ status: "failed", error: "couldn't blur the faces: no ffmpeg" }))).toEqual({ + text: "Failed: couldn't blur the faces: no ffmpeg", + tone: "red", + progress: null, + }); + expect(blurStatus(job({ status: "failed" }))?.text).toBe("Couldn’t blur the faces"); + expect(blurStatus(job({ status: "succeeded" }))).toBeNull(); + expect(blurStatus(null)).toBeNull(); + }); + + it("describes the copy", () => { + const now = Date.parse("2026-10-19T12:10:00Z"); + const copy = { + file: 3, + name: "review-faces-blurred.mp4", + download: "/x", + faces: 2, + who: "all", + size: 1536, + made_at: "2026-10-19T12:00:00Z", + }; + expect(copyLine(copy, now)).toBe("2 faces blurred · 1.5 KB · 10 minutes ago"); + expect(copyLine({ ...copy, faces: 1, size: null, made_at: null }, now)).toBe("1 face blurred"); + }); +}); diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 906887a94..1c1d36b07 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -282,6 +282,11 @@ export type { BatchRun, BatchSelection, BatchSummary, + BlurFacesData, + BlurFacesError, + BlurFacesErrors, + BlurFacesResponse, + BlurFacesResponses, BodyOauthRevoke, BodyOauthToken, BridgeStatus, @@ -1239,6 +1244,11 @@ export type { ExternalStartResponse, ExternalStartResponses, FaceAsset, + FaceBlur, + FaceBlurCopy, + FaceBlurCreate, + FaceBlurJob, + FaceBlurVisitors, FaceDismiss, FaceMerge, FaceMerged, @@ -1457,6 +1467,11 @@ export type { GetExtensionErrors, GetExtensionResponse, GetExtensionResponses, + GetFaceBlurData, + GetFaceBlurError, + GetFaceBlurErrors, + GetFaceBlurResponse, + GetFaceBlurResponses, GetFedoraStatusData, GetFedoraStatusResponse, GetFedoraStatusResponses, @@ -3745,6 +3760,11 @@ export type { SetElsewhereErrors, SetElsewhereResponse, SetElsewhereResponses, + SetFaceBlurVisitorsData, + SetFaceBlurVisitorsError, + SetFaceBlurVisitorsErrors, + SetFaceBlurVisitorsResponse, + SetFaceBlurVisitorsResponses, SetInstallData, SetInstallError, SetInstallErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 8a47cb19b..3120fa1ae 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -86,6 +86,9 @@ import type { BackupLinkData, BackupLinkErrors, BackupLinkResponses, + BlurFacesData, + BlurFacesErrors, + BlurFacesResponses, BridgeStatusData, BridgeStatusResponses, BrowseSourceData, @@ -678,6 +681,9 @@ import type { GetExtensionData, GetExtensionErrors, GetExtensionResponses, + GetFaceBlurData, + GetFaceBlurErrors, + GetFaceBlurResponses, GetFedoraStatusData, GetFedoraStatusResponses, GetFeedData, @@ -1801,6 +1807,9 @@ import type { SetElsewhereData, SetElsewhereErrors, SetElsewhereResponses, + SetFaceBlurVisitorsData, + SetFaceBlurVisitorsErrors, + SetFaceBlurVisitorsResponses, SetInstallData, SetInstallErrors, SetInstallResponses, @@ -6436,7 +6445,7 @@ export class Resources { * * The audio (or video) for the player, with byte ranges: a copy every browser plays when Settings › Video made one, * else the file itself (``/media`` always gives the file). Accepts a bearer token, a share link (``?s=``) or a signed - * link. + * link. Visitors get the copy with the faces blurred where an owner chose that (docs/video.md). */ public static getAudio( options: Options, @@ -9551,7 +9560,8 @@ export class Video { * * The video or audio file, with byte ranges; a document's or an image's file, to save. Accepts a bearer token, a * share link (``?s=``) or a signed link. With the asset's downloads off, 403 for anyone below its owners (players - * stream /recordings/{rid}/audio, which stays open). + * stream /recordings/{rid}/audio, which stays open). Visitors get the copy with the faces blurred where an owner chose + * that. */ public static getMedia( options: Options, @@ -9641,6 +9651,66 @@ export class Video { }); } + /** + * Get Face Blur + * + * The video's copy with the faces blurred for sharing, once one is made (with a signed link to save it, and whether + * visitors play it), and the latest job to make one: its status, how far ffmpeg got, why it waits or why it failed. + * Members of its namespace. + */ + public static getFaceBlur( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/resources/{rid}/blur-faces", + ...options, + }); + } + + /** + * Blur Faces + * + * Make a copy of the video with faces blurred, for sharing: every face found (`who` all, the default), the ones + * nobody has named (unnamed), or the face tracks in `tracks` (chosen). It runs in the background as a heavy step + * (quiet processing hours apply) and is kept as one of the asset's files, an attachment labelled "Faces blurred"; + * asking again replaces it, and asking while it runs gives the same job. The original stays as it is. Needs the + * right to process the asset; 400 for anything but a video, or no faces to blur. Audited as + * `recording.blur_faces`. + */ + public static blurFaces( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/resources/{rid}/blur-faces", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Set Face Blur Visitors + * + * Have visitors (public pages, embeds, share links, IIIF, podcast feeds) play the copy with the faces blurred + * instead of the original, or not; members play the original either way. While it's on and the copy is missing, + * visitors get nothing rather than the faces. Owners (the right to publish it); 400 before a copy is made. Audited + * as `recording.blur_faces.visitors`. + */ + public static setFaceBlurVisitors( + options: Options, + ): RequestResult { + return (options.client ?? client).put({ + url: "/api/v1/resources/{rid}/blur-faces/visitors", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * Delete Namespace Faces * diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index a798a6f02..1e8511219 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -8700,6 +8700,143 @@ export type FaceAsset = { [key: string]: unknown; }; +/** + * FaceBlur + */ +export type FaceBlur = { + /** + * the copy with the faces blurred, once one is made + */ + blurred?: FaceBlurCopy | null; + /** + * the latest job to make one + */ + job?: FaceBlurJob | null; + [key: string]: unknown; +}; + +/** + * FaceBlurCopy + * + * The copy with the faces blurred, kept as one of the asset's files (an attachment). + */ +export type FaceBlurCopy = { + /** + * File + * + * its id among the asset's files + */ + file: number; + /** + * Name + */ + name: string; + /** + * Size + */ + size?: number | null; + /** + * Download + * + * a signed link to save it + */ + download: string; + /** + * Faces + * + * how many faces (tracks) are blurred + */ + faces: number; + /** + * Who + * + * all, unnamed or chosen + */ + who: string; + /** + * Tracks + * + * the face tracks blurred + */ + tracks?: Array; + /** + * Made At + */ + made_at?: string | null; + /** + * Visitors + * + * visitors (public pages, embeds, share links, IIIF, feeds) play it instead of the original + */ + visitors?: boolean; + [key: string]: unknown; +}; + +/** + * FaceBlurCreate + */ +export type FaceBlurCreate = { + /** + * Who + * + * whose faces to blur: every face found, the ones nobody has named, or the tracks in `tracks` + */ + who?: "all" | "unnamed" | "chosen"; + /** + * Tracks + * + * with who=chosen: the face tracks to blur (the player's faces[].id) + */ + tracks?: Array; +}; + +/** + * FaceBlurJob + * + * The latest job to make the copy. + */ +export type FaceBlurJob = { + /** + * Id + */ + id: number; + /** + * Status + * + * queued, running, held, succeeded, failed or cancelled + */ + status: string; + /** + * Error + */ + error?: string | null; + /** + * Progress + * + * how far ffmpeg is, in percent, while it runs + */ + progress?: number | null; + /** + * Waits + * + * why it waits to start (quiet processing hours, or heavy steps at once) + */ + waits?: string | null; + [key: string]: unknown; +}; + +/** + * FaceBlurVisitors + */ +export type FaceBlurVisitors = { + /** + * On + * + * visitors play the copy with the faces blurred instead of the original + */ + on: boolean; +}; + /** * FaceDismiss */ @@ -45580,6 +45717,96 @@ export type DeleteFaceTrackResponses = { export type DeleteFaceTrackResponse = DeleteFaceTrackResponses[keyof DeleteFaceTrackResponses]; +export type GetFaceBlurData = { + body?: never; + path: { + /** + * Rid + */ + rid: number; + }; + query?: never; + url: "/api/v1/resources/{rid}/blur-faces"; +}; + +export type GetFaceBlurErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GetFaceBlurError = GetFaceBlurErrors[keyof GetFaceBlurErrors]; + +export type GetFaceBlurResponses = { + /** + * Successful Response + */ + 200: FaceBlur; +}; + +export type GetFaceBlurResponse = GetFaceBlurResponses[keyof GetFaceBlurResponses]; + +export type BlurFacesData = { + body: FaceBlurCreate; + path: { + /** + * Rid + */ + rid: number; + }; + query?: never; + url: "/api/v1/resources/{rid}/blur-faces"; +}; + +export type BlurFacesErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type BlurFacesError = BlurFacesErrors[keyof BlurFacesErrors]; + +export type BlurFacesResponses = { + /** + * Successful Response + */ + 202: JobQueued; +}; + +export type BlurFacesResponse = BlurFacesResponses[keyof BlurFacesResponses]; + +export type SetFaceBlurVisitorsData = { + body: FaceBlurVisitors; + path: { + /** + * Rid + */ + rid: number; + }; + query?: never; + url: "/api/v1/resources/{rid}/blur-faces/visitors"; +}; + +export type SetFaceBlurVisitorsErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type SetFaceBlurVisitorsError = SetFaceBlurVisitorsErrors[keyof SetFaceBlurVisitorsErrors]; + +export type SetFaceBlurVisitorsResponses = { + /** + * Successful Response + */ + 200: FaceBlur; +}; + +export type SetFaceBlurVisitorsResponse = SetFaceBlurVisitorsResponses[keyof SetFaceBlurVisitorsResponses]; + export type DeleteNamespaceFacesData = { body?: never; path: { diff --git a/nextjs-frontend/components/recording/hooks.ts b/nextjs-frontend/components/recording/hooks.ts index 438ef634f..1ad566675 100644 --- a/nextjs-frontend/components/recording/hooks.ts +++ b/nextjs-frontend/components/recording/hooks.ts @@ -16,11 +16,19 @@ import { Templates, Video, } from "@/app/openapi-client"; -import type { CommentCreate, FileUpdate, HighlightCreate, NoteCreate, Recording } from "@/app/openapi-client/types.gen"; +import type { + CommentCreate, + FaceBlurCreate, + FileUpdate, + HighlightCreate, + NoteCreate, + Recording, +} from "@/app/openapi-client/types.gen"; import type { EvidencePart } from "@/components/evidence/parts-dialog"; import type { FileRole } from "@/components/recording/files-model"; import { isActive, normalizeJob, visualNotes, type JobInfo } from "@/components/recording/jobs"; import { normalizePlayer } from "@/components/recording/model"; +import { blurring } from "@/components/recording/video/face-blur-model"; import { useToast } from "@/components/ui/toast"; import { ApiError, data, useApiClient } from "@/lib/api/browser"; import { useArchive } from "@/lib/hooks/session"; @@ -40,6 +48,7 @@ export const rk = { files: (id: number) => ["recording", id, "files"] as const, spec: (id: number) => ["recording", id, "spec"] as const, translation: (id: number) => ["recording", id, "files", "translation"] as const, + faceBlur: (id: number) => ["recording", id, "files", "face-blur"] as const, fileLines: (id: number, fid: number) => ["recording", id, "files", fid, "lines"] as const, job: (jid: number) => ["job", jid] as const, }; @@ -483,6 +492,46 @@ export function useSetLanguage(id: number) { }); } +/** A video's copy with the faces blurred for sharing, and the job making one: asked again every few seconds while it + * runs, so its progress shows (docs/video.md, Blurring faces for sharing). */ +export function useFaceBlur(id: number, enabled = true) { + const client = useApiClient(); + return useQuery({ + queryKey: rk.faceBlur(id), + enabled, + queryFn: () => data(Video.getFaceBlur({ client, path: { rid: id } })), + refetchInterval: (query) => (blurring(query.state.data) ? 3000 : false), + }); +} + +/** Make the copy with these faces blurred, in the background; and have visitors play it, or not. */ +export function useFaceBlurActions(id: number) { + const client = useApiClient(); + const qc = useQueryClient(); + const toast = useToast(); + const refresh = () => { + void qc.invalidateQueries({ queryKey: rk.faceBlur(id) }); + void qc.invalidateQueries({ queryKey: rk.jobs(id) }); + }; + const fail = (title: string) => (e: unknown) => + toast({ title, body: e instanceof ApiError ? e.message : "Please try again.", tone: "red" }); + return { + blur: useMutation({ + mutationFn: (body: FaceBlurCreate) => data(Video.blurFaces({ client, path: { rid: id }, body })), + onSuccess: refresh, + onError: fail("Couldn’t start blurring the faces"), + }), + visitors: useMutation({ + mutationFn: (on: boolean) => data(Video.setFaceBlurVisitors({ client, path: { rid: id }, body: { on } })), + onSuccess: (out, on) => { + qc.setQueryData(rk.faceBlur(id), out); + toast({ title: on ? "Visitors see the copy with faces blurred" : "Visitors see the original", tone: "green" }); + }, + onError: fail("Couldn’t change what visitors see"), + }), + }; +} + /** The asset's spec from its transcript (docs/notes.md, Specs from transcripts), checked again while it's being written. */ export function useSpec(id: number, enabled = true) { const client = useApiClient(); diff --git a/nextjs-frontend/components/recording/video/face-blur-model.ts b/nextjs-frontend/components/recording/video/face-blur-model.ts new file mode 100644 index 000000000..e184d58a8 --- /dev/null +++ b/nextjs-frontend/components/recording/video/face-blur-model.ts @@ -0,0 +1,50 @@ +/** + * Blur faces for sharing, in a video's People tab (docs/video.md, Blurring faces for sharing): what to ask for, and + * one line on the job making the copy. Pure functions. + */ +import type { FaceBlur, FaceBlurCopy, FaceBlurCreate } from "@/app/openapi-client/types.gen"; +import { bytes, plural, relative } from "@/lib/format"; + +const ACTIVE = ["queued", "running", "held"]; + +/** What to ask for: every face when all are ticked, else the ones ticked, in the order they first appear. */ +export function blurRequest(picked: ReadonlySet, all: readonly string[]): FaceBlurCreate { + return all.every((id) => picked.has(id)) + ? { who: "all" } + : { who: "chosen", tracks: all.filter((id) => picked.has(id)) }; +} + +/** Whether the latest job is still to finish (the tab asks again while it is). */ +export function blurring(s: FaceBlur | null | undefined): boolean { + return ACTIVE.includes(s?.job?.status ?? ""); +} + +export type BlurStatus = { text: string; tone: "intent" | "gate" | "red"; progress: number | null }; + +/** Where the latest job is: blurring (and how far), waiting (and why), or failed (and why); null when it's done. */ +export function blurStatus(s: FaceBlur | null | undefined): BlurStatus | null { + const j = s?.job; + if (!j) return null; + if (j.status === "running") + return { + text: j.progress != null ? `Blurring faces… ${j.progress}%` : "Blurring faces…", + tone: "intent", + progress: j.progress != null ? j.progress / 100 : null, + }; + if (j.status === "queued" || j.status === "held") + return { text: j.waits ? `Waiting to start: ${j.waits}` : "Waiting to start…", tone: "gate", progress: null }; + if (j.status === "failed") + return { text: j.error ? `Failed: ${j.error}` : "Couldn’t blur the faces", tone: "red", progress: null }; + return null; +} + +/** "2 faces blurred · 1.2 MB · 5 min ago": what the copy is. */ +export function copyLine(c: FaceBlurCopy, now = Date.now()): string { + return [ + plural(c.faces, "face") + " blurred", + c.size != null ? bytes(c.size) : null, + c.made_at ? relative(c.made_at, now) : null, + ] + .filter(Boolean) + .join(" · "); +} diff --git a/nextjs-frontend/components/recording/video/people-tab.tsx b/nextjs-frontend/components/recording/video/people-tab.tsx index 48f30ace2..854a1bc39 100644 --- a/nextjs-frontend/components/recording/video/people-tab.tsx +++ b/nextjs-frontend/components/recording/video/people-tab.tsx @@ -1,7 +1,7 @@ "use client"; import { useMutation, useQueryClient } from "@tanstack/react-query"; -import { Ellipsis, ScanFace, UserRoundX } from "lucide-react"; +import { Download, Ellipsis, ScanFace, UserRoundX } from "lucide-react"; import Link from "next/link"; import { useState, type FormEvent } from "react"; @@ -9,16 +9,17 @@ import { Video } from "@/app/openapi-client"; import { usePlayerApi } from "@/components/player/media"; import { useBlur } from "@/components/nsfw/nsfw"; import { useRec } from "@/components/recording/context"; -import { rk, useNamespaceFaces } from "@/components/recording/hooks"; +import { rk, useFaceBlur, useFaceBlurActions, useNamespaceFaces } from "@/components/recording/hooks"; import { facePages, pageRef } from "@/components/recording/document/model"; import type { FaceTrack } from "@/components/recording/model"; +import { blurRequest, blurStatus, copyLine } from "@/components/recording/video/face-blur-model"; import { screenTime } from "@/components/recording/video/model"; import { useFaceColors } from "@/components/recording/video/face-colors"; import { Button } from "@/components/ui/button"; import { Dialog } from "@/components/ui/dialog"; -import { Input, Switch } from "@/components/ui/field"; +import { Checkbox, Input, Switch } from "@/components/ui/field"; import { Menu, MenuContent, MenuItem, MenuLabel, MenuSeparator, MenuTrigger } from "@/components/ui/menu"; -import { EmptyState, Skeleton } from "@/components/ui/states"; +import { EmptyState, Progress, Skeleton } from "@/components/ui/states"; import { useToast } from "@/components/ui/toast"; import { ApiError, data, useApiClient } from "@/lib/api/browser"; import { tc } from "@/lib/format"; @@ -79,6 +80,7 @@ export function PeopleTab({ noFaces, onPage }: { noFaces?: boolean; onPage?: (pa return ( <> {pixelate} + {!onPage && } {model.facesMode === "detect" && ( This namespace is set to detect only: {onPage ? "boxes and pages" : "boxes and screen time"}, no names, no @@ -469,3 +471,153 @@ function PixelateSwitch({ id, ns, on }: { id: number; ns: string | null; on: boo ); } + +/** + * Blur faces for sharing (docs/video.md): one click, every face ticked, Enter to start; the job's progress, then the + * copy to download and, for owners, whether visitors play it instead of the original. The original stays as it is. + */ +function BlurFacesPanel() { + const { id, ns, model, canEdit } = useRec(); + const { can } = useArchive(); + const state = useFaceBlur(id); + const act = useFaceBlurActions(id); + const [picking, setPicking] = useState(false); + const status = blurStatus(state.data); + const copy = state.data?.blurred ?? null; + const owner = can("owner", ns); + const busy = status != null && status.tone !== "red"; + return ( +
+
+
+
Blur faces for sharing
+

+ A copy of this video with faces blurred, kept with its files. The original stays as it is. +

+
+ +
+ {status && ( +
+ + {status.text} + + {status.tone !== "red" && } +
+ )} + {copy && ( +
+
+
{copy.name}
+
{copyLine(copy)}
+
+ + Download + +
+ {!owner && {needRole("owner", ns)}} + act.visitors.mutate(v)} + /> +
+
+ )} + {picking && ( + setPicking(false)} + onBlur={(picked) => + act.blur.mutate( + blurRequest( + picked, + model.faces.map((f) => f.id), + ), + { onSuccess: () => setPicking(false) }, + ) + } + /> + )} +
+ ); +} + +/** Which faces to blur: all of them ticked and the button focused, so Enter starts it at once. */ +function BlurDialog({ + faces, + recognize, + pending, + onClose, + onBlur, +}: { + faces: FaceTrack[]; + recognize: boolean; + pending: boolean; + onClose: () => void; + onBlur: (picked: Set) => void; +}) { + const [picked, setPicked] = useState(() => new Set(faces.map((f) => f.id))); + const toggle = (fid: string, on: boolean) => + setPicked((p) => { + const next = new Set(p); + if (on) next.add(fid); + else next.delete(fid); + return next; + }); + return ( + !o && onClose()} + title="Blur faces for sharing" + description="Lens makes a copy of the video with these faces blurred wherever they were found, a little before and after too. It runs in the background; the copy is kept with the video’s files." + actions={ + <> + + + + } + > +
    + {faces.map((f, i) => ( +
  • + toggle(f.id, v)} + label={`${recognize ? f.name : `Face ${i + 1}`} · ${screenTime(f.screenMs)} on screen`} + /> +
  • + ))} +
+
+ ); +} diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 2fd8c1511..05736800e 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -2978,12 +2978,22 @@ export const FIELDS: FieldSpec[] = [ key: "heavy_steps", label: "Heavy steps", kind: "checks", - options: ["transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "embed"].map( - (s) => ({ - value: s, - label: s, - }), - ), + options: [ + "transcribe", + "read", + "diarize", + "shots", + "ocr", + "faces", + "objects", + "nsfw", + "blur_faces", + "describe", + "embed", + ].map((s) => ({ + value: s, + label: s, + })), }, { section: "workers", diff --git a/nextjs-frontend/components/ui/loop.tsx b/nextjs-frontend/components/ui/loop.tsx index 47303b1b4..613a65f95 100644 --- a/nextjs-frontend/components/ui/loop.tsx +++ b/nextjs-frontend/components/ui/loop.tsx @@ -28,6 +28,7 @@ export const STEP_TONE: Record = { spec: "gate", reply: "gate", translate: "gate", + blur_faces: "red", report: "gate", export: "gate", workflow: "green", @@ -54,6 +55,7 @@ export const STEP_LABEL: Record = { spec: "Spec", reply: "Reply", translate: "Translate", + blur_faces: "Blur faces", report: "Report", export: "Export", workflow: "Workflow", diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 5c81484b7..a7ff665d6 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -11734,7 +11734,7 @@ "resources" ], "summary": "Get Audio", - "description": "The audio (or video) for the player, with byte ranges: a copy every browser plays when Settings \u203a Video made one,\nelse the file itself (``/media`` always gives the file). Accepts a bearer token, a share link (``?s=``) or a signed\nlink.", + "description": "The audio (or video) for the player, with byte ranges: a copy every browser plays when Settings \u203a Video made one,\nelse the file itself (``/media`` always gives the file). Accepts a bearer token, a share link (``?s=``) or a signed\nlink. Visitors get the copy with the faces blurred where an owner chose that (docs/video.md).", "operationId": "get_audio", "parameters": [ { @@ -22286,7 +22286,7 @@ "video" ], "summary": "Get Media", - "description": "The video or audio file, with byte ranges; a document's or an image's file, to save. Accepts a bearer token, a\nshare link (``?s=``) or a signed link. With the asset's downloads off, 403 for anyone below its owners (players\nstream /recordings/{rid}/audio, which stays open).", + "description": "The video or audio file, with byte ranges; a document's or an image's file, to save. Accepts a bearer token, a\nshare link (``?s=``) or a signed link. With the asset's downloads off, 403 for anyone below its owners (players\nstream /recordings/{rid}/audio, which stays open). Visitors get the copy with the faces blurred where an owner chose\nthat.", "operationId": "get_media", "parameters": [ { @@ -22605,6 +22605,153 @@ } } }, + "/api/v1/resources/{rid}/blur-faces": { + "get": { + "tags": [ + "video" + ], + "summary": "Get Face Blur", + "description": "The video's copy with the faces blurred for sharing, once one is made (with a signed link to save it, and whether\nvisitors play it), and the latest job to make one: its status, how far ffmpeg got, why it waits or why it failed.\nMembers of its namespace.", + "operationId": "get_face_blur", + "parameters": [ + { + "name": "rid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Rid" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FaceBlur" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "video" + ], + "summary": "Blur Faces", + "description": "Make a copy of the video with faces blurred, for sharing: every face found (`who` all, the default), the ones\nnobody has named (unnamed), or the face tracks in `tracks` (chosen). It runs in the background as a heavy step\n(quiet processing hours apply) and is kept as one of the asset's files, an attachment labelled \"Faces blurred\";\nasking again replaces it, and asking while it runs gives the same job. The original stays as it is. Needs the\nright to process the asset; 400 for anything but a video, or no faces to blur. Audited as\n`recording.blur_faces`.", + "operationId": "blur_faces", + "parameters": [ + { + "name": "rid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Rid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FaceBlurCreate" + } + } + } + }, + "responses": { + "202": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobQueued" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/resources/{rid}/blur-faces/visitors": { + "put": { + "tags": [ + "video" + ], + "summary": "Set Face Blur Visitors", + "description": "Have visitors (public pages, embeds, share links, IIIF, podcast feeds) play the copy with the faces blurred\ninstead of the original, or not; members play the original either way. While it's on and the copy is missing,\nvisitors get nothing rather than the faces. Owners (the right to publish it); 400 before a copy is made. Audited\nas `recording.blur_faces.visitors`.", + "operationId": "set_face_blur_visitors", + "parameters": [ + { + "name": "rid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Rid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FaceBlurVisitors" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FaceBlur" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/namespaces/{name}/faces": { "get": { "tags": [ @@ -53123,6 +53270,208 @@ ], "title": "FaceAsset" }, + "FaceBlur": { + "properties": { + "blurred": { + "anyOf": [ + { + "$ref": "#/components/schemas/FaceBlurCopy" + }, + { + "type": "null" + } + ], + "description": "the copy with the faces blurred, once one is made" + }, + "job": { + "anyOf": [ + { + "$ref": "#/components/schemas/FaceBlurJob" + }, + { + "type": "null" + } + ], + "description": "the latest job to make one" + } + }, + "additionalProperties": true, + "type": "object", + "title": "FaceBlur" + }, + "FaceBlurCopy": { + "properties": { + "file": { + "type": "integer", + "title": "File", + "description": "its id among the asset's files" + }, + "name": { + "type": "string", + "title": "Name" + }, + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Size" + }, + "download": { + "type": "string", + "title": "Download", + "description": "a signed link to save it" + }, + "faces": { + "type": "integer", + "title": "Faces", + "description": "how many faces (tracks) are blurred" + }, + "who": { + "type": "string", + "title": "Who", + "description": "all, unnamed or chosen" + }, + "tracks": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Tracks", + "description": "the face tracks blurred" + }, + "made_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Made At" + }, + "visitors": { + "type": "boolean", + "title": "Visitors", + "description": "visitors (public pages, embeds, share links, IIIF, feeds) play it instead of the original", + "default": false + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "file", + "name", + "download", + "faces", + "who" + ], + "title": "FaceBlurCopy", + "description": "The copy with the faces blurred, kept as one of the asset's files (an attachment)." + }, + "FaceBlurCreate": { + "properties": { + "who": { + "type": "string", + "enum": [ + "all", + "unnamed", + "chosen" + ], + "title": "Who", + "description": "whose faces to blur: every face found, the ones nobody has named, or the tracks in `tracks`", + "default": "all" + }, + "tracks": { + "items": { + "type": "string" + }, + "type": "array", + "maxItems": 500, + "title": "Tracks", + "description": "with who=chosen: the face tracks to blur (the player's faces[].id)" + } + }, + "additionalProperties": false, + "type": "object", + "title": "FaceBlurCreate" + }, + "FaceBlurJob": { + "properties": { + "id": { + "type": "integer", + "title": "Id" + }, + "status": { + "type": "string", + "title": "Status", + "description": "queued, running, held, succeeded, failed or cancelled" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "progress": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Progress", + "description": "how far ffmpeg is, in percent, while it runs" + }, + "waits": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Waits", + "description": "why it waits to start (quiet processing hours, or heavy steps at once)" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "status" + ], + "title": "FaceBlurJob", + "description": "The latest job to make the copy." + }, + "FaceBlurVisitors": { + "properties": { + "on": { + "type": "boolean", + "title": "On", + "description": "visitors play the copy with the faces blurred instead of the original" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "on" + ], + "title": "FaceBlurVisitors" + }, "FaceDismiss": { "properties": { "kind": { diff --git a/threat-model.md b/threat-model.md index 32aef7a4d..545b29814 100644 --- a/threat-model.md +++ b/threat-model.md @@ -465,6 +465,7 @@ stored text for the consumer's format. | Evidence PDFs | `evidence.py`, `evidence_report.py` | the resource's file and what Lens holds about it (metadata, summary, speakers, emotion, entities, what is seen), for anyone who can read it; places only for members of its namespace; shared notes, highlights and comments only when picked, never private notes; what a model wrote is labelled | that a PDF handed on holds less than its reader could see (pick its parts) | *(documented, docs/api.md "never anyone's private notes")* | | Playlists | `domain/playlists.py`, `routes/playlists.py` | a shared playlist's clips (titles, notes, quotes, signed media links) only where its reader has a role on the clip's resource; its name and description to everyone signed in | that a shared playlist's name and description stay among the people who can read its clips | *(documented, docs/api.md "whoever made it")* | | IIIF notes layer (`/iiif//annotations/notes`) | `api/iiif.py`, `domain/iiif.py` | with `notes` in `iiif.layers` (off by default), shared notes and comments as Web Annotations, with their writers' names, only to someone with a role on the asset (their sign-in or the IIIF cookie); never unshared notes, never to public access, share, signed or IP-group links; sent `Cache-Control: private, no-store` | that a viewer that fetched the layer keeps it among the people with a role | *(documented, docs/iiif.md "Unshared notes are their writer's alone")* | +| Faces-blurred copy for visitors | `domain/face_blur.py`, `routes/recordings.py` (`serve_audio`), `api/iiif.py`, `routes/feeds.py` | once an owner turns it on, a copy of the video with the faces found blurred, served instead of the original to every request without a role in the namespace (public pages, embeds, share links, IIIF, podcast feeds); 404 to them when the copy is missing; members still get the original | that faces the detector missed are blurred | *(documented, docs/video.md "visitors get nothing rather than the faces")* | | Asset description | IIIF, public pages | "never a closed part"; anyone who may see the page sees it | that closing parts hides the description | *(documented, docs/access.md "never a closed part")* | | Embed player and `/s/` pages | `api/pages.py` | HTML built from DB values under a strict CSP; framable only per `server.embed_frame_ancestors` | that titles are inert outside this page | *(documented, docs/configuration.md "Content-Security-Policy")* | | Report pages | `api/pages.py` | links re-signed on read for every asset of the namespace in the page | that a report link opens only the report | *(inferred, Q10)* | diff --git a/threat-model.yaml b/threat-model.yaml index 9734c7cf5..28e4cfde5 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -1008,6 +1008,13 @@ outputs: invariants: [] downstream_must_not_assume: - {id: dmna-iiif-notes-kept-private, statement: "that a viewer that fetched the layer keeps it among the people with a role", provenance: {kind: documented, source: 'docs/iiif.md "Unshared notes are their writer''s alone"'}} + channel: "Faces-blurred copy for visitors (domain/face_blur.py, routes/recordings.py serve_audio, api/iiif.py, routes/feeds.py)" + taint: same-as-input + x-taint-note: "once an owner turns it on, a copy of the video with the faces found blurred, served instead of the original to every request without a role in the namespace (public pages, embeds, share links, IIIF, podcast feeds); 404 to them when the copy is missing; members still get the original" + taint_provenance: {kind: documented, source: 'docs/video.md "visitors get nothing rather than the faces"'} + invariants: [] + downstream_must_not_assume: + - {id: dmna-blur-catches-every-face, statement: "that faces the detector missed are blurred", provenance: {kind: documented, source: 'docs/video.md "visitors get nothing rather than the faces"'}} - component: authz channel: "Recording description (IIIF, public pages)" taint: same-as-input