From cf6862c26b46fb3cd4a745181ca5c808390ec082 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 08:12:40 +0000 Subject: [PATCH] feat(imports): date photos and videos by when they were taken, from the file itself Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QFUNzy1QZy5HhKAyJxTAYJ --- README.md | 5 +- changelog.d/file-dates.md | 6 + docs/api.md | 3 +- docs/processing.md | 18 +++ docs/requirements.md | 1 + fastapi_backend/app/api/mcp_tools.py | 5 +- .../app/api/v1/routes/recordings.py | 4 +- fastapi_backend/app/domain/ingest.py | 104 ++++++++++++--- fastapi_backend/app/domain/uploads.py | 14 +- fastapi_backend/app/schemas/recordings.py | 13 ++ fastapi_backend/tests/api/test_file_dates.py | 125 ++++++++++++++++++ nextjs-frontend/app/openapi-client/index.ts | 1 + .../app/openapi-client/types.gen.ts | 31 +++++ nextjs-frontend/openapi.json | 45 +++++++ 14 files changed, 344 insertions(+), 31 deletions(-) create mode 100644 changelog.d/file-dates.md create mode 100644 fastapi_backend/tests/api/test_file_dates.py diff --git a/README.md b/README.md index 336aa924..8ecb6bc3 100644 --- a/README.md +++ b/README.md @@ -143,8 +143,9 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p emails a light server can't convert itself. Google accounts connect with Connect with Google once Lens has a Google client, and Dropbox, OneDrive and Google Drive can sign in through rclone on the Lens server, with no app of your own to register. Upload audio, video, PDFs, - Office files, photos, JSON and zip or tar archives, and, with an Apache Tika server (an opt-in extra), documents - nothing else here can read; capture web pages one at a time, from a link list or a bookmarks file, by API or by asking + Office files, photos, JSON and zip or tar archives (photos and videos dated by when they were taken, from the file + itself, and photos placed by their GPS), and, with an Apache Tika server (an opt-in extra), documents nothing else + here can read; capture web pages one at a time, from a link list or a bookmarks file, by API or by asking the assistant, from private networks you allow in Settings too; and import videos and podcasts by their address (off by default), taking a video's own captions as its transcript when people wrote some (off by default too). Say what a file or a page is as it comes in (a podcast, an interview, a scan), in Import, by API or to the assistant, and its diff --git a/changelog.d/file-dates.md b/changelog.d/file-dates.md new file mode 100644 index 00000000..fdca39b6 --- /dev/null +++ b/changelog.d/file-dates.md @@ -0,0 +1,6 @@ +- **Photos and videos are dated by when they were taken.** An asset uploaded, scanned or imported is now dated by its + own file when its name gives no date: a photo by its EXIF (`DateTimeOriginal`, else `DateTime`), a video or a + recording by its creation time (QuickTime's `creationdate`, else `creation_time`), before the file's last change. A + photo's GPS fix is kept as its `place` (`lat`, `lon`), which `GET /recordings/{rid}` and MCP `get_recording` give its + namespace's members. A date in its name, or one its source gives, still wins, and a clock never set (1970, 1904) dates + nothing ([When an asset was made](docs/processing.md#when-an-asset-was-made)). diff --git a/docs/api.md b/docs/api.md index 50f5384a..bd2ff1a5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1216,7 +1216,8 @@ runs instead of the namespace's once a new asset is made, `content_type` is the type](processing.md#content-types) (one of the file's base type, else 400; its pipeline runs) and `collection` is where it goes (none of them with `recording`, which keeps its own). The same file uploaded again with a content type gives the asset it already is that one. The name loses any folders and its extension is lowercased; `modified` -(the file's last-modified time in milliseconds) dates the asset when its name doesn't. 400 for a type not in +(the file's last-modified time in milliseconds) dates the asset when neither its name nor the file itself does +([When an asset was made](processing.md#when-an-asset-was-made)). 400 for a type not in `uploads.extensions`, 413 over `uploads.max_mb`, 507 when the server's disk can't hold it with 512 MB to spare. `PUT /uploads/{uid}?offset=` sends the next piece as the raw request body (`application/octet-stream`), starting at diff --git a/docs/processing.md b/docs/processing.md index 999ee418..f8c1cb48 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -81,6 +81,24 @@ key terms with Universal-3 and Slam-1, or boosted words with the others. SenseVo they transcribe as before. The words go only to the engine the recording was going to anyway, and a provider's row in the [activity ledger](activity.md) says how many it was given. Run Transcribe again on an asset to use new names. +### When an asset was made + +An asset's date (`recorded_at`) is set once, when it arrives (scanned, uploaded, shared into a mounted folder or sent +to an import webhook), from the first of these that says it: + +1. A date in its file's name, such as `2024-03-05 interview.mp3` or `IMG_20240305_101500.jpg`. +2. The file itself: a photo's EXIF (`DateTimeOriginal`, with its `OffsetTimeOriginal` where the camera wrote one, else + `DateTime`), or a video's or a recording's creation time as ffprobe reads it (QuickTime's + `com.apple.quicktime.creationdate`, which keeps the phone's local time, else `creation_time`, in UTC). +3. When the file last changed (for an upload, the last-modified time the browser sends). + +Nothing is guessed: a clock that was never set (a year before 1971, such as QuickTime's 1904) or a date in the future +counts as no date. A date an imported transcript gives wins over all three, as an email's or a document's own date +does once it is read, and a file uploaded again into the namespace that has it keeps its asset's date. A file on a +[storage source](#storage-sources) is dated by the source (the date it lists, else when the file last changed there). A photo's GPS fix from its EXIF is kept as the asset's `place` (`{lat, lon}`, in degrees); +`GET /api/v1/recordings/{rid}` and the MCP tool `get_recording` give it to the namespace's members only, never to +visitors. It is read on this machine and nothing is sent anywhere. Assets that arrived before keep their dates. + ### Cleaning up transcripts Speech comes out of transcription as it was said, and the extractor, chapters, keywords and search all read it. With diff --git a/docs/requirements.md b/docs/requirements.md index 433ba068..9972da98 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -170,6 +170,7 @@ request, to another check that proves the same thing; loosening a requirement or | Connect with Google; Dropbox, OneDrive and Google Drive sign in through rclone on the server | [test_gmail_source.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_gmail_source.py) `test_connecting_the_account_in_the_browser`; [test_drive_google.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_drive_google.py) `test_drive_connects_with_google_for_rclone`; [test_rclone_signin.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_rclone_signin.py) `test_drive_signs_in_through_rclone_with_no_google_app`, `test_a_browser_on_the_server_reaches_rclone_itself`, `test_onedrive_finds_the_accounts_own_drive` | | Uploads of audio, PDFs, Office files, photos, JSON and archives | [test_uploads.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_uploads.py) `test_uploading_audio_in_chunks`, `test_json_uploaded_is_read_as_records`; [test_documents.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_documents.py) `test_a_pdf_becomes_a_document_with_its_pages_and_text`, `test_scans_and_images_are_read_by_ocr`; [test_conversion.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_conversion.py) `test_office_and_text_files_become_documents`; [test_anytopdf.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_anytopdf.py) `test_an_uploaded_zip_keeps_its_files`; [test_archives.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_archives.py) `test_reading_an_archive`; [journeys.mjs](https://github.com/adeelahmad/lense/blob/main/ci/clicks/journeys.mjs) `import-file`, `import-paste` | | Media browsers can't play (an AVI, HEVC, AMR) gets a copy they can, for the player only, off by default; files they play get none | [test_playable.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_playable.py) `test_a_camera_clip_gets_a_copy_every_browser_plays`, `test_a_file_browsers_play_gets_no_copy` | +| Photos and videos dated by when they were taken, from the file itself (a photo's EXIF date and GPS fix as `place`, a video's creation time) when no name or source dates them; a date already set is kept and none is invented | [test_file_dates.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_file_dates.py) `test_a_photo_is_dated_and_placed_by_its_exif`, `test_a_date_already_set_is_kept`, `test_a_video_is_dated_by_its_creation_time`, `test_a_scanned_video_is_dated_by_its_creation_time` | | An uploaded video, end to end | No check yet ([#486](https://github.com/adeelahmad/lense/issues/486)) | | Apache Tika (opt-in) reads documents nothing else here can | [test_tika.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_tika.py) `test_off_by_default_nothing_changes`, `test_what_nothing_here_can_convert_tika_reads_as_text` | | Web pages captured one at a time, from a link list or a bookmarks file, by API or by asking; private networks only when allowed | [test_webcapture.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_webcapture.py) `test_a_web_page_becomes_a_document`, `test_an_admin_allows_a_private_network_in_settings`, `test_private_addresses_are_refused`; [test_web_links.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_web_links.py) `test_one_or_several_addresses_as_a_list`, `test_a_bookmarks_file_becomes_pages_to_capture`; [test_setup_assistant.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_setup_assistant.py) `test_the_assistant_captures_web_pages` | diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index 53e42ad4..5f827013 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -905,8 +905,8 @@ def list_speakers(ctx: Context, namespace: str | None, query: str | None, limit: @tool( "get_recording", "Get an asset", - "One asset: what it is, its date, the people in it, summary, chapters (sections), the entities and " - "keywords most mentioned in it, and a url to it. Read its text with get_transcript.", + "One asset: what it is, its date, where a photo was taken (place), the people in it, summary, chapters (sections), " + "the entities and keywords most mentioned in it, and a url to it. Read its text with get_transcript.", Arg("recording_id", "integer", "the asset", required=True), ) def get_recording(ctx: Context, recording_id: int) -> dict[str, Any]: @@ -924,6 +924,7 @@ def get_recording(ctx: Context, recording_id: int) -> dict[str, Any]: "namespace": rec.namespace, "collection": " / ".join(c.name for c in rec.collection_path) or None, "recorded_at": raw.get("recorded_at"), + "place": rec.place.model_dump() if rec.place else None, # a photo's GPS fix, for members "duration": store.tc(raw["duration_ms"]) if raw.get("duration_ms") else None, "media": render.kind(raw), "pages": raw.get("pages"), diff --git a/fastapi_backend/app/api/v1/routes/recordings.py b/fastapi_backend/app/api/v1/routes/recordings.py index cea26c1f..eea5e740 100644 --- a/fastapi_backend/app/api/v1/routes/recordings.py +++ b/fastapi_backend/app/api/v1/routes/recordings.py @@ -311,7 +311,7 @@ def retag_recordings(body: RecordingsRetag, acl: Acl, user: Writer, db: Db) -> T def get_recording(rid: int, acl: Acl, db: Db, cfg: Cfg) -> Recording: r = acl.recording(rid) space = db.one("SELECT name FROM $s", s=R("space", r["space"])) or {} - d = {k: v for k, v in r.items() if k not in ("envelope", "stats", "summary", "access_parts", "enrichment")} + d = {k: v for k, v in r.items() if k not in ("envelope", "stats", "summary", "access_parts", "enrichment", "place")} a = acc.of(db, rid) d.update(access=a["access"], open=a["open"], featured=a["featured"], access_inherited=a["inherited"]) d.update( @@ -336,6 +336,8 @@ def get_recording(rid: int, acl: Acl, db: Db, cfg: Cfg) -> Recording: d["converted_from"] = _attached_to(acl, db, r.get("converted_from")) if r.get("enrichment") and acl.member(r): # where a photo was taken: members only d["enrichment"] = {k: v for k, v in r["enrichment"].items() if k != "photo_fp"} + if r.get("place") and acl.member(r): # its file's GPS fix, likewise + d["place"] = r["place"] d["nsfw"] = _nsfw(db, cfg, r, rid) d["enhanced"] = enhance.brief(r.get("enhanced"), rid) waiting = r.get("awaits_pdf") or None diff --git a/fastapi_backend/app/domain/ingest.py b/fastapi_backend/app/domain/ingest.py index c26a009d..c1e43c6e 100644 --- a/fastapi_backend/app/domain/ingest.py +++ b/fastapi_backend/app/domain/ingest.py @@ -7,6 +7,7 @@ import importlib.util import html import json +import math import pathlib import re import subprocess @@ -41,32 +42,30 @@ def parse_sv(raw): MEDIA_DEMUXERS = "mp3,mov,wav,flac,ogg,aac,matroska,amr,avi,asf,mpeg,mpegts,aiff" -def probe(path): - ch = 0 +def _ffprobe(path, entries): + """What ffprobe says of an audio or video file (the -show_entries asked for), as JSON; {} when it can't read it.""" try: out = subprocess.run( - [ - "ffprobe", - "-v", - "error", - "-format_whitelist", - MEDIA_DEMUXERS, - "-show_entries", - "format=duration:stream=channels,codec_type", - "-of", - "json", - str(path), - ], + ["ffprobe", "-v", "error", "-format_whitelist", MEDIA_DEMUXERS, "-show_entries", entries, "-of", "json", str(path)], capture_output=True, text=True, timeout=120, ) j = json.loads(out.stdout or "{}") + except (OSError, ValueError, subprocess.TimeoutExpired): + return {} + return j if isinstance(j, dict) else {} + + +def probe(path): + ch = 0 + j = _ffprobe(path, "format=duration:stream=channels,codec_type") + try: dur = float(j.get("format", {}).get("duration") or 0) ch = max([s.get("channels", 0) for s in j.get("streams", []) if s.get("codec_type") == "audio"] or [0]) if dur: return int(dur * 1000) or None, int(ch) or None - except (OSError, ValueError, subprocess.TimeoutExpired): + except (TypeError, ValueError): pass dur, wav_ch = _probe_wav(path) return dur, (int(ch) or None) or wav_ch @@ -110,14 +109,78 @@ def _fingerprint(f, size, block): DATE_RX = re.compile(r"((?:19|20)\d{2})[-_.]?(\d{2})[-_.]?(\d{2})(?:[ T_-]?(\d{2})[-_.:h]?(\d{2})(?:[-_.:m]?(\d{2}))?)?") -def recorded_at(path, mtime): +def recorded_at(path, mtime, taken_at=None): + """When an asset was made: a date in its file's name (someone named it so), else the date its file itself says it + was taken (taken()), else when the file last changed.""" for m in DATE_RX.finditer(pathlib.Path(path).stem): y, mo, d, h, mi, s = (int(x) if x else 0 for x in m.groups()) try: return dt.datetime(y, mo, d, h, mi, s).isoformat(timespec="seconds") except ValueError: continue - return dt.datetime.fromtimestamp(mtime).isoformat(timespec="seconds") + return taken_at or dt.datetime.fromtimestamp(mtime).isoformat(timespec="seconds") + + +OLDEST_YEAR = 1971 # a clock never set writes 1970, or 1904 (QuickTime's zero): not when anything was taken + + +def _taken_when(s): + """A date and time a file gives (ISO 8601, or EXIF's "2024:05:01 12:20:30"), as ISO 8601 to the second with its + offset where it has one; None for one that isn't a real date (unset, before 1971 or in the future).""" + s = str(s or "").strip().strip("\x00").strip() + if re.match(r"\d{4}:\d{2}:\d{2}", s): + s = s[:10].replace(":", "-") + s[10:] + try: + t = dt.datetime.fromisoformat(s) + except ValueError: + return None + if t.year < OLDEST_YEAR or t > dt.datetime.now(t.tzinfo) + dt.timedelta(days=1): + return None + return t.isoformat(timespec="seconds") + + +def _degrees(v, ref): + """An EXIF GPS coordinate (degrees, minutes, seconds) with its N/S or E/W, as signed degrees; None if unreadable.""" + try: + d, m, s = (float(x) for x in v) + except (TypeError, ValueError, ZeroDivisionError): + return None + x = d + m / 60 + s / 3600 + return -x if str(ref or "").upper().startswith(("S", "W")) else x + + +def _exif(path): + """A photo's EXIF: when it was taken (DateTimeOriginal, else DateTime) and its GPS fix, as taken() gives them.""" + from PIL import ExifTags, Image + + try: + with Image.open(path) as img: + exif = img.getexif() + sub, gps = exif.get_ifd(ExifTags.IFD.Exif), exif.get_ifd(ExifTags.IFD.GPSInfo) + except (OSError, ValueError, SyntaxError, Image.DecompressionBombError): + return None, None + orig = sub.get(ExifTags.Base.DateTimeOriginal) + when = _taken_when(f"{orig}{sub.get(ExifTags.Base.OffsetTimeOriginal) or ''}") if orig else None + when = when or _taken_when(exif.get(ExifTags.Base.DateTime)) + lat = _degrees(gps.get(ExifTags.GPS.GPSLatitude), gps.get(ExifTags.GPS.GPSLatitudeRef)) + lon = _degrees(gps.get(ExifTags.GPS.GPSLongitude), gps.get(ExifTags.GPS.GPSLongitudeRef)) + ok = lat is not None and lon is not None and math.isfinite(lat + lon) and abs(lat) <= 90 and abs(lon) <= 180 + place = {"lat": round(lat, 6), "lon": round(lon, 6)} if ok and (lat, lon) != (0, 0) else None # 0, 0: no fix + return when, place + + +def taken(path): + """When and where a photo, a video or a recording was taken, as its own file says: a photo's EXIF (DateTimeOriginal, + else DateTime, and its GPS fix), a video's or a recording's creation time (QuickTime's creationdate, which keeps + the local time, else creation_time). (ISO 8601 or None, {lat, lon} or None): nothing is guessed. Read it before + the file is encrypted.""" + ext = pathlib.Path(path).suffix.lower() + if ext in store.IMAGE_EXT: + return _exif(path) + if ext in store.MEDIA_EXT: + tags = (_ffprobe(path, "format_tags=creation_time,com.apple.quicktime.creationdate").get("format") or {}).get("tags") or {} + return _taken_when(tags.get("com.apple.quicktime.creationdate")) or _taken_when(tags.get("creation_time")), None + return None, None DECODE_SECONDS = 6 * 3600 # a recording is decoded far faster than it plays; past this ffmpeg is stuck @@ -230,7 +293,7 @@ def scan(db, cfg, only=None, log=print): d=store.clean({"path": str(p), "size": st.st_size, "mtime": st.st_mtime, "duration_ms": dur, "channels": ch}), ) continue - rid = db.next_id("recording") + rid, (when, place) = db.next_id("recording"), taken(p) db.q( "CREATE $r CONTENT $d", r=store.R("recording", rid), @@ -243,7 +306,8 @@ def scan(db, cfg, only=None, log=print): "fingerprint": fp, "fp_key": f"{nid}:{fp}", "title": p.stem, - "recorded_at": recorded_at(p, st.st_mtime), + "recorded_at": recorded_at(p, st.st_mtime, when), + "place": place, "duration_ms": dur, "channels": ch, "size": st.st_size, @@ -1077,7 +1141,7 @@ def import_transcript(db, cfg, ns, tpath, audio=None, title=None, speaker_names= title or (t.get("title") if t.get("form") == "chat" else None) or pathlib.Path(tpath).stem, # a Telegram chat's name fingerprint(src), str(src.resolve()), - recorded_at(src, st.st_mtime), + recorded_at(src, st.st_mtime, taken(src)[0]), audio, speaker_names, "import:" + pathlib.Path(tpath).suffix.lstrip("."), diff --git a/fastapi_backend/app/domain/uploads.py b/fastapi_backend/app/domain/uploads.py index 2821b9d8..4f45dd11 100644 --- a/fastapi_backend/app/domain/uploads.py +++ b/fastapi_backend/app/domain/uploads.py @@ -365,6 +365,7 @@ def finish(db, cfg, row, admin=False): ) copy = bool(dup and (has_file(db, cfg, dup) if kind or read else has_media(db, cfg, dup))) # it's here already, with its file probed = ingest.probe(dest) if not copy and (target or not (kind or read)) else None # read before the file is encrypted + taken = ingest.taken(dest) if not (copy or target or read or dup) else (None, None) # a new asset's: when and where if not copy: keyring.protect(db, cfg, sid, dest) job = None @@ -382,7 +383,7 @@ def finish(db, cfg, row, admin=False): _typed(db, rid, chosen) job = jobs.enqueue(db, rid, None, by=by, pipeline=row.get("pipeline")) elif kind: - rid, job = _document(db, dest, st, fp, sid, {**row, "content_type": chosen}, kind, dup, by) + rid, job = _document(db, dest, st, fp, sid, {**row, "content_type": chosen}, kind, dup, by, taken) else: dur, ch = probed media = store.clean( @@ -413,7 +414,8 @@ def finish(db, cfg, row, admin=False): "fingerprint": fp, "fp_key": f"{sid}:{fp}", "title": row.get("title") or pathlib.Path(row["filename"]).stem, - "recorded_at": ingest.recorded_at(dest, st.st_mtime), + "recorded_at": ingest.recorded_at(dest, st.st_mtime, taken[0]), + **({"place": taken[1]} if taken[1] else {}), "status": "new", "created_at": store.now(), }, @@ -458,10 +460,11 @@ def _as_text(dest): return chats.read(dest.read_text(encoding="utf-8-sig", errors="replace"), dest.name) -def _document(db, dest, st, fp, sid, row, kind, dup, by): +def _document(db, dest, st, fp, sid, row, kind, dup, by, taken): """A document or an image: a resource of its own, or the one the file was before (imported as a transcript, or whose file went), which takes it as this kind; its pages are drawn and read by the namespace's pipeline (or the - one chosen). One whose PDF an import webhook hands back (`await_pdf`) waits for it instead, with no job.""" + one chosen). One whose PDF an import webhook hands back (`await_pdf`) waits for it instead, with no job. A new one + is dated, and placed, by what its file says (`taken`, ingest.taken) when its name doesn't date it.""" d = {"path": str(dest), "source": kind, "size": st.st_size, "mtime": st.st_mtime, "media": {"kind": kind}, "status": "new"} if row.get("content_type"): d["content_type"] = row["content_type"] @@ -482,7 +485,8 @@ def _document(db, dest, st, fp, sid, row, kind, dup, by): "fingerprint": fp, "fp_key": f"{sid}:{fp}", "title": row.get("title") or pathlib.Path(row["filename"]).stem, - "recorded_at": ingest.recorded_at(dest, st.st_mtime), + "recorded_at": ingest.recorded_at(dest, st.st_mtime, taken[0]), + **({"place": taken[1]} if taken[1] else {}), "created_at": store.now(), }, ) diff --git a/fastapi_backend/app/schemas/recordings.py b/fastapi_backend/app/schemas/recordings.py index 9a410ea0..44a35e60 100644 --- a/fastapi_backend/app/schemas/recordings.py +++ b/fastapi_backend/app/schemas/recordings.py @@ -300,6 +300,13 @@ class EnrichedPlace(ResponseModel): said: str | None = Field(None, description="named in the text: the words that name it") +class GeoPoint(ResponseModel): + """Where a photo was taken, from its own file (its EXIF GPS fix), in WGS 84 degrees.""" + + lat: float = Field(description="latitude, north positive") + lon: float = Field(description="longitude, east positive") + + class EnrichedDate(ResponseModel): """A date or time the text gives.""" @@ -366,6 +373,12 @@ class Recording(ResponseModel): converted_from: ConvertedFrom | None = Field(None, description="made by a convert step: the asset it was made from") web: WebPage | None = Field(None, description="a web page captured as a document") enrichment: Enrichment | None = Field(None, description="the places and dates anytopdf found (analysis.anytopdf)") + recorded_at: str | None = Field( + None, + description="when it was made: a date in its file's name, else what its file says (a photo's EXIF, a video's " + "creation time), else its file's last change; a source's or a document's own date wins", + ) + place: GeoPoint | None = Field(None, description="a photo: where it was taken, from its EXIF GPS (members of its namespace only)") nsfw: NsfwState | None = Field(None, description="NSFW classification, where it's on in its namespace and it was classified") audio_cleanup: AudioCleanup | None = Field(None, description="how its clean_audio step cleaned up its sound before transcription") enhanced: Enhanced | None = Field(None, description="an image the enhance step looked at: what it found, and its enhanced copy") diff --git a/fastapi_backend/tests/api/test_file_dates.py b/fastapi_backend/tests/api/test_file_dates.py new file mode 100644 index 00000000..6af64234 --- /dev/null +++ b/fastapi_backend/tests/api/test_file_dates.py @@ -0,0 +1,125 @@ +"""An asset dated by when it was taken, as its own file says (docs/processing.md, Steps): a photo's EXIF date and GPS +fix, a video's creation time; a date its name gives wins, and nothing is guessed.""" + +from __future__ import annotations + +import datetime as dt +import io +import subprocess + +import pytest +from PIL import ExifTags, Image +from PIL.TiffImagePlugin import IFDRational + +from app.core import security +from app.domain import ingest, store +from tests.api.test_documents import _upload +from tests.api.test_mcp import tool +from tests.api.test_oauth import _bearer, _grant +from tests.helpers import login, make_user, quiet +from tests.video_helpers import needs_ffmpeg + +R = store.R +MODIFIED = 1_700_000_000_000 # the file's last change, as the browser sends it (2023-11-14) + + +@pytest.fixture +def app(cfg, db): + from app.main import create_app + + return create_app(cfg, db, background=False) + + +@pytest.fixture +def env(client, db): + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + return {"he": login(client, "ed@x.io", "editor password 1")} + + +def _photo(taken="2019:08:17 09:41:05", changed="2020:01:01 00:00:00", gps=True): + """A small JPEG with the EXIF a phone writes: when it was taken and changed, and where (the Eiffel Tower).""" + exif = Image.Exif() + if changed: + exif[ExifTags.Base.DateTime] = changed + if taken: + exif.get_ifd(ExifTags.IFD.Exif)[ExifTags.Base.DateTimeOriginal] = taken + if gps: + g = exif.get_ifd(ExifTags.IFD.GPSInfo) + g[ExifTags.GPS.GPSLatitudeRef], g[ExifTags.GPS.GPSLatitude] = "N", (IFDRational(48), IFDRational(51), IFDRational(2964, 100)) + g[ExifTags.GPS.GPSLongitudeRef], g[ExifTags.GPS.GPSLongitude] = "E", (IFDRational(2), IFDRational(17), IFDRational(4044, 100)) + out = io.BytesIO() + Image.new("RGB", (32, 24), "teal").save(out, format="JPEG", exif=exif) + return out.getvalue() + + +def _video(path, *metadata): + """A second of colour and tone, with the metadata given (key=value).""" + cmd = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", "color=c=blue:s=64x64:d=1"] + cmd += ["-f", "lavfi", "-i", "sine=frequency=440:duration=1", "-shortest", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac"] + for m in metadata: + cmd += ["-metadata", m] + subprocess.run([*cmd, *(["-movflags", "use_metadata_tags"] if path.suffix == ".mov" else []), str(path)], check=True) + return path + + +def _row(db, rid): + return db.one("SELECT recorded_at, place FROM $r", r=R("recording", rid)) + + +def test_a_photo_is_dated_and_placed_by_its_exif(client, new_client, env, db): + up = _upload(client, env["he"], _photo(), "IMG_4711.jpg", modified=MODIFIED) + rid = up["recording"] + # when it was taken (DateTimeOriginal, not DateTime nor the file's last change), and where + assert _row(db, rid) == {"recorded_at": "2019-08-17T09:41:05", "place": {"lat": 48.858233, "lon": 2.294567}} + rec = client.get(f"/api/v1/recordings/{rid}", headers=env["he"]).json() + assert (rec["recorded_at"], rec["place"]) == ("2019-08-17T09:41:05", {"lat": 48.858233, "lon": 2.294567}) + _, tokens = _grant(client, env["he"], scope="read") # an agent, through MCP + got = tool(client, _bearer(tokens), "get_recording", recording_id=rid) + assert (got["recorded_at"], got["place"]) == ("2019-08-17T09:41:05", {"lat": 48.858233, "lon": 2.294567}) + # where it was taken stays with its members: a link signed for a visitor doesn't say it + seen = new_client().get(security.sign_path(f"/api/v1/recordings/{rid}")) + assert seen.status_code == 200, seen.text + assert seen.json().get("place") is None and seen.json()["recorded_at"] == "2019-08-17T09:41:05" + # a photo with only DateTime is dated by that; one with no GPS has no place + rid = _upload(client, env["he"], _photo(taken=None, gps=False), "IMG_4712.jpg", modified=MODIFIED)["recording"] + assert _row(db, rid) == {"recorded_at": "2020-01-01T00:00:00", "place": None} + + +def test_a_date_already_set_is_kept(client, env, db): + # a date in its name is someone's: it wins over the EXIF (the place is still kept) + rid = _upload(client, env["he"], _photo(), "2024-03-05 beach.jpg", modified=MODIFIED)["recording"] + assert _row(db, rid) == {"recorded_at": "2024-03-05T00:00:00", "place": {"lat": 48.858233, "lon": 2.294567}} + # the same photo uploaded again is the same asset, its date untouched + db.q("UPDATE $r SET recorded_at = '2001-02-03T04:05:06'", r=R("recording", rid)) + again = _upload(client, env["he"], _photo(), "2024-03-05 beach.jpg", modified=MODIFIED) + assert (again["recording"], again["duplicate"]) == (rid, True) + assert _row(db, rid)["recorded_at"] == "2001-02-03T04:05:06" + # nothing is invented: a camera whose clock was never set says 1970, so the file's last change dates it + unset = _photo(taken="1970:01:01 00:00:00", changed=None, gps=False) + rid = _upload(client, env["he"], unset, "IMG_0001.jpg", modified=MODIFIED)["recording"] + assert _row(db, rid)["recorded_at"] == dt.datetime.fromtimestamp(MODIFIED / 1000).isoformat(timespec="seconds") + + +@needs_ffmpeg +def test_a_video_is_dated_by_its_creation_time(client, env, db, folder): + clip = _video(folder / "clip.mp4", "creation_time=2021-07-04T15:30:00.000000Z") + rid = _upload(client, env["he"], clip.read_bytes(), "clip.mp4", modified=MODIFIED)["recording"] + assert _row(db, rid) == {"recorded_at": "2021-07-04T15:30:00+00:00", "place": None} + assert client.get(f"/api/v1/recordings/{rid}", headers=env["he"]).json()["recorded_at"] == "2021-07-04T15:30:00+00:00" + # a phone's QuickTime creationdate keeps its local time, so it comes first + mov = _video(folder / "phone.mov", "creation_time=2021-07-04T15:30:00Z", "com.apple.quicktime.creationdate=2021-07-04T17:30:00+0200") + assert ingest.taken(mov) == ("2021-07-04T17:30:00+02:00", None) + # one that says nothing is dated as before; a QuickTime zero date (1904) says nothing + assert ingest.taken(_video(folder / "plain.mp4")) == (None, None) + assert ingest.taken(_video(folder / "zero.mp4", "creation_time=1904-01-01T00:00:00Z")) == (None, None) + + +@needs_ffmpeg +def test_a_scanned_video_is_dated_by_its_creation_time(db, cfg, folder): + shelf = folder / "shelf" + shelf.mkdir() + _video(shelf / "holiday.mp4", "creation_time=2022-12-24T18:00:00Z") + cfg["namespaces"]["pods"]["paths"] = [str(shelf)] + assert ingest.scan(db, cfg, only="pods", log=quiet)["new"] == 1 + rec = db.one("SELECT recorded_at, place FROM recording WHERE path = $p", p=str(shelf / "holiday.mp4")) + assert rec == {"recorded_at": "2022-12-24T18:00:00+00:00", "place": None} diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 18786aa5..663bc853 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -1303,6 +1303,7 @@ export type { ForgotPasswordRequest, ForgotPasswordResponse, ForgotPasswordResponses, + GeoPoint, GetActivityData, GetActivityError, GetActivityErrors, diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 54dfe4ab..257876a5 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -9593,6 +9593,27 @@ export type ForgotPasswordRequest = { email: string; }; +/** + * GeoPoint + * + * Where a photo was taken, from its own file (its EXIF GPS fix), in WGS 84 degrees. + */ +export type GeoPoint = { + /** + * Lat + * + * latitude, north positive + */ + lat: number; + /** + * Lon + * + * longitude, east positive + */ + lon: number; + [key: string]: unknown; +}; + /** * GoogleClient */ @@ -20633,6 +20654,16 @@ export type Recording = { * the places and dates anytopdf found (analysis.anytopdf) */ enrichment?: Enrichment | null; + /** + * Recorded At + * + * when it was made: a date in its file's name, else what its file says (a photo's EXIF, a video's creation time), else its file's last change; a source's or a document's own date wins + */ + recorded_at?: string | null; + /** + * a photo: where it was taken, from its EXIF GPS (members of its namespace only) + */ + place?: GeoPoint | null; /** * NSFW classification, where it's on in its namespace and it was classified */ diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 08377d42..49ddf508 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -53688,6 +53688,28 @@ ], "title": "ForgotPasswordRequest" }, + "GeoPoint": { + "properties": { + "lat": { + "type": "number", + "title": "Lat", + "description": "latitude, north positive" + }, + "lon": { + "type": "number", + "title": "Lon", + "description": "longitude, east positive" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "lat", + "lon" + ], + "title": "GeoPoint", + "description": "Where a photo was taken, from its own file (its EXIF GPS fix), in WGS 84 degrees." + }, "GoogleClient": { "properties": { "origin": { @@ -72404,6 +72426,29 @@ ], "description": "the places and dates anytopdf found (analysis.anytopdf)" }, + "recorded_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Recorded At", + "description": "when it was made: a date in its file's name, else what its file says (a photo's EXIF, a video's creation time), else its file's last change; a source's or a document's own date wins" + }, + "place": { + "anyOf": [ + { + "$ref": "#/components/schemas/GeoPoint" + }, + { + "type": "null" + } + ], + "description": "a photo: where it was taken, from its EXIF GPS (members of its namespace only)" + }, "nsfw": { "anyOf": [ {