From d7b842288aad804eab783131e0fb1f875974964d Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Thu, 15 Oct 2026 11:17:07 +0000 Subject: [PATCH] feat(sources): Google Calendar as a sensor, as a service account or a connected Google account A new source type, gcal, reads Google Calendar through the Calendar API with read-only access: each calendar is a folder and each event a file /.ics, read as text exactly as an iCal feed's events are, so browsing, choosing events, watching and importing work unchanged. It signs in as a service account (its key file; acting as a Workspace user through domain-wide delegation with impersonate) or as a connected Google account (an OAuth client's authorized user JSON). google_auth.py is the shared sign-in layer for Google sources: JWT bearer and refresh-token access tokens, cached in memory, every call to Google a row in the activity ledger, and Google's own token endpoint whatever a key file says. Events carry cancelled occurrences (EXDATE) and their video call, which the event text now shows for iCal feeds too. The web app's connection dialog gains the Google Calendar form; docs, changelog and threat model describe it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CGVfwJfXV4MiUhSyusia64 --- CHANGELOG.md | 8 +- README.md | 2 +- docs/assistant-email.md | 2 +- docs/processing.md | 29 +- docs/sensors.md | 2 +- fastapi_backend/app/domain/calendars.py | 16 +- fastapi_backend/app/domain/feeds.py | 19 +- fastapi_backend/app/domain/gcal.py | 403 ++++++++++++++++++ fastapi_backend/app/domain/google_auth.py | 218 ++++++++++ fastapi_backend/app/domain/sources.py | 29 +- fastapi_backend/app/schemas/sources.py | 2 +- .../tests/api/test_google_calendar_source.py | 284 ++++++++++++ fastapi_backend/tests/fake_google.py | 162 +++++++ .../__tests__/sources-model.test.ts | 48 +++ .../app/openapi-client/types.gen.ts | 4 +- .../components/sources/connection-fields.tsx | 130 ++++-- .../components/sources/source-icons.tsx | 2 + .../components/sources/source-model.ts | 59 ++- nextjs-frontend/openapi.json | 6 +- threat-model.json | 10 + threat-model.md | 1 + threat-model.yaml | 12 +- 22 files changed, 1380 insertions(+), 68 deletions(-) create mode 100644 fastapi_backend/app/domain/gcal.py create mode 100644 fastapi_backend/app/domain/google_auth.py create mode 100644 fastapi_backend/tests/api/test_google_calendar_source.py create mode 100644 fastapi_backend/tests/fake_google.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d7513f7c..1f2363a43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Google Calendar as a sensor.** Add a Google Calendar connection on the Sources page (or `POST /api/v1/sources` + with type `gcal`): each calendar is a folder and each event becomes text, as an iCal feed's do, read-only through + Google's Calendar API. It signs in as a Google account (an OAuth client's authorized user JSON) or as a service + account, which reads the calendars shared with it or, in Google Workspace, a user's calendars through domain-wide + delegation. Calls to Google show in the activity ledger + ([Google Calendar](docs/processing.md#email-imap-calendar-feeds-ical-and-google-calendar)). - **The assistant always answers.** When a question used up its tool steps (`ai.max_steps`), the assistant used to stop with "I ran out of steps before finishing". Now it's asked to answer from what it found, with the tools off, and says what it didn't get to check; a reply with nothing in it is asked again the same way. It also knows its step budget up @@ -828,7 +834,7 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version `POST …/{tid}/secret` and `GET …/{tid}/deliveries` (docs/api.md). - **Email and calendars as sources.** Sources can now be an email account (IMAP) or a calendar feed (iCal), next to S3, Drive, SFTP and the rest. Browse them, import chosen messages or events, or watch them like a folder - (docs/processing.md#email-imap-and-calendar-feeds-ical). + (docs/processing.md#email-imap-calendar-feeds-ical-and-google-calendar). - IMAP: mailboxes are folders and messages are `.eml` files named by their subjects. Each message becomes an email document, titled by its subject and dated when it was sent, and its attachments are kept and become resources of their own. Watching the whole account skips the bin, junk, drafts and Gmail's All Mail, Starred diff --git a/README.md b/README.md index a1c189969..1f6d68b5c 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Everything below is built and merged. Work that is still in a pull request is li ([Extending the assistant](docs/assistant.md#extending-the-assistant-tools-skills-hooks-and-plugins), [the MCP runner](docs/assistant.md#the-mcp-runner)) - **Bring in everything.** Watched folders on S3, Dropbox, Google Drive, OneDrive, SFTP, SMB, WebDAV or local disks - (a local folder in four clicks); IMAP mail and iCal calendars; a built-in MQTT broker, syslog and webhook streams; + (a local folder in four clicks); IMAP mail, iCal calendars and Google Calendar; a built-in MQTT broker, syslog and webhook streams; import webhooks for scanners and phone shortcuts. Upload audio, video, PDFs, Office files, photos, JSON and zip or tar archives, and, with an Apache Tika server (an opt-in extra), documents nothing else here can read; capture web pages one at a time, from a link list or a bookmarks file, by API or by asking the assistant, from private networks diff --git a/docs/assistant-email.md b/docs/assistant-email.md index b6c44e689..0e954b8ce 100644 --- a/docs/assistant-email.md +++ b/docs/assistant-email.md @@ -71,7 +71,7 @@ subject: newsletter = skip A rule's conditions are on `from`, `to` (To, Cc and the address it arrived at), `subject`, `list` (List-Id) and `mailbox`, joined by `;`, with patterns joined by `,`. Patterns work as in an email watch's routing rules -([Processing](processing.md#email-imap-and-calendar-feeds-ical)): one without `*` or `?` matches anywhere in the +([Processing](processing.md#email-imap-calendar-feeds-ical-and-google-calendar)): one without `*` or `?` matches anywhere in the text. `skip` leaves the email unanswered. When no rule matches, or a rule names a namespace that doesn't exist, Lens picks the namespace for each question, as in an open chat room ([Chat rooms](chat-rooms.md)). diff --git a/docs/processing.md b/docs/processing.md index adbfa7260..73875f7de 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -289,7 +289,7 @@ A folder scan and a routine run are each claimed first, so several processes doi ## Storage sources Admins add sources in the app (`/api/v1/sources`): S3 or S3-compatible, Dropbox, Google Drive, OneDrive, SFTP, SMB, -WebDAV, or a folder on this machine; or an email account (IMAP) or a calendar feed (iCal), below. A watched folder maps a path on a source to a namespace, with include/exclude +WebDAV, or a folder on this machine; or an email account (IMAP), a calendar feed (iCal) or Google Calendar, below. A watched folder maps a path on a source to a namespace, with include/exclude patterns, audio and/or transcripts, a polling interval, how long a file must be unchanged before it is picked up, and whether files already there are imported (backfill). New audio is queued for the full pipeline; new transcripts are imported and analysed. Audio stays where it is: it is copied to a cache for processing and streamed from the source for @@ -311,9 +311,9 @@ files, then Import (`POST /api/v1/import/source`). The listing marks files that - Folders on this machine can only be watched inside `sources.local_roots`, and the rclone binary can only be set in archive.yaml: the web app can neither open up the server's disk nor choose what runs. -### Email (IMAP) and calendar feeds (iCal) +### Email (IMAP), calendar feeds (iCal) and Google Calendar -Two kinds of source aren't storage and don't use rclone; their messages and events are shown as files, so browsing, +Three kinds of source aren't storage and don't use rclone; their messages and events are shown as files, so browsing, importing chosen ones and watching work as above. - **Email (IMAP)**: host, port, security (SSL/TLS, STARTTLS or none), user and password (an app password where the @@ -344,6 +344,29 @@ importing chosen ones and watching work as above. attendees, how it repeats, and its description), dated when it starts: an all-day event on its date, wherever you are. Time zones are read as IANA names, Windows' names (as Outlook writes them), or the calendar's own VTIMEZONE. An event that changes (its LAST-MODIFIED, or its size) is read again into the resource it already is, rather than made a second one. +- **Google Calendar** (`gcal`): read through Google's Calendar API with read-only access (`calendar.readonly`), so + nothing in the calendar is changed. Each calendar is a folder and each event a file `/.ics`, + named by its date and title, and read as text exactly as an iCal feed's events are; it also says which occurrences + of a repeating event were cancelled and the event's video call. A changed occurrence of a repeating event is an event + of its own, as Google keeps it; working-location entries ("Home", "Office") are left out. It signs in one of two ways, + each kept encrypted like a password and never shown again: + + - **Google account** (`token`): the "authorized user" JSON (`client_id`, `client_secret`, `refresh_token`) of an + OAuth client of your own Google Cloud project, with the Calendar API turned on there. `gcloud auth + application-default login --client-id-file=client.json --scopes=https://www.googleapis.com/auth/calendar.readonly,https://www.googleapis.com/auth/cloud-platform` + writes one (to `application_default_credentials.json` in gcloud's folder). + - **Service account** (`service_account_credentials`, its .json key file). On its own it reads the calendars shared + with its address: share them, and give their IDs in `calendars` (Google Calendar › Settings › Integrate + calendar). In Google Workspace it can read a user's calendars instead (`impersonate`), once an admin gave it + domain-wide delegation for the `calendar.readonly` scope (Admin console › Security › API controls). + + `calendars` (IDs, comma separated) limits the source to those; empty, it reads every calendar in the account's + list, and watching the whole account (no path) leaves out Google's own (holidays, birthdays, week numbers). Events + that ended more than `days_back` days ago (365) are left out; empty takes every event. A watch lists each calendar + again every poll (`events.list`, 2,500 events a call). Every call to Google is a row in the activity ledger + (`google.token`, `google.calendar`); Google throttling the account shows as working, slowly, and a scan it + interrupts is tried again at the next poll. The token endpoint is always Google's own, whatever a key file says. Without a Google Cloud project, + a Google calendar's secret address in iCal format (Settings › Integrate calendar) works as a calendar feed. - `.eml` and `.ics` files in any source, and uploaded through Import, can be read as text this way too. ## Pipelines and templates diff --git a/docs/sensors.md b/docs/sensors.md index edce29e9d..727286793 100644 --- a/docs/sensors.md +++ b/docs/sensors.md @@ -4,7 +4,7 @@ A **sensor** is anything that feeds Lens. There are two families, in one list (` | Family | Kinds | Channels | What it reports becomes | | --- | --- | --- | --- | -| Files | S3, Dropbox, Google Drive, OneDrive, SFTP, SMB, WebDAV, a local folder (through rclone); email (IMAP); calendar feeds (iCal) | the folders it watches | resources, run through their pipelines (as sources always have) | +| Files | S3, Dropbox, Google Drive, OneDrive, SFTP, SMB, WebDAV, a local folder (through rclone); email (IMAP); calendar feeds (iCal); Google Calendar | the folders it watches | resources, run through their pipelines (as sources always have) | | Streams | MQTT devices, syslog senders, webhooks | streams: an MQTT topic, a syslog program, a webhook's stream name | readings, kept with retention, and hourly summaries of anything numeric | File sensors are the storage sources: they are added and changed as sources (`/api/v1/sources`), and nothing about diff --git a/fastapi_backend/app/domain/calendars.py b/fastapi_backend/app/domain/calendars.py index cacc0c6cb..8203be6b0 100644 --- a/fastapi_backend/app/domain/calendars.py +++ b/fastapi_backend/app/domain/calendars.py @@ -350,18 +350,28 @@ def _span(event): return f"{_shown(s)} to {_shown(end)}" +def _skipped(event): + """The occurrences of a repeating event that don't happen (its EXDATEs), in order.""" + out = [when(p, x, event.get("zones")) for p, v in all_of(event, "EXDATE") for x in (v or "").split(",")] + return sorted((t for t in out if t is not None), key=iso) + + def event_text(event): - """An event as Markdown: its summary as the heading, when and where, who, how it repeats, then its description. - The details are 'Label — value' lines, so the transcript reader doesn't take them for speakers.""" + """An event as Markdown: its summary as the heading, when and where, who, how it repeats (and the occurrences that + don't happen), its video call, then its description. The details are 'Label — value' lines, so the transcript + reader doesn't take them for speakers.""" title = summary(event) or "(no title)" rows = [("When", _span(event))] rows.append(("Where", unescape(first(event, "LOCATION")[1] or "").strip())) p, v = first(event, "ORGANIZER") rows.append(("Organizer", _person(p, v) if v else "")) rows.append(("Attendees", ", ".join(_person(p, v) for p, v in all_of(event, "ATTENDEE")))) - rows.append(("Repeats", (first(event, "RRULE")[1] or "").strip())) + rule = (first(event, "RRULE")[1] or "").strip() + rows.append(("Repeats", rule)) + rows.append(("Except", "; ".join(_shown(t) for t in _skipped(event)) if rule else "")) status = (first(event, "STATUS")[1] or "").strip().lower() rows.append(("Status", status if status and status != "confirmed" else "")) + rows.append(("Meeting", (first(event, "CONFERENCE")[1] or "").strip())) rows.append(("Link", (first(event, "URL")[1] or "").strip())) head = [f"{k} — {v}" for k, v in rows if v] desc = unescape(first(event, "DESCRIPTION")[1] or "").strip() diff --git a/fastapi_backend/app/domain/feeds.py b/fastapi_backend/app/domain/feeds.py index 6560b0eb0..b02ead6e8 100644 --- a/fastapi_backend/app/domain/feeds.py +++ b/fastapi_backend/app/domain/feeds.py @@ -1,5 +1,5 @@ -"""Sources that aren't file storage: IMAP mailboxes and iCalendar feeds, shown as folders of files so browsing, -choosing files, watching and importing work for them as they do for rclone's storage (sources.py). +"""Sources that aren't file storage: IMAP mailboxes, iCalendar feeds and Google Calendar (gcal.py), shown as folders of +files so browsing, choosing files, watching and importing work for them as they do for rclone's storage (sources.py). - An IMAP account's folders are its mailboxes, and each message is a file, `/-.eml`: a UID stands for one message only while the mailbox's UIDVALIDITY stays the same, so a mailbox the server rebuilt @@ -34,8 +34,10 @@ import urllib.request from email import policy -from . import calendars, netguard +from . import calendars, gcal, netguard +# sources a module of their own handles: browse, list_files, fetch, immutable and test, as below +OWN = {"gcal": gcal} TYPES = { "imap": { "label": "Email (IMAP)", @@ -47,6 +49,7 @@ "fields": {"user": ""}, "secrets": ["url", "pass"], }, + **{k: m.TYPE for k, m in OWN.items()}, } SECURITY = ("ssl", "starttls", "none") # not taken when a whole account is watched: what can't be opened, the bin, junk and drafts, and the mailboxes that @@ -487,6 +490,8 @@ def check_path(path): def browse(cfg, src, path=""): + if src["type"] in OWN: + return OWN[src["type"]].browse(cfg, src, check_path(path)) return (_imap_browse if src["type"] == "imap" else _ical_browse)(cfg, src, check_path(path)) @@ -494,6 +499,8 @@ def list_files(cfg, src, path="", cursor=None): """Every file under `path`; for IMAP with a watch's `cursor` (a list of {mailbox, validity, uid}), only messages after it.""" path = check_path(path) + if src["type"] in OWN: + return OWN[src["type"]].list_files(cfg, src, path, cursor) if src["type"] == "imap": return _imap_files(cfg, src, path, {c["mailbox"]: c for c in cursor or []}) return _ical_browse(cfg, src, path) @@ -501,13 +508,19 @@ def list_files(cfg, src, path="", cursor=None): def fetch(cfg, src, path): """The file's bytes: a message as the server keeps it, an event as an .ics of its own.""" + if src["type"] in OWN: + return OWN[src["type"]].fetch(cfg, src, check_path(path)) return (_imap_fetch if src["type"] == "imap" else _ical_fetch)(cfg, src, check_path(path)) def immutable(src): """Whether a file, once fetched, never changes (an IMAP message), so a cached copy can be kept.""" + if src["type"] in OWN: + return OWN[src["type"]].immutable(src) return src["type"] == "imap" def test(cfg, src): + if src["type"] in OWN: + return OWN[src["type"]].test(cfg, src) (_imap_test if src["type"] == "imap" else _ical_test)(cfg, src) diff --git a/fastapi_backend/app/domain/gcal.py b/fastapi_backend/app/domain/gcal.py new file mode 100644 index 000000000..676663787 --- /dev/null +++ b/fastapi_backend/app/domain/gcal.py @@ -0,0 +1,403 @@ +"""Google Calendar as a source (a file sensor, as feeds.py's iCal feeds are), through the Calendar API: each calendar is +a folder and each event a file `/.ics`, so browsing, choosing events, watching and importing +work as they do for any source. + +It signs in as a service account or a connected Google account (google_auth.py), with read-only access +(calendar.readonly): nothing is changed in the calendar. `calendars` (calendar ids, comma separated) limits it to +those; otherwise it reads the account's own calendar list, which a service account only has when it acts as a +Workspace user (`impersonate`): share a calendar with the service account's address and give its id instead. +Watching the whole account (no path) takes every calendar in it but Google's own (holidays, birthdays, week +numbers). Events that ended more than `days_back` days ago are left out (empty: all of them). + +An event is written as an iCalendar event of its own (its title, when and where, organizer and attendees with their +answers, how it repeats and the occurrences cancelled, status, video call, its page in Google Calendar, and when it +was created and changed), so it is read as text exactly as an iCal feed's events are (calendars.py). A changed +occurrence of a repeating event is an event of its own, as Google keeps it. Working-location entries ("Home", +"Office") aren't events and are left out. +""" + +from __future__ import annotations + +import datetime as dt +import html +import re +import time +import urllib.parse +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +from . import calendars, google_auth + +API = "https://www.googleapis.com/calendar/v3" +SCOPES = ("https://www.googleapis.com/auth/calendar.readonly",) +TYPE = { + "label": "Google Calendar", + "fields": {"calendars": "", "impersonate": "", "days_back": "365"}, + "secrets": ["token", "service_account_credentials"], + "google": True, # signs in through google_auth.py +} +SYSTEM = "group.v.calendar.google.com" # Google's own calendars (holidays, birthdays, week numbers) end so +SKIP_TYPES = {"workingLocation"} +PAGE = 2500 # the most events.list gives at once +MAX_PAGES = 40 +LIST_SECONDS = 60 # a calendar listed for a scan is reused this long (the scan then reads each of its events) +EPOCH = "1970-01-01T00:00:00+00:00" +_LISTED: dict = {} # (source id, calendar id) -> (monotonic time, {event id.ics: (entry, body)}) +_CALENDARS: dict = {} # source id -> (monotonic time, its calendars) +PARTSTAT = {"needsAction": "NEEDS-ACTION", "accepted": "ACCEPTED", "declined": "DECLINED", "tentative": "TENTATIVE"} + + +def _params(src): + return {k: str(v).strip() for k, v in (src.get("params") or {}).items() if v is not None} + + +def _call(cfg, src, path, params=None, what=None): + creds = google_auth.credentials(cfg, src) + return google_auth.get(cfg, creds, SCOPES, API + path, params, api="calendar", what=what) + + +def _q(cal_id): + return urllib.parse.quote(cal_id, safe="@") + + +# ---------- calendars ---------- +def _chosen(src): + return [c.strip() for c in re.split(r"[,\s]+", _params(src).get("calendars", "")) if c.strip()] + + +def calendars_of(cfg, src): + """[{id, name, zone}] of the calendars this source reads: the chosen ones, else the account's calendar list (those + it can read events of), reused for LIST_SECONDS.""" + hit = _CALENDARS.get(src["id"]) + if hit and time.monotonic() - hit[0] < LIST_SECONDS and hit[1] == _chosen(src): + return hit[2] + out = _calendars(cfg, src) + _CALENDARS[src["id"]] = (time.monotonic(), _chosen(src), out) + return out + + +def _calendars(cfg, src): + chosen = _chosen(src) + if chosen: + out = [] + for cid in chosen: + try: + c = _call(cfg, src, f"/calendars/{_q(cid)}", what="calendars.get") + except FileNotFoundError: + raise ValueError( + f"there's no calendar {cid}, or it isn't shared with {google_auth.who(google_auth.credentials(cfg, src))}" + ) from None + out.append({"id": c.get("id") or cid, "name": c.get("summary") or cid, "zone": c.get("timeZone")}) + return out + out, token = [], None + for _ in range(MAX_PAGES): + q = {"minAccessRole": "reader", "maxResults": 250, **({"pageToken": token} if token else {})} + page = _call(cfg, src, "/users/me/calendarList", q, what="calendarList.list") + for c in page.get("items") or []: + if c.get("id") and not c.get("deleted"): + out.append( + { + "id": c["id"], + "name": c.get("summaryOverride") or c.get("summary") or c["id"], + "zone": c.get("timeZone"), + "primary": bool(c.get("primary")), + } + ) + token = page.get("nextPageToken") + if not token: + break + return sorted(out, key=lambda c: (not c.get("primary"), c["name"].lower())) + + +# ---------- an event as iCalendar ---------- +def _plain(v): + """A value written as it is (an address, a link, a rule): on one line, without control characters.""" + return re.sub(r"[\x00-\x1f\x7f]", "", str(v)).strip() + + +def _escape(v): + return ( + re.sub(r"[\x00-\x09\x0b-\x1f\x7f]", "", str(v)) + .replace("\\", "\\\\") + .replace(";", "\\;") + .replace(",", "\\,") + .replace("\r\n", "\n") + .replace("\n", "\\n") + ) + + +def _param(v): + v = re.sub(r'[\x00-\x1f"]', "", str(v)) + return f'"{v}"' if re.search(r"[;:,]", v) else v + + +def _fold(line): + """A content line folded at 75 characters, continuations starting with a space (RFC 5545 §3.1).""" + out = [line[:75]] + rest = line[75:] + while rest: + out.append(" " + rest[:74]) + rest = rest[74:] + return "\r\n".join(out) + + +def _zone(name): + try: + return ZoneInfo(name) if name else None + except (ZoneInfoNotFoundError, ValueError): + return None + + +def _instant(v): + try: + t = dt.datetime.fromisoformat(re.sub(r"(\.\d{6})\d+", r"\1", v.replace("Z", "+00:00"))) + except (ValueError, AttributeError): + return None + return t if t.tzinfo else t.replace(tzinfo=dt.timezone.utc) + + +def _when(name, t, zone): + """'DTSTART;TZID=Europe/Berlin:20261002T100000', 'DTSTART;VALUE=DATE:20261017' or 'DTSTART:20261002T080000Z' + for Google's {dateTime, timeZone} or {date}; None when there's neither.""" + t = t or {} + if t.get("date"): + return f"{name};VALUE=DATE:{t['date'].replace('-', '')}" + at = _instant(t.get("dateTime")) + if at is None: + return None + z = _zone(t.get("timeZone")) or _zone(zone) + if z is not None and str(z) not in ("UTC", "Etc/UTC"): + return f"{name};TZID={z}:{at.astimezone(z):%Y%m%dT%H%M%S}" + return f"{name}:{at.astimezone(dt.timezone.utc):%Y%m%dT%H%M%SZ}" + + +def _stamp(v): + at = _instant(v) + return at.astimezone(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") if at else None + + +def _text(desc): + """A description as text: Google Calendar's web app writes some as HTML (line breaks, links, lists).""" + if not re.search(r"<(br|/?p|/?div|/?ul|/?ol|li|a|b|i|u|span)\b", desc or "", re.I): + return desc or "" + t = re.sub( + r"]*href=\"([^\"]*)\"[^>]*>(.*?)", + lambda m: m.group(2) if html.unescape(m.group(1)) in (m.group(2), "") else f"{m.group(2)} ({m.group(1)})", + desc, + flags=re.I | re.S, + ) + t = re.sub(r"|

||", "\n", t, flags=re.I) + t = re.sub(r"]*>", "- ", t, flags=re.I) + t = html.unescape(re.sub(r"<[^>]+>", "", t)) + return re.sub(r"\n{3,}", "\n\n", t).strip() + + +def _person(name, p): + params = [] + if p.get("displayName"): + params.append(f"CN={_param(p['displayName'])}") + if name == "ATTENDEE": + if p.get("resource"): + params.append("CUTYPE=RESOURCE") + params.append(f"ROLE={'OPT-PARTICIPANT' if p.get('optional') else 'REQ-PARTICIPANT'}") + if PARTSTAT.get(p.get("responseStatus")): + params.append(f"PARTSTAT={PARTSTAT[p['responseStatus']]}") + return f"{name}{''.join(';' + x for x in params)}:mailto:{_plain(p['email'])}" + + +def _meeting(ev): + if ev.get("hangoutLink"): + return ev["hangoutLink"] + for e in (ev.get("conferenceData") or {}).get("entryPoints") or []: + if e.get("entryPointType") == "video" and e.get("uri"): + return e["uri"] + return None + + +def event_ics(ev, cal, cancelled=()): + """One of Google's events as an .ics of its own. `cancelled` are the RECURRENCE-ID values of the occurrences of a + repeating event that were cancelled, written as its EXDATEs.""" + zone = cal.get("zone") + lines = ["BEGIN:VEVENT", f"UID:{_plain(ev.get('iCalUID') or ev['id'])}"] + rid = _when("RECURRENCE-ID", ev.get("originalStartTime"), zone) + lines += [x for x in (rid, _when("DTSTART", ev.get("start"), zone), _when("DTEND", ev.get("end"), zone)) if x] + for name, value in (("SUMMARY", ev.get("summary")), ("LOCATION", ev.get("location")), ("DESCRIPTION", _text(ev.get("description")))): + if value: + lines.append(f"{name}:{_escape(value)}") + if (ev.get("organizer") or {}).get("email"): + lines.append(_person("ORGANIZER", ev["organizer"])) + lines += [_person("ATTENDEE", a) for a in ev.get("attendees") or [] if a.get("email")] + for rule in ev.get("recurrence") or []: # RRULE, EXRULE, RDATE and EXDATE lines, as iCalendar writes them + if re.match(r"^(RRULE|EXRULE|RDATE|EXDATE)[:;]", rule): + lines.append(_plain(rule)) + lines += [x.replace("RECURRENCE-ID", "EXDATE", 1) for x in sorted(cancelled)] + if ev.get("status"): + lines.append(f"STATUS:{_plain(ev['status']).upper()}") + if ev.get("transparency") == "transparent": + lines.append("TRANSP:TRANSPARENT") + if ev.get("visibility") in ("private", "confidential"): + lines.append(f"CLASS:{ev['visibility'].upper()}") + if _meeting(ev): + lines.append(f"CONFERENCE;VALUE=URI;FEATURE=VIDEO:{_plain(_meeting(ev))}") + if ev.get("htmlLink"): + lines.append(f"URL:{_plain(ev['htmlLink'])}") + for name, value in (("CREATED", ev.get("created")), ("LAST-MODIFIED", ev.get("updated"))): + if _stamp(value): + lines.append(f"{name}:{_stamp(value)}") + lines.append("END:VEVENT") + head = ["BEGIN:VCALENDAR", "VERSION:2.0", "PRODID:-//Lens//Google Calendar source//EN", f"X-WR-CALNAME:{_escape(cal['name'])}"] + return "\r\n".join([*head, *(_fold(l) for l in lines), "END:VCALENDAR", ""]) + + +# ---------- events as files ---------- +def _since(src): + days = _params(src).get("days_back", "") + if not days: + return None + try: + n = int(days) + except ValueError: + raise ValueError("days back is a number of days (empty for all events)") from None + if n <= 0: + return None + return (dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=n)).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _items(cfg, src, cal): + """Every event of a calendar Google gives (repeating ones once, with their changed occurrences), and its time zone.""" + out, token, zone, since = [], None, cal.get("zone"), _since(src) + for _ in range(MAX_PAGES): + q = { + "maxResults": PAGE, + "singleEvents": "false", + "showDeleted": "false", + **({"timeMin": since} if since else {}), + **({"pageToken": token} if token else {}), + } + try: + page = _call(cfg, src, f"/calendars/{_q(cal['id'])}/events", q, what="events.list") + except FileNotFoundError: + raise ValueError(f"there's no calendar {cal['id']}, or it isn't shared with this account") from None + zone = zone or page.get("timeZone") + out += page.get("items") or [] + token = page.get("nextPageToken") + if not token: + break + return out, zone + + +def _listing(cfg, src, cal): + """{event id.ics: (entry, body)} of a calendar, reused for LIST_SECONDS (a scan lists, then reads each event).""" + hit = _LISTED.get((src["id"], cal["id"])) + if hit and time.monotonic() - hit[0] < LIST_SECONDS: + return hit[1] + items, zone = _items(cfg, src, cal) + cal = {**cal, "zone": zone} + cancelled: dict[str, set] = {} + for ev in items: + if ev.get("status") == "cancelled" and ev.get("recurringEventId"): + rid = _when("RECURRENCE-ID", ev.get("originalStartTime"), zone) + if rid: + cancelled.setdefault(ev["recurringEventId"], set()).add(rid) + out = {} + for ev in items: + if ev.get("status") == "cancelled" or ev.get("eventType") in SKIP_TYPES or not re.fullmatch(r"[A-Za-z0-9_-]+", ev.get("id") or ""): + continue + body = event_ics(ev, cal, cancelled.get(ev["id"], ())).encode("utf-8") + parsed = calendars.parse(body.decode("utf-8"))["events"][0] + start = calendars.start(parsed) + title = calendars.summary(parsed) or "(no title)" + day = calendars.iso(start)[:10] + " " if start is not None else "" + name = re.sub(r"[\x00-\x1f\x7f/\\]", "", f"{day}{title}")[:150].strip() + rel = f"{ev['id']}.ics" + out[rel] = ( + { + "path": f"{cal['id']}/{rel}", + "rel": rel, + "name": f"{name}.ics", + "dir": False, + "size": len(body), + "modified": calendars.iso(calendars.changed(parsed)) or EPOCH, + "title": title, + "when": calendars.iso(start), + }, + body, + ) + _LISTED[(src["id"], cal["id"])] = (time.monotonic(), out) + return out + + +def events(cfg, src, cal, base=""): + """A calendar's events as files, their `rel` under `base` (the calendar's own folder when the whole account is + listed).""" + return [{**e, "rel": base + e["rel"]} for e, _ in _listing(cfg, src, cal).values()] + + +def _calendar(cfg, src, cal_id): + cal = next((c for c in calendars_of(cfg, src) if c["id"] == cal_id), None) + if cal is None and not _chosen(src): # one the account can read but doesn't list (shared, not added) + cal = {"id": cal_id, "name": cal_id, "zone": None} + if cal is None: + raise ValueError(f"there's no calendar {cal_id} in this connection") + return cal + + +# ---------- what feeds.py calls ---------- +def browse(cfg, src, path): + if not path: + return [ + {"path": c["id"], "rel": c["id"], "name": c["name"], "dir": True, "size": None, "modified": None} + for c in calendars_of(cfg, src) + ] + if "/" in path: + raise ValueError("a calendar has no folders") + return sorted(events(cfg, src, _calendar(cfg, src, path)), key=lambda e: e["when"] or "", reverse=True) + + +def list_files(cfg, src, path, cursor=None): + """A calendar's events, or every calendar's (no path) but Google's own.""" + if path: + return browse(cfg, src, path) + out = [] + for c in calendars_of(cfg, src): + if c["id"].endswith(SYSTEM) and not _chosen(src): + continue + out += events(cfg, src, c, f"{c['id']}/") + return out + + +def fetch(cfg, src, path): + cal_id, _, name = path.rpartition("/") + if not cal_id or not name.endswith(".ics"): + raise FileNotFoundError("that isn't an event (calendar id/event id.ics)") + hit = _LISTED.get((src["id"], cal_id)) + listed = hit[1] if hit and time.monotonic() - hit[0] < LIST_SECONDS else _listing(cfg, src, _calendar(cfg, src, cal_id)) + if name not in listed: + raise FileNotFoundError("that event isn't in the calendar any more") + return listed[name][1] + + +def immutable(src): + return False # an event can change + + +def forget(src): + """Drop what was listed for this source, so the next call asks Google again.""" + _CALENDARS.pop(src["id"], None) + for k in [k for k in _LISTED if k[0] == src["id"]]: + _LISTED.pop(k, None) + + +def test(cfg, src): + forget(src) + google_auth.credentials(cfg, src) # says what's missing before anything is called + _since(src) + found = calendars_of(cfg, src) + if not found: + creds = google_auth.credentials(cfg, src) + if creds["kind"] == "service_account" and not creds["subject"]: + raise RuntimeError( + f"signed in, but {creds['key']['client_email']} has no calendars of its own: share a calendar with that " + "address and enter its ID in Calendars, or act as a Workspace user" + ) + raise RuntimeError("signed in, but the account has no calendars") + events(cfg, src, found[0]) # reads events, as watching will diff --git a/fastapi_backend/app/domain/google_auth.py b/fastapi_backend/app/domain/google_auth.py new file mode 100644 index 000000000..d19e4742d --- /dev/null +++ b/fastapi_backend/app/domain/google_auth.py @@ -0,0 +1,218 @@ +"""Signing in to Google's APIs for a source (Google Calendar, gcal.py; other Google sources can use it too), and calling +them. + +A source signs in one of two ways, each kept sealed like a password: + +- As a **service account** (`service_account_credentials`, the .json key file Google Cloud gives): Lens signs a short + JWT with the account's private key and swaps it at Google's token endpoint for an access token (RFC 7523). In Google + Workspace it can act as a user of the domain (`impersonate`), once an admin gave it domain-wide delegation for the + scopes it asks for; otherwise it reads what was shared with its own address. +- As a **Google account** someone connected (`token`): an OAuth client's refresh token, kept as the "authorized user" + JSON ({type, client_id, client_secret, refresh_token}, as `gcloud auth application-default login --client-id-file` + writes it for a client of your own), so the source carries everything it needs to sign in. + +Access tokens last an hour; each is kept in memory until a minute before it ends, and never stored. The token endpoint +is always Google's own: a key file's `token_uri` is not followed. Every call to Google is a row in the activity ledger +(`google.token`, `google.`), with what was called, never what came back. +""" + +from __future__ import annotations + +import hashlib +import json +import time + +import httpx +import jwt + +from . import activity + +TOKEN_URL = "https://oauth2.googleapis.com/token" +TIMEOUT = 30 +_TOKENS: dict[str, tuple[float, str]] = {} # who and for what -> (when it ends, monotonic; the access token) +# what Google says when it throttles (403 with these reasons, or 429): the account works, it's only slowed down +THROTTLED = {"rateLimitExceeded", "userRateLimitExceeded", "quotaExceeded", "dailyLimitExceeded"} + + +class GoogleError(RuntimeError): + """Google refused or couldn't be reached; the message says why, for people.""" + + +def service_key(text): + """A service account's key file, checked and kept on one line. ValueError when it isn't one.""" + try: + key = json.loads(text) + except ValueError: + raise ValueError("the service account key isn't JSON: upload the .json key file Google Cloud gave you") from None + if not isinstance(key, dict) or key.get("type") != "service_account" or not key.get("client_email") or not key.get("private_key"): + raise ValueError("that isn't a service account key: it needs type service_account, client_email and private_key") + return json.dumps(key, separators=(",", ":")) + + +def authorized_user(text): + """A connected Google account as it's kept: {type: authorized_user, client_id, client_secret, refresh_token}. + ValueError when the text isn't one.""" + try: + o = json.loads(text) + except ValueError: + raise ValueError("the Google account isn't JSON: it's {client_id, client_secret, refresh_token}") from None + keys = ("client_id", "client_secret", "refresh_token") + if ( + not isinstance(o, dict) + or o.get("type", "authorized_user") != "authorized_user" + or not all(isinstance(o.get(k), str) and o[k] for k in keys) + ): + raise ValueError("that isn't a connected Google account: it needs client_id, client_secret and refresh_token") + return json.dumps({"type": "authorized_user", **{k: o[k].strip() for k in keys}}, separators=(",", ":")) + + +def check_secret(k, v): + """A Google source's secret as it's stored (sources.create and update call this).""" + if k == "service_account_credentials": + return service_key(v) + if k == "token": + return authorized_user(v) + return str(v) + + +def secret(cfg, src, k): + """A source's sealed secret, opened ("" when it has none).""" + from . import settings # settings imports sources, which imports what imports this + + sealed = (src.get("sealed") or {}).get(k) + return settings.unseal(cfg, sealed, f"source:{src['id']}:{k}") if sealed else "" + + +def credentials(cfg, src): + """What a source signs in with: {kind: service_account, key, subject} (its key file, acting as `impersonate` + when set), else {kind: authorized_user, user} (the Google account connected to it).""" + key = secret(cfg, src, "service_account_credentials") + if key: + subject = str((src.get("params") or {}).get("impersonate") or "").strip() + return {"kind": "service_account", "key": json.loads(key), "subject": subject or None} + user = secret(cfg, src, "token") + if user: + return {"kind": "authorized_user", "user": json.loads(user)} + raise GoogleError("connect a Google account, or upload a service account key, first") + + +def who(creds): + """The address a source signs in as, for messages: the service account's (or the user it acts as).""" + if creds["kind"] == "service_account": + return creds["subject"] or creds["key"]["client_email"] + return "the connected Google account" + + +def _cache_key(creds, scopes): + if creds["kind"] == "service_account": + k = creds["key"] + who_ = f"sa|{k['client_email']}|{k.get('private_key_id') or hashlib.sha256(k['private_key'].encode()).hexdigest()}|{creds['subject'] or ''}" + else: + who_ = f"user|{creds['user']['client_id']}|{creds['user']['refresh_token']}" + return hashlib.sha256(f"{who_}|{' '.join(sorted(scopes))}".encode()).hexdigest() + + +def _refused(r, creds): + try: + body = r.json() + except ValueError: + body = {} + err = body.get("error") if isinstance(body, dict) else None + said = (body.get("error_description") if isinstance(body, dict) else None) or err or f"HTTP {r.status_code}" + if creds["kind"] == "service_account" and err == "unauthorized_client" and creds["subject"]: + return GoogleError( + f"Google refused to let {creds['key']['client_email']} act as {creds['subject']}: {said}. A Workspace admin " + "gives it domain-wide delegation for these scopes (Admin console › Security › API controls)" + ) + if creds["kind"] == "authorized_user" and err == "invalid_grant": + return GoogleError(f"Google refused the connected account: {said}. Connect the Google account again") + return GoogleError(f"Google refused to sign in: {said}") + + +def access_token(cfg, creds, scopes, fresh=False): + """An access token for these scopes, from the cache unless it's about to end (or `fresh`).""" + key = _cache_key(creds, scopes) + hit = _TOKENS.get(key) + if hit and not fresh and hit[0] > time.monotonic(): + return hit[1] + if creds["kind"] == "service_account": + k, now = creds["key"], int(time.time()) + claims = {"iss": k["client_email"], "scope": " ".join(scopes), "aud": TOKEN_URL, "iat": now, "exp": now + 3600} + if creds["subject"]: + claims["sub"] = creds["subject"] + try: + assertion = jwt.encode( + claims, k["private_key"], algorithm="RS256", headers={"kid": k.get("private_key_id")} if k.get("private_key_id") else None + ) + except (ValueError, TypeError, jwt.PyJWTError) as e: + raise GoogleError(f"the service account's private key can't sign: {e}") from None + form = {"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", "assertion": assertion} + else: + u = creds["user"] + form = { + "grant_type": "refresh_token", + "client_id": u["client_id"], + "client_secret": u["client_secret"], + "refresh_token": u["refresh_token"], + } + with activity.call("google.token", cfg, detail={"as": creds["kind"]}): + try: + r = httpx.post(TOKEN_URL, data=form, headers={"Accept": "application/json"}, timeout=TIMEOUT) + except httpx.HTTPError as e: + raise GoogleError(f"couldn't reach Google to sign in: {e}") from None + if r.status_code >= 400: + raise _refused(r, creds) + try: + body = r.json() + token, lasts = body["access_token"], int(body.get("expires_in") or 3600) + except (ValueError, KeyError, TypeError): + raise GoogleError("Google's sign-in answered without an access token") from None + _TOKENS[key] = (time.monotonic() + max(lasts - 60, 30), token) + return token + + +def _reason(r): + try: + e = r.json().get("error") or {} + return (e.get("errors") or [{}])[0].get("reason") or e.get("status") or "", e.get("message") or "" + except (ValueError, AttributeError, IndexError, TypeError): + return "", "" + + +def _answer(r, creds): + """A reply from one of Google's APIs as JSON, or the error it means.""" + if r.status_code < 400: + try: + return r.json() + except ValueError: + raise GoogleError("Google answered with something that isn't JSON") from None + reason, message = _reason(r) + if r.status_code == 429 or reason in THROTTLED: + from . import sources # sources imports feeds, which imports what imports this + + raise sources.RateLimited(f"Google is rate limiting {who(creds)}: {message or reason or r.status_code}") + if r.status_code == 404: + raise FileNotFoundError(message or "Google has no such item, or it isn't shared with this account") + if r.status_code == 403 and reason in ("accessNotConfigured", "SERVICE_DISABLED"): + raise GoogleError(f"the API isn't turned on for this Google Cloud project: {message}") + raise GoogleError(f"Google answered {r.status_code}: {message or reason or 'refused'}") + + +def get(cfg, creds, scopes, url, params=None, api="api", what=None): + """GET one of Google's APIs as the source signs in, as JSON. Signs in again once if Google says the token ended. + sources.RateLimited when Google is throttling the account, FileNotFoundError when it has no such item, GoogleError + when it refused.""" + for attempt in (0, 1): + token = access_token(cfg, creds, scopes, fresh=attempt == 1) + with activity.call(f"google.{api}", cfg, detail={"call": what} if what else None) as row: + try: + r = httpx.get( + url, params=params, headers={"Authorization": f"Bearer {token}", "Accept": "application/json"}, timeout=TIMEOUT + ) + except httpx.HTTPError as e: + raise GoogleError(f"couldn't reach Google: {e}") from None + if r.status_code == 401 and attempt == 0: # the token ended early (or was revoked): sign in again, once + row.end(GoogleError("signed in again")) + continue + return _answer(r, creds) + raise GoogleError("Google didn't accept a fresh sign-in") # not reached: the second attempt returns or raises diff --git a/fastapi_backend/app/domain/sources.py b/fastapi_backend/app/domain/sources.py index 4764f8101..43d87a3e9 100644 --- a/fastapi_backend/app/domain/sources.py +++ b/fastapi_backend/app/domain/sources.py @@ -1,10 +1,10 @@ """Storage sources through rclone, and the folders on them that the archive watches. A source is one connection: S3 or S3-compatible, Dropbox, Google Drive, OneDrive, SFTP, SMB, WebDAV, or a folder on -this machine (only inside sources.local_roots); or, without rclone, an email account (IMAP) or a calendar feed (iCal), -whose messages and events are shown as files (feeds.py). A watch maps a folder on a source to a namespace: new audio, video, -documents (PDFs) and images become resources queued for the whole pipeline, their files staying on the source; new -transcripts are imported and queued for analysis. A watch on an email account can carry routing rules that send some +this machine (only inside sources.local_roots); or, without rclone, an email account (IMAP), a calendar feed (iCal) or +Google Calendar, whose messages and events are shown as files (feeds.py). A watch maps a folder on a source to a +namespace: new audio, video, documents (PDFs) and images become resources queued for the whole pipeline, their files +staying on the source; new transcripts are imported and queued for analysis. A watch on an email account can carry routing rules that send some messages to other namespaces, or skip them (mail_routes.py). Credentials are stored encrypted and handed to rclone in a private temporary config file per call; OAuth tokens rclone refreshes are saved. """ @@ -22,7 +22,7 @@ import subprocess import tempfile -from . import convert, deletion, feeds, ingest, jobs, mail_routes, settings, store +from . import convert, deletion, feeds, google_auth, ingest, jobs, mail_routes, settings, store R = store.R BACKENDS = { @@ -340,22 +340,19 @@ def _check_params(typ, params, secret_values): return spec -def _secret(k, v): - """A secret as it's stored: a service account's key file is checked and kept on one line, as rclone reads it.""" +def _secret(typ, k, v): + """A secret as it's stored: a service account's key file is checked and kept on one line, as rclone reads it; a + Google source's connected account is checked too (google_auth.py).""" + if BACKENDS[typ].get("google"): + return google_auth.check_secret(k, v) if k != "service_account_credentials": return str(v) - try: - key = json.loads(v) - except ValueError: - raise ValueError("the service account key isn't JSON: upload the .json key file Google Cloud gave you") from None - if not isinstance(key, dict) or key.get("type") != "service_account" or not key.get("client_email") or not key.get("private_key"): - raise ValueError("that isn't a service account key: it needs type service_account, client_email and private_key") - return json.dumps(key, separators=(",", ":")) + return google_auth.service_key(v) def create(db, cfg, name, typ, params=None, secret_values=None, user=None): spec = _check_params(typ, params, secret_values) - plain = {k: _secret(k, v) for k, v in (secret_values or {}).items() if isinstance(v, str) and v} + plain = {k: _secret(typ, k, v) for k, v in (secret_values or {}).items() if isinstance(v, str) and v} sid = db.next_id("storage_source") sealed = {k: settings.seal(cfg, v, f"source:{sid}:{k}") for k, v in plain.items()} db.q( @@ -383,7 +380,7 @@ def update(db, cfg, sid, name=None, params=None, secret_values=None): if v is None or v == "": sealed.pop(k, None) elif isinstance(v, str): - sealed[k] = settings.seal(cfg, _secret(k, v), f"source:{sid}:{k}") + sealed[k] = settings.seal(cfg, _secret(src["type"], k, v), f"source:{sid}:{k}") db.q( "UPDATE $r MERGE $d", r=R("storage_source", sid), diff --git a/fastapi_backend/app/schemas/sources.py b/fastapi_backend/app/schemas/sources.py index 0aa5a8d20..545cf234b 100644 --- a/fastapi_backend/app/schemas/sources.py +++ b/fastapi_backend/app/schemas/sources.py @@ -8,7 +8,7 @@ from app.schemas.common import Created, Ok, RequestModel, ResponseModel -SourceType = Literal["s3", "dropbox", "drive", "onedrive", "sftp", "smb", "webdav", "local", "imap", "ical"] +SourceType = Literal["s3", "dropbox", "drive", "onedrive", "sftp", "smb", "webdav", "local", "imap", "ical", "gcal"] WatchKinds = Literal["audio", "transcripts", "both", "documents", "all"] KINDS_HELP = ( "what to pick up: audio (and video), transcripts, both of those (PDFs read as transcripts), documents (PDFs and " diff --git a/fastapi_backend/tests/api/test_google_calendar_source.py b/fastapi_backend/tests/api/test_google_calendar_source.py new file mode 100644 index 000000000..1fb97910f --- /dev/null +++ b/fastapi_backend/tests/api/test_google_calendar_source.py @@ -0,0 +1,284 @@ +"""Google Calendar as a source: signing in as a service account (alone, or acting as a Workspace user) or as a +connected Google account, its calendars as folders and their events as files, browsed, watched and imported like an +iCal feed's. Everything runs against tests/fake_google.py, never Google.""" + +from __future__ import annotations + +import json + +import pytest + +from app.domain import calendars, gcal, google_auth, sources +from tests import fake_google +from tests.helpers import drain, login, make_user + +TEAM = "harbour@group.calendar.google.com" +HOLIDAYS = "en.irish#holiday@group.v.calendar.google.com" +MARA = "mara@harbour.example" +MEET = "https://meet.google.com/abc-defg-hij" + + +def _at(when, zone="Europe/Berlin"): + return {"date": when} if len(when) == 10 else {"dateTime": when, "timeZone": zone} + + +def _event(eid, summary, start, end, updated="2026-09-25T12:00:00.000Z", **more): + return { + "id": eid, + "iCalUID": f"{eid.split('_')[0]}@google.com", + "status": "confirmed", + "summary": summary, + "start": _at(start), + "end": _at(end), + "created": "2026-09-01T08:00:00.000Z", + "updated": updated, + "htmlLink": f"https://www.google.com/calendar/event?eid={eid}", + **more, + } + + +def _team_events(updated="2026-09-25T12:00:00.000Z", description=None): + return [ + _event( + "standup", + "Harbour stand-up", + "2026-10-02T10:00:00+02:00", + "2026-10-02T10:30:00+02:00", + updated, + location="Pier 4, meeting room", + description=description + or 'Agenda: the night crossing
the new buoys and the keeper\'s report', + organizer={"email": "mara@example.org", "displayName": "Mara Keane"}, + attendees=[ + {"email": "tom@example.org", "displayName": "Tom Byrne", "responseStatus": "accepted"}, + {"email": "ann@example.org", "responseStatus": "needsAction", "optional": True}, + ], + recurrence=["RRULE:FREQ=WEEKLY;BYDAY=FR"], + hangoutLink=MEET, + ), + _event( + "standup_20261009T080000Z", + "Harbour stand-up (moved)", + "2026-10-09T11:30:00+02:00", + "2026-10-09T12:00:00+02:00", + recurringEventId="standup", + originalStartTime=_at("2026-10-09T10:00:00+02:00"), + ), + { + "id": "standup_20261016T080000Z", + "status": "cancelled", + "recurringEventId": "standup", + "originalStartTime": _at("2026-10-16T10:00:00+02:00"), + }, + _event("regatta", "Autumn regatta", "2026-10-17", "2026-10-19", description="Two days on the water."), + _event("office", "Office", "2026-10-05", "2026-10-06", eventType="workingLocation"), + _event("old", "Launch of the old boat", "2020-03-01T09:00:00Z", "2020-03-01T10:00:00Z"), + ] + + +@pytest.fixture +def google(monkeypatch): + g = fake_google.Google() + monkeypatch.setattr(google_auth, "TOKEN_URL", g.base + "/token") + monkeypatch.setattr(gcal, "API", g.base + "/calendar/v3") + google_auth._TOKENS.clear() + gcal._LISTED.clear() + gcal._CALENDARS.clear() + g.calendars[TEAM] = {"summary": "Harbour team", "timeZone": "Europe/Berlin", "events": _team_events(), "readers": set()} + g.calendars[HOLIDAYS] = {"summary": "Holidays in Ireland", "timeZone": "Europe/Dublin", "events": [], "readers": {MARA}} + g.calendars[MARA] = { + "summary": MARA, + "timeZone": "Europe/Dublin", + "events": [ # without a time zone of its own: the calendar's + { + **_event("dentist", "Dentist", "2026-10-20T15:00:00+01:00", "2026-10-20T15:30:00+01:00", visibility="private"), + "start": {"dateTime": "2026-10-20T15:00:00+01:00"}, + "end": {"dateTime": "2026-10-20T15:30:00+01:00"}, + } + ], + "readers": {MARA}, + } + g.lists[MARA] = [MARA, TEAM, HOLIDAYS] + yield g + g.close() + + +@pytest.fixture +def admin(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +def _service_account(google, readers=(TEAM,)): + key, public = fake_google.service_account_key() + google.service_accounts[key["client_email"]] = public + for c in readers: + google.calendars[c]["readers"].add(key["client_email"]) + return key + + +def _segments(db, rid): + return " ".join(s["text"] for s in db.rows("SELECT idx, text FROM segment WHERE recording = $r ORDER BY idx", r=rid)) + + +def test_a_shared_calendar_read_as_a_service_account(google, client, db, cfg, admin): + h = admin + key = _service_account(google) + assert client.get("/api/v1/sources/backends", headers=h).json()["gcal"] == { + "label": "Google Calendar", + "fields": {"calendars": "", "impersonate": "", "days_back": "365"}, + "secrets": ["token", "service_account_credentials"], + "oauth": None, + } + # on its own, a service account has no calendars: it's told what to do + body = {"name": "team", "type": "gcal", "secrets": {"service_account_credentials": json.dumps(key)}} + r = client.post("/api/v1/sources", headers=h, json=body) + assert r.status_code == 200, r.text + sid, health = r.json()["id"], r.json()["health"] + assert not health["ok"] and f"{key['client_email']} has no calendars of its own: share a calendar" in health["error"] + r = client.patch(f"/api/v1/sources/{sid}", headers=h, json={"params": {"calendars": TEAM}}) + assert r.json()["health"]["ok"], r.text + # the key is never shown again; its token_uri is never followed + src = client.get("/api/v1/sources", headers=h).json()[0] + assert src["secrets"] == {"token": {"secret": True, "set": False}, "service_account_credentials": {"secret": True, "set": True}} + assert "PRIVATE KEY" not in json.dumps(src) + assert [c for c in google.calls if c[0] == "token"] # at Google's own token endpoint (here, the fake) + + top = client.get(f"/api/v1/sources/{sid}/browse", headers=h).json() + assert [(e["path"], e["name"], e["dir"]) for e in top] == [(TEAM, "Harbour team", True)] + entries = client.get(f"/api/v1/sources/{sid}/browse", headers=h, params={"path": TEAM}).json() + # the cancelled occurrence, the working location and the event from 2020 aren't files + assert [e["name"] for e in entries] == [ + "2026-10-17 Autumn regatta.ics", + "2026-10-09 Harbour stand-up (moved).ics", + "2026-10-02 Harbour stand-up.ics", + ] + standup = entries[2] + assert (standup["path"], standup["title"], standup["when"], standup["modified"]) == ( + f"{TEAM}/standup.ics", + "Harbour stand-up", + "2026-10-02T10:00:00+02:00", + "2026-09-25T12:00:00+00:00", + ) + assert client.get(f"/api/v1/sources/{sid}/browse", headers=h, params={"path": "nobody@example.org"}).status_code == 400 + + # watched with backfill: every event becomes a text resource, titled and dated by the event + r = client.post( + "/api/v1/watches", headers=h, json={"source": sid, "path": TEAM, "namespace": "pods", "stable_seconds": 0, "backfill": True} + ) + assert r.status_code == 200, r.text + wid = r.json()["id"] + assert client.post("/api/v1/watches/preview", headers=h, json={"source": sid, "path": TEAM}).json()["transcripts"] == 3 + assert sources.poll_watch(db, cfg, wid)["new"] == 3 + drain(db, cfg) + recs = {r["title"]: r for r in db.rows("SELECT record::id(id) AS id, title, source, recorded_at, status, remote FROM recording")} + assert set(recs) == {"Harbour stand-up", "Harbour stand-up (moved)", "Autumn regatta"} + one = recs["Harbour stand-up"] + assert (one["source"], one["recorded_at"], one["status"]) == ("transcript", "2026-10-02T10:00:00+02:00", "analyzed") + assert one["remote"] == {"source": sid, "path": standup["path"]} + text = _segments(db, one["id"]) + for said in ( + "Pier 4, meeting room", + "Mara Keane (mara@example.org)", + "Tom Byrne (tom@example.org), ann@example.org", + "FREQ=WEEKLY;BYDAY=FR", + "Friday 16 October 2026, 10:00 (Europe/Berlin)", # the cancelled occurrence + MEET, + "the new buoys and the keeper's report (https://example.org/report)", + ): + assert said in text, said + assert "
" not in text and " text") == "plain text" diff --git a/fastapi_backend/tests/fake_google.py b/fastapi_backend/tests/fake_google.py new file mode 100644 index 000000000..bc00165d9 --- /dev/null +++ b/fastapi_backend/tests/fake_google.py @@ -0,0 +1,162 @@ +"""Google's token endpoint and Calendar API, enough for tests: a service account signs in with a JWT (checked against +its public key: issuer, audience, scope, and the user it acts as) and a connected account with its refresh token; the +access tokens they get read the calendars and events the test sets up, in pages.""" + +from __future__ import annotations + +import http.server +import json +import secrets +import threading +import urllib.parse + +import jwt +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + + +def service_account_key(email="lens-reader@harbour-project.iam.gserviceaccount.com"): + """A service account key file as Google Cloud gives it, with a fresh RSA key; and that key's public half.""" + private = rsa.generate_private_key(public_exponent=65537, key_size=2048) + pem = private.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()).decode() + key = { + "type": "service_account", + "project_id": "harbour-project", + "private_key_id": "k1", + "private_key": pem, + "client_email": email, + "client_id": "1234567890", + "token_uri": "https://evil.example/token", # never followed: Lens always signs in at Google's own + } + return key, private.public_key() + + +class Google: + def __init__(self): + self.service_accounts: dict[str, object] = {} # client_email -> public key + self.delegated: dict[str, set] = {} # client_email -> users it may act as + self.users: dict[str, dict] = {} # refresh token -> {client_id, client_secret, account} + self.tokens: dict[str, str] = {} # access token -> account it reads as + self.calendars: dict[str, dict] = {} # calendar id -> {summary, timeZone, events: [...], readers: {account}} + self.lists: dict[str, list] = {} # account -> calendar ids in its calendar list + self.calls: list[tuple[str, dict]] = [] + self.page = 2500 + self.expire_next = False # answer the next API call 401, as when a token ends early + self.throttle = False + google = self + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def _json(self, obj, status=200): + data = json.dumps(obj).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _error(self, status, reason, message): + return self._json( + {"error": {"code": status, "message": message, "errors": [{"reason": reason, "message": message}]}}, status + ) + + def do_POST(self): + form = dict(urllib.parse.parse_qsl(self.rfile.read(int(self.headers["Content-Length"])).decode())) + google.calls.append(("token", form)) + if form.get("grant_type") == "urn:ietf:params:oauth:grant-type:jwt-bearer": + try: + head = jwt.get_unverified_header(form["assertion"]) + claims = jwt.decode(form["assertion"], options={"verify_signature": False}) + public = google.service_accounts[claims["iss"]] + claims = jwt.decode(form["assertion"], public, algorithms=["RS256"], audience=google.base + "/token") + except (KeyError, jwt.PyJWTError) as e: + return self._json({"error": "invalid_grant", "error_description": f"Invalid JWT: {e}"}, 400) + assert head.get("kid") == "k1" + who = claims.get("sub") or claims["iss"] + if claims.get("sub") and claims["sub"] not in google.delegated.get(claims["iss"], set()): + return self._json( + { + "error": "unauthorized_client", + "error_description": "Client is unauthorized to retrieve access tokens using this method, or client not authorized for any of the scopes requested.", + }, + 401, + ) + if claims.get("scope") != "https://www.googleapis.com/auth/calendar.readonly": + return self._json({"error": "invalid_scope"}, 400) + elif form.get("grant_type") == "refresh_token": + u = google.users.get(form.get("refresh_token")) + if not u: + return self._json({"error": "invalid_grant", "error_description": "Token has been expired or revoked."}, 400) + if (form.get("client_id"), form.get("client_secret")) != (u["client_id"], u["client_secret"]): + return self._json({"error": "invalid_client", "error_description": "The OAuth client was not found."}, 401) + who = u["account"] + else: + return self._json({"error": "unsupported_grant_type"}, 400) + token = secrets.token_urlsafe(16) + google.tokens[token] = who + return self._json({"access_token": token, "expires_in": 3599, "token_type": "Bearer"}) + + def do_GET(self): + u = urllib.parse.urlsplit(self.path) + q = dict(urllib.parse.parse_qsl(u.query)) + path = urllib.parse.unquote(u.path) + google.calls.append((path, q)) + who = google.tokens.get((self.headers.get("Authorization") or "").removeprefix("Bearer ")) + if google.expire_next: + google.expire_next = False + who = None + if not who: + return self._error(401, "authError", "Invalid Credentials") + if google.throttle: + return self._error(403, "rateLimitExceeded", "Rate Limit Exceeded") + parts = path.removeprefix("/calendar/v3/").split("/") + if parts[:3] == ["users", "me", "calendarList"]: + ids = google.lists.get(who, []) + items = [ + { + "id": i, + "summary": google.calendars[i]["summary"], + "timeZone": google.calendars[i]["timeZone"], + "accessRole": "owner", + **({"primary": True} if i == who else {}), + } + for i in ids + ] + return self._json(self._paged(items, q, 250)) + if parts[0] == "calendars" and len(parts) >= 2: + cal = google.calendars.get(parts[1]) + if not cal or who not in cal["readers"]: + return self._error(404, "notFound", "Not Found") + if len(parts) == 2: + return self._json({"id": parts[1], "summary": cal["summary"], "timeZone": cal["timeZone"]}) + if parts[2:] == ["events"]: + assert q.get("singleEvents") == "false" and q.get("showDeleted") == "false" + items = [ + e + for e in cal["events"] + if not q.get("timeMin") + or (e.get("end", {}).get("dateTime") or e.get("end", {}).get("date") or "9") >= q["timeMin"][:10] + or e.get("recurrence") + ] + return self._json({"summary": cal["summary"], "timeZone": cal["timeZone"], **self._paged(items, q, google.page)}) + return self._error(404, "notFound", "Not Found") + + def _paged(self, items, q, size): + start = int(q.get("pageToken") or 0) + size = min(size, int(q.get("maxResults") or size)) + out = {"items": items[start : start + size]} + if start + size < len(items): + out["nextPageToken"] = str(start + size) + return out + + self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base = f"http://127.0.0.1:{self.server.server_address[1]}" + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + def api_calls(self, what=None): + return [c for c in self.calls if c[0] != "token" and (what is None or c[0].endswith(what))] + + def close(self): + self.server.shutdown() diff --git a/nextjs-frontend/__tests__/sources-model.test.ts b/nextjs-frontend/__tests__/sources-model.test.ts index 5cd885f57..a44434b19 100644 --- a/nextjs-frontend/__tests__/sources-model.test.ts +++ b/nextjs-frontend/__tests__/sources-model.test.ts @@ -6,6 +6,7 @@ import { healthOf, imapHost, nextScan, + parseGoogleAccount, parseRcloneToken, parseServiceAccountKey, pickupText, @@ -206,6 +207,53 @@ describe("connections", () => { expect(suggestName("ical", cal.params)).toBe("Calendar feed"); expect([fileKind("Harbour report.eml"), fileKind("2026-10-02 Stand-up.ics")]).toEqual(["document", "transcript"]); }); + it("signs Google Calendar in as a Google account or a service account", () => { + const GCAL: BackendSpec = { + label: "Google Calendar", + fields: { calendars: "", impersonate: "", days_back: "365" }, + secrets: ["token", "service_account_credentials"], + }; + const account = '{"type":"authorized_user","client_id":"c","client_secret":"s","refresh_token":"r","quota":"x"}'; + expect(parseGoogleAccount(account)).toEqual({ + ok: true, + json: '{"client_id":"c","client_secret":"s","refresh_token":"r"}', + }); + expect(parseGoogleAccount("").ok).toBe(null); + expect(parseGoogleAccount("{oops").ok).toBe(false); + expect(parseGoogleAccount('{"refresh_token":"r"}').ok).toBe(false); + + const blank = emptyForm(GCAL); + expect(blank.driveAuth).toBe("token"); + expect(Object.keys(validateForm("gcal", GCAL, blank, false))).toEqual(["token"]); + const connected = { ...blank, secrets: { token: account } }; + expect(validateForm("gcal", GCAL, connected, false)).toEqual({}); + expect(validateForm("gcal", GCAL, blank, true, (k) => k === "token")).toEqual({}); + expect( + validateForm("gcal", GCAL, { ...connected, params: { ...blank.params, days_back: "a year" } }, false), + ).toHaveProperty("days_back"); + // an account replaces a saved key, and a key file replaces a connected account + expect(buildPayload(GCAL, connected, "gcal")).toEqual({ + params: { calendars: "", impersonate: "", days_back: "365" }, + secrets: { + token: '{"client_id":"c","client_secret":"s","refresh_token":"r"}', + service_account_credentials: null, + }, + }); + const key = JSON.stringify({ + type: "service_account", + client_email: "lens@p.iam.gserviceaccount.com", + private_key: "K", + }); + const sa = { ...blank, driveAuth: "service_account" as const, secrets: { service_account_credentials: key } }; + expect(validateForm("gcal", GCAL, sa, false)).toEqual({}); + expect(buildPayload(GCAL, sa, "gcal").secrets).toEqual({ service_account_credentials: key, token: null }); + + expect(sourceSubtitle({ type: "gcal", params: { calendars: "" } })).toBe("Google Calendar · every calendar"); + expect(sourceSubtitle({ type: "gcal", params: { calendars: "a@x, b@x", impersonate: "mara@x.io" } })).toBe( + "Google Calendar · as mara@x.io · 2 calendars", + ); + expect(suggestName("gcal", blank.params)).toBe("Google Calendar"); + }); it("confirms deletes by name, loosely", () => { expect(confirms(" dropbox · callrecorder", "Dropbox · CallRecorder")).toBe(true); expect(confirms("Dropbox", "Dropbox · CallRecorder")).toBe(false); diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index b0d2a2f70..04b8ca942 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -16925,7 +16925,7 @@ export type Source = { /** * Type */ - type: "s3" | "dropbox" | "drive" | "onedrive" | "sftp" | "smb" | "webdav" | "local" | "imap" | "ical"; + type: "s3" | "dropbox" | "drive" | "onedrive" | "sftp" | "smb" | "webdav" | "local" | "imap" | "ical" | "gcal"; /** * Label */ @@ -16969,7 +16969,7 @@ export type SourceCreate = { /** * Type */ - type: "s3" | "dropbox" | "drive" | "onedrive" | "sftp" | "smb" | "webdav" | "local" | "imap" | "ical"; + type: "s3" | "dropbox" | "drive" | "onedrive" | "sftp" | "smb" | "webdav" | "local" | "imap" | "ical" | "gcal"; /** * Params */ diff --git a/nextjs-frontend/components/sources/connection-fields.tsx b/nextjs-frontend/components/sources/connection-fields.tsx index 553a8261d..9dccb0e76 100644 --- a/nextjs-frontend/components/sources/connection-fields.tsx +++ b/nextjs-frontend/components/sources/connection-fields.tsx @@ -162,6 +162,39 @@ export function ConnectionFields({ ) ) : null; + /** A Google service account's .json key file (Google Drive, Google Calendar). */ + const serviceAccountKey = (share: string) => { + const keyText = form.secrets.service_account_credentials; + const key = keyText ? parseServiceAccountKey(keyText) : null; + return ( + + {({ id, describedBy }) => ( + { + const f = e.target.files?.[0]; + setSecret("service_account_credentials", f ? await f.text() : undefined); + }} + /> + )} + + ); + }; + switch (type) { case "s3": return ( @@ -182,35 +215,7 @@ export function ConnectionFields({ case "dropbox": return token; case "drive": { - const keyText = form.secrets.service_account_credentials; - const key = keyText ? parseServiceAccountKey(keyText) : null; - const serviceAccount = ( - - {({ id, describedBy }) => ( - { - const f = e.target.files?.[0]; - setSecret("service_account_credentials", f ? await f.text() : undefined); - }} - /> - )} - - ); + const serviceAccount = serviceAccountKey("Share the folders Lens should read with that address."); return (
); + case "gcal": + return ( +
+ onChange({ ...form, driveAuth: v as ConnForm["driveAuth"] })} + items={[ + { value: "token", label: "Google account" }, + { value: "service_account", label: "Service account" }, + ]} + /> + {form.driveAuth === "service_account" ? ( + <> + {serviceAccountKey("Share the calendars Lens should read with that address, and enter their IDs below.")} + {text("impersonate", { + optional: true, + placeholder: "you@company.com", + hint: "Workspace only: read this user’s calendars (needs domain-wide delegation)", + })} + + ) : editing && isSet("token") && form.secrets.token === undefined ? ( +
+ A Google account is connected (never shown again). + +
+ ) : ( + + {({ id, describedBy, invalid }) => ( +