From 6da350aa4a1818fd18bad5e8849eaf46f413887c Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Thu, 15 Oct 2026 11:32:13 +0000 Subject: [PATCH 1/3] feat(sources): Connect with Google for Google Calendar connections A Google Calendar connection signs in to a Google account from a button: Lens saves the connection, opens Google's consent screen for read-only calendar access, and Google sends the browser back to Sources with the account connected and its calendars open. The OAuth client is the Google sign-in provider's, an earlier connection's, or one entered in the form, which shows the redirect URI to register. A connection whose sign-in stopped working gets "Connect Google account" on its row. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CGVfwJfXV4MiUhSyusia64 --- CHANGELOG.md | 6 + docs/authentication.md | 4 +- docs/processing.md | 15 +- fastapi_backend/app/api/v1/routes/external.py | 29 ++- fastapi_backend/app/api/v1/routes/sources.py | 41 ++++- fastapi_backend/app/domain/gcal.py | 2 +- fastapi_backend/app/domain/google_auth.py | 143 +++++++++++++++ fastapi_backend/app/schemas/sources.py | 18 ++ .../tests/api/test_google_calendar_source.py | 75 +++++++- fastapi_backend/tests/fake_google.py | 48 +++++ .../__tests__/add-connection.test.tsx | 70 +++++++- .../__tests__/sources-model.test.ts | 13 +- nextjs-frontend/app/openapi-client/index.ts | 11 ++ nextjs-frontend/app/openapi-client/sdk.gen.ts | 40 +++++ .../app/openapi-client/types.gen.ts | 98 +++++++++++ .../components/sources/connection-dialog.tsx | 38 ++++ .../components/sources/connection-fields.tsx | 165 +++++++++++++++--- .../components/sources/source-model.ts | 11 +- .../components/sources/sources-page.tsx | 47 ++++- nextjs-frontend/openapi.json | 165 ++++++++++++++++++ threat-model.json | 10 ++ threat-model.md | 1 + threat-model.yaml | 12 +- 23 files changed, 1018 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2c2abe6c..521325be7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version account, which reads the calendars shared with it or, in Google Workspace, a user's calendars through domain-wide delegation. Calls to Google show in the activity ledger ([Google Calendar](docs/processing.md#email-imap-calendar-feeds-ical-and-google-calendar)). +- **Connect with Google.** A Google Calendar connection signs in to a Google account from a button now: Google's consent + screen asks for read-only calendar access and sends you back to Sources with the account connected and its calendars + open. It uses the Google sign-in provider's OAuth client, an earlier connection's, or one entered in the form, which + also shows the redirect URI to register. A connection whose sign-in stopped working has **Connect Google account** + on its row; pasting the authorized user JSON still works + ([Google Calendar](docs/processing.md#email-imap-calendar-feeds-ical-and-google-calendar)). ## 0.10.0 October 15, 2026 {id="0.10.0"} diff --git a/docs/authentication.md b/docs/authentication.md index f9de67554..7e63d4239 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -64,7 +64,9 @@ OpenID Connect provider (Authentik, Keycloak, Okta, ...; give its issuer address `/.well-known/openid-configuration` there). Each needs a client id and secret from the provider, and the redirect URI the dialog shows: `/api/v1/auth/external//callback`, registered for every address people open Lens at. The secret is kept encrypted (like other settings secrets) and never shown again. The sign-in page then -has a "Continue with ..." button for each provider that's turned on. +has a "Continue with ..." button for each provider that's turned on. Google's client also connects Google accounts to +Google Calendar sources (Connect with Google, [Google Calendar](processing.md#email-imap-calendar-feeds-ical-and-google-calendar)), +through the same redirect URI. * **Who it signs in.** An outside account that's connected signs in its Lens account. One that isn't yet is connected to the account with the same email, when the provider vouches for the email: Google and OpenID Connect providers say diff --git a/docs/processing.md b/docs/processing.md index 73875f7de..f94f51ae6 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -351,8 +351,19 @@ importing chosen ones and watching work as above. of its own, as Google keeps it; working-location entries ("Home", "Office") are left out. It signs in one of two ways, each kept encrypted like a password and never shown again: - - **Google account** (`token`): the "authorized user" JSON (`client_id`, `client_secret`, `refresh_token`) of an - OAuth client of your own Google Cloud project, with the Calendar API turned on there. `gcloud auth + - **Google account** (`token`): **Connect with Google** in the connection form saves the connection and opens + Google's consent screen in this browser. Once you allow read-only calendar access, Google sends you back to + Sources with the account connected (its address shows on the connection) and its calendars open, to pick one to + watch. It goes through an OAuth client of a Google Cloud project with the Calendar API turned on: the Google + sign-in provider's (Settings › Sign-in) when there is one, else an earlier Google connection's, else one you + enter in the form: a web application client whose authorized redirect URIs include + `/api/v1/auth/external/google/callback` (the address Google sign-in uses too; the form shows + it). While the client's OAuth consent screen is in Testing, Google ends its refresh tokens after 7 days and warns + that the app isn't verified; publish it to keep the connection. A connection whose sign-in stopped working shows + **Connect Google account** on its row. An agent starts the same flow + (`POST /api/v1/sources/{id}/google/connect` returns the consent address), but the consent itself is a person's, + in the browser that asked. What is kept is the "authorized user" JSON (`client_id`, `client_secret`, + `refresh_token`), which can also be pasted instead (Or paste an authorized user JSON): `gcloud auth application-default login --client-id-file=client.json --scopes=https://www.googleapis.com/auth/calendar.readonly,https://www.googleapis.com/auth/cloud-platform` writes one (to `application_default_credentials.json` in gcloud's folder). - **Service account** (`service_account_credentials`, its .json key file). On its own it reads the calendars shared diff --git a/fastapi_backend/app/api/v1/routes/external.py b/fastapi_backend/app/api/v1/routes/external.py index 85cf532f5..e37674fd2 100644 --- a/fastapi_backend/app/api/v1/routes/external.py +++ b/fastapi_backend/app/api/v1/routes/external.py @@ -15,7 +15,7 @@ from fastapi.responses import RedirectResponse from app.api.deps import AdminWriter, Cfg, CurrentUser, Db, Writer, domain_errors, web_origin -from app.domain import auth, external_login, passkeys +from app.domain import auth, external_login, google_auth, passkeys, sources from app.schemas.auth import ( ExternalIdentity, ExternalProvider, @@ -96,6 +96,8 @@ def external_callback( flow = external_login.claim(db, state, request.cookies.get(COOKIE)) except ValueError as e: return _back(web_origin(request), error=str(e), next="/login") + if flow.get("mode") == "source": + return _source_connected(db, cfg, flow, code, state, error, error_description) origin, connect = flow["origin"], flow.get("mode") == "connect" away = "/account" if connect else "/login" if error or not code: @@ -114,6 +116,31 @@ def external_callback( return _back(origin, ticket=passkeys.issue_ticket(db, u["id"], f"external:{key}"), next=flow.get("next") or "/") +def _source_connected(db: Db, cfg: Cfg, flow: dict, code: str, state: str, error: str, error_description: str) -> RedirectResponse: + """A Google account connected to a source (google_auth.connect_start): ends on the Sources page, which shows how it + went and offers the source's calendars to watch.""" + sid = flow.get("source") + if error or not code: + why = "you cancelled" if error == "access_denied" else (error_description or error or "no code came back") + q = {"google_error": f"connecting the Google account didn't finish: {why}", "source": sid} + else: + try: + sid, email = google_auth.connect_finish(db, cfg, flow, code, state) + sources.test(db, cfg, sid) + who = {"id": flow.get("account"), "email": flow.get("by")} + auth.audit(db, who, "source.google_connect", f"storage_source:{sid}", [email] if email else None) + q = {"google": "connected", "source": sid, "account": email} + except KeyError: + q = {"google_error": "that connection was removed meanwhile"} + except (ValueError, RuntimeError) as e: + q = {"google_error": str(e), "source": sid} + r = RedirectResponse( + f"{flow['origin']}/sources?{urllib.parse.urlencode({k: v for k, v in q.items() if v not in (None, '')})}", status_code=303 + ) + r.delete_cookie(COOKIE, path="/api/v1/auth/external") + return r + + # ---------- your outside accounts ---------- def _signed_in(user: CurrentUser) -> None: if user.via != "access": diff --git a/fastapi_backend/app/api/v1/routes/sources.py b/fastapi_backend/app/api/v1/routes/sources.py index ea7d36bdc..b2a34136b 100644 --- a/fastapi_backend/app/api/v1/routes/sources.py +++ b/fastapi_backend/app/api/v1/routes/sources.py @@ -5,16 +5,20 @@ import logging from collections import Counter -from fastapi import APIRouter, BackgroundTasks, HTTPException, Request +from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, Response -from app.api.deps import Acl, AdminReader, AdminWriter, Cfg, CurrentUser, Db -from app.domain import auth, sources, store +from app.api.deps import Acl, AdminReader, AdminWriter, Cfg, CurrentUser, Db, web_origin +from app.api.v1.routes.external import _cookie +from app.domain import auth, google_auth, sources, store from app.domain.store import DB from app.schemas.admin import Started from app.schemas.common import Created, Ok from app.schemas.sources import ( Backend, BrowseEntry, + GoogleClient, + GoogleConnect, + GoogleConnectStart, RoutedMessage, RoutePreviewRequest, Source, @@ -94,6 +98,37 @@ def test_source(sid: int, user: AdminWriter, db: Db, cfg: Cfg) -> SourceHealth: return sources.test(db, cfg, sid) +@router.get("/sources/google-client") +def google_client(user: AdminReader, request: Request, db: Db, cfg: Cfg) -> GoogleClient: + """The OAuth client a Google account would be connected through (never its secret), and the redirect URI it needs.""" + try: + c = google_auth.client(db, cfg) + except ValueError: + c = {"origin": None, "client_id": None} + return GoogleClient(origin=c["origin"], client_id=c["client_id"], redirect_uri=web_origin(request) + google_auth.CALLBACK) + + +@router.post("/sources/{sid}/google/connect") +def connect_google( + sid: int, body: GoogleConnect, user: AdminWriter, request: Request, response: Response, db: Db, cfg: Cfg +) -> GoogleConnectStart: + """Connect a Google account to a Google source (Google Calendar): open the returned address in this browser, consent + there, and Google sends you back to the Sources page with the account connected. Its refresh token is kept sealed on + the source. Without a client here, the Google sign-in provider's (Settings › Sign-in) or an earlier connection's is + used. Audited as `source.google_connect`.""" + try: + src = sources.get(db, sid) + except KeyError: + raise HTTPException(404, "not found") from None + origin = web_origin(request) + try: + url, browser = google_auth.connect_start(db, cfg, src, origin, (body.client_id, body.client_secret), user.as_audit()) + except ValueError as e: + raise HTTPException(400, str(e)) from None + _cookie(response, origin, browser) + return GoogleConnectStart(url=url) + + @router.get("/sources/{sid}/browse") def browse_source(sid: int, user: AdminReader, db: Db, cfg: Cfg, path: str = "") -> list[BrowseEntry]: """The folders and files at one path of a source (for local sources, no path lists the allowed roots), with the diff --git a/fastapi_backend/app/domain/gcal.py b/fastapi_backend/app/domain/gcal.py index 676663787..40f33f83d 100644 --- a/fastapi_backend/app/domain/gcal.py +++ b/fastapi_backend/app/domain/gcal.py @@ -31,7 +31,7 @@ SCOPES = ("https://www.googleapis.com/auth/calendar.readonly",) TYPE = { "label": "Google Calendar", - "fields": {"calendars": "", "impersonate": "", "days_back": "365"}, + "fields": {"calendars": "", "impersonate": "", "days_back": "365", "account": ""}, # account: the one connected "secrets": ["token", "service_account_credentials"], "google": True, # signs in through google_auth.py } diff --git a/fastapi_backend/app/domain/google_auth.py b/fastapi_backend/app/domain/google_auth.py index d19e4742d..04aceeaf6 100644 --- a/fastapi_backend/app/domain/google_auth.py +++ b/fastapi_backend/app/domain/google_auth.py @@ -18,9 +18,13 @@ from __future__ import annotations +import base64 +import datetime as dt import hashlib import json +import secrets import time +import urllib.parse import httpx import jwt @@ -216,3 +220,142 @@ def get(cfg, creds, scopes, url, params=None, api="api", what=None): continue return _answer(r, creds) raise GoogleError("Google didn't accept a fresh sign-in") # not reached: the second attempt returns or raises + + +# ---------- connecting a Google account to a source (OAuth consent in the browser) ---------- +AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth" +IDENTITY = ("openid", "email") # asked for too, to show which account is connected +FLOW_MINUTES = 10 +CALLBACK = "/api/v1/auth/external/google/callback" # the Google sign-in provider's own: one redirect URI for both + + +def scopes_for(src): + """The scopes a source's type reads with (its module's SCOPES).""" + from . import feeds # feeds imports gcal, which imports this + + mod = feeds.OWN.get(src["type"]) + if mod is None or not getattr(mod, "SCOPES", None): + raise ValueError(f"a {src['type']} source doesn't sign in with a Google account") + return tuple(mod.SCOPES) + + +def client(db, cfg, client_id=None, client_secret=None): + """The OAuth client a Google account is connected through: {client_id, client_secret, origin}. The one given, else + the Google sign-in provider's (Settings › Sign-in), else the one a Google source was connected through last. + ValueError when there's none.""" + if client_id or client_secret: + if not (client_id and client_secret): + raise ValueError("enter both the client ID and the client secret of the Google OAuth client") + return {"client_id": str(client_id).strip(), "client_secret": str(client_secret).strip(), "origin": "given"} + from . import external_login, settings, sources, store + + row = db.one("SELECT client_id, client_secret FROM $r", r=store.R("login_provider", "google")) + if row and row.get("client_id") and row.get("client_secret"): + secret_ = settings.unseal(cfg, row["client_secret"], external_login._context("google")) + return {"client_id": row["client_id"], "client_secret": secret_, "origin": "sign-in"} + google_types = [k for k, v in sources.BACKENDS.items() if v.get("google")] + rows = db.rows( + "SELECT record::id(id) AS id, created_at FROM storage_source WHERE type IN $t AND sealed.token != NONE ORDER BY created_at DESC", + t=google_types, + ) + for sid in [r["id"] for r in rows]: + user = json.loads(secret(cfg, sources.get(db, sid), "token") or "{}") + if user.get("client_id") and user.get("client_secret"): + return {"client_id": user["client_id"], "client_secret": user["client_secret"], "origin": "connection"} + raise ValueError( + "Lens needs a Google OAuth client to connect an account: add Google in Settings › Sign-in, or enter a client " + "ID and secret (Google Cloud › APIs & Services › Credentials, a web application client)" + ) + + +def connect_start(db, cfg, src, origin, given=None, by=None): + """Google's consent page for connecting an account to source `src`, and the browser secret to keep in a cookie + (the sign-in cookie: the flow is an external_flow, used once, ten minutes, bound to this browser).""" + from . import auth, settings, store + + scopes = scopes_for(src) + c = client(db, cfg, *(given or (None, None))) + state, verifier, browser = secrets.token_urlsafe(24), secrets.token_urlsafe(48), secrets.token_urlsafe(24) + challenge = base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip("=") + redirect_uri = origin.rstrip("/") + CALLBACK + expires = (dt.datetime.now(dt.timezone.utc) + dt.timedelta(minutes=FLOW_MINUTES)).isoformat(timespec="seconds") + db.q("DELETE external_flow WHERE expires_at < $n", n=store.now()) + db.q( + "CREATE $r CONTENT $d", + r=store.R("external_flow", auth.sha(state)), + d={ + "provider": "google", + "mode": "source", + "source": src["id"], + "scopes": list(scopes), + "client_id": c["client_id"], + "client_secret": settings.seal(cfg, c["client_secret"], f"external_flow:{auth.sha(state)}"), + "verifier": verifier, + "browser": auth.sha(browser), + "redirect_uri": redirect_uri, + "origin": origin.rstrip("/"), + "account": (by or {}).get("id"), # who started it, for the audit log + "by": (by or {}).get("email"), + "expires_at": expires, + }, + ) + q = { + "response_type": "code", + "client_id": c["client_id"], + "redirect_uri": redirect_uri, + "scope": " ".join([*IDENTITY, *scopes]), + "state": state, + "code_challenge": challenge, + "code_challenge_method": "S256", + "access_type": "offline", # a refresh token, so the source reads on its own + "prompt": "consent", # Google gives a refresh token only when it asks + "include_granted_scopes": "false", + } + return AUTH_URL + "?" + urllib.parse.urlencode(q), browser + + +def connect_finish(db, cfg, flow, code, state): + """Swap the code Google sent back for the account's refresh token and keep it on the source (its `token`; a + service account key it had is dropped). Returns the source's id and the account's address.""" + from . import auth, settings, sources + + src = sources.get(db, flow["source"]) # KeyError: removed meanwhile + client_secret = settings.unseal(cfg, flow["client_secret"], f"external_flow:{auth.sha(state)}") + form = { + "grant_type": "authorization_code", + "code": code, + "redirect_uri": flow["redirect_uri"], + "client_id": flow["client_id"], + "client_secret": client_secret, + "code_verifier": flow["verifier"], + } + with activity.call("google.token", cfg, detail={"as": "consent"}): + try: + r = httpx.post(TOKEN_URL, data=form, headers={"Accept": "application/json"}, timeout=TIMEOUT) + except httpx.HTTPError as e: + raise GoogleError(f"couldn't reach Google: {e}") from None + try: + body = r.json() + except ValueError: + body = {} + if r.status_code >= 400 or not isinstance(body, dict): + said = (body.get("error_description") or body.get("error")) if isinstance(body, dict) else None + raise GoogleError(f"Google refused: {said or r.status_code}; check the client ID and secret") + if not body.get("refresh_token"): + raise GoogleError("Google gave no refresh token; remove Lens from your Google account's third-party access and connect again") + granted = set(str(body.get("scope") or "").split()) + missing = [s for s in flow.get("scopes") or [] if granted and s not in granted] + if missing: + raise GoogleError("Google wasn't given access to what this connection reads: tick it on Google's consent screen") + email = "" + if body.get("id_token"): + try: # straight from Google's token endpoint over TLS: no signature to check (OpenID Connect §3.1.3.7) + email = str(jwt.decode(body["id_token"], options={"verify_signature": False}).get("email") or "") + except jwt.PyJWTError: + email = "" + user = json.dumps( + {"type": "authorized_user", "client_id": flow["client_id"], "client_secret": client_secret, "refresh_token": body["refresh_token"]} + ) + params = {"account": email} if "account" in sources.BACKENDS[src["type"]]["fields"] else None + sources.update(db, cfg, src["id"], params=params, secret_values={"token": user, "service_account_credentials": None}) + return src["id"], email diff --git a/fastapi_backend/app/schemas/sources.py b/fastapi_backend/app/schemas/sources.py index 545cf234b..db8992d6c 100644 --- a/fastapi_backend/app/schemas/sources.py +++ b/fastapi_backend/app/schemas/sources.py @@ -69,6 +69,24 @@ class SourceUpdated(Ok): health: SourceHealth +class GoogleClient(ResponseModel): + origin: Literal["sign-in", "connection"] | None = Field( + description="where the OAuth client for connecting a Google account comes from: the Google sign-in provider, " + "an earlier Google connection, or none (enter one to connect)" + ) + client_id: str | None = None + redirect_uri: str = Field(description="the address to add to the OAuth client's authorized redirect URIs") + + +class GoogleConnect(RequestModel): + client_id: str | None = Field(default=None, max_length=300, description="an OAuth client of your own (else the one Lens has)") + client_secret: str | None = Field(default=None, max_length=300) + + +class GoogleConnectStart(ResponseModel): + url: str = Field(description="Google's consent page; open it in this browser") + + class ImportedAs(ResponseModel): recording: int namespace: str diff --git a/fastapi_backend/tests/api/test_google_calendar_source.py b/fastapi_backend/tests/api/test_google_calendar_source.py index 1fb97910f..f4546485d 100644 --- a/fastapi_backend/tests/api/test_google_calendar_source.py +++ b/fastapi_backend/tests/api/test_google_calendar_source.py @@ -5,6 +5,7 @@ from __future__ import annotations import json +import urllib.parse import pytest @@ -81,6 +82,7 @@ def google(monkeypatch): g = fake_google.Google() monkeypatch.setattr(google_auth, "TOKEN_URL", g.base + "/token") monkeypatch.setattr(gcal, "API", g.base + "/calendar/v3") + monkeypatch.setattr(google_auth, "AUTH_URL", g.base + "/auth") google_auth._TOKENS.clear() gcal._LISTED.clear() gcal._CALENDARS.clear() @@ -126,7 +128,7 @@ def test_a_shared_calendar_read_as_a_service_account(google, client, db, cfg, ad key = _service_account(google) assert client.get("/api/v1/sources/backends", headers=h).json()["gcal"] == { "label": "Google Calendar", - "fields": {"calendars": "", "impersonate": "", "days_back": "365"}, + "fields": {"calendars": "", "impersonate": "", "days_back": "365", "account": ""}, "secrets": ["token", "service_account_credentials"], "oauth": None, } @@ -282,3 +284,74 @@ def test_an_event_as_icalendar(): bare = {**_event("x3", "Call", "2026-10-02T08:00:00Z", "2026-10-02T09:00:00Z"), "start": {"dateTime": "2026-10-02T08:00:00Z"}} assert "DTSTART:20261002T080000Z" in gcal.event_ics(bare, {**cal, "zone": "Nowhere/Atlantis"}) assert gcal._text("plain text") == "plain text" + + +def _back(r): + assert r.status_code == 303, r.text + u = urllib.parse.urlsplit(r.headers["location"]) + return u.path, dict(urllib.parse.parse_qsl(u.query)) + + +def test_connecting_a_google_account_in_the_browser(google, client, new_client, db, cfg, admin): + h = admin + sid = client.post("/api/v1/sources", headers=h, json={"type": "gcal", "name": "Mara"}).json()["id"] + # no OAuth client anywhere: Lens says what it needs, and the redirect URI to give Google + got = client.get("/api/v1/sources/google-client", headers=h).json() + assert got == {"origin": None, "client_id": None, "redirect_uri": "http://127.0.0.1/api/v1/auth/external/google/callback"} + r = client.post(f"/api/v1/sources/{sid}/google/connect", headers=h, json={}) + assert r.status_code == 400 and "needs a Google OAuth client" in r.json()["detail"] + r = client.post(f"/api/v1/sources/{sid}/google/connect", headers=h, json={"client_id": "only-an-id"}) + assert r.status_code == 400 and "both the client ID and the client secret" in r.json()["detail"] + + # the Google sign-in provider's client is used, through its own redirect URI + google.clients.add(("lens-web", "web secret 1")) + r = client.post("/api/v1/auth/providers", headers=h, json={"kind": "google", "client_id": "lens-web", "client_secret": "web secret 1"}) + assert r.status_code == 200, r.text + assert client.get("/api/v1/sources/google-client", headers=h).json()["origin"] == "sign-in" + url = client.post(f"/api/v1/sources/{sid}/google/connect", headers=h, json={}).json()["url"] + q = dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query)) + assert q["client_id"] == "lens-web" and q["scope"].split() == ["openid", "email", *gcal.SCOPES] + path, back = _back(client.get(google.consent(url, MARA), follow_redirects=False)) + assert path == "/sources" and back == {"google": "connected", "source": str(sid), "account": MARA} + src = client.get("/api/v1/sources", headers=h).json()[0] + assert src["health"]["ok"] and src["params"]["account"] == MARA and src["secrets"]["token"]["set"] + assert "web secret 1" not in json.dumps(src) + assert [e["name"] for e in sources.browse(db, cfg, sid)] == [MARA, "Harbour team", "Holidays in Ireland"] + audit = db.rows("SELECT email, action, target, detail FROM audit_log WHERE action = 'source.google_connect'") + assert audit == [{"email": "root@x.io", "action": "source.google_connect", "target": f"storage_source:{sid}", "detail": [MARA]}] + + # cancelled on Google's page; calendars unticked there; a code used twice; another browser + url = client.post(f"/api/v1/sources/{sid}/google/connect", headers=h, json={}).json()["url"] + state = dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query))["state"] + _, back = _back(client.get(f"/api/v1/auth/external/google/callback?error=access_denied&state={state}", follow_redirects=False)) + assert "you cancelled" in back["google_error"] and back["source"] == str(sid) + url = client.post(f"/api/v1/sources/{sid}/google/connect", headers=h, json={}).json()["url"] + _, back = _back(client.get(google.consent(url, MARA, granted="openid email"), follow_redirects=False)) + assert "wasn't given access" in back["google_error"] + url = client.post(f"/api/v1/sources/{sid}/google/connect", headers=h, json={}).json()["url"] + callback = google.consent(url, MARA) + other = new_client() + r = other.get(callback, follow_redirects=False) + assert r.status_code == 303 and "another+browser" in r.headers["location"] + + # a service account key gives way to the connected account; with no sign-in provider, the last connection's client + key = _service_account(google) + r = client.post( + "/api/v1/sources", + headers=h, + json={"type": "gcal", "params": {"calendars": TEAM}, "secrets": {"service_account_credentials": json.dumps(key)}}, + ) + sid2 = r.json()["id"] + client.delete("/api/v1/auth/providers/google", headers=h) + assert client.get("/api/v1/sources/google-client", headers=h).json() == { + "origin": "connection", + "client_id": "lens-web", + "redirect_uri": "http://127.0.0.1/api/v1/auth/external/google/callback", + } + google.calendars[TEAM]["readers"].add(MARA) + url = client.post(f"/api/v1/sources/{sid2}/google/connect", headers=h, json={}).json()["url"] + _, back = _back(client.get(google.consent(url, MARA), follow_redirects=False)) + assert back["google"] == "connected" + src = next(s for s in client.get("/api/v1/sources", headers=h).json() if s["id"] == sid2) + assert src["secrets"] == {"token": {"secret": True, "set": True}, "service_account_credentials": {"secret": True, "set": False}} + assert google_auth.credentials(cfg, sources.get(db, sid2))["kind"] == "authorized_user" diff --git a/fastapi_backend/tests/fake_google.py b/fastapi_backend/tests/fake_google.py index bc00165d9..7db13db1e 100644 --- a/fastapi_backend/tests/fake_google.py +++ b/fastapi_backend/tests/fake_google.py @@ -4,6 +4,8 @@ from __future__ import annotations +import base64 +import hashlib import http.server import json import secrets @@ -36,6 +38,8 @@ def __init__(self): self.service_accounts: dict[str, object] = {} # client_email -> public key self.delegated: dict[str, set] = {} # client_email -> users it may act as self.users: dict[str, dict] = {} # refresh token -> {client_id, client_secret, account} + self.clients: set[tuple[str, str]] = set() # OAuth clients (id, secret) that may swap codes + self.codes: dict[str, dict] = {} # authorization code -> what the consent screen gave self.tokens: dict[str, str] = {} # access token -> account it reads as self.calendars: dict[str, dict] = {} # calendar id -> {summary, timeZone, events: [...], readers: {account}} self.lists: dict[str, list] = {} # account -> calendar ids in its calendar list @@ -85,6 +89,34 @@ def do_POST(self): ) if claims.get("scope") != "https://www.googleapis.com/auth/calendar.readonly": return self._json({"error": "invalid_scope"}, 400) + elif form.get("grant_type") == "authorization_code": + c = google.codes.pop(form.get("code"), None) + if (form.get("client_id"), form.get("client_secret")) not in google.clients: + return self._json({"error": "invalid_client", "error_description": "The OAuth client was not found."}, 401) + if not c or form.get("redirect_uri") != c["redirect_uri"] or form.get("client_id") != c["client_id"]: + return self._json({"error": "invalid_grant", "error_description": "Bad Request"}, 400) + verifier = form.get("code_verifier") or "" + if base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip("=") != c["challenge"]: + return self._json({"error": "invalid_grant", "error_description": "Invalid code verifier."}, 400) + refresh = secrets.token_urlsafe(16) + google.users[refresh] = {"client_id": c["client_id"], "client_secret": form["client_secret"], "account": c["account"]} + token = secrets.token_urlsafe(16) + google.tokens[token] = c["account"] + id_token = jwt.encode( + {"iss": "https://accounts.google.com", "sub": "1", "email": c["account"]}, + "a key Lens never checks: the id token comes straight from Google", + algorithm="HS256", + ) + return self._json( + { + "access_token": token, + "expires_in": 3599, + "refresh_token": refresh, + "id_token": id_token, + "scope": c["granted"], + "token_type": "Bearer", + } + ) elif form.get("grant_type") == "refresh_token": u = google.users.get(form.get("refresh_token")) if not u: @@ -155,6 +187,22 @@ def _paged(self, items, q, size): self.base = f"http://127.0.0.1:{self.server.server_address[1]}" threading.Thread(target=self.server.serve_forever, daemon=True).start() + def consent(self, url, account, granted=None): + """What Google's consent page does for `account`: the address it sends the browser back to, with a code. The + scopes granted are those asked for, unless `granted` says which (Google lets people untick some).""" + q = dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query)) + assert url.startswith(self.base + "/auth?") and q["access_type"] == "offline" and q["prompt"] == "consent" + assert q["code_challenge_method"] == "S256" and "openid" in q["scope"].split() + code = secrets.token_urlsafe(12) + self.codes[code] = { + "client_id": q["client_id"], + "redirect_uri": q["redirect_uri"], + "challenge": q["code_challenge"], + "account": account, + "granted": q["scope"] if granted is None else granted, + } + return q["redirect_uri"] + "?" + urllib.parse.urlencode({"code": code, "state": q["state"]}) + def api_calls(self, what=None): return [c for c in self.calls if c[0] != "token" and (what is None or c[0].endswith(what))] diff --git a/nextjs-frontend/__tests__/add-connection.test.tsx b/nextjs-frontend/__tests__/add-connection.test.tsx index 4d0a479b1..f6bb1101f 100644 --- a/nextjs-frontend/__tests__/add-connection.test.tsx +++ b/nextjs-frontend/__tests__/add-connection.test.tsx @@ -5,18 +5,34 @@ import "@testing-library/jest-dom"; import { Sources } from "@/app/openapi-client"; import { ConnectionDialog } from "@/components/sources/connection-dialog"; import { TooltipProvider } from "@/components/ui/tooltip"; +import { navigate } from "@/lib/navigate"; jest.mock("@/app/openapi-client", () => ({ - Sources: { createSource: jest.fn(), updateSource: jest.fn(), testSource: jest.fn(), deleteSource: jest.fn() }, + Sources: { + createSource: jest.fn(), + updateSource: jest.fn(), + testSource: jest.fn(), + deleteSource: jest.fn(), + googleClient: jest.fn(), + connectGoogle: jest.fn(), + }, })); jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); jest.mock("@/components/ui/toast", () => ({ useToast: () => jest.fn() })); +jest.mock("@/lib/navigate", () => ({ navigate: { assign: jest.fn(), replace: jest.fn() } })); const ok = (data: unknown) => Promise.resolve({ data, response: { ok: true, status: 200 } }); const BACKENDS = { local: { label: "This machine", fields: {}, secrets: [] }, smb: { label: "SMB", fields: { host: "", user: "", port: "", domain: "" }, secrets: ["pass"] }, + gcal: { + label: "Google Calendar", + fields: { calendars: "", impersonate: "", days_back: "365", account: "" }, + secrets: ["token", "service_account_credentials"], + }, }; +const CALLBACK = "https://lens.example/api/v1/auth/external/google/callback"; +const CONSENT = "https://accounts.google.com/o/oauth2/v2/auth?client_id=c"; function show(onSaved: (id: number) => void) { const qc = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }); @@ -95,3 +111,55 @@ describe("adding a connection", () => { }); }); }); + +describe("connecting a Google account", () => { + beforeEach(() => { + (Sources.createSource as jest.Mock) + .mockReset() + .mockImplementation(() => + ok({ id: 7, health: { ok: false, error: "connect a Google account, or upload a service account key, first" } }), + ); + (Sources.connectGoogle as jest.Mock).mockReset().mockImplementation(() => ok({ url: CONSENT })); + (navigate.assign as jest.Mock).mockReset(); + }); + + it("saves the connection and goes to Google's consent screen in one click", async () => { + (Sources.googleClient as jest.Mock).mockImplementation(() => + ok({ origin: "sign-in", client_id: "c", redirect_uri: CALLBACK }), + ); + show(jest.fn()); + fireEvent.click(screen.getByRole("radio", { name: /Google Calendar/ })); + expect(await screen.findByText(/Through Google sign-in’s OAuth client/)).toBeInTheDocument(); + expect(screen.queryByLabelText("OAuth client ID")).toBeNull(); // nothing to fill in: Lens has a client + fireEvent.change(screen.getByLabelText(/Calendars/), { target: { value: "team@group.calendar.google.com" } }); + fireEvent.click(screen.getByRole("button", { name: "Connect with Google" })); + await waitFor(() => expect(navigate.assign).toHaveBeenCalledWith(CONSENT)); + expect((Sources.createSource as jest.Mock).mock.calls[0][0].body).toEqual({ + type: "gcal", + name: "Google Calendar", + params: { calendars: "team@group.calendar.google.com", impersonate: "", days_back: "365", account: "" }, + secrets: {}, + }); + expect((Sources.connectGoogle as jest.Mock).mock.calls[0][0]).toMatchObject({ path: { sid: 7 }, body: {} }); + }); + + it("asks for an OAuth client, and shows the redirect URI to register, only when Lens has none", async () => { + (Sources.googleClient as jest.Mock).mockImplementation(() => + ok({ origin: null, client_id: null, redirect_uri: CALLBACK }), + ); + show(jest.fn()); + fireEvent.click(screen.getByRole("radio", { name: /Google Calendar/ })); + const id = await screen.findByLabelText("OAuth client ID"); + expect(screen.getByText(CALLBACK)).toBeInTheDocument(); + const connect = screen.getByRole("button", { name: "Connect with Google" }); + expect(connect).toBeDisabled(); + fireEvent.change(id, { target: { value: " 123.apps.googleusercontent.com " } }); + fireEvent.change(screen.getByLabelText("Client secret"), { target: { value: "shh" } }); + fireEvent.click(connect); + await waitFor(() => expect(navigate.assign).toHaveBeenCalledWith(CONSENT)); + expect((Sources.connectGoogle as jest.Mock).mock.calls[0][0].body).toEqual({ + client_id: "123.apps.googleusercontent.com", + client_secret: "shh", + }); + }); +}); diff --git a/nextjs-frontend/__tests__/sources-model.test.ts b/nextjs-frontend/__tests__/sources-model.test.ts index a44434b19..9c0616033 100644 --- a/nextjs-frontend/__tests__/sources-model.test.ts +++ b/nextjs-frontend/__tests__/sources-model.test.ts @@ -210,7 +210,7 @@ describe("connections", () => { it("signs Google Calendar in as a Google account or a service account", () => { const GCAL: BackendSpec = { label: "Google Calendar", - fields: { calendars: "", impersonate: "", days_back: "365" }, + fields: { calendars: "", impersonate: "", days_back: "365", account: "" }, secrets: ["token", "service_account_credentials"], }; const account = '{"type":"authorized_user","client_id":"c","client_secret":"s","refresh_token":"r","quota":"x"}'; @@ -233,7 +233,7 @@ describe("connections", () => { ).toHaveProperty("days_back"); // an account replaces a saved key, and a key file replaces a connected account expect(buildPayload(GCAL, connected, "gcal")).toEqual({ - params: { calendars: "", impersonate: "", days_back: "365" }, + params: { calendars: "", impersonate: "", days_back: "365", account: "" }, secrets: { token: '{"client_id":"c","client_secret":"s","refresh_token":"r"}', service_account_credentials: null, @@ -247,11 +247,20 @@ describe("connections", () => { const sa = { ...blank, driveAuth: "service_account" as const, secrets: { service_account_credentials: key } }; expect(validateForm("gcal", GCAL, sa, false)).toEqual({}); expect(buildPayload(GCAL, sa, "gcal").secrets).toEqual({ service_account_credentials: key, token: null }); + // a key file also forgets which account was connected + const was = { ...sa, params: { ...sa.params, account: "mara@harbour.example" } }; + expect(buildPayload(GCAL, was, "gcal").params.account).toBe(""); + expect(buildPayload(GCAL, { ...was, driveAuth: "token" as const, secrets: {} }, "gcal").params.account).toBe( + "mara@harbour.example", + ); expect(sourceSubtitle({ type: "gcal", params: { calendars: "" } })).toBe("Google Calendar · every calendar"); expect(sourceSubtitle({ type: "gcal", params: { calendars: "a@x, b@x", impersonate: "mara@x.io" } })).toBe( "Google Calendar · as mara@x.io · 2 calendars", ); + expect(sourceSubtitle({ type: "gcal", params: { calendars: "", account: "mara@x.io" } })).toBe( + "Google Calendar · as mara@x.io · every calendar", + ); expect(suggestName("gcal", blank.params)).toBe("Google Calendar"); }); it("confirms deletes by name, loosely", () => { diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index af490230d..45804e92a 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -329,6 +329,11 @@ export type { ConnectData, ConnectError, ConnectErrors, + ConnectGoogleData, + ConnectGoogleError, + ConnectGoogleErrors, + ConnectGoogleResponse, + ConnectGoogleResponses, ConnectResponse, ConnectResponses, Consent, @@ -1265,6 +1270,12 @@ export type { GetWorkflowErrors, GetWorkflowResponse, GetWorkflowResponses, + GoogleClient, + GoogleClientData, + GoogleClientResponse, + GoogleClientResponses, + GoogleConnect, + GoogleConnectStart, Gpu, Graph, GraphAsOfData, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index e3bfc4787..65aa98191 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -123,6 +123,9 @@ import type { CombineBatchResponses, ConnectData, ConnectErrors, + ConnectGoogleData, + ConnectGoogleErrors, + ConnectGoogleResponses, ConnectResponses, ConsentData, ConsentErrors, @@ -633,6 +636,8 @@ import type { GetWorkflowData, GetWorkflowErrors, GetWorkflowResponses, + GoogleClientData, + GoogleClientResponses, GraphAsOfData, GraphAsOfErrors, GraphAsOfResponses, @@ -7999,6 +8004,41 @@ export class Sources { }); } + /** + * Google Client + * + * The OAuth client a Google account would be connected through (never its secret), and the redirect URI it needs. + */ + public static googleClient( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/sources/google-client", + ...options, + }); + } + + /** + * Connect Google + * + * Connect a Google account to a Google source (Google Calendar): open the returned address in this browser, consent + * there, and Google sends you back to the Sources page with the account connected. Its refresh token is kept sealed on + * the source. Without a client here, the Google sign-in provider's (Settings › Sign-in) or an earlier connection's is + * used. Audited as `source.google_connect`. + */ + public static connectGoogle( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/sources/{sid}/google/connect", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * Browse Source * diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 04b8ca942..ed556b51f 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -5769,6 +5769,58 @@ export type ForgotPasswordRequest = { email: string; }; +/** + * GoogleClient + */ +export type GoogleClient = { + /** + * Origin + * + * where the OAuth client for connecting a Google account comes from: the Google sign-in provider, an earlier Google connection, or none (enter one to connect) + */ + origin: "sign-in" | "connection" | null; + /** + * Client Id + */ + client_id?: string | null; + /** + * Redirect Uri + * + * the address to add to the OAuth client's authorized redirect URIs + */ + redirect_uri: string; + [key: string]: unknown; +}; + +/** + * GoogleConnect + */ +export type GoogleConnect = { + /** + * Client Id + * + * an OAuth client of your own (else the one Lens has) + */ + client_id?: string | null; + /** + * Client Secret + */ + client_secret?: string | null; +}; + +/** + * GoogleConnectStart + */ +export type GoogleConnectStart = { + /** + * Url + * + * Google's consent page; open it in this browser + */ + url: string; + [key: string]: unknown; +}; + /** * Gpu */ @@ -32496,6 +32548,52 @@ export type TestSourceResponses = { export type TestSourceResponse = TestSourceResponses[keyof TestSourceResponses]; +export type GoogleClientData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/sources/google-client"; +}; + +export type GoogleClientResponses = { + /** + * Successful Response + */ + 200: GoogleClient; +}; + +export type GoogleClientResponse = GoogleClientResponses[keyof GoogleClientResponses]; + +export type ConnectGoogleData = { + body: GoogleConnect; + path: { + /** + * Sid + */ + sid: number; + }; + query?: never; + url: "/api/v1/sources/{sid}/google/connect"; +}; + +export type ConnectGoogleErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ConnectGoogleError = ConnectGoogleErrors[keyof ConnectGoogleErrors]; + +export type ConnectGoogleResponses = { + /** + * Successful Response + */ + 200: GoogleConnectStart; +}; + +export type ConnectGoogleResponse = ConnectGoogleResponses[keyof ConnectGoogleResponses]; + export type BrowseSourceData = { body?: never; path: { diff --git a/nextjs-frontend/components/sources/connection-dialog.tsx b/nextjs-frontend/components/sources/connection-dialog.tsx index 28ab5b8cd..b82cb4061 100644 --- a/nextjs-frontend/components/sources/connection-dialog.tsx +++ b/nextjs-frontend/components/sources/connection-dialog.tsx @@ -23,6 +23,7 @@ import { Dialog } from "@/components/ui/dialog"; import { Field, Input } from "@/components/ui/field"; import { useToast } from "@/components/ui/toast"; import { data, useApiClient } from "@/lib/api/browser"; +import { navigate } from "@/lib/navigate"; import { cn } from "@/lib/utils"; type Step = "type" | "details" | "test"; @@ -213,6 +214,41 @@ export function ConnectionDialog({ }, }); + // Connect with Google: the connection is saved as it is (made first when new), then this browser goes to Google's + // consent screen and comes back to Sources with the account connected + const connectGoogle = useMutation({ + mutationFn: async (oauthClient: { client_id?: string; client_secret?: string }) => { + let id = createdId; + const body = buildPayload(backends[type], form, type); + if (id == null) { + const name = form.name.trim() || suggestName(type, form.params); + id = (await data(Sources.createSource({ client, body: { type, name, ...body } }))).id; + setCreatedId(id); + setCreatedName(name); + } else + await data( + Sources.updateSource({ + client, + path: { sid: id }, + body: { ...body, ...(editing && form.name.trim() ? { name: form.name.trim() } : {}) }, + }), + ); + return (await data(Sources.connectGoogle({ client, path: { sid: id }, body: oauthClient }))).url; + }, + onSuccess: (url) => navigate.assign(url), + onError: (e: Error) => { + refresh(); + setErrors({ _: e.message }); + }, + }); + + const goConnect = (oauthClient: { client_id?: string; client_secret?: string }) => { + const e = validateForm(type, spec, form, saved, isSet); + delete e.token; // what Google's consent screen is for + setErrors(e); + if (!Object.keys(e).length) connectGoogle.mutate(oauthClient); + }; + const goTest = () => { const e = validateForm(type, spec, form, saved, isSet); setErrors(e); @@ -316,6 +352,8 @@ export function ConnectionDialog({ saved={saved} isSet={isSet} replaceToken={replaceToken} + onConnectGoogle={goConnect} + connectingGoogle={connectGoogle.isPending || connectGoogle.isSuccess} /> ) : null} {errors._ && ( diff --git a/nextjs-frontend/components/sources/connection-fields.tsx b/nextjs-frontend/components/sources/connection-fields.tsx index 9dccb0e76..397e1e521 100644 --- a/nextjs-frontend/components/sources/connection-fields.tsx +++ b/nextjs-frontend/components/sources/connection-fields.tsx @@ -1,7 +1,9 @@ "use client"; +import { useQuery } from "@tanstack/react-query"; import { useState } from "react"; +import { Sources } from "@/app/openapi-client"; import { DRIVE_SCOPES, FIELD_LABEL, @@ -17,7 +19,9 @@ import { import { TokenPaste } from "@/components/sources/token-paste"; import { Button } from "@/components/ui/button"; import { Field, Input, SecretField, Select, Textarea } from "@/components/ui/field"; +import { CodeBlock } from "@/components/ui/states"; import { Segmented } from "@/components/ui/tabs"; +import { data, useApiClient } from "@/lib/api/browser"; const SERVICE: Partial> = { dropbox: "Dropbox", @@ -35,6 +39,8 @@ export function ConnectionFields({ saved, isSet, replaceToken, + onConnectGoogle, + connectingGoogle, }: { type: SourceType; spec: BackendSpec; @@ -45,8 +51,21 @@ export function ConnectionFields({ saved: boolean; isSet: (key: string) => boolean; replaceToken?: boolean; + /** Connect with Google (Google Calendar): save the connection and go to Google's consent screen, through this OAuth + * client when one is given, else the one Lens has. */ + onConnectGoogle?: (client: { client_id?: string; client_secret?: string }) => void; + connectingGoogle?: boolean; }) { + const api = useApiClient(); const [pasting, setPasting] = useState(Boolean(replaceToken)); + const [ownClient, setOwnClient] = useState(false); + const [oauthClient, setOauthClient] = useState({ client_id: "", client_secret: "" }); + const googleClient = useQuery({ + queryKey: ["google-client"], + queryFn: () => data(Sources.googleClient({ client: api })), + enabled: type === "gcal", + staleTime: 60_000, + }); const editing = saved; const p = form.params; const set = (k: string, v: string) => onChange({ ...form, params: { ...p, [k]: v } }); @@ -195,6 +214,124 @@ export function ConnectionFields({ ); }; + /** Google Calendar as a Google account: Connect with Google (Google's consent screen), or an authorized user JSON + * pasted under the fold. */ + const googleAccount = (() => { + const g = googleClient.data; + const connected = editing && isSet("token") && form.secrets.token === undefined; + const needClient = Boolean(g) && (g?.origin == null || ownClient); + const pasted = form.secrets.token ?? ""; + return ( +
+ {connected && ( +

+ {p.account ? ( + <> + Connected as {p.account}. + + ) : ( + "A Google account is connected." + )} +

+ )} + {needClient && g && ( + <> +
+ + {({ id }) => ( + setOauthClient({ ...oauthClient, client_id: e.target.value })} + /> + )} + + + {({ id }) => ( + setOauthClient({ ...oauthClient, client_secret: e.target.value })} + /> + )} + +
+
+ + In Google Cloud › APIs & Services: enable the Google Calendar API, then under Credentials create an + OAuth client ID (a web application) with this authorized redirect URI. Lens keeps the client with the + connection. + + +
+ + )} +
+ + {g?.origin && !ownClient && ( + + Through {g.origin === "sign-in" ? "Google sign-in’s" : "an earlier connection’s"} OAuth client ·{" "} + + + )} +
+ {errors.token && !pasted.trim() && ( +

+ {errors.token} +

+ )} +
+ Or paste an authorized user JSON +
+ + {({ id, describedBy, invalid }) => ( +