From f4dab72f8dbdbc2456b24a30401ad3a940dd3b47 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Thu, 15 Oct 2026 11:24:08 +0000 Subject: [PATCH] feat(sources): connect Google Drive by signing in with Google Google Drive connections no longer need rclone run on another computer and a token pasted back. Sign in with Google opens Google's consent page in a small window; when it comes back the connection is made, tested and named after the account, and the dialog goes on to pick a folder whose files Lens imports. It uses the admin's own Google app, the one people can sign in to Lens with (login_provider google), through the same external_flow rows and callback, so nothing more is registered at Google and signing in to Lens can stay off. The app can be set up from the Drive dialog. Tokens are sealed as the source's token in rclone's shape, and rclone gets that app's client id and secret to renew them. POST /sources/signin hands agents a link for their person (not tied to a browser when started from an API token); source_oauth.fresh_token gives code that calls Google itself a working access token. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VrczAC5nKz1zkCUvRKXG5C --- CHANGELOG.md | 7 + docs/authentication.md | 4 + docs/processing.md | 15 ++ fastapi_backend/app/api/v1/routes/external.py | 34 ++- fastapi_backend/app/api/v1/routes/sources.py | 51 +++- fastapi_backend/app/domain/external_login.py | 17 +- fastapi_backend/app/domain/source_oauth.py | 252 ++++++++++++++++++ fastapi_backend/app/domain/sources.py | 37 ++- fastapi_backend/app/schemas/sources.py | 28 ++ .../tests/api/test_google_calendar_source.py | 1 + .../tests/api/test_source_signin.py | 196 ++++++++++++++ fastapi_backend/tests/fake_oidc.py | 21 +- .../__tests__/drive-signin.test.tsx | 147 ++++++++++ .../__tests__/sources-model.test.ts | 23 ++ nextjs-frontend/app/(auth)/connected/page.tsx | 10 + nextjs-frontend/app/openapi-client/index.ts | 9 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 27 +- .../app/openapi-client/types.gen.ts | 143 ++++++++++ .../components/settings/sign-in-providers.tsx | 1 + .../components/sources/connected-return.tsx | 60 +++++ .../components/sources/connection-dialog.tsx | 76 +++++- .../components/sources/connection-fields.tsx | 50 +++- .../components/sources/google-app-setup.tsx | 102 +++++++ nextjs-frontend/components/sources/signin.ts | 83 ++++++ .../components/sources/source-model.ts | 29 +- .../components/sources/sources-page.tsx | 54 +++- .../components/storage/add-connection.tsx | 3 + nextjs-frontend/openapi.json | 232 +++++++++++++++- nextjs-frontend/proxy.ts | 11 +- threat-model.json | 10 + threat-model.md | 1 + threat-model.yaml | 11 +- 32 files changed, 1698 insertions(+), 47 deletions(-) create mode 100644 fastapi_backend/app/domain/source_oauth.py create mode 100644 fastapi_backend/tests/api/test_source_signin.py create mode 100644 nextjs-frontend/__tests__/drive-signin.test.tsx create mode 100644 nextjs-frontend/app/(auth)/connected/page.tsx create mode 100644 nextjs-frontend/components/sources/connected-return.tsx create mode 100644 nextjs-frontend/components/sources/google-app-setup.tsx create mode 100644 nextjs-frontend/components/sources/signin.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 52b0a3654..b9da68035 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version shows only Projects, Areas, Resources or Archive, and each note says which notes it's inside. Jump to (⌘K) finds notes as well. `GET /api/v1/notes/search`; `find_notes` gives agents the words around each match ([Finding notes](docs/notes.md#finding-notes)). +- **Connect Google Drive by signing in with Google.** Sensors › Add connection › Google Drive › Google account now + has **Sign in with Google**: Google's page opens in a small window, and the connection is made, tested and named + after the account, ready to pick a folder whose files Lens imports. Nothing to run or paste. It uses your own + Google app, the one people can sign in to Lens with; set it up once in the dialog (client ID and secret, with the + redirect URI shown) or in Settings › Sign-in, and signing in to Lens with Google can stay off. Agents get the same + link from `POST /api/v1/sources/signin` to hand to their person. Pasting a token from rclone and service accounts + work as before ([Storage sources](docs/processing.md#storage-sources)). ## 0.10.0 October 15, 2026 {id="0.10.0"} diff --git a/docs/authentication.md b/docs/authentication.md index d687e9919..ab639ba46 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -83,6 +83,10 @@ has a "Continue with ..." button for each provider that's turned on. Removing a provider disconnects the accounts connected through it. Audited as `login_provider.add`, `.change`, `.remove`, `external.connect` and `external.disconnect`; sign-ins as `login` with `external:`. +The Google app also connects Google Drive (Sensors › Add connection › Google Drive › Sign in with Google): the same +client id, secret and redirect URI, with Drive's scope asked for, so it works with signing in to Lens turned off +([Storage sources](processing.md#storage-sources)). + ## The web app: NextAuth with API tokens ``` diff --git a/docs/processing.md b/docs/processing.md index ab7c8d7d1..1d9a01b6d 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -302,6 +302,21 @@ files, then Import (`POST /api/v1/import/source`). The listing marks files that - Credentials are encrypted in the database (AES-GCM, key from `ARCHIVE_SECRET_KEY` or `data_dir/secret.key`) and given to rclone in a private temporary config file per call. For Dropbox, Drive and OneDrive, paste the token from `rclone authorize dropbox` (or drive, onedrive); tokens rclone refreshes are saved back. +- **Google Drive signs in with Google** in the web app, with nothing to paste: choose Google account, then Sign in + with Google. A small window opens Google's consent page; when Google sends it back, the connection is made, tested + and named after the account (Google Drive · you@gmail.com), and the dialog goes on to pick a folder to watch. It + uses your own Google app, the one people can sign in to Lens with (Settings › Sign-in): set it up once, in the + dialog or there, with signing in to Lens left off if you like. In Google Cloud console › APIs & Services, enable + the Google Drive API; create an OAuth client ID (Web application) whose redirect URI is the address the dialog + shows (`/api/v1/auth/external/google/callback`, the same as sign-in's); and publish the app on + the OAuth consent screen, or Google ends its sign-ins after 7 days. Google accepts `https://` redirect addresses + and `http://localhost`. The token is kept encrypted, and rclone renews it with that app's client id and secret. + A connection whose sign-in ran out shows Sign in again. Without a Google app, paste a token from rclone as before. +- Agents connect Drive the same way: `POST /api/v1/sources/signin` with `{"type": "drive", "params": {...}}` (or + `{"source": }` to sign one in again) answers `{url}`. From the web app the link is tied to that browser; from + an API token or app it isn't, so an agent can hand it to its person, who opens it once within ten minutes and ends + on a page saying it's connected (`/connected`). Then `POST /api/v1/watches` imports its files. Code that calls + Google itself gets a fresh access token from `source_oauth.fresh_token`. Audited as `source.signin`. - Google Drive can also sign in as a service account: choose Service account and upload its JSON key file (`service_account_credentials`). Share the folders Lens should read with the account's address, or give a shared drive ID (`team_drive`); in Google Workspace, `impersonate` reads a user's own Drive through domain-wide delegation. diff --git a/fastapi_backend/app/api/v1/routes/external.py b/fastapi_backend/app/api/v1/routes/external.py index 85cf532f5..67f59293e 100644 --- a/fastapi_backend/app/api/v1/routes/external.py +++ b/fastapi_backend/app/api/v1/routes/external.py @@ -5,6 +5,10 @@ cookie tying the sign-in to this browser. The provider sends it back to the callback, which ends on the web app's /external-signin page with a one-time ticket in the URL fragment; the web app's server swaps it for a session at POST /auth/ticket, as for passkeys. + +Connecting a storage source by signing in (source_oauth.py, POST /sources/signin) comes back through the same callback, +so the app registered with the provider needs no second address; it ends on the web app's page it started from, with +?connected= or ?connect_error=. """ from __future__ import annotations @@ -15,7 +19,7 @@ from fastapi.responses import RedirectResponse from app.api.deps import AdminWriter, Cfg, CurrentUser, Db, Writer, domain_errors, web_origin -from app.domain import auth, external_login, passkeys +from app.domain import auth, external_login, passkeys, source_oauth from app.schemas.auth import ( ExternalIdentity, ExternalProvider, @@ -30,7 +34,8 @@ COOKIE = "lens_external" -def _cookie(response: Response, origin: str, value: str) -> None: +def flow_cookie(response: Response, origin: str, value: str) -> None: + """The cookie that ties a round trip through a provider to the browser that started it.""" response.set_cookie( COOKIE, value, @@ -50,7 +55,7 @@ def _redirect(response: Response, request: Request, db: Db, cfg: Cfg, key: str, raise HTTPException(404, "no such sign-in provider") from None except ValueError as e: raise HTTPException(400, str(e)) from None - _cookie(response, origin, browser) + flow_cookie(response, origin, browser) return ExternalRedirect(url=url) @@ -96,6 +101,8 @@ def external_callback( flow = external_login.claim(db, state, request.cookies.get(COOKIE)) except ValueError as e: return _back(web_origin(request), error=str(e), next="/login") + if flow.get("mode") == "source": + return _source_back(db, cfg, flow, code, error, error_description) origin, connect = flow["origin"], flow.get("mode") == "connect" away = "/account" if connect else "/login" if error or not code: @@ -114,6 +121,27 @@ def external_callback( return _back(origin, ticket=passkeys.issue_ticket(db, u["id"], f"external:{key}"), next=flow.get("next") or "/") +def _source_back(db: Db, cfg: Cfg, flow: dict, code: str, error: str, error_description: str) -> RedirectResponse: + """A source signed in (or not): back to the page it started from, with ?connected= or ?connect_error=.""" + if error or not code: + why = "you cancelled" if error == "access_denied" else (error_description or error or "no code came back") + q = {"connect_error": f"signing in didn't finish: {why}"} + else: + try: + sid = source_oauth.finish(db, cfg, flow, code) + except ValueError as e: + q = {"connect_error": str(e)} + else: + q = {"connected": str(sid)} + auth.audit( + db, {"id": flow.get("account"), "email": (flow.get("source") or {}).get("by")}, "source.signin", f"storage_source:{sid}" + ) + page = flow.get("next") or "/sources" + r = RedirectResponse(f"{flow['origin']}{page}{'&' if '?' in page else '?'}{urllib.parse.urlencode(q)}", status_code=303) + r.delete_cookie(COOKIE, path="/api/v1/auth/external") + return r + + # ---------- your outside accounts ---------- def _signed_in(user: CurrentUser) -> None: if user.via != "access": diff --git a/fastapi_backend/app/api/v1/routes/sources.py b/fastapi_backend/app/api/v1/routes/sources.py index 283b30f52..1f8b1f9fc 100644 --- a/fastapi_backend/app/api/v1/routes/sources.py +++ b/fastapi_backend/app/api/v1/routes/sources.py @@ -5,10 +5,11 @@ import logging from collections import Counter -from fastapi import APIRouter, BackgroundTasks, HTTPException, Request +from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, Response -from app.api.deps import Acl, AdminReader, AdminWriter, Cfg, CurrentUser, Db -from app.domain import auth, sources, store +from app.api.deps import Acl, AdminReader, AdminWriter, Cfg, CurrentUser, Db, web_origin +from app.api.v1.routes.external import flow_cookie +from app.domain import auth, source_oauth, sources, store from app.domain.store import DB from app.schemas.admin import Started from app.schemas.common import Created, Ok @@ -21,6 +22,8 @@ SourceCreate, SourceCreated, SourceHealth, + SourceSignIn, + SourceSignInLink, SourceUpdate, SourceUpdated, Watch, @@ -47,10 +50,13 @@ def _watch(db: DB, wid: int) -> None: @router.get("/sources/backends") -def list_backends(user: CurrentUser) -> dict[str, Backend]: - """The kinds of storage a source can be, with their settings and secrets.""" +def list_backends(user: CurrentUser, db: Db) -> dict[str, Backend]: + """The kinds of storage a source can be, with their settings and secrets, and how each signs in in the browser.""" keys = {"label", "fields", "secrets", "oauth"} - return {k: Backend.model_validate({f: x for f, x in v.items() if f in keys}) for k, v in sources.BACKENDS.items()} + return { + k: Backend.model_validate({**{f: x for f, x in v.items() if f in keys}, "signin": source_oauth.signin(db, k)}) + for k, v in sources.BACKENDS.items() + } @router.get("/sources") @@ -80,6 +86,39 @@ def update_source(sid: int, body: SourceUpdate, user: AdminWriter, db: Db, cfg: return SourceUpdated(health=sources.test(db, cfg, sid)) +@router.post("/sources/signin") +def start_signin(body: SourceSignIn, user: AdminWriter, request: Request, response: Response, db: Db, cfg: Cfg) -> SourceSignInLink: + """Connect Google Drive by signing in with Google (through the Google app in Settings › Sign-in), or sign a + connection in again (`source`): open the returned address in a browser. The provider sends it back to the web app + page `next`, with ?connected= (tested; next, watch one of its folders) or ?connect_error=. Started from + the web app it's tied to that browser; from an API token or app it isn't, so an agent can hand the link to its + person. Either way it works once, within ten minutes. Audited as `source.signin` when it finishes.""" + if body.source is None and body.type is None: + raise HTTPException(400, "say which type of connection to make, or which connection to sign in again") + origin, bound = web_origin(request), user.via == "access" + try: + url, browser = source_oauth.start( + db, + cfg, + origin, + body.type, + body.params, + body.name, + body.source, + account=user.id, + by=user.email, + bound=bound, + next_path=body.next, + ) + except KeyError: + raise HTTPException(404, "not found") from None + except ValueError as e: + raise HTTPException(400, str(e)) from None + if browser: + flow_cookie(response, origin, browser) + return SourceSignInLink(url=url, bound=bool(browser)) + + @router.delete("/sources/{sid}") def delete_source(sid: int, user: AdminWriter, db: Db) -> Ok: """Remove a source and its watched folders.""" diff --git a/fastapi_backend/app/domain/external_login.py b/fastapi_backend/app/domain/external_login.py index 4c1a1277f..13761be1f 100644 --- a/fastapi_backend/app/domain/external_login.py +++ b/fastapi_backend/app/domain/external_login.py @@ -190,6 +190,11 @@ def callback_path(key): return f"/api/v1/auth/external/{key}/callback" +def client_secret(cfg, row): + """A provider's client secret, unsealed, for its token endpoint.""" + return settings.unseal(cfg, row["client_secret"], _context(row["key"])) + + # ---------- signing in ---------- def _endpoints(row): kind = row["kind"] @@ -246,7 +251,7 @@ def start(db, cfg, key, origin, mode="login", account=None, next_path="/"): "origin": origin.rstrip("/"), "mode": mode, "account": account, - "next": _safe_next(next_path), + "next": safe_next(next_path), "expires_at": _later(FLOW_MINUTES * 60), } ), @@ -268,7 +273,7 @@ def start(db, cfg, key, origin, mode="login", account=None, next_path="/"): return ep["authorize"] + sep + urllib.parse.urlencode(q), browser -def _safe_next(path): +def safe_next(path): """A path on this site. Browsers drop tabs and line breaks from a URL ("/\\t/evil.example" is "//evil.example").""" path = path or "/" ok = path.startswith("/") and not path.startswith("//") and "\\" not in path and not re.search(r"[\x00-\x1f\x7f]", path) @@ -276,11 +281,15 @@ def _safe_next(path): def claim(db, state, browser): - """The flow a callback answers, used up. ExternalLoginError when it's gone, expired, or started in another browser.""" + """The flow a callback answers, used up. ExternalLoginError when it's gone, expired, or started in another browser. + A flow an agent started through the API for its person to open (connecting a source, source_oauth.py) isn't tied + to a browser: its link alone finishes it, once.""" rows = db.rows("DELETE $r RETURN BEFORE", r=R("external_flow", auth.sha(state))) if state else [] row = rows[0] if rows else None if not row or (row.get("expires_at") or "") < store.now(): raise ExternalLoginError("that sign-in took too long or was already used; try again") + if row.get("unbound") and row.get("mode") == "source": + return row if not browser or auth.sha(browser) != row.get("browser"): raise ExternalLoginError("that sign-in was started in another browser; start it again here") return row @@ -289,7 +298,7 @@ def claim(db, state, browser): def _profile(row, cfg, flow, code): """{sub, email, trusted, name} from the provider, for the code it sent back.""" ep = _endpoints(row) - secret = settings.unseal(cfg, row["client_secret"], _context(row["key"])) + secret = client_secret(cfg, row) try: r = httpx.post( ep["token"], diff --git a/fastapi_backend/app/domain/source_oauth.py b/fastapi_backend/app/domain/source_oauth.py new file mode 100644 index 000000000..4bd7fcdcc --- /dev/null +++ b/fastapi_backend/app/domain/source_oauth.py @@ -0,0 +1,252 @@ +"""Connecting a storage source by signing in at the service in the browser, rather than pasting a token from rclone. + +Google Drive signs in through the Google app an admin set up in Settings › Sign-in (external_login.py, kind google): +its client id and secret, and its callback address, so nothing more is registered at Google. The app may stay off for +signing in to Lens; connecting a source only needs it to exist. The round trip is an external_flow row like a +sign-in's (PKCE, ten minutes, used once, tied by a cookie to the browser that started it), in mode "source". An agent +can start one through the API for its person to open in their own browser: that flow isn't tied to a browser, so the +link alone finishes it, once. + +The callback swaps the code for tokens at Google and keeps them sealed as the source's `token`, in rclone's own shape +(access_token, token_type, refresh_token, expiry). The source's `oauth` says which app signed it in and as whom; rclone +is handed that app's client id and secret (sources._config), because Google refreshes a token only for the app that +issued it. A token pasted from rclone, or a service account key, replaces it (sources.update drops `oauth`). + +Code that talks to Google itself rather than through rclone gets an access token from fresh_token, which refreshes it +when it is about to run out. +""" + +from __future__ import annotations + +import base64 +import datetime as dt +import hashlib +import json +import re +import secrets +import urllib.parse + +import httpx + +from . import auth, external_login, settings, sources, store + +R = store.R +GOOGLE_SCOPE = "https://www.googleapis.com/auth/" +DRIVE_SCOPES = ("drive", "drive.readonly", "drive.file", "drive.metadata.readonly") +TIMEOUT = 15 + + +def _drive_scope(params): + asked = [s.strip() for s in str(params.get("scope") or "drive.readonly").split(",") if s.strip()] + if not asked or any(s not in DRIVE_SCOPES for s in asked): + raise ValueError(f"the Drive scope is one of {', '.join(DRIVE_SCOPES)}") + return " ".join(GOOGLE_SCOPE + s for s in asked) + + +# the source types that sign in here: the kind of sign-in provider whose app they use, and the scopes they ask for +TYPES = {"drive": {"kind": "google", "scope": _drive_scope}} + + +def app(db, typ): + """The sign-in provider (with its sealed secret) that a source type signs in through, or None when it isn't set up. + Google's provider key is its kind: an install has one Google app.""" + t = TYPES.get(typ) + if not t: + return None + try: + row = external_login.provider(db, t["kind"]) + except KeyError: + return None + return row if row.get("client_id") and row.get("client_secret") else None + + +def signin(db, typ): + """How a source type signs in in the browser, for the connection form; None when it only takes a pasted token or + keys. `ready` says whether its app is set up; `callback_path` is the address to register with the service.""" + t = TYPES.get(typ) + if not t: + return None + return { + "kind": t["kind"], + "label": external_login.LABELS[t["kind"]], + "ready": app(db, typ) is not None, + "callback_path": external_login.callback_path(t["kind"]), + } + + +def _later(seconds): + return (dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=seconds)).isoformat(timespec="seconds") + + +def start(db, cfg, origin, typ=None, params=None, name=None, sid=None, account=None, by=None, bound=True, next_path="/connected"): + """The service's sign-in page for connecting a new source of type `typ` (with `params`, named `name`), or for + signing source `sid` in again; and, for a flow tied to this browser (`bound`), the secret to keep in its cookie + (else None). ValueError when the type can't sign in here or its app isn't set up; KeyError for no such source.""" + if sid is not None: + src = sources.get(db, sid) + typ, params = src["type"], {**(src.get("params") or {}), **(params or {})} + if typ not in TYPES: + raise ValueError(f"{sources.BACKENDS[typ]['label'] if typ in sources.BACKENDS else typ} doesn't sign in here") + params = {**sources.BACKENDS[typ]["fields"], **(params or {})} + sources._check_params(typ, params, None) + scope = TYPES[typ]["scope"](params) + row = app(db, typ) + if not row: + label = external_login.LABELS[TYPES[typ]["kind"]] + raise ValueError(f"set up your {label} app first (Settings › Sign-in, or the connection form): its client id and secret") + ep = external_login.ENDPOINTS[row["kind"]] + state, verifier = secrets.token_urlsafe(24), secrets.token_urlsafe(48) + browser = secrets.token_urlsafe(24) if bound else None + challenge = base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip("=") + redirect_uri = origin.rstrip("/") + external_login.callback_path(row["key"]) + db.q("DELETE external_flow WHERE expires_at < $n", n=store.now()) + db.q( + "CREATE $r CONTENT $d", + r=R("external_flow", auth.sha(state)), + d=store.clean( + { + "provider": row["key"], + "verifier": verifier, + "browser": auth.sha(browser) if browser else None, + "unbound": None if bound else True, + "redirect_uri": redirect_uri, + "origin": origin.rstrip("/"), + "mode": "source", + "account": account, + "next": external_login.safe_next(next_path), + "source": store.clean({"id": sid, "type": typ, "params": params, "name": (name or "").strip()[:80] or None, "by": by}), + "expires_at": _later(external_login.FLOW_MINUTES * 60), + } + ), + ) + q = { + "response_type": "code", + "client_id": row["client_id"], + "redirect_uri": redirect_uri, + "scope": f"openid email {scope}", # who signed in, to show beside the connection + "state": state, + "code_challenge": challenge, + "code_challenge_method": "S256", + "access_type": "offline", # a refresh token, so the connection keeps working + "prompt": "consent", # Google only sends one when asked for consent + } + return ep["authorize"] + "?" + urllib.parse.urlencode(q), browser + + +def _token(cfg, row, form): + """What the provider's token endpoint answers to `form` (with the app's client id and secret).""" + try: + r = httpx.post( + external_login.ENDPOINTS[row["kind"]]["token"], + data={**form, "client_id": row["client_id"], "client_secret": external_login.client_secret(cfg, row)}, + headers={"Accept": "application/json"}, + timeout=TIMEOUT, + ) + tok = r.json() + except (httpx.HTTPError, ValueError): + raise external_login.ExternalLoginError(f"{external_login.LABELS[row['kind']]} didn't answer; try again") from None + if r.status_code >= 400 or not isinstance(tok, dict) or not tok.get("access_token"): + why = (tok.get("error_description") or tok.get("error")) if isinstance(tok, dict) else None + raise external_login.ExternalLoginError( + f"{external_login.LABELS[row['kind']]} refused: {why or r.status_code}; check the app's client id and secret" + ) + return tok + + +def _expiry(tok): + seconds = tok.get("expires_in") + seconds = int(seconds) if isinstance(seconds, (int, float, str)) and str(seconds).isdigit() else 3600 + when = dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=seconds) + return when.isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _email(row, access): + """The signed-in account's address, to show beside the connection; None when the provider won't say.""" + try: + info = httpx.get( + external_login.ENDPOINTS[row["kind"]]["userinfo"], + headers={"Authorization": f"Bearer {access}", "Accept": "application/json"}, + timeout=TIMEOUT, + ).json() + except (httpx.HTTPError, ValueError): + return None + email = info.get("email") if isinstance(info, dict) else None + return str(email)[:200] if email else None + + +def finish(db, cfg, flow, code): + """Swap the code the provider sent back for tokens and connect the source the flow is for: made now when it's new, + else signed in again. Returns the source's id. ExternalLoginError (a ValueError) says why it didn't work.""" + want = flow.get("source") or {} + row = app(db, want.get("type")) + if not row or row["key"] != flow.get("provider"): + raise external_login.ExternalLoginError("the app this sign-in went through was removed; set it up again and sign in again") + tok = _token( + cfg, + row, + {"grant_type": "authorization_code", "code": code, "redirect_uri": flow["redirect_uri"], "code_verifier": flow["verifier"]}, + ) + if not tok.get("refresh_token"): + raise external_login.ExternalLoginError( + f"{external_login.LABELS[row['kind']]} sent no refresh token, so the connection would stop within the hour; sign in again" + ) + email = _email(row, tok["access_token"]) + token = json.dumps( + { + "access_token": tok["access_token"], + "token_type": tok.get("token_type") or "Bearer", + "refresh_token": tok["refresh_token"], + "expiry": _expiry(tok), + }, + separators=(",", ":"), + ) + typ = want["type"] + others = {k: None for k in sources.BACKENDS[typ]["secrets"] if k != "token"} # signed in: no service account + if want.get("id") is not None: + sid = want["id"] + try: + sources.update(db, cfg, sid, params=want.get("params"), secret_values={"token": token, **others}) + except KeyError: + raise external_login.ExternalLoginError("that connection was deleted while you were signing in") from None + else: + label = sources.BACKENDS[typ]["label"] + name = want.get("name") or (f"{label} · {email}" if email else label) + sid = sources.create(db, cfg, name, typ, want.get("params"), {"token": token}, want.get("by")) + db.q( + "UPDATE $r SET oauth = $o", + r=R("storage_source", sid), + o=store.clean({"provider": row["key"], "email": email, "at": store.now()}), + ) + sources.test(db, cfg, sid) + return sid + + +def _when(text): + """An rclone token's expiry (Go writes RFC 3339 with up to nine digits of seconds), or None.""" + m = re.fullmatch(r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)(\.\d+)?(Z|[+-]\d\d:\d\d)", str(text or "")) + if not m: + return None + return dt.datetime.fromisoformat(m[1] + (m[2] or "")[:7] + ("+00:00" if m[3] == "Z" else m[3])) + + +def fresh_token(db, cfg, src, margin=60): + """A working access token for a source signed in here, refreshed at the provider when it runs out within `margin` + seconds (the new one is saved). For code that calls the service itself; rclone refreshes its own. ValueError when + the source wasn't signed in here, or the provider refuses (sign in again).""" + oauth = src.get("oauth") or {} + row = app(db, src["type"]) + if not oauth or not row or row["key"] != oauth.get("provider") or "token" not in (src.get("sealed") or {}): + raise ValueError(f"{src.get('name') or 'this connection'} wasn't signed in through an app set up here; sign in again") + context = f"source:{src['id']}:token" + tok = json.loads(settings.unseal(cfg, src["sealed"]["token"], context)) + when = _when(tok.get("expiry")) + if when and when > dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=margin): + return tok["access_token"] + new = _token(cfg, row, {"grant_type": "refresh_token", "refresh_token": tok.get("refresh_token") or ""}) + tok.update(access_token=new["access_token"], token_type=new.get("token_type") or tok.get("token_type") or "Bearer", expiry=_expiry(new)) + if new.get("refresh_token"): + tok["refresh_token"] = new["refresh_token"] + sealed = settings.seal(cfg, json.dumps(tok, separators=(",", ":")), context) + db.q("UPDATE $r SET sealed.token = $t", r=R("storage_source", src["id"]), t=sealed) + src["sealed"]["token"] = sealed + return tok["access_token"] diff --git a/fastapi_backend/app/domain/sources.py b/fastapi_backend/app/domain/sources.py index a4d846086..88a8f4b26 100644 --- a/fastapi_backend/app/domain/sources.py +++ b/fastapi_backend/app/domain/sources.py @@ -2,9 +2,10 @@ A source is one connection: S3 or S3-compatible, Dropbox, Google Drive, OneDrive, SFTP, SMB, WebDAV, or a folder on this machine (only inside sources.local_roots); or, without rclone, an email account (IMAP), a calendar feed (iCal) or -Google Calendar, whose messages and events are shown as files (feeds.py). A watch maps a folder on a source to a -namespace: new audio, video, documents (PDFs) and images become resources queued for the whole pipeline, their files -staying on the source; new transcripts are imported and queued for analysis. A watch on an email account can carry routing rules that send some +Google Calendar, whose messages and events are shown as files (feeds.py). Google Drive can also be connected by signing in +with Google in the browser, through the Google app set up in Settings › Sign-in (source_oauth.py). A watch maps a folder +on a source to a namespace: new audio, video, documents (PDFs) and images become resources queued for the whole +pipeline, their files staying on the source; new transcripts are imported and queued for analysis. A watch on an email account can carry routing rules that send some messages to other namespaces, or skip them (mail_routes.py). Credentials are stored encrypted and handed to rclone in a private temporary config file per call; OAuth tokens rclone refreshes are saved. """ @@ -22,7 +23,7 @@ import subprocess import tempfile -from . import convert, deletion, feeds, google_auth, ingest, jobs, mail_routes, settings, store +from . import convert, deletion, external_login, feeds, google_auth, ingest, jobs, mail_routes, settings, store R = store.R BACKENDS = { @@ -114,9 +115,14 @@ def obscure(cfg, value): def get(db, sid): - row = db.one("SELECT record::id(id) AS id, name, type, params, sealed, health, created_at FROM $r", r=R("storage_source", sid)) + row = db.one("SELECT record::id(id) AS id, name, type, params, sealed, health, created_at, oauth FROM $r", r=R("storage_source", sid)) if not row or row["type"] not in BACKENDS: # a stream sensor (sensors.py) shares the table, but isn't a source raise KeyError(sid) + if (row.get("oauth") or {}).get("provider"): # signed in through an app set up here: rclone refreshes with that app + try: + row["app"] = external_login.provider(db, row["oauth"]["provider"]) + except KeyError: + pass # the app was removed: the token stops refreshing, and the connection asks to sign in again return row @@ -130,6 +136,10 @@ def _config(cfg, src): if k in spec["secrets"]: v = settings.unseal(cfg, sealed, f"source:{src['id']}:{k}") lines.append(f"{k} = {_one_line(obscure(cfg, v) if k in OBSCURED else v)}") + app = src.get("app") + if app and app.get("client_id") and app.get("client_secret"): # Google refreshes a token only for the app it gave it to + lines.append(f"client_id = {_one_line(app['client_id'])}") + lines.append(f"client_secret = {_one_line(external_login.client_secret(cfg, app))}") return name, "\n".join(lines) + "\n" @@ -387,6 +397,9 @@ def update(db, cfg, sid, name=None, params=None, secret_values=None): d=store.clean({"name": name, "params": {**(src.get("params") or {}), **(params or {})} if params else None}), ) db.q("UPDATE $r SET sealed = $s", r=R("storage_source", sid), s=sealed) # SET, not MERGE: a MERGE can't drop a removed secret + sv = secret_values or {} + if "token" in sv or sv.get("service_account_credentials"): # a pasted token or a key: no longer signed in here + db.q("UPDATE $r SET oauth = NONE", r=R("storage_source", sid)) def remove(db, sid): @@ -399,6 +412,15 @@ def remove(db, sid): ) +def _signed_in(oauth): + """Who a connection signed in as in the browser (source_oauth.py): {provider, email, at}; None for pasted tokens + and keys.""" + if not (oauth or {}).get("provider"): + return None + kind = oauth["provider"] if oauth["provider"] in external_login.LABELS else "oidc" + return {"provider": external_login.LABELS[kind], "email": oauth.get("email"), "at": oauth.get("at")} + + def view(src): spec = BACKENDS[src["type"]] return { @@ -409,6 +431,7 @@ def view(src): "params": src.get("params") or {}, "secrets": {k: {"secret": True, "set": k in (src.get("sealed") or {})} for k in spec["secrets"]}, "oauth": spec.get("oauth"), + "signed_in": _signed_in(src.get("oauth")), "health": src.get("health"), "created_at": src.get("created_at"), } @@ -420,7 +443,9 @@ def list_sources(db): counts[w["source"]] = counts.get(w["source"], 0) + 1 return [ {**view(s), "watches": counts.get(s["id"], 0)} - for s in db.rows("SELECT record::id(id) AS id, name, type, params, sealed, health, created_at FROM storage_source ORDER BY id") + for s in db.rows( + "SELECT record::id(id) AS id, name, type, params, sealed, health, created_at, oauth FROM storage_source ORDER BY id" + ) if s["type"] in BACKENDS ] diff --git a/fastapi_backend/app/schemas/sources.py b/fastapi_backend/app/schemas/sources.py index bd0c110c6..5811e9718 100644 --- a/fastapi_backend/app/schemas/sources.py +++ b/fastapi_backend/app/schemas/sources.py @@ -16,11 +16,19 @@ ) +class SignInInfo(ResponseModel): + kind: str = Field(description="the kind of sign-in provider whose app it signs in through (google)") + label: str + ready: bool = Field(description="that app is set up (Settings › Sign-in), so POST /sources/signin works") + callback_path: str = Field(description="the address, on this site, to register with the service as the app's redirect URI") + + class Backend(ResponseModel): label: str fields: dict[str, str] = Field(description="parameters and their defaults") secrets: list[str] oauth: str | None = Field(default=None, description="the command that produces the token, for OAuth backends") + signin: SignInInfo | None = Field(default=None, description="signing in in the browser instead of pasting a token") class SourceHealth(ResponseModel): @@ -35,6 +43,12 @@ class SecretFlag(ResponseModel): set: bool +class SignedIn(ResponseModel): + provider: str = Field(description="the service it signed in with (Google)") + email: str | None = Field(default=None, description="the account it signed in as") + at: str | None = None + + class Source(ResponseModel): id: int name: str @@ -43,6 +57,7 @@ class Source(ResponseModel): params: dict[str, Any] secrets: dict[str, SecretFlag] oauth: str | None = None + signed_in: SignedIn | None = Field(default=None, description="signed in in the browser (POST /sources/signin), as whom") health: SourceHealth | None = None created_at: str | None = None watches: int = 0 @@ -61,6 +76,19 @@ class SourceUpdate(RequestModel): secrets: dict[str, str | None] | None = Field(default=None, description="null or empty removes a secret") +class SourceSignIn(RequestModel): + type: SourceType | None = Field(default=None, description="the kind of connection to make (drive); leave out with `source`") + source: int | None = Field(default=None, description="a connection to sign in again, instead of making a new one") + name: str | None = Field(default=None, description="the new connection's name; by default the service and the account") + params: dict[str, Any] | None = Field(default=None, description="its options, as for POST /sources (scope, root_folder_id...)") + next: str = Field(default="/connected", description="the web app's page to come back to (by default one that only says how it went)") + + +class SourceSignInLink(ResponseModel): + url: str = Field(description="the service's sign-in page: open it in a browser; it works once, within ten minutes") + bound: bool = Field(description="tied to this browser (a cookie came with it); else the link alone finishes it") + + class SourceCreated(Created): health: SourceHealth diff --git a/fastapi_backend/tests/api/test_google_calendar_source.py b/fastapi_backend/tests/api/test_google_calendar_source.py index 1fb97910f..4e4c46500 100644 --- a/fastapi_backend/tests/api/test_google_calendar_source.py +++ b/fastapi_backend/tests/api/test_google_calendar_source.py @@ -129,6 +129,7 @@ def test_a_shared_calendar_read_as_a_service_account(google, client, db, cfg, ad "fields": {"calendars": "", "impersonate": "", "days_back": "365"}, "secrets": ["token", "service_account_credentials"], "oauth": None, + "signin": None, } # on its own, a service account has no calendars: it's told what to do body = {"name": "team", "type": "gcal", "secrets": {"service_account_credentials": json.dumps(key)}} diff --git a/fastapi_backend/tests/api/test_source_signin.py b/fastapi_backend/tests/api/test_source_signin.py new file mode 100644 index 000000000..837a1b02c --- /dev/null +++ b/fastapi_backend/tests/api/test_source_signin.py @@ -0,0 +1,196 @@ +"""Connecting Google Drive by signing in with Google in the browser (source_oauth.py), against a fake Google.""" + +from __future__ import annotations + +import json +import urllib.parse + +import pytest + +from app.domain import external_login, settings, source_oauth, sources +from tests.fake_oidc import Provider +from tests.helpers import login, make_user + +WEB = {"x-forwarded-host": "localhost:3000", "x-forwarded-proto": "http"} +ORIGIN = "http://localhost:3000" +ADA = {"sub": "ada-1", "email": "ada@gmail.com", "email_verified": True} +APP = {"kind": "google", "client_id": "lens", "client_secret": "s3cret", "enabled": False} + + +@pytest.fixture +def google(monkeypatch): + p = Provider() + monkeypatch.setitem( + external_login.ENDPOINTS, + "google", + {"authorize": p.base + "/authorize", "token": p.base + "/token", "userinfo": p.base + "/userinfo", "scope": "openid email profile"}, + ) + yield p + p.close() + + +@pytest.fixture +def admin(app, client, db, monkeypatch): + monkeypatch.setattr(sources, "test", lambda *a, **k: {"ok": True}) # no Drive to list in tests + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +def _query(url): + return dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query)) + + +def _back(r): + assert r.status_code == 303, r.text + url = urllib.parse.urlsplit(r.headers["location"]) + return f"{url.scheme}://{url.netloc}{url.path}", dict(urllib.parse.parse_qsl(url.query)) + + +def _sign_in(client, google, headers, body, profile=ADA): + r = client.post("/api/v1/sources/signin", json=body, headers={**WEB, **headers}) + assert r.status_code == 200, r.text + code, state = google.issue(r.json()["url"], profile) + return client.get("/api/v1/auth/external/google/callback", params={"code": code, "state": state}, headers=WEB, follow_redirects=False) + + +def test_drive_signs_in_with_google_and_rclone_refreshes_with_that_app(client, db, cfg, admin, google): + drive = client.get("/api/v1/sources/backends", headers=admin).json()["drive"] + assert drive["signin"] == { + "kind": "google", + "label": "Google", + "ready": False, + "callback_path": "/api/v1/auth/external/google/callback", + } + assert client.get("/api/v1/sources/backends", headers=admin).json()["s3"]["signin"] is None + # no Google app yet: it says where to set one up + r = client.post("/api/v1/sources/signin", json={"type": "drive"}, headers={**WEB, **admin}) + assert r.status_code == 400 and "Google app" in r.json()["detail"] + # the app, with signing in to Lens with Google left off + assert client.post("/api/v1/auth/providers", json=APP, headers=admin).status_code == 200 + assert client.get("/api/v1/auth/external").json() == [] + assert client.get("/api/v1/sources/backends", headers=admin).json()["drive"]["signin"]["ready"] + + body = {"type": "drive", "params": {"root_folder_id": "1AbC"}, "next": "/import?tab=source"} + r = client.post("/api/v1/sources/signin", json=body, headers={**WEB, **admin}) + assert r.status_code == 200 and r.json()["bound"] and "lens_external" in r.headers["set-cookie"] + q = _query(r.json()["url"]) + assert q["redirect_uri"] == ORIGIN + "/api/v1/auth/external/google/callback" # the sign-in's own: nothing more to register + assert q["scope"] == "openid email https://www.googleapis.com/auth/drive.readonly" + assert q["access_type"] == "offline" and q["prompt"] == "consent" and q["code_challenge_method"] == "S256" + code, state = google.issue(r.json()["url"], ADA) + page, back = _back( + client.get("/api/v1/auth/external/google/callback", params={"code": code, "state": state}, headers=WEB, follow_redirects=False) + ) + assert page == ORIGIN + "/import" and back["tab"] == "source" and back["connected"] + sid = int(back["connected"]) + + s = next(s for s in client.get("/api/v1/sources", headers=admin).json() if s["id"] == sid) + assert s["name"] == "Google Drive · ada@gmail.com" and s["type"] == "drive" + assert s["signed_in"]["provider"] == "Google" and s["signed_in"]["email"] == "ada@gmail.com" + assert s["secrets"]["token"]["set"] and not s["secrets"]["service_account_credentials"]["set"] + assert s["params"]["root_folder_id"] == "1AbC" and s["params"]["scope"] == "drive.readonly" + refresh = next(iter(google.refresh)) + assert refresh not in json.dumps(db.rows("SELECT * FROM storage_source")) # sealed + + # rclone gets the app's client id and secret: Google refreshes a token only for the app that issued it + _, conf = sources._config(cfg, sources.get(db, sid)) + assert "client_id = lens\n" in conf and "client_secret = s3cret\n" in conf and "root_folder_id = 1AbC\n" in conf + tok = json.loads(next(l for l in conf.splitlines() if l.startswith("token = ")).split(" = ", 1)[1]) + assert tok["refresh_token"] == refresh and tok["token_type"] == "Bearer" and tok["expiry"].endswith("Z") + assert "source.signin" in db.values("SELECT VALUE action FROM audit_log") + + # the link is used once + again = client.get("/api/v1/auth/external/google/callback", params={"code": code, "state": state}, headers=WEB, follow_redirects=False) + assert "already used" in urllib.parse.unquote_plus(again.headers["location"]) + # cancelling at Google comes back with why, and makes nothing + r = client.post("/api/v1/sources/signin", json={"type": "drive"}, headers={**WEB, **admin}) + state = _query(r.json()["url"])["state"] + _, back = _back( + client.get( + "/api/v1/auth/external/google/callback", params={"error": "access_denied", "state": state}, headers=WEB, follow_redirects=False + ) + ) + assert back == {"connect_error": "signing in didn't finish: you cancelled"} + assert len(client.get("/api/v1/sources", headers=admin).json()) == 1 + + +def test_a_link_started_in_one_browser_cant_be_finished_in_another(client, new_client, db, admin, google): + client.post("/api/v1/auth/providers", json=APP, headers=admin) + r = client.post("/api/v1/sources/signin", json={"type": "drive"}, headers={**WEB, **admin}) + code, state = google.issue(r.json()["url"], ADA) + other = new_client() + r = other.get("/api/v1/auth/external/google/callback", params={"code": code, "state": state}, headers=WEB, follow_redirects=False) + assert "another browser" in urllib.parse.unquote_plus(r.headers["location"]) + assert client.get("/api/v1/sources", headers=admin).json() == [] + + +def test_an_agent_gets_a_link_its_person_opens(client, new_client, db, admin, google): + """From an API token the link isn't tied to a browser: the person it's handed to finishes it in theirs, once.""" + client.post("/api/v1/auth/providers", json=APP, headers=admin) + token = client.post("/api/v1/tokens", json={"name": "agent", "scope": "write"}, headers=admin).json()["token"] + agent = {"Authorization": f"Bearer {token}"} + r = client.post("/api/v1/sources/signin", json={"type": "drive", "name": "Ada's Drive"}, headers={**WEB, **agent}) + assert r.status_code == 200 and not r.json()["bound"] and "set-cookie" not in r.headers + code, state = google.issue(r.json()["url"], ADA) + person = new_client() # another browser, with no cookie from the start + _, back = _back( + person.get("/api/v1/auth/external/google/callback", params={"code": code, "state": state}, headers=WEB, follow_redirects=False) + ) + s = client.get("/api/v1/sources", headers=admin).json() + assert back == {"connected": str(s[0]["id"])} and s[0]["name"] == "Ada's Drive" + # a read-only token can't connect anything + ro = client.post("/api/v1/tokens", json={"name": "ro", "scope": "read"}, headers=admin).json()["token"] + r = client.post("/api/v1/sources/signin", json={"type": "drive"}, headers={**WEB, "Authorization": f"Bearer {ro}"}) + assert r.status_code == 403 + + +def test_signing_in_again_replaces_a_service_account_and_a_pasted_token_replaces_the_sign_in(client, db, cfg, admin, google): + client.post("/api/v1/auth/providers", json=APP, headers=admin) + key = { + "type": "service_account", + "private_key": "-----BEGIN " + "PRIVATE KEY-----\nMIIEv...\n-----END " + "PRIVATE KEY-----\n", # split: not a real key + "client_email": "lens@lens-test.iam.gserviceaccount.com", + } + body = {"name": "Team drive", "type": "drive", "secrets": {"service_account_credentials": json.dumps(key)}} + sid = client.post("/api/v1/sources", json=body, headers=admin).json()["id"] + _, back = _back(_sign_in(client, google, admin, {"source": sid, "params": {"scope": "drive"}})) + assert back == {"connected": str(sid)} + s = client.get("/api/v1/sources", headers=admin).json()[0] + assert s["name"] == "Team drive" and s["params"]["scope"] == "drive" and s["signed_in"]["email"] == "ada@gmail.com" + assert s["secrets"]["token"]["set"] and not s["secrets"]["service_account_credentials"]["set"] + assert google.posts[-1]["grant_type"] == "authorization_code" and google.posts[-1]["client_secret"] == "s3cret" + # editing other options keeps the sign-in + client.patch(f"/api/v1/sources/{sid}", json={"params": {"root_folder_id": "1X"}}, headers=admin) + assert client.get("/api/v1/sources", headers=admin).json()[0]["signed_in"] + # a token pasted from rclone is rclone's own: its app refreshes it, not ours + pasted = json.dumps({"access_token": "a", "token_type": "Bearer", "refresh_token": "r", "expiry": "2030-01-01T00:00:00Z"}) + client.patch(f"/api/v1/sources/{sid}", json={"secrets": {"token": pasted}}, headers=admin) + assert client.get("/api/v1/sources", headers=admin).json()[0]["signed_in"] is None + assert "client_id" not in sources._config(cfg, sources.get(db, sid))[1] + # unknown sources and types that don't sign in here + assert client.post("/api/v1/sources/signin", json={"source": 999}, headers={**WEB, **admin}).status_code == 404 + r = client.post("/api/v1/sources/signin", json={"type": "dropbox"}, headers={**WEB, **admin}) + assert r.status_code == 400 and "doesn't sign in here" in r.json()["detail"] + r = client.post("/api/v1/sources/signin", json={"type": "drive", "params": {"scope": "gmail"}}, headers={**WEB, **admin}) + assert r.status_code == 400 and "scope" in r.json()["detail"] + assert client.post("/api/v1/sources/signin", json={}, headers={**WEB, **admin}).status_code == 400 + + +def test_fresh_token_refreshes_it_when_it_runs_out(client, db, cfg, admin, google): + client.post("/api/v1/auth/providers", json=APP, headers=admin) + _sign_in(client, google, admin, {"type": "drive"}) + src = sources.get(db, client.get("/api/v1/sources", headers=admin).json()[0]["id"]) + first = json.loads(settings.unseal(cfg, src["sealed"]["token"], f"source:{src['id']}:token")) + calls = len(google.posts) + assert source_oauth.fresh_token(db, cfg, src) == first["access_token"] and len(google.posts) == calls # still good + newer = source_oauth.fresh_token(db, cfg, src, margin=4000) # as if it ran out within the hour + assert newer != first["access_token"] and google.posts[-1]["grant_type"] == "refresh_token" + kept = json.loads(settings.unseal(cfg, sources.get(db, src["id"])["sealed"]["token"], f"source:{src['id']}:token")) + assert kept["access_token"] == newer and kept["refresh_token"] == first["refresh_token"] # saved, refresh token kept + google.refresh.clear() # revoked at Google + with pytest.raises(ValueError, match="expired or revoked"): + source_oauth.fresh_token(db, cfg, sources.get(db, src["id"]), margin=4000) + # the app removed: nothing refreshes it any more + client.delete("/api/v1/auth/providers/google", headers=admin) + with pytest.raises(ValueError, match="sign in again"): + source_oauth.fresh_token(db, cfg, sources.get(db, src["id"])) diff --git a/fastapi_backend/tests/fake_oidc.py b/fastapi_backend/tests/fake_oidc.py index 2a0808aa3..3e492752d 100644 --- a/fastapi_backend/tests/fake_oidc.py +++ b/fastapi_backend/tests/fake_oidc.py @@ -1,6 +1,7 @@ """A tiny OpenID Connect provider for tests (also answers like GitHub's API): discovery, the token endpoint (checks the -client secret, redirect URI and PKCE verifier) and userinfo. Tests play the browser: they read the state and PKCE -challenge from the sign-in address Lens hands out and call `issue` for the code the provider would send back.""" +client secret, redirect URI and PKCE verifier; hands out refresh tokens and swaps them) and userinfo. Tests play the +browser: they read the state and PKCE challenge from the sign-in address Lens hands out and call `issue` for the code +the provider would send back.""" from __future__ import annotations @@ -18,6 +19,8 @@ def __init__(self, client_id="lens", client_secret="s3cret"): self.client_id, self.client_secret = client_id, client_secret self.codes: dict[str, dict] = {} self.tokens: dict[str, dict] = {} + self.refresh: dict[str, dict] = {} # refresh tokens it gave out, and whose they are + self.posts: list[dict] = [] # every form sent to the token endpoint self.emails: list[dict] = [] # GitHub's /user/emails self.auto: dict | None = None # for a browser: GET /authorize signs in as this profile straight away provider = self @@ -61,17 +64,25 @@ def do_GET(self): def do_POST(self): form = dict(urllib.parse.parse_qsl(self.rfile.read(int(self.headers["Content-Length"])).decode())) + provider.posts.append(form) c = provider.codes.pop(form.get("code"), None) if form.get("client_id") != provider.client_id or form.get("client_secret") != provider.client_secret: return self._json({"error": "invalid_client", "error_description": "bad client credentials"}, 401) + if form.get("grant_type") == "refresh_token": + who = provider.refresh.get(form.get("refresh_token") or "") + if who is None: + return self._json({"error": "invalid_grant", "error_description": "Token has been expired or revoked."}, 400) + token = secrets.token_urlsafe(16) + provider.tokens[token] = who + return self._json({"access_token": token, "token_type": "Bearer", "expires_in": 3599}) if not c or form.get("redirect_uri") != c["redirect_uri"]: return self._json({"error": "invalid_grant"}, 400) verifier = form.get("code_verifier") or "" if base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip("=") != c["challenge"]: return self._json({"error": "invalid_grant", "error_description": "PKCE check failed"}, 400) - token = secrets.token_urlsafe(16) - provider.tokens[token] = c["profile"] - return self._json({"access_token": token, "token_type": "Bearer"}) + token, refresh = secrets.token_urlsafe(16), secrets.token_urlsafe(16) + provider.tokens[token] = provider.refresh[refresh] = c["profile"] + return self._json({"access_token": token, "token_type": "Bearer", "expires_in": 3599, "refresh_token": refresh}) self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) self.base = f"http://127.0.0.1:{self.server.server_address[1]}" diff --git a/nextjs-frontend/__tests__/drive-signin.test.tsx b/nextjs-frontend/__tests__/drive-signin.test.tsx new file mode 100644 index 000000000..f09806301 --- /dev/null +++ b/nextjs-frontend/__tests__/drive-signin.test.tsx @@ -0,0 +1,147 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import "@testing-library/jest-dom"; + +import { Auth, Sources } from "@/app/openapi-client"; +import { ConnectionDialog } from "@/components/sources/connection-dialog"; +import { TooltipProvider } from "@/components/ui/tooltip"; +import { navigate } from "@/lib/navigate"; + +jest.mock("@/app/openapi-client", () => ({ + Sources: { + createSource: jest.fn(), + updateSource: jest.fn(), + testSource: jest.fn(), + deleteSource: jest.fn(), + startSignin: jest.fn(), + listSources: jest.fn(), + }, + Auth: { addLoginProvider: jest.fn() }, +})); +jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); +jest.mock("@/components/ui/toast", () => ({ useToast: () => jest.fn() })); +jest.mock("@/lib/navigate", () => ({ navigate: { assign: jest.fn(), replace: jest.fn() } })); + +/** The same-site channel the /connected page answers on, kept in this test's memory. */ +class Channel { + static open: Channel[] = []; + onmessage: ((e: { data: unknown }) => void) | null = null; + constructor(public name: string) { + Channel.open.push(this); + } + postMessage(data: unknown) { + for (const c of Channel.open) if (c !== this && c.name === this.name) c.onmessage?.({ data }); + } + close() { + Channel.open = Channel.open.filter((c) => c !== this); + } +} + +const ok = (data: unknown) => Promise.resolve({ data, response: { ok: true, status: 200 } }); +const SIGNIN = { kind: "google", label: "Google", ready: true, callback_path: "/api/v1/auth/external/google/callback" }; +const DRIVE = { + label: "Google Drive", + fields: { scope: "drive.readonly", root_folder_id: "", team_drive: "", impersonate: "" }, + secrets: ["token", "service_account_credentials"], + oauth: "rclone authorize drive", +}; + +function show(onSaved: (id: number) => void, signin: typeof SIGNIN | null = SIGNIN) { + const qc = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }); + return render( + + + undefined} + backends={{ drive: { ...DRIVE, signin } }} + onSaved={onSaved} + /> + + , + ); +} + +describe("connecting Google Drive by signing in with Google", () => { + const popup = { location: { href: "" }, close: jest.fn() }; + beforeEach(() => { + Channel.open = []; + (globalThis as { BroadcastChannel?: unknown }).BroadcastChannel = Channel; + jest.spyOn(window, "open").mockReturnValue(popup as unknown as Window); + (Sources.startSignin as jest.Mock) + .mockReset() + .mockImplementation(() => ok({ url: "https://google.test/auth", bound: true })); + (Sources.listSources as jest.Mock).mockReset().mockImplementation(() => + ok([ + { + id: 7, + name: "Google Drive · ada@gmail.com", + type: "drive", + params: {}, + secrets: {}, + signed_in: { provider: "Google", email: "ada@gmail.com" }, + health: { ok: true }, + }, + ]), + ); + (Sources.updateSource as jest.Mock).mockReset().mockImplementation(() => ok({ health: { ok: true } })); + }); + + it("signs in in a small window, then tests and names the connection it made", async () => { + const saved = jest.fn(); + show(saved); + fireEvent.click(screen.getByRole("radio", { name: /Google Drive/ })); + fireEvent.change(screen.getByLabelText(/Root folder ID/), { target: { value: "1AbC" } }); + expect(screen.queryByRole("button", { name: "Test connection" })).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Sign in with Google" })); + await waitFor(() => expect(popup.location.href).toBe("https://google.test/auth")); + const body = (Sources.startSignin as jest.Mock).mock.calls[0][0].body; + expect(body).toMatchObject({ + type: "drive", + name: null, + params: { root_folder_id: "1AbC", scope: "drive.readonly" }, + }); + expect(screen.getByRole("button", { name: "Waiting for Google…" })).toHaveAttribute("aria-disabled", "true"); + // the /connected page, in the window, says it worked + const tag = new URLSearchParams(body.next.split("?")[1]).get("tag"); + act(() => new Channel("lens-connect").postMessage({ tag, connected: 7 })); + expect(await screen.findByLabelText("Name")).toHaveValue("Google Drive · ada@gmail.com"); + expect(screen.getByText("Listed the top folder")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Done" })); + await waitFor(() => expect(saved).toHaveBeenCalledWith(7)); + expect(Sources.createSource).not.toHaveBeenCalled(); + expect(Sources.updateSource).not.toHaveBeenCalled(); // the name it was made with was kept + }); + + it("goes with the whole page where popups are blocked, and shows why it didn't work", async () => { + (window.open as jest.Mock).mockReturnValue(null); + show(jest.fn()); + fireEvent.click(screen.getByRole("radio", { name: /Google Drive/ })); + fireEvent.click(screen.getByRole("button", { name: "Sign in with Google" })); + await waitFor(() => expect(navigate.assign).toHaveBeenCalledWith("https://google.test/auth")); + expect((Sources.startSignin as jest.Mock).mock.calls[0][0].body.next).toBe("/"); + }); + + it("asks for the Google app first, and keeps signing in to Lens with Google off", async () => { + (Auth.addLoginProvider as jest.Mock).mockReset().mockImplementation(() => ok({ key: "google" })); + show(jest.fn(), { ...SIGNIN, ready: false }); + fireEvent.click(screen.getByRole("radio", { name: /Google Drive/ })); + expect(screen.getByText(/http:\/\/localhost\/api\/v1\/auth\/external\/google\/callback/)).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Sign in with Google" })).toHaveAttribute("aria-disabled", "true"); + fireEvent.change(screen.getByLabelText("Client ID"), { target: { value: "123.apps.googleusercontent.com" } }); + fireEvent.change(screen.getByLabelText("Client secret"), { target: { value: "GOCSPX-x" } }); + fireEvent.click(screen.getByRole("button", { name: "Save app" })); + await waitFor(() => + expect((Auth.addLoginProvider as jest.Mock).mock.calls[0][0].body).toEqual({ + kind: "google", + client_id: "123.apps.googleusercontent.com", + client_secret: "GOCSPX-x", + enabled: false, + }), + ); + // or the token rclone printed, as before + fireEvent.click(screen.getByRole("button", { name: "Or paste a token from rclone" })); + expect(screen.getByText(/Run this on a computer with a browser/)).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Test connection" })).toBeInTheDocument(); + }); +}); diff --git a/nextjs-frontend/__tests__/sources-model.test.ts b/nextjs-frontend/__tests__/sources-model.test.ts index 2754e8060..3f708adb2 100644 --- a/nextjs-frontend/__tests__/sources-model.test.ts +++ b/nextjs-frontend/__tests__/sources-model.test.ts @@ -180,6 +180,29 @@ describe("connections", () => { }; expect(emptyForm(DRIVE, saved).driveAuth).toBe("service_account"); }); + it("signs Google Drive in with Google when the Google app is set up, and keeps a pasted token as pasted", () => { + const DRIVE: BackendSpec = { + label: "Google Drive", + fields: { scope: "drive.readonly", root_folder_id: "" }, + secrets: ["token", "service_account_credentials"], + oauth: "rclone authorize drive", + signin: { kind: "google", label: "Google", ready: true, callback_path: "/api/v1/auth/external/google/callback" }, + }; + // signing in with Google: no token to paste + expect(validateForm("drive", DRIVE, emptyForm(DRIVE), false)).toEqual({}); + expect(Object.keys(validateForm("drive", DRIVE, { ...emptyForm(DRIVE), drivePaste: true }, false))).toEqual([ + "token", + ]); + const token = { secret: true, set: true }; + const none = { secret: true, set: false }; + const pasted = { name: "d", params: {}, secrets: { token, service_account_credentials: none } }; + expect(emptyForm(DRIVE, pasted).drivePaste).toBe(true); + const signedIn = { ...pasted, signed_in: { provider: "Google", email: "ada@gmail.com" } }; + expect(emptyForm(DRIVE, signedIn).drivePaste).toBe(false); + expect(sourceSubtitle({ ...signedIn, type: "drive", params: { scope: "drive", root_folder_id: "1x" } })).toBe( + "Drive · ada@gmail.com · full access · one folder", + ); + }); it("takes email accounts and calendar feeds", () => { const IMAP: BackendSpec = { label: "Email (IMAP)", diff --git a/nextjs-frontend/app/(auth)/connected/page.tsx b/nextjs-frontend/app/(auth)/connected/page.tsx new file mode 100644 index 000000000..c76cead8b --- /dev/null +++ b/nextjs-frontend/app/(auth)/connected/page.tsx @@ -0,0 +1,10 @@ +import type { Metadata } from "next"; + +import { ConnectedReturn } from "@/components/sources/connected-return"; + +export const metadata: Metadata = { title: "Connected" }; + +/** Where signing a connection in at its service (Google Drive) comes back to, in the window that sign-in opened. */ +export default function ConnectedPage() { + return ; +} diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index c5089f60c..a6699d3a2 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -2734,6 +2734,8 @@ export type { ShareLink, ShareServer, ShareSite, + SignedIn, + SignInInfo, SigninLink, SigninLinkAnswer, SigninLinkData, @@ -2783,6 +2785,8 @@ export type { SourceImport, SourceImportRequest, SourceImportResult, + SourceSignIn, + SourceSignInLink, SourceUpdate, SourceUpdated, SparqlQuery, @@ -2814,6 +2818,11 @@ export type { SplitSegmentResponse, SplitSegmentResponses, Started, + StartSigninData, + StartSigninError, + StartSigninErrors, + StartSigninResponse, + StartSigninResponses, StartUpgradeData, StartUpgradeError, StartUpgradeErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 8fe7b4f38..cb4ce8a4e 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -1352,6 +1352,9 @@ import type { SplitSegmentData, SplitSegmentErrors, SplitSegmentResponses, + StartSigninData, + StartSigninErrors, + StartSigninResponses, StartUpgradeData, StartUpgradeErrors, StartUpgradeResponses, @@ -7935,7 +7938,7 @@ export class Sources { /** * List Backends * - * The kinds of storage a source can be, with their settings and secrets. + * The kinds of storage a source can be, with their settings and secrets, and how each signs in in the browser. */ public static listBackends( options?: Options, @@ -8006,6 +8009,28 @@ export class Sources { }); } + /** + * Start Signin + * + * Connect Google Drive by signing in with Google (through the Google app in Settings › Sign-in), or sign a + * connection in again (`source`): open the returned address in a browser. The provider sends it back to the web app + * page `next`, with ?connected= (tested; next, watch one of its folders) or ?connect_error=. Started from + * the web app it's tied to that browser; from an API token or app it isn't, so an agent can hand the link to its + * person. Either way it works once, within ten minutes. Audited as `source.signin` when it finishes. + */ + public static startSignin( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/sources/signin", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * Test Source */ diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 9c5374968..a2d463fc1 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -914,6 +914,10 @@ export type Backend = { * the command that produces the token, for OAuth backends */ oauth?: string | null; + /** + * signing in in the browser instead of pasting a token + */ + signin?: SignInInfo | null; [key: string]: unknown; }; @@ -16577,6 +16581,58 @@ export type SharedContext = { [key: string]: unknown; }; +/** + * SignInInfo + */ +export type SignInInfo = { + /** + * Kind + * + * the kind of sign-in provider whose app it signs in through (google) + */ + kind: string; + /** + * Label + */ + label: string; + /** + * Ready + * + * that app is set up (Settings › Sign-in), so POST /sources/signin works + */ + ready: boolean; + /** + * Callback Path + * + * the address, on this site, to register with the service as the app's redirect URI + */ + callback_path: string; + [key: string]: unknown; +}; + +/** + * SignedIn + */ +export type SignedIn = { + /** + * Provider + * + * the service it signed in with (Google) + */ + provider: string; + /** + * Email + * + * the account it signed in as + */ + email?: string | null; + /** + * At + */ + at?: string | null; + [key: string]: unknown; +}; + /** * SigninLink */ @@ -17016,6 +17072,10 @@ export type Source = { * Oauth */ oauth?: string | null; + /** + * signed in in the browser (POST /sources/signin), as whom + */ + signed_in?: SignedIn | null; health?: SourceHealth | null; /** * Created At @@ -17173,6 +17233,64 @@ export type SourceImportResult = { [key: string]: unknown; }; +/** + * SourceSignIn + */ +export type SourceSignIn = { + /** + * Type + * + * the kind of connection to make (drive); leave out with `source` + */ + type?: + "s3" | "dropbox" | "drive" | "onedrive" | "sftp" | "smb" | "webdav" | "local" | "imap" | "ical" | "gcal" | null; + /** + * Source + * + * a connection to sign in again, instead of making a new one + */ + source?: number | null; + /** + * Name + * + * the new connection's name; by default the service and the account + */ + name?: string | null; + /** + * Params + * + * its options, as for POST /sources (scope, root_folder_id...) + */ + params?: { + [key: string]: unknown; + } | null; + /** + * Next + * + * the web app's page to come back to (by default one that only says how it went) + */ + next?: string; +}; + +/** + * SourceSignInLink + */ +export type SourceSignInLink = { + /** + * Url + * + * the service's sign-in page: open it in a browser; it works once, within ten minutes + */ + url: string; + /** + * Bound + * + * tied to this browser (a cookie came with it); else the link alone finishes it + */ + bound: boolean; + [key: string]: unknown; +}; + /** * SourceUpdate */ @@ -32590,6 +32708,31 @@ export type UpdateSourceResponses = { export type UpdateSourceResponse = UpdateSourceResponses[keyof UpdateSourceResponses]; +export type StartSigninData = { + body: SourceSignIn; + path?: never; + query?: never; + url: "/api/v1/sources/signin"; +}; + +export type StartSigninErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type StartSigninError = StartSigninErrors[keyof StartSigninErrors]; + +export type StartSigninResponses = { + /** + * Successful Response + */ + 200: SourceSignInLink; +}; + +export type StartSigninResponse = StartSigninResponses[keyof StartSigninResponses]; + export type TestSourceData = { body?: never; path: { diff --git a/nextjs-frontend/components/settings/sign-in-providers.tsx b/nextjs-frontend/components/settings/sign-in-providers.tsx index a7c028b1e..3fe0c6f90 100644 --- a/nextjs-frontend/components/settings/sign-in-providers.tsx +++ b/nextjs-frontend/components/settings/sign-in-providers.tsx @@ -88,6 +88,7 @@ export function SignInProviders() {
{p.people === 1 ? "1 person connected" : `${p.people} people connected`} + {p.kind === "google" && " · also connects Google Drive"}
| null>(null); + + useEffect(() => { + const q = new URLSearchParams(window.location.search); + const connected = q.get("connected"); + const now = connected + ? { connected: Number(connected) } + : { error: q.get("connect_error") || "Signing in didn’t finish." }; + setGot(now); + const tag = q.get("tag"); + if (tag && typeof BroadcastChannel !== "undefined") { + const channel = new BroadcastChannel(CHANNEL); + channel.postMessage({ tag, ...now } satisfies Connected); + channel.close(); + window.close(); // only closes a window a page opened; otherwise this page stays and says how it went + } + }, []); + + if (!got) return {null}; + return ( + + {got.connected ? "Pick a folder to import" : "Back to Lens"} + + } + > + {got.error ? ( + {sentence(got.error)} + ) : ( +

+ Lens can read it now. You can close this window, or pick a folder whose files Lens should import. +

+ )} +
+ ); +} diff --git a/nextjs-frontend/components/sources/connection-dialog.tsx b/nextjs-frontend/components/sources/connection-dialog.tsx index f292d4f39..517483708 100644 --- a/nextjs-frontend/components/sources/connection-dialog.tsx +++ b/nextjs-frontend/components/sources/connection-dialog.tsx @@ -6,6 +6,7 @@ import { useEffect, useState, type ReactNode } from "react"; import { Sources } from "@/app/openapi-client"; import type { Source, SourceHealth } from "@/app/openapi-client/types.gen"; import { ConnectionFields } from "@/components/sources/connection-fields"; +import { signIn } from "@/components/sources/signin"; import { TYPE_ICON } from "@/components/sources/source-icons"; import { TYPE_NAME, @@ -53,7 +54,8 @@ function StepTrail({ step, editing }: { step: Step; editing: boolean }) { /** SO2: add a connection (choose type → details → test, named there), or edit one (details → test). A type with * nothing to fill in (a folder on this machine) is added, tested and named at once: the test step only shows if it - * fails. */ + * fails. Google Drive signs in with Google in a small window instead of testing a pasted token: the connection is + * made (and tested) when Google sends that window back, and the dialog goes on to the test step. */ export function ConnectionDialog({ open, onOpenChange, @@ -84,8 +86,12 @@ export function ConnectionDialog({ const [createdName, setCreatedName] = useState(""); const [result, setResult] = useState<{ health: SourceHealth; - ms: number; + ms: number | null; } | null>(null); + /** Signed in with Google here, as this address ("" when Google didn't say); null when not. */ + const [signedInAs, setSignedInAs] = useState(null); + /** Waiting for the Google window; aborting it cancels. */ + const [waiting, setWaiting] = useState(null); /** Secrets saved during this dialog (the source prop doesn't know about them yet). */ const [savedKeys, setSavedKeys] = useState>(new Set()); const spec = backends[type]; @@ -111,7 +117,10 @@ export function ConnectionDialog({ setCreatedId(source?.id ?? null); setResult(null); setSavedKeys(new Set()); + setSignedInAs(source?.signed_in ? (source.signed_in.email ?? "") : null); + setWaiting(null); }, [open]); + useEffect(() => () => waiting?.abort(), [waiting]); const refresh = () => { void qc.invalidateQueries({ queryKey: ["sources"] }); @@ -213,6 +222,45 @@ export function ConnectionDialog({ }, }); + // Google Drive with a Google account signs in with Google, unless a token is pasted or it's already signed in and + // only its options change + const viaGoogle = type === "drive" && form.driveAuth === "token" && !form.drivePaste && Boolean(spec?.signin); + const needSignIn = viaGoogle && !(saved && isSet("token") && signedInAs != null); + + const startSignIn = async () => { + const e = validateForm(type, spec, form, saved, isSet); + setErrors(e); + if (Object.keys(e).length) return; + const ctl = new AbortController(); + setWaiting(ctl); + try { + const { params } = buildPayload(spec, form, type); + const got = await signIn( + client, + createdId != null ? { source: createdId, params } : { type, params, name: form.name.trim() || null }, + ctl.signal, + ); + if (got.error) { + if (got.error !== "cancelled") setErrors({ _: got.error }); + return; + } + const id = got.connected!; + const made = (await data(Sources.listSources({ client }))).find((s) => s.id === id); + setCreatedId(id); + setCreatedName(made?.name ?? ""); + setSignedInAs(made?.signed_in ? (made.signed_in.email ?? "") : ""); + setSavedKeys((prev) => new Set([...prev, "token", "-service_account_credentials"])); + setForm((f) => ({ ...f, secrets: {}, tokenText: "", name: editing ? f.name : (made?.name ?? f.name) })); + setResult({ health: made?.health ?? { ok: false, error: "Not tested yet" }, ms: null }); + setStep("test"); + refresh(); + } catch (err) { + setErrors({ _: err instanceof Error ? err.message : String(err) }); + } finally { + setWaiting(null); + } + }; + const goTest = () => { const e = validateForm(type, spec, form, saved, isSet); setErrors(e); @@ -316,6 +364,9 @@ export function ConnectionDialog({ saved={saved} isSet={isSet} replaceToken={replaceToken} + signedIn={signedInAs} + onSignIn={() => void startSignIn()} + signingIn={Boolean(waiting)} /> ) : null} {errors._ && ( @@ -335,12 +386,23 @@ export function ConnectionDialog({ Back )} - - + {needSignIn ? ( + + ) : ( + + )} ); } else if (step === "test") { @@ -364,7 +426,7 @@ export function ConnectionDialog({ ? "Listed the top folder" : "Couldn’t list the top folder"} - {result ? `${result.ms} ms` : ""} + {result?.ms != null ? `${result.ms} ms` : ""} {result && !ok && (
diff --git a/nextjs-frontend/components/sources/connection-fields.tsx b/nextjs-frontend/components/sources/connection-fields.tsx index 9dccb0e76..b76510126 100644 --- a/nextjs-frontend/components/sources/connection-fields.tsx +++ b/nextjs-frontend/components/sources/connection-fields.tsx @@ -14,6 +14,7 @@ import { imapHost, parseServiceAccountKey, } from "@/components/sources/source-model"; +import { GoogleAppSetup } from "@/components/sources/google-app-setup"; import { TokenPaste } from "@/components/sources/token-paste"; import { Button } from "@/components/ui/button"; import { Field, Input, SecretField, Select, Textarea } from "@/components/ui/field"; @@ -35,6 +36,9 @@ export function ConnectionFields({ saved, isSet, replaceToken, + signedIn, + onSignIn, + signingIn, }: { type: SourceType; spec: BackendSpec; @@ -45,6 +49,11 @@ export function ConnectionFields({ saved: boolean; isSet: (key: string) => boolean; replaceToken?: boolean; + /** Signed in with Google here, as this address ("" when Google didn't say); null when not. */ + signedIn?: string | null; + /** Sign in (again) with Google in the browser; shown while a saved connection can also just be saved. */ + onSignIn?: () => void; + signingIn?: boolean; }) { const [pasting, setPasting] = useState(Boolean(replaceToken)); const editing = saved; @@ -216,6 +225,45 @@ export function ConnectionFields({ return token; case "drive": { const serviceAccount = serviceAccountKey("Share the folders Lens should read with that address."); + const google = spec.signin; + const link = (label: string, paste: boolean) => ( + + ); + // a Google account signs in with Google here (through the admin's Google app), or pastes rclone's token + const googleAccount = + form.drivePaste || !google ? ( +
+ {token} + {google && link("Sign in with Google instead", false)} +
+ ) : google.ready ? ( +
+
+ + {signedIn != null + ? `Signed in with Google${signedIn ? ` as ${signedIn}` : ""}. Sign in again to use another account.` + : "You sign in with Google in a small window. Lens keeps the token encrypted and renews it."} + + {signedIn != null && onSignIn && ( + + )} +
+ {link("Paste a token from rclone instead", true)} +
+ ) : ( +
+ + {link("Or paste a token from rclone", true)} +
+ ); return (
- {form.driveAuth === "service_account" ? serviceAccount : token} + {form.driveAuth === "service_account" ? serviceAccount : googleAccount} {select("scope", DRIVE_SCOPES)} {text("root_folder_id", { mono: true, diff --git a/nextjs-frontend/components/sources/google-app-setup.tsx b/nextjs-frontend/components/sources/google-app-setup.tsx new file mode 100644 index 000000000..3ecc08ac8 --- /dev/null +++ b/nextjs-frontend/components/sources/google-app-setup.tsx @@ -0,0 +1,102 @@ +"use client"; + +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { useState } from "react"; + +import { Auth } from "@/app/openapi-client"; +import type { SignInInfo } from "@/app/openapi-client/types.gen"; +import { Button } from "@/components/ui/button"; +import { Field, Input } from "@/components/ui/field"; +import { CodeBlock } from "@/components/ui/states"; +import { data, useApiClient } from "@/lib/api/browser"; + +/** + * Signing in with Google needs the admin's own Google app (an OAuth client), once. It's the app people can sign in to + * Lens with too (Settings › Sign-in), saved here with that left off, so the redirect URI is the sign-in's own. + */ +export function GoogleAppSetup({ signin }: { signin: SignInInfo }) { + const client = useApiClient(); + const qc = useQueryClient(); + const [id, setId] = useState(""); + const [secret, setSecret] = useState(""); + const save = useMutation({ + mutationFn: () => + data( + Auth.addLoginProvider({ + client, + body: { kind: "google", client_id: id.trim(), client_secret: secret.trim(), enabled: false }, + }), + ), + onSuccess: () => { + void qc.invalidateQueries({ queryKey: ["source-backends"] }); + void qc.invalidateQueries({ queryKey: ["login-providers"] }); + }, + }); + const redirect = + typeof window === "undefined" ? signin.callback_path : `${window.location.origin}${signin.callback_path}`; + const ready = Boolean(id.trim() && secret.trim()); + const enter = (e: React.KeyboardEvent) => { + if (e.key === "Enter" && ready) { + e.preventDefault(); + save.mutate(); + } + }; + return ( +
+

To sign in with Google, add your Google app once

+
    +
  1. In Google Cloud console › APIs & Services, enable the Google Drive API.
  2. +
  3. + On the OAuth consent screen, publish the app. While it’s in testing, Google signs Lens out after 7 days. +
  4. +
  5. Credentials › Create OAuth client ID › Web application, with this redirect URI:
  6. +
+ +
+ + {(f) => ( + setId(e.target.value)} + onKeyDown={enter} + autoComplete="off" + spellCheck={false} + /> + )} + + + {(f) => ( + setSecret(e.target.value)} + onKeyDown={enter} + autoComplete="new-password" + /> + )} + +
+ {save.error && ( +

+ {save.error.message} +

+ )} +
+ + + Signing in to Lens with Google stays off until you turn it on in Settings › Sign-in. + +
+
+ ); +} diff --git a/nextjs-frontend/components/sources/signin.ts b/nextjs-frontend/components/sources/signin.ts new file mode 100644 index 000000000..b2354d3be --- /dev/null +++ b/nextjs-frontend/components/sources/signin.ts @@ -0,0 +1,83 @@ +"use client"; + +import { useEffect, useRef } from "react"; + +import { Sources } from "@/app/openapi-client"; +import type { Client } from "@/app/openapi-client/client"; +import type { SourceSignIn } from "@/app/openapi-client/types.gen"; +import { data } from "@/lib/api/browser"; +import { navigate } from "@/lib/navigate"; + +/** What the /connected page tells the window that opened the sign-in, over this channel (same site only). */ +export const CHANNEL = "lens-connect"; +export type Connected = { tag: string; connected?: number; error?: string }; + +/** + * Connect a source by signing in at the service (POST /sources/signin): Google's page opens in a small window, so the + * dialog behind it keeps its place, and this resolves with the connection's id (or why it didn't work) when that + * window comes back to /connected. It waits until then, or until `signal` aborts (the person cancels): a window that + * went through Google can look closed to this page while it's still open. Where popups are blocked the whole page goes + * instead, and comes back to this page with ?connected= or ?connect_error= (useSignInReturn). + */ +export async function signIn( + client: Client, + body: Omit, + signal?: AbortSignal, +): Promise { + const tag = Math.random().toString(36).slice(2, 10); + // opened now, in the click: a window opened after the request returns would be blocked as a popup + const popup = window.open("", "lens-connect", "popup,width=520,height=680"); + const q = new URLSearchParams(window.location.search); + q.delete("connected"); + q.delete("connect_error"); + const here = `${window.location.pathname}${q.size ? `?${q}` : ""}`; + let url: string; + try { + ({ url } = await data( + Sources.startSignin({ client, body: { ...body, next: popup ? `/connected?tag=${tag}` : here } }), + )); + } catch (e) { + popup?.close(); + throw e; + } + if (!popup) { + navigate.assign(url); + return new Promise(() => undefined); // the page is leaving + } + popup.location.href = url; + return new Promise((resolve) => { + const channel = typeof BroadcastChannel === "undefined" ? null : new BroadcastChannel(CHANNEL); + const done = (got: Connected) => { + channel?.close(); + resolve(got); + }; + if (channel) + channel.onmessage = (e: MessageEvent) => { + if (e.data?.tag === tag) done(e.data); + }; + signal?.addEventListener("abort", () => { + popup.close(); + done({ tag, error: "cancelled" }); + }); + }); +} + +/** + * Back from signing in with the whole page (popups blocked): the page it started on has ?connected= or + * ?connect_error=. Hands either over once and drops both from the address. + */ +export function useSignInReturn(on: (got: { connected?: number; error?: string }) => void, enabled = true) { + const seen = useRef(false); + useEffect(() => { + if (!enabled || seen.current) return; + const q = new URLSearchParams(window.location.search); + const connected = q.get("connected"); + const error = q.get("connect_error"); + if (!connected && !error) return; + seen.current = true; + q.delete("connected"); + q.delete("connect_error"); + window.history.replaceState(window.history.state, "", `${window.location.pathname}${q.size ? `?${q}` : ""}`); + on(error ? { error } : { connected: Number(connected) }); + }, [enabled]); +} diff --git a/nextjs-frontend/components/sources/source-model.ts b/nextjs-frontend/components/sources/source-model.ts index 686806b06..fc2a332c0 100644 --- a/nextjs-frontend/components/sources/source-model.ts +++ b/nextjs-frontend/components/sources/source-model.ts @@ -2,7 +2,7 @@ * Storage connections and watched folders as the Sources screens show them: labels, one-line summaries, health in * the backend's own words, the rclone token paste check, and what a watched folder's settings mean in words. */ -import type { Source, Watch, WatchPreview } from "@/app/openapi-client/types.gen"; +import type { SignInInfo, Source, Watch, WatchPreview } from "@/app/openapi-client/types.gen"; import { plural } from "@/lib/format"; export type SourceType = Source["type"]; @@ -130,8 +130,8 @@ export function imapHost(email: string): string { const str = (v: unknown) => (v == null ? "" : String(v)).trim(); -/** "S3 · AWS eu-central-1", "SFTP · recorder@calls-gw:22", "Drive · read only"... */ -export function sourceSubtitle(s: Pick): string { +/** "S3 · AWS eu-central-1", "SFTP · recorder@calls-gw:22", "Drive · ada@gmail.com · read only"... */ +export function sourceSubtitle(s: Pick & { signed_in?: Source["signed_in"] }): string { const p = (s.params ?? {}) as Record; switch (s.type) { case "s3": { @@ -141,7 +141,14 @@ export function sourceSubtitle(s: Pick): string { case "dropbox": return "Dropbox · token"; case "drive": - return `Drive · ${str(p.scope) === "drive" ? "full access" : "read only"}${str(p.root_folder_id) ? " · one folder" : ""}`; + return [ + "Drive", + str(s.signed_in?.email), + str(p.scope) === "drive" ? "full access" : "read only", + str(p.root_folder_id) ? "one folder" : "", + ] + .filter(Boolean) + .join(" · "); case "onedrive": return `OneDrive · token${str(p.drive_type) ? ` · ${str(p.drive_type)}` : ""}`; case "sftp": @@ -416,6 +423,8 @@ export type BackendSpec = { fields: Record; secrets: string[]; oauth?: string | null; + /** Signs in in the browser (Google Drive, through the Google app in Settings › Sign-in), instead of a pasted token. */ + signin?: SignInInfo | null; }; export type ConnForm = { @@ -430,6 +439,8 @@ export type ConnForm = { /** Google Drive and Google Calendar sign in as a Google account (a token), or as a service account (its JSON key * file). */ driveAuth: "token" | "service_account"; + /** A Google account's token pasted from `rclone authorize`, rather than signing in with Google here. */ + drivePaste: boolean; }; export type ServiceAccountKey = { ok: true; email: string } | { ok: false; error: string }; @@ -475,7 +486,10 @@ export function parseServiceAccountKey(text: string): ServiceAccountKey { return { ok: true, email: o.client_email }; } -export function emptyForm(spec: BackendSpec, source?: Pick | null): ConnForm { +export function emptyForm( + spec: BackendSpec, + source?: (Pick & { signed_in?: Source["signed_in"] }) | null, +): ConnForm { const params: Record = {}; for (const [k, def] of Object.entries(spec.fields)) params[k] = str((source?.params as Record | undefined)?.[k] ?? def); @@ -488,6 +502,8 @@ export function emptyForm(spec: BackendSpec, source?: Pick + void sources.refetch().then((r) => { + const made = ((r.data ?? []) as Source[]).find((x) => x.id === id); + if (!made) return; + toast({ tone: "green", title: "Connected", body: made.name }); + setSelected(id); + if (!((watches.data ?? []) as Watch[]).some((w) => w.source === id)) setBrowsing(made); + }); + useSignInReturn( + (got) => + got.connected + ? connected(got.connected) + : toast({ tone: "red", title: "Couldn’t connect", body: got.error ?? "Signing in didn’t finish." }), + admin, + ); + // Sign in again: Google's window has to open in the click itself, so this isn't a mutation (those start a tick later) + const [againId, setAgainId] = useState(null); + const signInAgain = (s: Source) => { + setAgainId(s.id); + signIn(client, { source: s.id }) + .then((got) => { + if (got.error && got.error !== "cancelled") toast({ tone: "red", title: "Couldn’t sign in", body: got.error }); + else if (got.connected) toast({ tone: "green", title: "Signed in again", body: s.name }); + }) + .catch((e: Error) => toast({ tone: "red", title: "Couldn’t sign in", body: e.message })) + .finally(() => { + setAgainId(null); + void qc.invalidateQueries({ queryKey: ["sources"] }); + }); + }; + const test = useMutation({ mutationFn: (s: Source) => data(Sources.testSource({ client, path: { sid: s.id } })), onSuccess: (h, s) => { @@ -164,12 +197,19 @@ export function SourcesPage() { const actions = (s: Source) => { const bad = s.health && !s.health.ok; const oauth = Boolean(s.oauth); + const signedIn = Boolean(s.signed_in); return (
e.stopPropagation()}> {bad && oauth && /token|auth|expired|unauthori[sz]ed|401|invalid_grant/i.test(s.health?.error ?? "") ? ( - + signedIn ? ( + + ) : ( + + ) ) : bad || !s.health ? (