diff --git a/CHANGELOG.md b/CHANGELOG.md index 52b0a3654..b9da68035 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version shows only Projects, Areas, Resources or Archive, and each note says which notes it's inside. Jump to (⌘K) finds notes as well. `GET /api/v1/notes/search`; `find_notes` gives agents the words around each match ([Finding notes](docs/notes.md#finding-notes)). +- **Connect Google Drive by signing in with Google.** Sensors › Add connection › Google Drive › Google account now + has **Sign in with Google**: Google's page opens in a small window, and the connection is made, tested and named + after the account, ready to pick a folder whose files Lens imports. Nothing to run or paste. It uses your own + Google app, the one people can sign in to Lens with; set it up once in the dialog (client ID and secret, with the + redirect URI shown) or in Settings › Sign-in, and signing in to Lens with Google can stay off. Agents get the same + link from `POST /api/v1/sources/signin` to hand to their person. Pasting a token from rclone and service accounts + work as before ([Storage sources](docs/processing.md#storage-sources)). ## 0.10.0 October 15, 2026 {id="0.10.0"} diff --git a/docs/authentication.md b/docs/authentication.md index d687e9919..ab639ba46 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -83,6 +83,10 @@ has a "Continue with ..." button for each provider that's turned on. Removing a provider disconnects the accounts connected through it. Audited as `login_provider.add`, `.change`, `.remove`, `external.connect` and `external.disconnect`; sign-ins as `login` with `external:`. +The Google app also connects Google Drive (Sensors › Add connection › Google Drive › Sign in with Google): the same +client id, secret and redirect URI, with Drive's scope asked for, so it works with signing in to Lens turned off +([Storage sources](processing.md#storage-sources)). + ## The web app: NextAuth with API tokens ``` diff --git a/docs/processing.md b/docs/processing.md index ab7c8d7d1..1d9a01b6d 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -302,6 +302,21 @@ files, then Import (`POST /api/v1/import/source`). The listing marks files that - Credentials are encrypted in the database (AES-GCM, key from `ARCHIVE_SECRET_KEY` or `data_dir/secret.key`) and given to rclone in a private temporary config file per call. For Dropbox, Drive and OneDrive, paste the token from `rclone authorize dropbox` (or drive, onedrive); tokens rclone refreshes are saved back. +- **Google Drive signs in with Google** in the web app, with nothing to paste: choose Google account, then Sign in + with Google. A small window opens Google's consent page; when Google sends it back, the connection is made, tested + and named after the account (Google Drive · you@gmail.com), and the dialog goes on to pick a folder to watch. It + uses your own Google app, the one people can sign in to Lens with (Settings › Sign-in): set it up once, in the + dialog or there, with signing in to Lens left off if you like. In Google Cloud console › APIs & Services, enable + the Google Drive API; create an OAuth client ID (Web application) whose redirect URI is the address the dialog + shows (`/api/v1/auth/external/google/callback`, the same as sign-in's); and publish the app on + the OAuth consent screen, or Google ends its sign-ins after 7 days. Google accepts `https://` redirect addresses + and `http://localhost`. The token is kept encrypted, and rclone renews it with that app's client id and secret. + A connection whose sign-in ran out shows Sign in again. Without a Google app, paste a token from rclone as before. +- Agents connect Drive the same way: `POST /api/v1/sources/signin` with `{"type": "drive", "params": {...}}` (or + `{"source": }` to sign one in again) answers `{url}`. From the web app the link is tied to that browser; from + an API token or app it isn't, so an agent can hand it to its person, who opens it once within ten minutes and ends + on a page saying it's connected (`/connected`). Then `POST /api/v1/watches` imports its files. Code that calls + Google itself gets a fresh access token from `source_oauth.fresh_token`. Audited as `source.signin`. - Google Drive can also sign in as a service account: choose Service account and upload its JSON key file (`service_account_credentials`). Share the folders Lens should read with the account's address, or give a shared drive ID (`team_drive`); in Google Workspace, `impersonate` reads a user's own Drive through domain-wide delegation. diff --git a/fastapi_backend/app/api/v1/routes/external.py b/fastapi_backend/app/api/v1/routes/external.py index 85cf532f5..67f59293e 100644 --- a/fastapi_backend/app/api/v1/routes/external.py +++ b/fastapi_backend/app/api/v1/routes/external.py @@ -5,6 +5,10 @@ cookie tying the sign-in to this browser. The provider sends it back to the callback, which ends on the web app's /external-signin page with a one-time ticket in the URL fragment; the web app's server swaps it for a session at POST /auth/ticket, as for passkeys. + +Connecting a storage source by signing in (source_oauth.py, POST /sources/signin) comes back through the same callback, +so the app registered with the provider needs no second address; it ends on the web app's page it started from, with +?connected= or ?connect_error=. """ from __future__ import annotations @@ -15,7 +19,7 @@ from fastapi.responses import RedirectResponse from app.api.deps import AdminWriter, Cfg, CurrentUser, Db, Writer, domain_errors, web_origin -from app.domain import auth, external_login, passkeys +from app.domain import auth, external_login, passkeys, source_oauth from app.schemas.auth import ( ExternalIdentity, ExternalProvider, @@ -30,7 +34,8 @@ COOKIE = "lens_external" -def _cookie(response: Response, origin: str, value: str) -> None: +def flow_cookie(response: Response, origin: str, value: str) -> None: + """The cookie that ties a round trip through a provider to the browser that started it.""" response.set_cookie( COOKIE, value, @@ -50,7 +55,7 @@ def _redirect(response: Response, request: Request, db: Db, cfg: Cfg, key: str, raise HTTPException(404, "no such sign-in provider") from None except ValueError as e: raise HTTPException(400, str(e)) from None - _cookie(response, origin, browser) + flow_cookie(response, origin, browser) return ExternalRedirect(url=url) @@ -96,6 +101,8 @@ def external_callback( flow = external_login.claim(db, state, request.cookies.get(COOKIE)) except ValueError as e: return _back(web_origin(request), error=str(e), next="/login") + if flow.get("mode") == "source": + return _source_back(db, cfg, flow, code, error, error_description) origin, connect = flow["origin"], flow.get("mode") == "connect" away = "/account" if connect else "/login" if error or not code: @@ -114,6 +121,27 @@ def external_callback( return _back(origin, ticket=passkeys.issue_ticket(db, u["id"], f"external:{key}"), next=flow.get("next") or "/") +def _source_back(db: Db, cfg: Cfg, flow: dict, code: str, error: str, error_description: str) -> RedirectResponse: + """A source signed in (or not): back to the page it started from, with ?connected= or ?connect_error=.""" + if error or not code: + why = "you cancelled" if error == "access_denied" else (error_description or error or "no code came back") + q = {"connect_error": f"signing in didn't finish: {why}"} + else: + try: + sid = source_oauth.finish(db, cfg, flow, code) + except ValueError as e: + q = {"connect_error": str(e)} + else: + q = {"connected": str(sid)} + auth.audit( + db, {"id": flow.get("account"), "email": (flow.get("source") or {}).get("by")}, "source.signin", f"storage_source:{sid}" + ) + page = flow.get("next") or "/sources" + r = RedirectResponse(f"{flow['origin']}{page}{'&' if '?' in page else '?'}{urllib.parse.urlencode(q)}", status_code=303) + r.delete_cookie(COOKIE, path="/api/v1/auth/external") + return r + + # ---------- your outside accounts ---------- def _signed_in(user: CurrentUser) -> None: if user.via != "access": diff --git a/fastapi_backend/app/api/v1/routes/sources.py b/fastapi_backend/app/api/v1/routes/sources.py index 283b30f52..1f8b1f9fc 100644 --- a/fastapi_backend/app/api/v1/routes/sources.py +++ b/fastapi_backend/app/api/v1/routes/sources.py @@ -5,10 +5,11 @@ import logging from collections import Counter -from fastapi import APIRouter, BackgroundTasks, HTTPException, Request +from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, Response -from app.api.deps import Acl, AdminReader, AdminWriter, Cfg, CurrentUser, Db -from app.domain import auth, sources, store +from app.api.deps import Acl, AdminReader, AdminWriter, Cfg, CurrentUser, Db, web_origin +from app.api.v1.routes.external import flow_cookie +from app.domain import auth, source_oauth, sources, store from app.domain.store import DB from app.schemas.admin import Started from app.schemas.common import Created, Ok @@ -21,6 +22,8 @@ SourceCreate, SourceCreated, SourceHealth, + SourceSignIn, + SourceSignInLink, SourceUpdate, SourceUpdated, Watch, @@ -47,10 +50,13 @@ def _watch(db: DB, wid: int) -> None: @router.get("/sources/backends") -def list_backends(user: CurrentUser) -> dict[str, Backend]: - """The kinds of storage a source can be, with their settings and secrets.""" +def list_backends(user: CurrentUser, db: Db) -> dict[str, Backend]: + """The kinds of storage a source can be, with their settings and secrets, and how each signs in in the browser.""" keys = {"label", "fields", "secrets", "oauth"} - return {k: Backend.model_validate({f: x for f, x in v.items() if f in keys}) for k, v in sources.BACKENDS.items()} + return { + k: Backend.model_validate({**{f: x for f, x in v.items() if f in keys}, "signin": source_oauth.signin(db, k)}) + for k, v in sources.BACKENDS.items() + } @router.get("/sources") @@ -80,6 +86,39 @@ def update_source(sid: int, body: SourceUpdate, user: AdminWriter, db: Db, cfg: return SourceUpdated(health=sources.test(db, cfg, sid)) +@router.post("/sources/signin") +def start_signin(body: SourceSignIn, user: AdminWriter, request: Request, response: Response, db: Db, cfg: Cfg) -> SourceSignInLink: + """Connect Google Drive by signing in with Google (through the Google app in Settings › Sign-in), or sign a + connection in again (`source`): open the returned address in a browser. The provider sends it back to the web app + page `next`, with ?connected= (tested; next, watch one of its folders) or ?connect_error=. Started from + the web app it's tied to that browser; from an API token or app it isn't, so an agent can hand the link to its + person. Either way it works once, within ten minutes. Audited as `source.signin` when it finishes.""" + if body.source is None and body.type is None: + raise HTTPException(400, "say which type of connection to make, or which connection to sign in again") + origin, bound = web_origin(request), user.via == "access" + try: + url, browser = source_oauth.start( + db, + cfg, + origin, + body.type, + body.params, + body.name, + body.source, + account=user.id, + by=user.email, + bound=bound, + next_path=body.next, + ) + except KeyError: + raise HTTPException(404, "not found") from None + except ValueError as e: + raise HTTPException(400, str(e)) from None + if browser: + flow_cookie(response, origin, browser) + return SourceSignInLink(url=url, bound=bool(browser)) + + @router.delete("/sources/{sid}") def delete_source(sid: int, user: AdminWriter, db: Db) -> Ok: """Remove a source and its watched folders.""" diff --git a/fastapi_backend/app/domain/external_login.py b/fastapi_backend/app/domain/external_login.py index 4c1a1277f..13761be1f 100644 --- a/fastapi_backend/app/domain/external_login.py +++ b/fastapi_backend/app/domain/external_login.py @@ -190,6 +190,11 @@ def callback_path(key): return f"/api/v1/auth/external/{key}/callback" +def client_secret(cfg, row): + """A provider's client secret, unsealed, for its token endpoint.""" + return settings.unseal(cfg, row["client_secret"], _context(row["key"])) + + # ---------- signing in ---------- def _endpoints(row): kind = row["kind"] @@ -246,7 +251,7 @@ def start(db, cfg, key, origin, mode="login", account=None, next_path="/"): "origin": origin.rstrip("/"), "mode": mode, "account": account, - "next": _safe_next(next_path), + "next": safe_next(next_path), "expires_at": _later(FLOW_MINUTES * 60), } ), @@ -268,7 +273,7 @@ def start(db, cfg, key, origin, mode="login", account=None, next_path="/"): return ep["authorize"] + sep + urllib.parse.urlencode(q), browser -def _safe_next(path): +def safe_next(path): """A path on this site. Browsers drop tabs and line breaks from a URL ("/\\t/evil.example" is "//evil.example").""" path = path or "/" ok = path.startswith("/") and not path.startswith("//") and "\\" not in path and not re.search(r"[\x00-\x1f\x7f]", path) @@ -276,11 +281,15 @@ def _safe_next(path): def claim(db, state, browser): - """The flow a callback answers, used up. ExternalLoginError when it's gone, expired, or started in another browser.""" + """The flow a callback answers, used up. ExternalLoginError when it's gone, expired, or started in another browser. + A flow an agent started through the API for its person to open (connecting a source, source_oauth.py) isn't tied + to a browser: its link alone finishes it, once.""" rows = db.rows("DELETE $r RETURN BEFORE", r=R("external_flow", auth.sha(state))) if state else [] row = rows[0] if rows else None if not row or (row.get("expires_at") or "") < store.now(): raise ExternalLoginError("that sign-in took too long or was already used; try again") + if row.get("unbound") and row.get("mode") == "source": + return row if not browser or auth.sha(browser) != row.get("browser"): raise ExternalLoginError("that sign-in was started in another browser; start it again here") return row @@ -289,7 +298,7 @@ def claim(db, state, browser): def _profile(row, cfg, flow, code): """{sub, email, trusted, name} from the provider, for the code it sent back.""" ep = _endpoints(row) - secret = settings.unseal(cfg, row["client_secret"], _context(row["key"])) + secret = client_secret(cfg, row) try: r = httpx.post( ep["token"], diff --git a/fastapi_backend/app/domain/source_oauth.py b/fastapi_backend/app/domain/source_oauth.py new file mode 100644 index 000000000..4bd7fcdcc --- /dev/null +++ b/fastapi_backend/app/domain/source_oauth.py @@ -0,0 +1,252 @@ +"""Connecting a storage source by signing in at the service in the browser, rather than pasting a token from rclone. + +Google Drive signs in through the Google app an admin set up in Settings › Sign-in (external_login.py, kind google): +its client id and secret, and its callback address, so nothing more is registered at Google. The app may stay off for +signing in to Lens; connecting a source only needs it to exist. The round trip is an external_flow row like a +sign-in's (PKCE, ten minutes, used once, tied by a cookie to the browser that started it), in mode "source". An agent +can start one through the API for its person to open in their own browser: that flow isn't tied to a browser, so the +link alone finishes it, once. + +The callback swaps the code for tokens at Google and keeps them sealed as the source's `token`, in rclone's own shape +(access_token, token_type, refresh_token, expiry). The source's `oauth` says which app signed it in and as whom; rclone +is handed that app's client id and secret (sources._config), because Google refreshes a token only for the app that +issued it. A token pasted from rclone, or a service account key, replaces it (sources.update drops `oauth`). + +Code that talks to Google itself rather than through rclone gets an access token from fresh_token, which refreshes it +when it is about to run out. +""" + +from __future__ import annotations + +import base64 +import datetime as dt +import hashlib +import json +import re +import secrets +import urllib.parse + +import httpx + +from . import auth, external_login, settings, sources, store + +R = store.R +GOOGLE_SCOPE = "https://www.googleapis.com/auth/" +DRIVE_SCOPES = ("drive", "drive.readonly", "drive.file", "drive.metadata.readonly") +TIMEOUT = 15 + + +def _drive_scope(params): + asked = [s.strip() for s in str(params.get("scope") or "drive.readonly").split(",") if s.strip()] + if not asked or any(s not in DRIVE_SCOPES for s in asked): + raise ValueError(f"the Drive scope is one of {', '.join(DRIVE_SCOPES)}") + return " ".join(GOOGLE_SCOPE + s for s in asked) + + +# the source types that sign in here: the kind of sign-in provider whose app they use, and the scopes they ask for +TYPES = {"drive": {"kind": "google", "scope": _drive_scope}} + + +def app(db, typ): + """The sign-in provider (with its sealed secret) that a source type signs in through, or None when it isn't set up. + Google's provider key is its kind: an install has one Google app.""" + t = TYPES.get(typ) + if not t: + return None + try: + row = external_login.provider(db, t["kind"]) + except KeyError: + return None + return row if row.get("client_id") and row.get("client_secret") else None + + +def signin(db, typ): + """How a source type signs in in the browser, for the connection form; None when it only takes a pasted token or + keys. `ready` says whether its app is set up; `callback_path` is the address to register with the service.""" + t = TYPES.get(typ) + if not t: + return None + return { + "kind": t["kind"], + "label": external_login.LABELS[t["kind"]], + "ready": app(db, typ) is not None, + "callback_path": external_login.callback_path(t["kind"]), + } + + +def _later(seconds): + return (dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=seconds)).isoformat(timespec="seconds") + + +def start(db, cfg, origin, typ=None, params=None, name=None, sid=None, account=None, by=None, bound=True, next_path="/connected"): + """The service's sign-in page for connecting a new source of type `typ` (with `params`, named `name`), or for + signing source `sid` in again; and, for a flow tied to this browser (`bound`), the secret to keep in its cookie + (else None). ValueError when the type can't sign in here or its app isn't set up; KeyError for no such source.""" + if sid is not None: + src = sources.get(db, sid) + typ, params = src["type"], {**(src.get("params") or {}), **(params or {})} + if typ not in TYPES: + raise ValueError(f"{sources.BACKENDS[typ]['label'] if typ in sources.BACKENDS else typ} doesn't sign in here") + params = {**sources.BACKENDS[typ]["fields"], **(params or {})} + sources._check_params(typ, params, None) + scope = TYPES[typ]["scope"](params) + row = app(db, typ) + if not row: + label = external_login.LABELS[TYPES[typ]["kind"]] + raise ValueError(f"set up your {label} app first (Settings › Sign-in, or the connection form): its client id and secret") + ep = external_login.ENDPOINTS[row["kind"]] + state, verifier = secrets.token_urlsafe(24), secrets.token_urlsafe(48) + browser = secrets.token_urlsafe(24) if bound else None + challenge = base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip("=") + redirect_uri = origin.rstrip("/") + external_login.callback_path(row["key"]) + db.q("DELETE external_flow WHERE expires_at < $n", n=store.now()) + db.q( + "CREATE $r CONTENT $d", + r=R("external_flow", auth.sha(state)), + d=store.clean( + { + "provider": row["key"], + "verifier": verifier, + "browser": auth.sha(browser) if browser else None, + "unbound": None if bound else True, + "redirect_uri": redirect_uri, + "origin": origin.rstrip("/"), + "mode": "source", + "account": account, + "next": external_login.safe_next(next_path), + "source": store.clean({"id": sid, "type": typ, "params": params, "name": (name or "").strip()[:80] or None, "by": by}), + "expires_at": _later(external_login.FLOW_MINUTES * 60), + } + ), + ) + q = { + "response_type": "code", + "client_id": row["client_id"], + "redirect_uri": redirect_uri, + "scope": f"openid email {scope}", # who signed in, to show beside the connection + "state": state, + "code_challenge": challenge, + "code_challenge_method": "S256", + "access_type": "offline", # a refresh token, so the connection keeps working + "prompt": "consent", # Google only sends one when asked for consent + } + return ep["authorize"] + "?" + urllib.parse.urlencode(q), browser + + +def _token(cfg, row, form): + """What the provider's token endpoint answers to `form` (with the app's client id and secret).""" + try: + r = httpx.post( + external_login.ENDPOINTS[row["kind"]]["token"], + data={**form, "client_id": row["client_id"], "client_secret": external_login.client_secret(cfg, row)}, + headers={"Accept": "application/json"}, + timeout=TIMEOUT, + ) + tok = r.json() + except (httpx.HTTPError, ValueError): + raise external_login.ExternalLoginError(f"{external_login.LABELS[row['kind']]} didn't answer; try again") from None + if r.status_code >= 400 or not isinstance(tok, dict) or not tok.get("access_token"): + why = (tok.get("error_description") or tok.get("error")) if isinstance(tok, dict) else None + raise external_login.ExternalLoginError( + f"{external_login.LABELS[row['kind']]} refused: {why or r.status_code}; check the app's client id and secret" + ) + return tok + + +def _expiry(tok): + seconds = tok.get("expires_in") + seconds = int(seconds) if isinstance(seconds, (int, float, str)) and str(seconds).isdigit() else 3600 + when = dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=seconds) + return when.isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _email(row, access): + """The signed-in account's address, to show beside the connection; None when the provider won't say.""" + try: + info = httpx.get( + external_login.ENDPOINTS[row["kind"]]["userinfo"], + headers={"Authorization": f"Bearer {access}", "Accept": "application/json"}, + timeout=TIMEOUT, + ).json() + except (httpx.HTTPError, ValueError): + return None + email = info.get("email") if isinstance(info, dict) else None + return str(email)[:200] if email else None + + +def finish(db, cfg, flow, code): + """Swap the code the provider sent back for tokens and connect the source the flow is for: made now when it's new, + else signed in again. Returns the source's id. ExternalLoginError (a ValueError) says why it didn't work.""" + want = flow.get("source") or {} + row = app(db, want.get("type")) + if not row or row["key"] != flow.get("provider"): + raise external_login.ExternalLoginError("the app this sign-in went through was removed; set it up again and sign in again") + tok = _token( + cfg, + row, + {"grant_type": "authorization_code", "code": code, "redirect_uri": flow["redirect_uri"], "code_verifier": flow["verifier"]}, + ) + if not tok.get("refresh_token"): + raise external_login.ExternalLoginError( + f"{external_login.LABELS[row['kind']]} sent no refresh token, so the connection would stop within the hour; sign in again" + ) + email = _email(row, tok["access_token"]) + token = json.dumps( + { + "access_token": tok["access_token"], + "token_type": tok.get("token_type") or "Bearer", + "refresh_token": tok["refresh_token"], + "expiry": _expiry(tok), + }, + separators=(",", ":"), + ) + typ = want["type"] + others = {k: None for k in sources.BACKENDS[typ]["secrets"] if k != "token"} # signed in: no service account + if want.get("id") is not None: + sid = want["id"] + try: + sources.update(db, cfg, sid, params=want.get("params"), secret_values={"token": token, **others}) + except KeyError: + raise external_login.ExternalLoginError("that connection was deleted while you were signing in") from None + else: + label = sources.BACKENDS[typ]["label"] + name = want.get("name") or (f"{label} · {email}" if email else label) + sid = sources.create(db, cfg, name, typ, want.get("params"), {"token": token}, want.get("by")) + db.q( + "UPDATE $r SET oauth = $o", + r=R("storage_source", sid), + o=store.clean({"provider": row["key"], "email": email, "at": store.now()}), + ) + sources.test(db, cfg, sid) + return sid + + +def _when(text): + """An rclone token's expiry (Go writes RFC 3339 with up to nine digits of seconds), or None.""" + m = re.fullmatch(r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)(\.\d+)?(Z|[+-]\d\d:\d\d)", str(text or "")) + if not m: + return None + return dt.datetime.fromisoformat(m[1] + (m[2] or "")[:7] + ("+00:00" if m[3] == "Z" else m[3])) + + +def fresh_token(db, cfg, src, margin=60): + """A working access token for a source signed in here, refreshed at the provider when it runs out within `margin` + seconds (the new one is saved). For code that calls the service itself; rclone refreshes its own. ValueError when + the source wasn't signed in here, or the provider refuses (sign in again).""" + oauth = src.get("oauth") or {} + row = app(db, src["type"]) + if not oauth or not row or row["key"] != oauth.get("provider") or "token" not in (src.get("sealed") or {}): + raise ValueError(f"{src.get('name') or 'this connection'} wasn't signed in through an app set up here; sign in again") + context = f"source:{src['id']}:token" + tok = json.loads(settings.unseal(cfg, src["sealed"]["token"], context)) + when = _when(tok.get("expiry")) + if when and when > dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=margin): + return tok["access_token"] + new = _token(cfg, row, {"grant_type": "refresh_token", "refresh_token": tok.get("refresh_token") or ""}) + tok.update(access_token=new["access_token"], token_type=new.get("token_type") or tok.get("token_type") or "Bearer", expiry=_expiry(new)) + if new.get("refresh_token"): + tok["refresh_token"] = new["refresh_token"] + sealed = settings.seal(cfg, json.dumps(tok, separators=(",", ":")), context) + db.q("UPDATE $r SET sealed.token = $t", r=R("storage_source", src["id"]), t=sealed) + src["sealed"]["token"] = sealed + return tok["access_token"] diff --git a/fastapi_backend/app/domain/sources.py b/fastapi_backend/app/domain/sources.py index a4d846086..88a8f4b26 100644 --- a/fastapi_backend/app/domain/sources.py +++ b/fastapi_backend/app/domain/sources.py @@ -2,9 +2,10 @@ A source is one connection: S3 or S3-compatible, Dropbox, Google Drive, OneDrive, SFTP, SMB, WebDAV, or a folder on this machine (only inside sources.local_roots); or, without rclone, an email account (IMAP), a calendar feed (iCal) or -Google Calendar, whose messages and events are shown as files (feeds.py). A watch maps a folder on a source to a -namespace: new audio, video, documents (PDFs) and images become resources queued for the whole pipeline, their files -staying on the source; new transcripts are imported and queued for analysis. A watch on an email account can carry routing rules that send some +Google Calendar, whose messages and events are shown as files (feeds.py). Google Drive can also be connected by signing in +with Google in the browser, through the Google app set up in Settings › Sign-in (source_oauth.py). A watch maps a folder +on a source to a namespace: new audio, video, documents (PDFs) and images become resources queued for the whole +pipeline, their files staying on the source; new transcripts are imported and queued for analysis. A watch on an email account can carry routing rules that send some messages to other namespaces, or skip them (mail_routes.py). Credentials are stored encrypted and handed to rclone in a private temporary config file per call; OAuth tokens rclone refreshes are saved. """ @@ -22,7 +23,7 @@ import subprocess import tempfile -from . import convert, deletion, feeds, google_auth, ingest, jobs, mail_routes, settings, store +from . import convert, deletion, external_login, feeds, google_auth, ingest, jobs, mail_routes, settings, store R = store.R BACKENDS = { @@ -114,9 +115,14 @@ def obscure(cfg, value): def get(db, sid): - row = db.one("SELECT record::id(id) AS id, name, type, params, sealed, health, created_at FROM $r", r=R("storage_source", sid)) + row = db.one("SELECT record::id(id) AS id, name, type, params, sealed, health, created_at, oauth FROM $r", r=R("storage_source", sid)) if not row or row["type"] not in BACKENDS: # a stream sensor (sensors.py) shares the table, but isn't a source raise KeyError(sid) + if (row.get("oauth") or {}).get("provider"): # signed in through an app set up here: rclone refreshes with that app + try: + row["app"] = external_login.provider(db, row["oauth"]["provider"]) + except KeyError: + pass # the app was removed: the token stops refreshing, and the connection asks to sign in again return row @@ -130,6 +136,10 @@ def _config(cfg, src): if k in spec["secrets"]: v = settings.unseal(cfg, sealed, f"source:{src['id']}:{k}") lines.append(f"{k} = {_one_line(obscure(cfg, v) if k in OBSCURED else v)}") + app = src.get("app") + if app and app.get("client_id") and app.get("client_secret"): # Google refreshes a token only for the app it gave it to + lines.append(f"client_id = {_one_line(app['client_id'])}") + lines.append(f"client_secret = {_one_line(external_login.client_secret(cfg, app))}") return name, "\n".join(lines) + "\n" @@ -387,6 +397,9 @@ def update(db, cfg, sid, name=None, params=None, secret_values=None): d=store.clean({"name": name, "params": {**(src.get("params") or {}), **(params or {})} if params else None}), ) db.q("UPDATE $r SET sealed = $s", r=R("storage_source", sid), s=sealed) # SET, not MERGE: a MERGE can't drop a removed secret + sv = secret_values or {} + if "token" in sv or sv.get("service_account_credentials"): # a pasted token or a key: no longer signed in here + db.q("UPDATE $r SET oauth = NONE", r=R("storage_source", sid)) def remove(db, sid): @@ -399,6 +412,15 @@ def remove(db, sid): ) +def _signed_in(oauth): + """Who a connection signed in as in the browser (source_oauth.py): {provider, email, at}; None for pasted tokens + and keys.""" + if not (oauth or {}).get("provider"): + return None + kind = oauth["provider"] if oauth["provider"] in external_login.LABELS else "oidc" + return {"provider": external_login.LABELS[kind], "email": oauth.get("email"), "at": oauth.get("at")} + + def view(src): spec = BACKENDS[src["type"]] return { @@ -409,6 +431,7 @@ def view(src): "params": src.get("params") or {}, "secrets": {k: {"secret": True, "set": k in (src.get("sealed") or {})} for k in spec["secrets"]}, "oauth": spec.get("oauth"), + "signed_in": _signed_in(src.get("oauth")), "health": src.get("health"), "created_at": src.get("created_at"), } @@ -420,7 +443,9 @@ def list_sources(db): counts[w["source"]] = counts.get(w["source"], 0) + 1 return [ {**view(s), "watches": counts.get(s["id"], 0)} - for s in db.rows("SELECT record::id(id) AS id, name, type, params, sealed, health, created_at FROM storage_source ORDER BY id") + for s in db.rows( + "SELECT record::id(id) AS id, name, type, params, sealed, health, created_at, oauth FROM storage_source ORDER BY id" + ) if s["type"] in BACKENDS ] diff --git a/fastapi_backend/app/schemas/sources.py b/fastapi_backend/app/schemas/sources.py index bd0c110c6..5811e9718 100644 --- a/fastapi_backend/app/schemas/sources.py +++ b/fastapi_backend/app/schemas/sources.py @@ -16,11 +16,19 @@ ) +class SignInInfo(ResponseModel): + kind: str = Field(description="the kind of sign-in provider whose app it signs in through (google)") + label: str + ready: bool = Field(description="that app is set up (Settings › Sign-in), so POST /sources/signin works") + callback_path: str = Field(description="the address, on this site, to register with the service as the app's redirect URI") + + class Backend(ResponseModel): label: str fields: dict[str, str] = Field(description="parameters and their defaults") secrets: list[str] oauth: str | None = Field(default=None, description="the command that produces the token, for OAuth backends") + signin: SignInInfo | None = Field(default=None, description="signing in in the browser instead of pasting a token") class SourceHealth(ResponseModel): @@ -35,6 +43,12 @@ class SecretFlag(ResponseModel): set: bool +class SignedIn(ResponseModel): + provider: str = Field(description="the service it signed in with (Google)") + email: str | None = Field(default=None, description="the account it signed in as") + at: str | None = None + + class Source(ResponseModel): id: int name: str @@ -43,6 +57,7 @@ class Source(ResponseModel): params: dict[str, Any] secrets: dict[str, SecretFlag] oauth: str | None = None + signed_in: SignedIn | None = Field(default=None, description="signed in in the browser (POST /sources/signin), as whom") health: SourceHealth | None = None created_at: str | None = None watches: int = 0 @@ -61,6 +76,19 @@ class SourceUpdate(RequestModel): secrets: dict[str, str | None] | None = Field(default=None, description="null or empty removes a secret") +class SourceSignIn(RequestModel): + type: SourceType | None = Field(default=None, description="the kind of connection to make (drive); leave out with `source`") + source: int | None = Field(default=None, description="a connection to sign in again, instead of making a new one") + name: str | None = Field(default=None, description="the new connection's name; by default the service and the account") + params: dict[str, Any] | None = Field(default=None, description="its options, as for POST /sources (scope, root_folder_id...)") + next: str = Field(default="/connected", description="the web app's page to come back to (by default one that only says how it went)") + + +class SourceSignInLink(ResponseModel): + url: str = Field(description="the service's sign-in page: open it in a browser; it works once, within ten minutes") + bound: bool = Field(description="tied to this browser (a cookie came with it); else the link alone finishes it") + + class SourceCreated(Created): health: SourceHealth diff --git a/fastapi_backend/tests/api/test_google_calendar_source.py b/fastapi_backend/tests/api/test_google_calendar_source.py index 1fb97910f..4e4c46500 100644 --- a/fastapi_backend/tests/api/test_google_calendar_source.py +++ b/fastapi_backend/tests/api/test_google_calendar_source.py @@ -129,6 +129,7 @@ def test_a_shared_calendar_read_as_a_service_account(google, client, db, cfg, ad "fields": {"calendars": "", "impersonate": "", "days_back": "365"}, "secrets": ["token", "service_account_credentials"], "oauth": None, + "signin": None, } # on its own, a service account has no calendars: it's told what to do body = {"name": "team", "type": "gcal", "secrets": {"service_account_credentials": json.dumps(key)}} diff --git a/fastapi_backend/tests/api/test_source_signin.py b/fastapi_backend/tests/api/test_source_signin.py new file mode 100644 index 000000000..837a1b02c --- /dev/null +++ b/fastapi_backend/tests/api/test_source_signin.py @@ -0,0 +1,196 @@ +"""Connecting Google Drive by signing in with Google in the browser (source_oauth.py), against a fake Google.""" + +from __future__ import annotations + +import json +import urllib.parse + +import pytest + +from app.domain import external_login, settings, source_oauth, sources +from tests.fake_oidc import Provider +from tests.helpers import login, make_user + +WEB = {"x-forwarded-host": "localhost:3000", "x-forwarded-proto": "http"} +ORIGIN = "http://localhost:3000" +ADA = {"sub": "ada-1", "email": "ada@gmail.com", "email_verified": True} +APP = {"kind": "google", "client_id": "lens", "client_secret": "s3cret", "enabled": False} + + +@pytest.fixture +def google(monkeypatch): + p = Provider() + monkeypatch.setitem( + external_login.ENDPOINTS, + "google", + {"authorize": p.base + "/authorize", "token": p.base + "/token", "userinfo": p.base + "/userinfo", "scope": "openid email profile"}, + ) + yield p + p.close() + + +@pytest.fixture +def admin(app, client, db, monkeypatch): + monkeypatch.setattr(sources, "test", lambda *a, **k: {"ok": True}) # no Drive to list in tests + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +def _query(url): + return dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query)) + + +def _back(r): + assert r.status_code == 303, r.text + url = urllib.parse.urlsplit(r.headers["location"]) + return f"{url.scheme}://{url.netloc}{url.path}", dict(urllib.parse.parse_qsl(url.query)) + + +def _sign_in(client, google, headers, body, profile=ADA): + r = client.post("/api/v1/sources/signin", json=body, headers={**WEB, **headers}) + assert r.status_code == 200, r.text + code, state = google.issue(r.json()["url"], profile) + return client.get("/api/v1/auth/external/google/callback", params={"code": code, "state": state}, headers=WEB, follow_redirects=False) + + +def test_drive_signs_in_with_google_and_rclone_refreshes_with_that_app(client, db, cfg, admin, google): + drive = client.get("/api/v1/sources/backends", headers=admin).json()["drive"] + assert drive["signin"] == { + "kind": "google", + "label": "Google", + "ready": False, + "callback_path": "/api/v1/auth/external/google/callback", + } + assert client.get("/api/v1/sources/backends", headers=admin).json()["s3"]["signin"] is None + # no Google app yet: it says where to set one up + r = client.post("/api/v1/sources/signin", json={"type": "drive"}, headers={**WEB, **admin}) + assert r.status_code == 400 and "Google app" in r.json()["detail"] + # the app, with signing in to Lens with Google left off + assert client.post("/api/v1/auth/providers", json=APP, headers=admin).status_code == 200 + assert client.get("/api/v1/auth/external").json() == [] + assert client.get("/api/v1/sources/backends", headers=admin).json()["drive"]["signin"]["ready"] + + body = {"type": "drive", "params": {"root_folder_id": "1AbC"}, "next": "/import?tab=source"} + r = client.post("/api/v1/sources/signin", json=body, headers={**WEB, **admin}) + assert r.status_code == 200 and r.json()["bound"] and "lens_external" in r.headers["set-cookie"] + q = _query(r.json()["url"]) + assert q["redirect_uri"] == ORIGIN + "/api/v1/auth/external/google/callback" # the sign-in's own: nothing more to register + assert q["scope"] == "openid email https://www.googleapis.com/auth/drive.readonly" + assert q["access_type"] == "offline" and q["prompt"] == "consent" and q["code_challenge_method"] == "S256" + code, state = google.issue(r.json()["url"], ADA) + page, back = _back( + client.get("/api/v1/auth/external/google/callback", params={"code": code, "state": state}, headers=WEB, follow_redirects=False) + ) + assert page == ORIGIN + "/import" and back["tab"] == "source" and back["connected"] + sid = int(back["connected"]) + + s = next(s for s in client.get("/api/v1/sources", headers=admin).json() if s["id"] == sid) + assert s["name"] == "Google Drive · ada@gmail.com" and s["type"] == "drive" + assert s["signed_in"]["provider"] == "Google" and s["signed_in"]["email"] == "ada@gmail.com" + assert s["secrets"]["token"]["set"] and not s["secrets"]["service_account_credentials"]["set"] + assert s["params"]["root_folder_id"] == "1AbC" and s["params"]["scope"] == "drive.readonly" + refresh = next(iter(google.refresh)) + assert refresh not in json.dumps(db.rows("SELECT * FROM storage_source")) # sealed + + # rclone gets the app's client id and secret: Google refreshes a token only for the app that issued it + _, conf = sources._config(cfg, sources.get(db, sid)) + assert "client_id = lens\n" in conf and "client_secret = s3cret\n" in conf and "root_folder_id = 1AbC\n" in conf + tok = json.loads(next(l for l in conf.splitlines() if l.startswith("token = ")).split(" = ", 1)[1]) + assert tok["refresh_token"] == refresh and tok["token_type"] == "Bearer" and tok["expiry"].endswith("Z") + assert "source.signin" in db.values("SELECT VALUE action FROM audit_log") + + # the link is used once + again = client.get("/api/v1/auth/external/google/callback", params={"code": code, "state": state}, headers=WEB, follow_redirects=False) + assert "already used" in urllib.parse.unquote_plus(again.headers["location"]) + # cancelling at Google comes back with why, and makes nothing + r = client.post("/api/v1/sources/signin", json={"type": "drive"}, headers={**WEB, **admin}) + state = _query(r.json()["url"])["state"] + _, back = _back( + client.get( + "/api/v1/auth/external/google/callback", params={"error": "access_denied", "state": state}, headers=WEB, follow_redirects=False + ) + ) + assert back == {"connect_error": "signing in didn't finish: you cancelled"} + assert len(client.get("/api/v1/sources", headers=admin).json()) == 1 + + +def test_a_link_started_in_one_browser_cant_be_finished_in_another(client, new_client, db, admin, google): + client.post("/api/v1/auth/providers", json=APP, headers=admin) + r = client.post("/api/v1/sources/signin", json={"type": "drive"}, headers={**WEB, **admin}) + code, state = google.issue(r.json()["url"], ADA) + other = new_client() + r = other.get("/api/v1/auth/external/google/callback", params={"code": code, "state": state}, headers=WEB, follow_redirects=False) + assert "another browser" in urllib.parse.unquote_plus(r.headers["location"]) + assert client.get("/api/v1/sources", headers=admin).json() == [] + + +def test_an_agent_gets_a_link_its_person_opens(client, new_client, db, admin, google): + """From an API token the link isn't tied to a browser: the person it's handed to finishes it in theirs, once.""" + client.post("/api/v1/auth/providers", json=APP, headers=admin) + token = client.post("/api/v1/tokens", json={"name": "agent", "scope": "write"}, headers=admin).json()["token"] + agent = {"Authorization": f"Bearer {token}"} + r = client.post("/api/v1/sources/signin", json={"type": "drive", "name": "Ada's Drive"}, headers={**WEB, **agent}) + assert r.status_code == 200 and not r.json()["bound"] and "set-cookie" not in r.headers + code, state = google.issue(r.json()["url"], ADA) + person = new_client() # another browser, with no cookie from the start + _, back = _back( + person.get("/api/v1/auth/external/google/callback", params={"code": code, "state": state}, headers=WEB, follow_redirects=False) + ) + s = client.get("/api/v1/sources", headers=admin).json() + assert back == {"connected": str(s[0]["id"])} and s[0]["name"] == "Ada's Drive" + # a read-only token can't connect anything + ro = client.post("/api/v1/tokens", json={"name": "ro", "scope": "read"}, headers=admin).json()["token"] + r = client.post("/api/v1/sources/signin", json={"type": "drive"}, headers={**WEB, "Authorization": f"Bearer {ro}"}) + assert r.status_code == 403 + + +def test_signing_in_again_replaces_a_service_account_and_a_pasted_token_replaces_the_sign_in(client, db, cfg, admin, google): + client.post("/api/v1/auth/providers", json=APP, headers=admin) + key = { + "type": "service_account", + "private_key": "-----BEGIN " + "PRIVATE KEY-----\nMIIEv...\n-----END " + "PRIVATE KEY-----\n", # split: not a real key + "client_email": "lens@lens-test.iam.gserviceaccount.com", + } + body = {"name": "Team drive", "type": "drive", "secrets": {"service_account_credentials": json.dumps(key)}} + sid = client.post("/api/v1/sources", json=body, headers=admin).json()["id"] + _, back = _back(_sign_in(client, google, admin, {"source": sid, "params": {"scope": "drive"}})) + assert back == {"connected": str(sid)} + s = client.get("/api/v1/sources", headers=admin).json()[0] + assert s["name"] == "Team drive" and s["params"]["scope"] == "drive" and s["signed_in"]["email"] == "ada@gmail.com" + assert s["secrets"]["token"]["set"] and not s["secrets"]["service_account_credentials"]["set"] + assert google.posts[-1]["grant_type"] == "authorization_code" and google.posts[-1]["client_secret"] == "s3cret" + # editing other options keeps the sign-in + client.patch(f"/api/v1/sources/{sid}", json={"params": {"root_folder_id": "1X"}}, headers=admin) + assert client.get("/api/v1/sources", headers=admin).json()[0]["signed_in"] + # a token pasted from rclone is rclone's own: its app refreshes it, not ours + pasted = json.dumps({"access_token": "a", "token_type": "Bearer", "refresh_token": "r", "expiry": "2030-01-01T00:00:00Z"}) + client.patch(f"/api/v1/sources/{sid}", json={"secrets": {"token": pasted}}, headers=admin) + assert client.get("/api/v1/sources", headers=admin).json()[0]["signed_in"] is None + assert "client_id" not in sources._config(cfg, sources.get(db, sid))[1] + # unknown sources and types that don't sign in here + assert client.post("/api/v1/sources/signin", json={"source": 999}, headers={**WEB, **admin}).status_code == 404 + r = client.post("/api/v1/sources/signin", json={"type": "dropbox"}, headers={**WEB, **admin}) + assert r.status_code == 400 and "doesn't sign in here" in r.json()["detail"] + r = client.post("/api/v1/sources/signin", json={"type": "drive", "params": {"scope": "gmail"}}, headers={**WEB, **admin}) + assert r.status_code == 400 and "scope" in r.json()["detail"] + assert client.post("/api/v1/sources/signin", json={}, headers={**WEB, **admin}).status_code == 400 + + +def test_fresh_token_refreshes_it_when_it_runs_out(client, db, cfg, admin, google): + client.post("/api/v1/auth/providers", json=APP, headers=admin) + _sign_in(client, google, admin, {"type": "drive"}) + src = sources.get(db, client.get("/api/v1/sources", headers=admin).json()[0]["id"]) + first = json.loads(settings.unseal(cfg, src["sealed"]["token"], f"source:{src['id']}:token")) + calls = len(google.posts) + assert source_oauth.fresh_token(db, cfg, src) == first["access_token"] and len(google.posts) == calls # still good + newer = source_oauth.fresh_token(db, cfg, src, margin=4000) # as if it ran out within the hour + assert newer != first["access_token"] and google.posts[-1]["grant_type"] == "refresh_token" + kept = json.loads(settings.unseal(cfg, sources.get(db, src["id"])["sealed"]["token"], f"source:{src['id']}:token")) + assert kept["access_token"] == newer and kept["refresh_token"] == first["refresh_token"] # saved, refresh token kept + google.refresh.clear() # revoked at Google + with pytest.raises(ValueError, match="expired or revoked"): + source_oauth.fresh_token(db, cfg, sources.get(db, src["id"]), margin=4000) + # the app removed: nothing refreshes it any more + client.delete("/api/v1/auth/providers/google", headers=admin) + with pytest.raises(ValueError, match="sign in again"): + source_oauth.fresh_token(db, cfg, sources.get(db, src["id"])) diff --git a/fastapi_backend/tests/fake_oidc.py b/fastapi_backend/tests/fake_oidc.py index 2a0808aa3..3e492752d 100644 --- a/fastapi_backend/tests/fake_oidc.py +++ b/fastapi_backend/tests/fake_oidc.py @@ -1,6 +1,7 @@ """A tiny OpenID Connect provider for tests (also answers like GitHub's API): discovery, the token endpoint (checks the -client secret, redirect URI and PKCE verifier) and userinfo. Tests play the browser: they read the state and PKCE -challenge from the sign-in address Lens hands out and call `issue` for the code the provider would send back.""" +client secret, redirect URI and PKCE verifier; hands out refresh tokens and swaps them) and userinfo. Tests play the +browser: they read the state and PKCE challenge from the sign-in address Lens hands out and call `issue` for the code +the provider would send back.""" from __future__ import annotations @@ -18,6 +19,8 @@ def __init__(self, client_id="lens", client_secret="s3cret"): self.client_id, self.client_secret = client_id, client_secret self.codes: dict[str, dict] = {} self.tokens: dict[str, dict] = {} + self.refresh: dict[str, dict] = {} # refresh tokens it gave out, and whose they are + self.posts: list[dict] = [] # every form sent to the token endpoint self.emails: list[dict] = [] # GitHub's /user/emails self.auto: dict | None = None # for a browser: GET /authorize signs in as this profile straight away provider = self @@ -61,17 +64,25 @@ def do_GET(self): def do_POST(self): form = dict(urllib.parse.parse_qsl(self.rfile.read(int(self.headers["Content-Length"])).decode())) + provider.posts.append(form) c = provider.codes.pop(form.get("code"), None) if form.get("client_id") != provider.client_id or form.get("client_secret") != provider.client_secret: return self._json({"error": "invalid_client", "error_description": "bad client credentials"}, 401) + if form.get("grant_type") == "refresh_token": + who = provider.refresh.get(form.get("refresh_token") or "") + if who is None: + return self._json({"error": "invalid_grant", "error_description": "Token has been expired or revoked."}, 400) + token = secrets.token_urlsafe(16) + provider.tokens[token] = who + return self._json({"access_token": token, "token_type": "Bearer", "expires_in": 3599}) if not c or form.get("redirect_uri") != c["redirect_uri"]: return self._json({"error": "invalid_grant"}, 400) verifier = form.get("code_verifier") or "" if base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip("=") != c["challenge"]: return self._json({"error": "invalid_grant", "error_description": "PKCE check failed"}, 400) - token = secrets.token_urlsafe(16) - provider.tokens[token] = c["profile"] - return self._json({"access_token": token, "token_type": "Bearer"}) + token, refresh = secrets.token_urlsafe(16), secrets.token_urlsafe(16) + provider.tokens[token] = provider.refresh[refresh] = c["profile"] + return self._json({"access_token": token, "token_type": "Bearer", "expires_in": 3599, "refresh_token": refresh}) self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) self.base = f"http://127.0.0.1:{self.server.server_address[1]}" diff --git a/nextjs-frontend/__tests__/drive-signin.test.tsx b/nextjs-frontend/__tests__/drive-signin.test.tsx new file mode 100644 index 000000000..f09806301 --- /dev/null +++ b/nextjs-frontend/__tests__/drive-signin.test.tsx @@ -0,0 +1,147 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import "@testing-library/jest-dom"; + +import { Auth, Sources } from "@/app/openapi-client"; +import { ConnectionDialog } from "@/components/sources/connection-dialog"; +import { TooltipProvider } from "@/components/ui/tooltip"; +import { navigate } from "@/lib/navigate"; + +jest.mock("@/app/openapi-client", () => ({ + Sources: { + createSource: jest.fn(), + updateSource: jest.fn(), + testSource: jest.fn(), + deleteSource: jest.fn(), + startSignin: jest.fn(), + listSources: jest.fn(), + }, + Auth: { addLoginProvider: jest.fn() }, +})); +jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); +jest.mock("@/components/ui/toast", () => ({ useToast: () => jest.fn() })); +jest.mock("@/lib/navigate", () => ({ navigate: { assign: jest.fn(), replace: jest.fn() } })); + +/** The same-site channel the /connected page answers on, kept in this test's memory. */ +class Channel { + static open: Channel[] = []; + onmessage: ((e: { data: unknown }) => void) | null = null; + constructor(public name: string) { + Channel.open.push(this); + } + postMessage(data: unknown) { + for (const c of Channel.open) if (c !== this && c.name === this.name) c.onmessage?.({ data }); + } + close() { + Channel.open = Channel.open.filter((c) => c !== this); + } +} + +const ok = (data: unknown) => Promise.resolve({ data, response: { ok: true, status: 200 } }); +const SIGNIN = { kind: "google", label: "Google", ready: true, callback_path: "/api/v1/auth/external/google/callback" }; +const DRIVE = { + label: "Google Drive", + fields: { scope: "drive.readonly", root_folder_id: "", team_drive: "", impersonate: "" }, + secrets: ["token", "service_account_credentials"], + oauth: "rclone authorize drive", +}; + +function show(onSaved: (id: number) => void, signin: typeof SIGNIN | null = SIGNIN) { + const qc = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }); + return render( + + + undefined} + backends={{ drive: { ...DRIVE, signin } }} + onSaved={onSaved} + /> + + , + ); +} + +describe("connecting Google Drive by signing in with Google", () => { + const popup = { location: { href: "" }, close: jest.fn() }; + beforeEach(() => { + Channel.open = []; + (globalThis as { BroadcastChannel?: unknown }).BroadcastChannel = Channel; + jest.spyOn(window, "open").mockReturnValue(popup as unknown as Window); + (Sources.startSignin as jest.Mock) + .mockReset() + .mockImplementation(() => ok({ url: "https://google.test/auth", bound: true })); + (Sources.listSources as jest.Mock).mockReset().mockImplementation(() => + ok([ + { + id: 7, + name: "Google Drive · ada@gmail.com", + type: "drive", + params: {}, + secrets: {}, + signed_in: { provider: "Google", email: "ada@gmail.com" }, + health: { ok: true }, + }, + ]), + ); + (Sources.updateSource as jest.Mock).mockReset().mockImplementation(() => ok({ health: { ok: true } })); + }); + + it("signs in in a small window, then tests and names the connection it made", async () => { + const saved = jest.fn(); + show(saved); + fireEvent.click(screen.getByRole("radio", { name: /Google Drive/ })); + fireEvent.change(screen.getByLabelText(/Root folder ID/), { target: { value: "1AbC" } }); + expect(screen.queryByRole("button", { name: "Test connection" })).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Sign in with Google" })); + await waitFor(() => expect(popup.location.href).toBe("https://google.test/auth")); + const body = (Sources.startSignin as jest.Mock).mock.calls[0][0].body; + expect(body).toMatchObject({ + type: "drive", + name: null, + params: { root_folder_id: "1AbC", scope: "drive.readonly" }, + }); + expect(screen.getByRole("button", { name: "Waiting for Google…" })).toHaveAttribute("aria-disabled", "true"); + // the /connected page, in the window, says it worked + const tag = new URLSearchParams(body.next.split("?")[1]).get("tag"); + act(() => new Channel("lens-connect").postMessage({ tag, connected: 7 })); + expect(await screen.findByLabelText("Name")).toHaveValue("Google Drive · ada@gmail.com"); + expect(screen.getByText("Listed the top folder")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Done" })); + await waitFor(() => expect(saved).toHaveBeenCalledWith(7)); + expect(Sources.createSource).not.toHaveBeenCalled(); + expect(Sources.updateSource).not.toHaveBeenCalled(); // the name it was made with was kept + }); + + it("goes with the whole page where popups are blocked, and shows why it didn't work", async () => { + (window.open as jest.Mock).mockReturnValue(null); + show(jest.fn()); + fireEvent.click(screen.getByRole("radio", { name: /Google Drive/ })); + fireEvent.click(screen.getByRole("button", { name: "Sign in with Google" })); + await waitFor(() => expect(navigate.assign).toHaveBeenCalledWith("https://google.test/auth")); + expect((Sources.startSignin as jest.Mock).mock.calls[0][0].body.next).toBe("/"); + }); + + it("asks for the Google app first, and keeps signing in to Lens with Google off", async () => { + (Auth.addLoginProvider as jest.Mock).mockReset().mockImplementation(() => ok({ key: "google" })); + show(jest.fn(), { ...SIGNIN, ready: false }); + fireEvent.click(screen.getByRole("radio", { name: /Google Drive/ })); + expect(screen.getByText(/http:\/\/localhost\/api\/v1\/auth\/external\/google\/callback/)).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Sign in with Google" })).toHaveAttribute("aria-disabled", "true"); + fireEvent.change(screen.getByLabelText("Client ID"), { target: { value: "123.apps.googleusercontent.com" } }); + fireEvent.change(screen.getByLabelText("Client secret"), { target: { value: "GOCSPX-x" } }); + fireEvent.click(screen.getByRole("button", { name: "Save app" })); + await waitFor(() => + expect((Auth.addLoginProvider as jest.Mock).mock.calls[0][0].body).toEqual({ + kind: "google", + client_id: "123.apps.googleusercontent.com", + client_secret: "GOCSPX-x", + enabled: false, + }), + ); + // or the token rclone printed, as before + fireEvent.click(screen.getByRole("button", { name: "Or paste a token from rclone" })); + expect(screen.getByText(/Run this on a computer with a browser/)).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Test connection" })).toBeInTheDocument(); + }); +}); diff --git a/nextjs-frontend/__tests__/sources-model.test.ts b/nextjs-frontend/__tests__/sources-model.test.ts index 2754e8060..3f708adb2 100644 --- a/nextjs-frontend/__tests__/sources-model.test.ts +++ b/nextjs-frontend/__tests__/sources-model.test.ts @@ -180,6 +180,29 @@ describe("connections", () => { }; expect(emptyForm(DRIVE, saved).driveAuth).toBe("service_account"); }); + it("signs Google Drive in with Google when the Google app is set up, and keeps a pasted token as pasted", () => { + const DRIVE: BackendSpec = { + label: "Google Drive", + fields: { scope: "drive.readonly", root_folder_id: "" }, + secrets: ["token", "service_account_credentials"], + oauth: "rclone authorize drive", + signin: { kind: "google", label: "Google", ready: true, callback_path: "/api/v1/auth/external/google/callback" }, + }; + // signing in with Google: no token to paste + expect(validateForm("drive", DRIVE, emptyForm(DRIVE), false)).toEqual({}); + expect(Object.keys(validateForm("drive", DRIVE, { ...emptyForm(DRIVE), drivePaste: true }, false))).toEqual([ + "token", + ]); + const token = { secret: true, set: true }; + const none = { secret: true, set: false }; + const pasted = { name: "d", params: {}, secrets: { token, service_account_credentials: none } }; + expect(emptyForm(DRIVE, pasted).drivePaste).toBe(true); + const signedIn = { ...pasted, signed_in: { provider: "Google", email: "ada@gmail.com" } }; + expect(emptyForm(DRIVE, signedIn).drivePaste).toBe(false); + expect(sourceSubtitle({ ...signedIn, type: "drive", params: { scope: "drive", root_folder_id: "1x" } })).toBe( + "Drive · ada@gmail.com · full access · one folder", + ); + }); it("takes email accounts and calendar feeds", () => { const IMAP: BackendSpec = { label: "Email (IMAP)", diff --git a/nextjs-frontend/app/(auth)/connected/page.tsx b/nextjs-frontend/app/(auth)/connected/page.tsx new file mode 100644 index 000000000..c76cead8b --- /dev/null +++ b/nextjs-frontend/app/(auth)/connected/page.tsx @@ -0,0 +1,10 @@ +import type { Metadata } from "next"; + +import { ConnectedReturn } from "@/components/sources/connected-return"; + +export const metadata: Metadata = { title: "Connected" }; + +/** Where signing a connection in at its service (Google Drive) comes back to, in the window that sign-in opened. */ +export default function ConnectedPage() { + return ; +} diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index c5089f60c..a6699d3a2 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -2734,6 +2734,8 @@ export type { ShareLink, ShareServer, ShareSite, + SignedIn, + SignInInfo, SigninLink, SigninLinkAnswer, SigninLinkData, @@ -2783,6 +2785,8 @@ export type { SourceImport, SourceImportRequest, SourceImportResult, + SourceSignIn, + SourceSignInLink, SourceUpdate, SourceUpdated, SparqlQuery, @@ -2814,6 +2818,11 @@ export type { SplitSegmentResponse, SplitSegmentResponses, Started, + StartSigninData, + StartSigninError, + StartSigninErrors, + StartSigninResponse, + StartSigninResponses, StartUpgradeData, StartUpgradeError, StartUpgradeErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 8fe7b4f38..cb4ce8a4e 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -1352,6 +1352,9 @@ import type { SplitSegmentData, SplitSegmentErrors, SplitSegmentResponses, + StartSigninData, + StartSigninErrors, + StartSigninResponses, StartUpgradeData, StartUpgradeErrors, StartUpgradeResponses, @@ -7935,7 +7938,7 @@ export class Sources { /** * List Backends * - * The kinds of storage a source can be, with their settings and secrets. + * The kinds of storage a source can be, with their settings and secrets, and how each signs in in the browser. */ public static listBackends( options?: Options, @@ -8006,6 +8009,28 @@ export class Sources { }); } + /** + * Start Signin + * + * Connect Google Drive by signing in with Google (through the Google app in Settings › Sign-in), or sign a + * connection in again (`source`): open the returned address in a browser. The provider sends it back to the web app + * page `next`, with ?connected= (tested; next, watch one of its folders) or ?connect_error=. Started from + * the web app it's tied to that browser; from an API token or app it isn't, so an agent can hand the link to its + * person. Either way it works once, within ten minutes. Audited as `source.signin` when it finishes. + */ + public static startSignin( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/sources/signin", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * Test Source */ diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 9c5374968..a2d463fc1 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -914,6 +914,10 @@ export type Backend = { * the command that produces the token, for OAuth backends */ oauth?: string | null; + /** + * signing in in the browser instead of pasting a token + */ + signin?: SignInInfo | null; [key: string]: unknown; }; @@ -16577,6 +16581,58 @@ export type SharedContext = { [key: string]: unknown; }; +/** + * SignInInfo + */ +export type SignInInfo = { + /** + * Kind + * + * the kind of sign-in provider whose app it signs in through (google) + */ + kind: string; + /** + * Label + */ + label: string; + /** + * Ready + * + * that app is set up (Settings › Sign-in), so POST /sources/signin works + */ + ready: boolean; + /** + * Callback Path + * + * the address, on this site, to register with the service as the app's redirect URI + */ + callback_path: string; + [key: string]: unknown; +}; + +/** + * SignedIn + */ +export type SignedIn = { + /** + * Provider + * + * the service it signed in with (Google) + */ + provider: string; + /** + * Email + * + * the account it signed in as + */ + email?: string | null; + /** + * At + */ + at?: string | null; + [key: string]: unknown; +}; + /** * SigninLink */ @@ -17016,6 +17072,10 @@ export type Source = { * Oauth */ oauth?: string | null; + /** + * signed in in the browser (POST /sources/signin), as whom + */ + signed_in?: SignedIn | null; health?: SourceHealth | null; /** * Created At @@ -17173,6 +17233,64 @@ export type SourceImportResult = { [key: string]: unknown; }; +/** + * SourceSignIn + */ +export type SourceSignIn = { + /** + * Type + * + * the kind of connection to make (drive); leave out with `source` + */ + type?: + "s3" | "dropbox" | "drive" | "onedrive" | "sftp" | "smb" | "webdav" | "local" | "imap" | "ical" | "gcal" | null; + /** + * Source + * + * a connection to sign in again, instead of making a new one + */ + source?: number | null; + /** + * Name + * + * the new connection's name; by default the service and the account + */ + name?: string | null; + /** + * Params + * + * its options, as for POST /sources (scope, root_folder_id...) + */ + params?: { + [key: string]: unknown; + } | null; + /** + * Next + * + * the web app's page to come back to (by default one that only says how it went) + */ + next?: string; +}; + +/** + * SourceSignInLink + */ +export type SourceSignInLink = { + /** + * Url + * + * the service's sign-in page: open it in a browser; it works once, within ten minutes + */ + url: string; + /** + * Bound + * + * tied to this browser (a cookie came with it); else the link alone finishes it + */ + bound: boolean; + [key: string]: unknown; +}; + /** * SourceUpdate */ @@ -32590,6 +32708,31 @@ export type UpdateSourceResponses = { export type UpdateSourceResponse = UpdateSourceResponses[keyof UpdateSourceResponses]; +export type StartSigninData = { + body: SourceSignIn; + path?: never; + query?: never; + url: "/api/v1/sources/signin"; +}; + +export type StartSigninErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type StartSigninError = StartSigninErrors[keyof StartSigninErrors]; + +export type StartSigninResponses = { + /** + * Successful Response + */ + 200: SourceSignInLink; +}; + +export type StartSigninResponse = StartSigninResponses[keyof StartSigninResponses]; + export type TestSourceData = { body?: never; path: { diff --git a/nextjs-frontend/components/settings/sign-in-providers.tsx b/nextjs-frontend/components/settings/sign-in-providers.tsx index a7c028b1e..3fe0c6f90 100644 --- a/nextjs-frontend/components/settings/sign-in-providers.tsx +++ b/nextjs-frontend/components/settings/sign-in-providers.tsx @@ -88,6 +88,7 @@ export function SignInProviders() {
{p.people === 1 ? "1 person connected" : `${p.people} people connected`} + {p.kind === "google" && " · also connects Google Drive"}
| null>(null); + + useEffect(() => { + const q = new URLSearchParams(window.location.search); + const connected = q.get("connected"); + const now = connected + ? { connected: Number(connected) } + : { error: q.get("connect_error") || "Signing in didn’t finish." }; + setGot(now); + const tag = q.get("tag"); + if (tag && typeof BroadcastChannel !== "undefined") { + const channel = new BroadcastChannel(CHANNEL); + channel.postMessage({ tag, ...now } satisfies Connected); + channel.close(); + window.close(); // only closes a window a page opened; otherwise this page stays and says how it went + } + }, []); + + if (!got) return {null}; + return ( + + {got.connected ? "Pick a folder to import" : "Back to Lens"} + + } + > + {got.error ? ( + {sentence(got.error)} + ) : ( +

+ Lens can read it now. You can close this window, or pick a folder whose files Lens should import. +

+ )} +
+ ); +} diff --git a/nextjs-frontend/components/sources/connection-dialog.tsx b/nextjs-frontend/components/sources/connection-dialog.tsx index f292d4f39..517483708 100644 --- a/nextjs-frontend/components/sources/connection-dialog.tsx +++ b/nextjs-frontend/components/sources/connection-dialog.tsx @@ -6,6 +6,7 @@ import { useEffect, useState, type ReactNode } from "react"; import { Sources } from "@/app/openapi-client"; import type { Source, SourceHealth } from "@/app/openapi-client/types.gen"; import { ConnectionFields } from "@/components/sources/connection-fields"; +import { signIn } from "@/components/sources/signin"; import { TYPE_ICON } from "@/components/sources/source-icons"; import { TYPE_NAME, @@ -53,7 +54,8 @@ function StepTrail({ step, editing }: { step: Step; editing: boolean }) { /** SO2: add a connection (choose type → details → test, named there), or edit one (details → test). A type with * nothing to fill in (a folder on this machine) is added, tested and named at once: the test step only shows if it - * fails. */ + * fails. Google Drive signs in with Google in a small window instead of testing a pasted token: the connection is + * made (and tested) when Google sends that window back, and the dialog goes on to the test step. */ export function ConnectionDialog({ open, onOpenChange, @@ -84,8 +86,12 @@ export function ConnectionDialog({ const [createdName, setCreatedName] = useState(""); const [result, setResult] = useState<{ health: SourceHealth; - ms: number; + ms: number | null; } | null>(null); + /** Signed in with Google here, as this address ("" when Google didn't say); null when not. */ + const [signedInAs, setSignedInAs] = useState(null); + /** Waiting for the Google window; aborting it cancels. */ + const [waiting, setWaiting] = useState(null); /** Secrets saved during this dialog (the source prop doesn't know about them yet). */ const [savedKeys, setSavedKeys] = useState>(new Set()); const spec = backends[type]; @@ -111,7 +117,10 @@ export function ConnectionDialog({ setCreatedId(source?.id ?? null); setResult(null); setSavedKeys(new Set()); + setSignedInAs(source?.signed_in ? (source.signed_in.email ?? "") : null); + setWaiting(null); }, [open]); + useEffect(() => () => waiting?.abort(), [waiting]); const refresh = () => { void qc.invalidateQueries({ queryKey: ["sources"] }); @@ -213,6 +222,45 @@ export function ConnectionDialog({ }, }); + // Google Drive with a Google account signs in with Google, unless a token is pasted or it's already signed in and + // only its options change + const viaGoogle = type === "drive" && form.driveAuth === "token" && !form.drivePaste && Boolean(spec?.signin); + const needSignIn = viaGoogle && !(saved && isSet("token") && signedInAs != null); + + const startSignIn = async () => { + const e = validateForm(type, spec, form, saved, isSet); + setErrors(e); + if (Object.keys(e).length) return; + const ctl = new AbortController(); + setWaiting(ctl); + try { + const { params } = buildPayload(spec, form, type); + const got = await signIn( + client, + createdId != null ? { source: createdId, params } : { type, params, name: form.name.trim() || null }, + ctl.signal, + ); + if (got.error) { + if (got.error !== "cancelled") setErrors({ _: got.error }); + return; + } + const id = got.connected!; + const made = (await data(Sources.listSources({ client }))).find((s) => s.id === id); + setCreatedId(id); + setCreatedName(made?.name ?? ""); + setSignedInAs(made?.signed_in ? (made.signed_in.email ?? "") : ""); + setSavedKeys((prev) => new Set([...prev, "token", "-service_account_credentials"])); + setForm((f) => ({ ...f, secrets: {}, tokenText: "", name: editing ? f.name : (made?.name ?? f.name) })); + setResult({ health: made?.health ?? { ok: false, error: "Not tested yet" }, ms: null }); + setStep("test"); + refresh(); + } catch (err) { + setErrors({ _: err instanceof Error ? err.message : String(err) }); + } finally { + setWaiting(null); + } + }; + const goTest = () => { const e = validateForm(type, spec, form, saved, isSet); setErrors(e); @@ -316,6 +364,9 @@ export function ConnectionDialog({ saved={saved} isSet={isSet} replaceToken={replaceToken} + signedIn={signedInAs} + onSignIn={() => void startSignIn()} + signingIn={Boolean(waiting)} /> ) : null} {errors._ && ( @@ -335,12 +386,23 @@ export function ConnectionDialog({ Back )} - - + {needSignIn ? ( + + ) : ( + + )} ); } else if (step === "test") { @@ -364,7 +426,7 @@ export function ConnectionDialog({ ? "Listed the top folder" : "Couldn’t list the top folder"} - {result ? `${result.ms} ms` : ""} + {result?.ms != null ? `${result.ms} ms` : ""} {result && !ok && (
diff --git a/nextjs-frontend/components/sources/connection-fields.tsx b/nextjs-frontend/components/sources/connection-fields.tsx index 9dccb0e76..b76510126 100644 --- a/nextjs-frontend/components/sources/connection-fields.tsx +++ b/nextjs-frontend/components/sources/connection-fields.tsx @@ -14,6 +14,7 @@ import { imapHost, parseServiceAccountKey, } from "@/components/sources/source-model"; +import { GoogleAppSetup } from "@/components/sources/google-app-setup"; import { TokenPaste } from "@/components/sources/token-paste"; import { Button } from "@/components/ui/button"; import { Field, Input, SecretField, Select, Textarea } from "@/components/ui/field"; @@ -35,6 +36,9 @@ export function ConnectionFields({ saved, isSet, replaceToken, + signedIn, + onSignIn, + signingIn, }: { type: SourceType; spec: BackendSpec; @@ -45,6 +49,11 @@ export function ConnectionFields({ saved: boolean; isSet: (key: string) => boolean; replaceToken?: boolean; + /** Signed in with Google here, as this address ("" when Google didn't say); null when not. */ + signedIn?: string | null; + /** Sign in (again) with Google in the browser; shown while a saved connection can also just be saved. */ + onSignIn?: () => void; + signingIn?: boolean; }) { const [pasting, setPasting] = useState(Boolean(replaceToken)); const editing = saved; @@ -216,6 +225,45 @@ export function ConnectionFields({ return token; case "drive": { const serviceAccount = serviceAccountKey("Share the folders Lens should read with that address."); + const google = spec.signin; + const link = (label: string, paste: boolean) => ( + + ); + // a Google account signs in with Google here (through the admin's Google app), or pastes rclone's token + const googleAccount = + form.drivePaste || !google ? ( +
+ {token} + {google && link("Sign in with Google instead", false)} +
+ ) : google.ready ? ( +
+
+ + {signedIn != null + ? `Signed in with Google${signedIn ? ` as ${signedIn}` : ""}. Sign in again to use another account.` + : "You sign in with Google in a small window. Lens keeps the token encrypted and renews it."} + + {signedIn != null && onSignIn && ( + + )} +
+ {link("Paste a token from rclone instead", true)} +
+ ) : ( +
+ + {link("Or paste a token from rclone", true)} +
+ ); return (
- {form.driveAuth === "service_account" ? serviceAccount : token} + {form.driveAuth === "service_account" ? serviceAccount : googleAccount} {select("scope", DRIVE_SCOPES)} {text("root_folder_id", { mono: true, diff --git a/nextjs-frontend/components/sources/google-app-setup.tsx b/nextjs-frontend/components/sources/google-app-setup.tsx new file mode 100644 index 000000000..3ecc08ac8 --- /dev/null +++ b/nextjs-frontend/components/sources/google-app-setup.tsx @@ -0,0 +1,102 @@ +"use client"; + +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { useState } from "react"; + +import { Auth } from "@/app/openapi-client"; +import type { SignInInfo } from "@/app/openapi-client/types.gen"; +import { Button } from "@/components/ui/button"; +import { Field, Input } from "@/components/ui/field"; +import { CodeBlock } from "@/components/ui/states"; +import { data, useApiClient } from "@/lib/api/browser"; + +/** + * Signing in with Google needs the admin's own Google app (an OAuth client), once. It's the app people can sign in to + * Lens with too (Settings › Sign-in), saved here with that left off, so the redirect URI is the sign-in's own. + */ +export function GoogleAppSetup({ signin }: { signin: SignInInfo }) { + const client = useApiClient(); + const qc = useQueryClient(); + const [id, setId] = useState(""); + const [secret, setSecret] = useState(""); + const save = useMutation({ + mutationFn: () => + data( + Auth.addLoginProvider({ + client, + body: { kind: "google", client_id: id.trim(), client_secret: secret.trim(), enabled: false }, + }), + ), + onSuccess: () => { + void qc.invalidateQueries({ queryKey: ["source-backends"] }); + void qc.invalidateQueries({ queryKey: ["login-providers"] }); + }, + }); + const redirect = + typeof window === "undefined" ? signin.callback_path : `${window.location.origin}${signin.callback_path}`; + const ready = Boolean(id.trim() && secret.trim()); + const enter = (e: React.KeyboardEvent) => { + if (e.key === "Enter" && ready) { + e.preventDefault(); + save.mutate(); + } + }; + return ( +
+

To sign in with Google, add your Google app once

+
    +
  1. In Google Cloud console › APIs & Services, enable the Google Drive API.
  2. +
  3. + On the OAuth consent screen, publish the app. While it’s in testing, Google signs Lens out after 7 days. +
  4. +
  5. Credentials › Create OAuth client ID › Web application, with this redirect URI:
  6. +
+ +
+ + {(f) => ( + setId(e.target.value)} + onKeyDown={enter} + autoComplete="off" + spellCheck={false} + /> + )} + + + {(f) => ( + setSecret(e.target.value)} + onKeyDown={enter} + autoComplete="new-password" + /> + )} + +
+ {save.error && ( +

+ {save.error.message} +

+ )} +
+ + + Signing in to Lens with Google stays off until you turn it on in Settings › Sign-in. + +
+
+ ); +} diff --git a/nextjs-frontend/components/sources/signin.ts b/nextjs-frontend/components/sources/signin.ts new file mode 100644 index 000000000..b2354d3be --- /dev/null +++ b/nextjs-frontend/components/sources/signin.ts @@ -0,0 +1,83 @@ +"use client"; + +import { useEffect, useRef } from "react"; + +import { Sources } from "@/app/openapi-client"; +import type { Client } from "@/app/openapi-client/client"; +import type { SourceSignIn } from "@/app/openapi-client/types.gen"; +import { data } from "@/lib/api/browser"; +import { navigate } from "@/lib/navigate"; + +/** What the /connected page tells the window that opened the sign-in, over this channel (same site only). */ +export const CHANNEL = "lens-connect"; +export type Connected = { tag: string; connected?: number; error?: string }; + +/** + * Connect a source by signing in at the service (POST /sources/signin): Google's page opens in a small window, so the + * dialog behind it keeps its place, and this resolves with the connection's id (or why it didn't work) when that + * window comes back to /connected. It waits until then, or until `signal` aborts (the person cancels): a window that + * went through Google can look closed to this page while it's still open. Where popups are blocked the whole page goes + * instead, and comes back to this page with ?connected= or ?connect_error= (useSignInReturn). + */ +export async function signIn( + client: Client, + body: Omit, + signal?: AbortSignal, +): Promise { + const tag = Math.random().toString(36).slice(2, 10); + // opened now, in the click: a window opened after the request returns would be blocked as a popup + const popup = window.open("", "lens-connect", "popup,width=520,height=680"); + const q = new URLSearchParams(window.location.search); + q.delete("connected"); + q.delete("connect_error"); + const here = `${window.location.pathname}${q.size ? `?${q}` : ""}`; + let url: string; + try { + ({ url } = await data( + Sources.startSignin({ client, body: { ...body, next: popup ? `/connected?tag=${tag}` : here } }), + )); + } catch (e) { + popup?.close(); + throw e; + } + if (!popup) { + navigate.assign(url); + return new Promise(() => undefined); // the page is leaving + } + popup.location.href = url; + return new Promise((resolve) => { + const channel = typeof BroadcastChannel === "undefined" ? null : new BroadcastChannel(CHANNEL); + const done = (got: Connected) => { + channel?.close(); + resolve(got); + }; + if (channel) + channel.onmessage = (e: MessageEvent) => { + if (e.data?.tag === tag) done(e.data); + }; + signal?.addEventListener("abort", () => { + popup.close(); + done({ tag, error: "cancelled" }); + }); + }); +} + +/** + * Back from signing in with the whole page (popups blocked): the page it started on has ?connected= or + * ?connect_error=. Hands either over once and drops both from the address. + */ +export function useSignInReturn(on: (got: { connected?: number; error?: string }) => void, enabled = true) { + const seen = useRef(false); + useEffect(() => { + if (!enabled || seen.current) return; + const q = new URLSearchParams(window.location.search); + const connected = q.get("connected"); + const error = q.get("connect_error"); + if (!connected && !error) return; + seen.current = true; + q.delete("connected"); + q.delete("connect_error"); + window.history.replaceState(window.history.state, "", `${window.location.pathname}${q.size ? `?${q}` : ""}`); + on(error ? { error } : { connected: Number(connected) }); + }, [enabled]); +} diff --git a/nextjs-frontend/components/sources/source-model.ts b/nextjs-frontend/components/sources/source-model.ts index 686806b06..fc2a332c0 100644 --- a/nextjs-frontend/components/sources/source-model.ts +++ b/nextjs-frontend/components/sources/source-model.ts @@ -2,7 +2,7 @@ * Storage connections and watched folders as the Sources screens show them: labels, one-line summaries, health in * the backend's own words, the rclone token paste check, and what a watched folder's settings mean in words. */ -import type { Source, Watch, WatchPreview } from "@/app/openapi-client/types.gen"; +import type { SignInInfo, Source, Watch, WatchPreview } from "@/app/openapi-client/types.gen"; import { plural } from "@/lib/format"; export type SourceType = Source["type"]; @@ -130,8 +130,8 @@ export function imapHost(email: string): string { const str = (v: unknown) => (v == null ? "" : String(v)).trim(); -/** "S3 · AWS eu-central-1", "SFTP · recorder@calls-gw:22", "Drive · read only"... */ -export function sourceSubtitle(s: Pick): string { +/** "S3 · AWS eu-central-1", "SFTP · recorder@calls-gw:22", "Drive · ada@gmail.com · read only"... */ +export function sourceSubtitle(s: Pick & { signed_in?: Source["signed_in"] }): string { const p = (s.params ?? {}) as Record; switch (s.type) { case "s3": { @@ -141,7 +141,14 @@ export function sourceSubtitle(s: Pick): string { case "dropbox": return "Dropbox · token"; case "drive": - return `Drive · ${str(p.scope) === "drive" ? "full access" : "read only"}${str(p.root_folder_id) ? " · one folder" : ""}`; + return [ + "Drive", + str(s.signed_in?.email), + str(p.scope) === "drive" ? "full access" : "read only", + str(p.root_folder_id) ? "one folder" : "", + ] + .filter(Boolean) + .join(" · "); case "onedrive": return `OneDrive · token${str(p.drive_type) ? ` · ${str(p.drive_type)}` : ""}`; case "sftp": @@ -416,6 +423,8 @@ export type BackendSpec = { fields: Record; secrets: string[]; oauth?: string | null; + /** Signs in in the browser (Google Drive, through the Google app in Settings › Sign-in), instead of a pasted token. */ + signin?: SignInInfo | null; }; export type ConnForm = { @@ -430,6 +439,8 @@ export type ConnForm = { /** Google Drive and Google Calendar sign in as a Google account (a token), or as a service account (its JSON key * file). */ driveAuth: "token" | "service_account"; + /** A Google account's token pasted from `rclone authorize`, rather than signing in with Google here. */ + drivePaste: boolean; }; export type ServiceAccountKey = { ok: true; email: string } | { ok: false; error: string }; @@ -475,7 +486,10 @@ export function parseServiceAccountKey(text: string): ServiceAccountKey { return { ok: true, email: o.client_email }; } -export function emptyForm(spec: BackendSpec, source?: Pick | null): ConnForm { +export function emptyForm( + spec: BackendSpec, + source?: (Pick & { signed_in?: Source["signed_in"] }) | null, +): ConnForm { const params: Record = {}; for (const [k, def] of Object.entries(spec.fields)) params[k] = str((source?.params as Record | undefined)?.[k] ?? def); @@ -488,6 +502,8 @@ export function emptyForm(spec: BackendSpec, source?: Pick + void sources.refetch().then((r) => { + const made = ((r.data ?? []) as Source[]).find((x) => x.id === id); + if (!made) return; + toast({ tone: "green", title: "Connected", body: made.name }); + setSelected(id); + if (!((watches.data ?? []) as Watch[]).some((w) => w.source === id)) setBrowsing(made); + }); + useSignInReturn( + (got) => + got.connected + ? connected(got.connected) + : toast({ tone: "red", title: "Couldn’t connect", body: got.error ?? "Signing in didn’t finish." }), + admin, + ); + // Sign in again: Google's window has to open in the click itself, so this isn't a mutation (those start a tick later) + const [againId, setAgainId] = useState(null); + const signInAgain = (s: Source) => { + setAgainId(s.id); + signIn(client, { source: s.id }) + .then((got) => { + if (got.error && got.error !== "cancelled") toast({ tone: "red", title: "Couldn’t sign in", body: got.error }); + else if (got.connected) toast({ tone: "green", title: "Signed in again", body: s.name }); + }) + .catch((e: Error) => toast({ tone: "red", title: "Couldn’t sign in", body: e.message })) + .finally(() => { + setAgainId(null); + void qc.invalidateQueries({ queryKey: ["sources"] }); + }); + }; + const test = useMutation({ mutationFn: (s: Source) => data(Sources.testSource({ client, path: { sid: s.id } })), onSuccess: (h, s) => { @@ -164,12 +197,19 @@ export function SourcesPage() { const actions = (s: Source) => { const bad = s.health && !s.health.ok; const oauth = Boolean(s.oauth); + const signedIn = Boolean(s.signed_in); return (
e.stopPropagation()}> {bad && oauth && /token|auth|expired|unauthori[sz]ed|401|invalid_grant/i.test(s.health?.error ?? "") ? ( - + signedIn ? ( + + ) : ( + + ) ) : bad || !s.health ? (