From 676d9b15521d0a067e548789e2339af4a11cecee Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 19:50:49 +0000 Subject: [PATCH 1/2] fix(sources): let Google Drive sign in with a service account key file Drive connections only took a token pasted from rclone authorize, so a service account JSON key had nowhere to go (the API refused it as an unknown option). Drive now takes service_account_credentials (checked and stored on one line, as rclone reads it), plus team_drive and impersonate. The connection dialog gets a Google account / Service account switch with a key file upload. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01G2TAAG5yQ66RzoXMi3MZdV --- docs/processing.md | 3 + fastapi_backend/app/domain/sources.py | 22 +++++-- fastapi_backend/tests/api/test_sources.py | 37 ++++++++++++ .../__tests__/sources-model.test.ts | 41 +++++++++++++ .../components/sources/connection-dialog.tsx | 2 +- .../components/sources/connection-fields.tsx | 58 ++++++++++++++++++- .../components/sources/source-model.ts | 54 +++++++++++++++-- 7 files changed, 206 insertions(+), 11 deletions(-) diff --git a/docs/processing.md b/docs/processing.md index db1c6eb5c..bcfbe7a15 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -262,6 +262,9 @@ files, then Import (`POST /api/v1/import/source`). The listing marks files that - Credentials are encrypted in the database (AES-GCM, key from `ARCHIVE_SECRET_KEY` or `data_dir/secret.key`) and given to rclone in a private temporary config file per call. For Dropbox, Drive and OneDrive, paste the token from `rclone authorize dropbox` (or drive, onedrive); tokens rclone refreshes are saved back. +- Google Drive can also sign in as a service account: choose Service account and upload its JSON key file + (`service_account_credentials`). Share the folders Lens should read with the account's address, or give a shared + drive ID (`team_drive`); in Google Workspace, `impersonate` reads a user's own Drive through domain-wide delegation. - Folders on this machine can only be watched inside `sources.local_roots`, and the rclone binary can only be set in archive.yaml: the web app can neither open up the server's disk nor choose what runs. diff --git a/fastapi_backend/app/domain/sources.py b/fastapi_backend/app/domain/sources.py index 835b02223..5ee4519f2 100644 --- a/fastapi_backend/app/domain/sources.py +++ b/fastapi_backend/app/domain/sources.py @@ -34,8 +34,8 @@ "dropbox": {"label": "Dropbox", "fields": {}, "secrets": ["token"], "oauth": "rclone authorize dropbox"}, "drive": { "label": "Google Drive", - "fields": {"scope": "drive.readonly", "root_folder_id": ""}, - "secrets": ["token"], + "fields": {"scope": "drive.readonly", "root_folder_id": "", "team_drive": "", "impersonate": ""}, + "secrets": ["token", "service_account_credentials"], # signs in with a token, or as a service account "oauth": "rclone authorize drive", }, "onedrive": { @@ -314,10 +314,24 @@ def _check_params(typ, params, secret_values): return spec +def _secret(k, v): + """A secret as it's stored: a service account's key file is checked and kept on one line, as rclone reads it.""" + if k != "service_account_credentials": + return str(v) + try: + key = json.loads(v) + except ValueError: + raise ValueError("the service account key isn't JSON: upload the .json key file Google Cloud gave you") from None + if not isinstance(key, dict) or key.get("type") != "service_account" or not key.get("client_email") or not key.get("private_key"): + raise ValueError("that isn't a service account key: it needs type service_account, client_email and private_key") + return json.dumps(key, separators=(",", ":")) + + def create(db, cfg, name, typ, params=None, secret_values=None, user=None): spec = _check_params(typ, params, secret_values) + plain = {k: _secret(k, v) for k, v in (secret_values or {}).items() if isinstance(v, str) and v} sid = db.next_id("storage_source") - sealed = {k: settings.seal(cfg, str(v), f"source:{sid}:{k}") for k, v in (secret_values or {}).items() if isinstance(v, str) and v} + sealed = {k: settings.seal(cfg, v, f"source:{sid}:{k}") for k, v in plain.items()} db.q( "CREATE $r CONTENT $d", r=R("storage_source", sid), @@ -343,7 +357,7 @@ def update(db, cfg, sid, name=None, params=None, secret_values=None): if v is None or v == "": sealed.pop(k, None) elif isinstance(v, str): - sealed[k] = settings.seal(cfg, v, f"source:{sid}:{k}") + sealed[k] = settings.seal(cfg, _secret(k, v), f"source:{sid}:{k}") db.q( "UPDATE $r MERGE $d", r=R("storage_source", sid), diff --git a/fastapi_backend/tests/api/test_sources.py b/fastapi_backend/tests/api/test_sources.py index 319a41858..28516ef67 100644 --- a/fastapi_backend/tests/api/test_sources.py +++ b/fastapi_backend/tests/api/test_sources.py @@ -283,3 +283,40 @@ def test_importing_chosen_files_of_a_source(client, new_client, db, cfg, folder, later = client.post(url, json={**body, "paths": [str(inbox / "later.txt")], "collection": shelf}, headers=h).json()["results"] assert later[0]["status"] == "queued", later assert db.one("SELECT collection FROM $r", r=R("recording", later[0]["recording"]))["collection"] == shelf + + +def test_google_drive_as_a_service_account(client, db, cfg, monkeypatch): + """A Drive connection signs in with a service account's key file: checked, sealed, and handed to rclone on one line.""" + monkeypatch.setattr(sources, "test", lambda *a, **k: {"ok": True}) # no Google to reach in tests + make_user(db, "root@x.io", "root password 1", admin=True) + h = login(client, "root@x.io", "root password 1") + assert "service_account_credentials" in client.get("/api/v1/sources/backends", headers=h).json()["drive"]["secrets"] + key = { + "type": "service_account", + "project_id": "lens-test", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEv...\n-----END PRIVATE KEY-----\n", + "client_email": "lens@lens-test.iam.gserviceaccount.com", + } + body = { + "name": "Drive", + "type": "drive", + "params": {"team_drive": "0ABC"}, + "secrets": {"service_account_credentials": json.dumps(key, indent=2)}, + } + r = client.post("/api/v1/sources", json=body, headers=h) + assert r.status_code == 200, r.text + sid = r.json()["id"] + drive = next(s for s in client.get("/api/v1/sources", headers=h).json() if s["id"] == sid) + assert drive["secrets"]["service_account_credentials"]["set"] and not drive["secrets"]["token"]["set"] + assert "lens-test.iam" not in str(db.rows("SELECT * FROM storage_source")) + + _, conf = sources._config(cfg, sources.get(db, sid)) + line = next(l for l in conf.splitlines() if l.startswith("service_account_credentials = ")) + assert json.loads(line.split(" = ", 1)[1]) == key # one line rclone can read back, the key's own \n kept + assert "team_drive = 0ABC" in conf and "token" not in conf + + for bad in ("not json", json.dumps({"type": "authorized_user", "client_email": "x", "private_key": "y"})): + r = client.post("/api/v1/sources", json={**body, "secrets": {"service_account_credentials": bad}}, headers=h) + assert r.status_code == 400 and "service account" in r.text + r = client.patch(f"/api/v1/sources/{sid}", json={"secrets": {"service_account_credentials": "{}"}}, headers=h) + assert r.status_code == 400 diff --git a/nextjs-frontend/__tests__/sources-model.test.ts b/nextjs-frontend/__tests__/sources-model.test.ts index 2b45d8275..fde33954d 100644 --- a/nextjs-frontend/__tests__/sources-model.test.ts +++ b/nextjs-frontend/__tests__/sources-model.test.ts @@ -7,6 +7,7 @@ import { imapHost, nextScan, parseRcloneToken, + parseServiceAccountKey, pickupText, sourceSubtitle, splitPatterns, @@ -132,6 +133,46 @@ describe("connections", () => { token: '{"access_token":"a"}', }); }); + it("signs Google Drive in with a token or a service account key file", () => { + const DRIVE: BackendSpec = { + label: "Google Drive", + fields: { scope: "drive.readonly", root_folder_id: "", team_drive: "", impersonate: "" }, + secrets: ["token", "service_account_credentials"], + oauth: "rclone authorize drive", + }; + const key = JSON.stringify({ + type: "service_account", + client_email: "lens@p.iam.gserviceaccount.com", + private_key: "K", + }); + expect(parseServiceAccountKey(key)).toEqual({ ok: true, email: "lens@p.iam.gserviceaccount.com" }); + expect(parseServiceAccountKey("nope").ok).toBe(false); + expect(parseServiceAccountKey('{"access_token":"a"}').ok).toBe(false); + + const sa = { ...emptyForm(DRIVE), driveAuth: "service_account" as const }; + expect(Object.keys(validateForm("drive", DRIVE, sa, false))).toEqual(["service_account_credentials"]); + const withKey = { ...sa, secrets: { service_account_credentials: key } }; + expect(validateForm("drive", DRIVE, withKey, false)).toEqual({}); + expect(validateForm("drive", DRIVE, sa, true, (k) => k === "service_account_credentials")).toEqual({}); + expect(buildPayload(DRIVE, withKey, "drive").secrets).toEqual({ service_account_credentials: key, token: null }); + + // a pasted token replaces a saved key, and the form opens on whichever is saved + const tok = { + ...emptyForm(DRIVE), + secrets: { service_account_credentials: key }, + tokenText: '{"access_token":"a"}', + }; + expect(buildPayload(DRIVE, tok, "drive").secrets).toEqual({ + token: '{"access_token":"a"}', + service_account_credentials: null, + }); + const saved = { + name: "d", + params: {}, + secrets: { token: { secret: true, set: false }, service_account_credentials: { secret: true, set: true } }, + }; + expect(emptyForm(DRIVE, saved).driveAuth).toBe("service_account"); + }); it("takes email accounts and calendar feeds", () => { const IMAP: BackendSpec = { label: "Email (IMAP)", diff --git a/nextjs-frontend/components/sources/connection-dialog.tsx b/nextjs-frontend/components/sources/connection-dialog.tsx index b6d402864..1704fdd80 100644 --- a/nextjs-frontend/components/sources/connection-dialog.tsx +++ b/nextjs-frontend/components/sources/connection-dialog.tsx @@ -213,7 +213,7 @@ export function ConnectionDialog({ ? `Edit “${source?.name}”` : TYPE_NAME[type] === "S3 / compatible" ? "S3 or S3-compatible" - : spec?.oauth + : spec?.oauth && type !== "drive" ? `${TYPE_NAME[type]} · paste a token` : (spec?.label ?? TYPE_NAME[type]) : step === "test" diff --git a/nextjs-frontend/components/sources/connection-fields.tsx b/nextjs-frontend/components/sources/connection-fields.tsx index c79b15793..553a8261d 100644 --- a/nextjs-frontend/components/sources/connection-fields.tsx +++ b/nextjs-frontend/components/sources/connection-fields.tsx @@ -12,6 +12,7 @@ import { type ConnForm, type SourceType, imapHost, + parseServiceAccountKey, } from "@/components/sources/source-model"; import { TokenPaste } from "@/components/sources/token-paste"; import { Button } from "@/components/ui/button"; @@ -180,10 +181,48 @@ export function ConnectionFields({ ); case "dropbox": return token; - case "drive": + case "drive": { + const keyText = form.secrets.service_account_credentials; + const key = keyText ? parseServiceAccountKey(keyText) : null; + const serviceAccount = ( + + {({ id, describedBy }) => ( + { + const f = e.target.files?.[0]; + setSecret("service_account_credentials", f ? await f.text() : undefined); + }} + /> + )} + + ); return (
- {token} + onChange({ ...form, driveAuth: v as ConnForm["driveAuth"] })} + items={[ + { value: "token", label: "Google account" }, + { value: "service_account", label: "Service account" }, + ]} + /> + {form.driveAuth === "service_account" ? serviceAccount : token} {select("scope", DRIVE_SCOPES)} {text("root_folder_id", { mono: true, @@ -191,8 +230,23 @@ export function ConnectionFields({ optional: true, hint: "From the folder’s URL; limits the connection to that folder", })} + {form.driveAuth === "service_account" && ( +
+ {text("team_drive", { + mono: true, + optional: true, + hint: "Read a shared drive the account is a member of", + })} + {text("impersonate", { + optional: true, + placeholder: "you@company.com", + hint: "Workspace only: needs domain-wide delegation", + })} +
+ )}
); + } case "onedrive": return (
diff --git a/nextjs-frontend/components/sources/source-model.ts b/nextjs-frontend/components/sources/source-model.ts index f8db2c3d6..ee6868841 100644 --- a/nextjs-frontend/components/sources/source-model.ts +++ b/nextjs-frontend/components/sources/source-model.ts @@ -58,6 +58,9 @@ export const FIELD_LABEL: Record = { token: "Token", scope: "Scope", root_folder_id: "Root folder ID", + team_drive: "Shared drive ID", + impersonate: "Act as user", + service_account_credentials: "Service account key", drive_id: "Drive ID", drive_type: "Drive type", host: "Host", @@ -410,8 +413,34 @@ export type ConnForm = { sftpAuth: "pass" | "key_pem"; /** What was pasted from `rclone authorize` (OAuth types). */ tokenText: string; + /** Google Drive signs in with a token, or as a service account (its JSON key file). */ + driveAuth: "token" | "service_account"; }; +export type ServiceAccountKey = { ok: true; email: string } | { ok: false; error: string }; + +/** A Google Cloud service account key file: what it signs in as, or why it isn't one. */ +export function parseServiceAccountKey(text: string): ServiceAccountKey { + let o: { type?: unknown; client_email?: unknown; private_key?: unknown } | null; + try { + o = JSON.parse(text); + } catch { + return { ok: false, error: "That file isn’t JSON. Choose the .json key file Google Cloud gave you." }; + } + if ( + !o || + typeof o !== "object" || + o.type !== "service_account" || + typeof o.client_email !== "string" || + !o.private_key + ) + return { + ok: false, + error: "That isn’t a service account key. In Google Cloud: IAM › Service accounts › Keys › Add key › JSON.", + }; + return { ok: true, email: o.client_email }; +} + export function emptyForm(spec: BackendSpec, source?: Pick | null): ConnForm { const params: Record = {}; for (const [k, def] of Object.entries(spec.fields)) @@ -423,6 +452,8 @@ export function emptyForm(spec: BackendSpec, source?: Pick (form.secrets[k] === undefined ? editing && isSet(k) : Boolean(form.secrets[k])); - if (spec.oauth) { + const serviceAccount = type === "drive" && form.driveAuth === "service_account"; + if (serviceAccount) { + const k = form.secrets.service_account_credentials; + const key = k ? parseServiceAccountKey(k) : null; + if (key?.ok === false) e.service_account_credentials = key.error; + else if (!secret("service_account_credentials")) + e.service_account_credentials = "Choose the service account’s JSON key file."; + } else if (spec.oauth) { const t = parseRcloneToken(form.tokenText); if (t.ok === false) e.token = t.error; else if (t.ok === null && !(editing && isSet("token"))) e.token = "Paste the token rclone printed."; @@ -499,7 +537,10 @@ export function buildPayload( for (const k of spec.secrets) { if (k === "token") continue; const v = form.secrets[k]; - if (type === "sftp" && k !== form.sftpAuth) { + const other = + (type === "sftp" && k !== form.sftpAuth) || + (type === "drive" && k === "service_account_credentials" && form.driveAuth !== "service_account"); + if (other) { // Signing in one way drops the other secret only when it was explicitly cleared. if (v === "") secrets[k] = null; continue; @@ -507,9 +548,14 @@ export function buildPayload( if (v === undefined) continue; secrets[k] = v === "" ? null : type === "sftp" && k === "key_pem" ? v.replace(/\r\n/g, "\n") : v; } - if (spec.oauth) { + if (type === "drive" && form.driveAuth === "service_account") { + if (secrets.service_account_credentials) secrets.token = null; // rclone would sign in as the account anyway + } else if (spec.oauth) { const t = parseRcloneToken(form.tokenText); - if (t.ok) secrets.token = t.token; + if (t.ok) { + secrets.token = t.token; + if (type === "drive") secrets.service_account_credentials = null; // or rclone keeps using the account + } } return { params, secrets }; } From 795d7c026485d6d6fa0952291822acb1b03a86f3 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 19:57:51 +0000 Subject: [PATCH 2/2] test(sources): keep the fake service account key from tripping detect-private-key Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01G2TAAG5yQ66RzoXMi3MZdV --- fastapi_backend/tests/api/test_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastapi_backend/tests/api/test_sources.py b/fastapi_backend/tests/api/test_sources.py index 28516ef67..5c8115e73 100644 --- a/fastapi_backend/tests/api/test_sources.py +++ b/fastapi_backend/tests/api/test_sources.py @@ -294,7 +294,7 @@ def test_google_drive_as_a_service_account(client, db, cfg, monkeypatch): key = { "type": "service_account", "project_id": "lens-test", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEv...\n-----END PRIVATE KEY-----\n", + "private_key": "-----BEGIN " + "PRIVATE KEY-----\nMIIEv...\n-----END " + "PRIVATE KEY-----\n", # split: not a real key "client_email": "lens@lens-test.iam.gserviceaccount.com", } body = {