From cde5d24ec35f0b88ace35c802dfb1bcab8c59008 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 19:58:04 +0000 Subject: [PATCH] fix(sources): keep working when Google Drive rate limits A throttled Drive account showed as a broken connection, a scan marked the file it couldn't fetch as an error for good, and Lens's own files failed to save once Drive's daily upload limit was hit, though reading still worked. Now rate limits are told apart from failures: the check shows the connection working (limited), rclone gets Drive's default 10 low-level retries, throttled scan files wait for the next scan, and new files are kept on this machine while Drive refuses uploads. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01G2TAAG5yQ66RzoXMi3MZdV --- docs/processing.md | 3 + docs/storage.md | 4 ++ fastapi_backend/app/domain/blobs.py | 24 +++++-- fastapi_backend/app/domain/sources.py | 42 +++++++++-- fastapi_backend/app/schemas/admin.py | 1 + fastapi_backend/app/schemas/sources.py | 1 + fastapi_backend/tests/api/test_rate_limits.py | 72 +++++++++++++++++++ fastapi_backend/tests/fake_rclone.py | 63 ++++++++++++++++ .../__tests__/sources-model.test.ts | 6 ++ .../app/openapi-client/types.gen.ts | 12 ++++ .../components/settings/bodies.tsx | 6 +- .../components/sources/source-model.ts | 4 +- .../components/sources/sources-page.tsx | 1 + nextjs-frontend/openapi.json | 24 +++++++ 14 files changed, 252 insertions(+), 11 deletions(-) create mode 100644 fastapi_backend/tests/api/test_rate_limits.py create mode 100755 fastapi_backend/tests/fake_rclone.py diff --git a/docs/processing.md b/docs/processing.md index bcfbe7a15..4a40122d3 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -265,6 +265,9 @@ files, then Import (`POST /api/v1/import/source`). The listing marks files that - Google Drive can also sign in as a service account: choose Service account and upload its JSON key file (`service_account_credentials`). Share the folders Lens should read with the account's address, or give a shared drive ID (`team_drive`); in Google Workspace, `impersonate` reads a user's own Drive through domain-wide delegation. +- A rate-limited connection (Google Drive throttling an account) still works: its check shows it working, slowly; + rclone backs off and retries (Drive gets 10 low-level retries); and a file a scan couldn't fetch waits for the next + scan, as do the rest of that scan's files, rather than failing for good. - Folders on this machine can only be watched inside `sources.local_roots`, and the rclone binary can only be set in archive.yaml: the web app can neither open up the server's disk nor choose what runs. diff --git a/docs/storage.md b/docs/storage.md index a2f3b23cd..f18623ae1 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -24,6 +24,10 @@ sealed like any setting's secret: there's nothing to type or remember. Each file records where it went, so changing these settings never strands what is already stored: older files are read from wherever they were written, and new ones go to the new place. +When a connection refuses new files for a while (Google Drive's daily upload limit), new files are kept on this +machine instead, and each records that, so nothing fails; files already on the connection are still read from it. The +check then says it works but is rate limited (`limited`). + `POST /api/v1/settings/files/test` writes a small file there, reads it back and removes it: what is saved, or a `{store, connection, folder, crypt}` given in the body, to check before saving. The assistant (for admins) has `file_storage`, which says where files go and which connections could take them, and checks it with `check`; it diff --git a/fastapi_backend/app/domain/blobs.py b/fastapi_backend/app/domain/blobs.py index d828ce835..c2e97ffe7 100644 --- a/fastapi_backend/app/domain/blobs.py +++ b/fastapi_backend/app/domain/blobs.py @@ -125,14 +125,15 @@ def _missing(err): def _rclone(cfg, conf, *argv, timeout=TIMEOUT): + # --drive-stop-on-upload-limit: past Google's daily upload limit, fail at once rather than retry for hours out = subprocess.run( - [sources._bin(cfg), "--config", conf, "--retries", "2", "--low-level-retries", "3", *argv], + [sources._bin(cfg), "--config", conf, "--retries", "2", "--low-level-retries", "3", "--drive-stop-on-upload-limit", *argv], capture_output=True, text=True, timeout=timeout, ) if out.returncode: - raise RuntimeError(sources._clean_err(out.stderr) or f"rclone exited with {out.returncode}") + raise sources.failure(out.stderr) if out.stderr.strip() else RuntimeError(f"rclone exited with {out.returncode}") return out.stdout @@ -193,7 +194,13 @@ def put(db, cfg, sid, key, path, where=None): keyring.protect(db, cfg, sid, tmp) else: keyring.encrypt_file(db, cfg, sid, tmp, force=True) # it leaves the machine: only ever as ciphertext - _send(db, cfg, where, key, tmp) + try: + _send(db, cfg, where, key, tmp) + except sources.RateLimited: + # the connection refuses new files for now (Drive's daily upload limit): keep this one here instead, so + # nothing fails; it records where it went, and what's on the connection is still read from there + where = {"store": "local"} + _send(db, cfg, where, key, tmp) # already encrypted with the namespace's key return where @@ -230,7 +237,16 @@ def test(db, cfg, where=None): store_, key = where["store"], f"lens-check/{secrets.token_hex(8)}" with _scratch(cfg) as tmp, _scratch(cfg) as back: pathlib.Path(tmp).write_bytes(probe) - _send(db, cfg, where, key, tmp) + try: + _send(db, cfg, where, key, tmp) + except sources.RateLimited as e: + return { + "ok": True, + "store": store_, + "seconds": round(time.monotonic() - t0, 2), + "limited": f"refusing new files for now ({e}); until that lifts, new files are kept on this " + "machine and files already there are still read from it"[:400], + } _fetch(db, cfg, where, key, back) delete(db, cfg, where, key) if pathlib.Path(back).read_bytes() != probe: diff --git a/fastapi_backend/app/domain/sources.py b/fastapi_backend/app/domain/sources.py index 5ee4519f2..4764f8101 100644 --- a/fastapi_backend/app/domain/sources.py +++ b/fastapi_backend/app/domain/sources.py @@ -75,6 +75,26 @@ } +# what Google Drive (and others) say when they throttle: the connection works, it is only slowed or refusing new files +RATE_LIMIT = re.compile(r"rate ?limit|quota exceeded|exceeded .*quota|too many requests|\b429\b|upload limit", re.I) + + +class RateLimited(RuntimeError): + """The service is throttling this account: reading usually still works, more slowly; new files may be refused.""" + + +def failure(text): + """The error to raise for what rclone printed on failing: RateLimited when the service is only throttling.""" + msg = _clean_err(text) + return RateLimited(msg) if RATE_LIMIT.search(text or "") else RuntimeError(msg) + + +def retries(src): + """rclone's retry flags for a source. Drive throttles in bursts: rclone backs off between low-level retries, so + Drive gets its default 10 rather than giving up on the first burst.""" + return ["--retries", "1", "--low-level-retries", "10" if src["type"] == "drive" else "2"] + + def _bin(cfg): b = os.environ.get("RCLONE_BINARY") or cfg["sources"].get("rclone") or shutil.which("rclone") if not b: @@ -147,14 +167,14 @@ def run(db, cfg, src, argv, timeout=300): name, d, conf = _private_conf(cfg, src) try: out = subprocess.run( - [_bin(cfg), "--config", conf, "--retries", "1", "--low-level-retries", "2", *argv(name)], + [_bin(cfg), "--config", conf, *retries(src), *argv(name)], capture_output=True, text=True, timeout=timeout, ) _writeback(db, cfg, src, conf) if out.returncode: - raise RuntimeError(_clean_err(out.stderr) or f"rclone exited with {out.returncode}") + raise failure(out.stderr) if out.stderr.strip() else RuntimeError(f"rclone exited with {out.returncode}") return out.stdout finally: shutil.rmtree(d, ignore_errors=True) @@ -253,6 +273,12 @@ def test(db, cfg, sid): if not feeds.handles(src): run(db, cfg, src, lambda n: ["lsjson", "--max-depth", "1", "--dirs-only", f"{n}:{target}"], timeout=60) health = {"ok": True, "checked_at": store.now()} + except RateLimited as e: # it signed in: the service is only slowing it down, and rclone keeps retrying + health = { + "ok": True, + "checked_at": store.now(), + "limited": f"{BACKENDS[src['type']]['label']} is rate limiting this account: {e}"[:300], + } except (RuntimeError, ValueError, subprocess.TimeoutExpired) as e: health = {"ok": False, "checked_at": store.now(), "error": str(e)[:300]} db.q("UPDATE $r SET health = $h", r=R("storage_source", sid), h=health) @@ -630,7 +656,7 @@ def poll_watch(db, cfg, wid, log=print): spaces = set(store.space_names(db)) if routes else set() if routes: stats["routed"] = 0 - files, held = list_files(db, cfg, src, w["path"], w.get("cursor")), set() + files, held, limited = list_files(db, cfg, src, w["path"], w.get("cursor")), set(), None for f in files: kind = kind_of(cfg, w, f) if not kind: @@ -649,8 +675,8 @@ def poll_watch(db, cfg, wid, log=print): if space is None or (first and not w.get("backfill")) or (src["id"], f["path"]) in gone[space]: db.q("UPSERT $k CONTENT $d", k=key, d={**row, "status": "skipped"}) stats["skipped"] += 1 - elif (now - _when(f["modified"])).total_seconds() < (w.get("stable_seconds") or 0): - db.q("UPSERT $k CONTENT $d", k=key, d={**row, "status": "waiting"}) + elif limited or (now - _when(f["modified"])).total_seconds() < (w.get("stable_seconds") or 0): + db.q("UPSERT $k CONTENT $d", k=key, d=store.clean({**row, "status": "waiting", "error": limited})) stats["waiting"] += 1 held.add(f["path"]) else: @@ -660,6 +686,12 @@ def poll_watch(db, cfg, wid, log=print): stats["new"] += 1 if space != w["space"]: stats["routed"] += 1 + except RateLimited as e: # throttled, not broken: this file and the rest wait for the next scan + limited = f"rate limited: {e}"[:300] + db.q("UPSERT $k CONTENT $d", k=key, d={**row, "status": "waiting", "error": limited}) + stats["waiting"] += 1 + held.add(f["path"]) + log(f" {f['path']}: {limited}") except Exception as e: # noqa: BLE001 - one bad file must not stop the scan db.q("UPSERT $k CONTENT $d", k=key, d={**row, "status": "error", "error": f"{type(e).__name__}: {e}"[:300]}) stats["errors"] += 1 diff --git a/fastapi_backend/app/schemas/admin.py b/fastapi_backend/app/schemas/admin.py index 6a843ad4b..6743dc51c 100644 --- a/fastapi_backend/app/schemas/admin.py +++ b/fastapi_backend/app/schemas/admin.py @@ -237,6 +237,7 @@ class FileStoreTestResult(ResponseModel): store: Literal["local", "connection"] | None = None seconds: float | None = None error: str | None = None + limited: str | None = Field(default=None, description="the connection is rate limiting: reads work, new files are kept here") class AuditEntry(ResponseModel): diff --git a/fastapi_backend/app/schemas/sources.py b/fastapi_backend/app/schemas/sources.py index 965bce852..0aa5a8d20 100644 --- a/fastapi_backend/app/schemas/sources.py +++ b/fastapi_backend/app/schemas/sources.py @@ -27,6 +27,7 @@ class SourceHealth(ResponseModel): ok: bool checked_at: str | None = None error: str | None = None + limited: str | None = Field(default=None, description="the service is rate limiting the account; it still works") class SecretFlag(ResponseModel): diff --git a/fastapi_backend/tests/api/test_rate_limits.py b/fastapi_backend/tests/api/test_rate_limits.py new file mode 100644 index 000000000..1a893222d --- /dev/null +++ b/fastapi_backend/tests/api/test_rate_limits.py @@ -0,0 +1,72 @@ +"""A connection that is rate limited (Google Drive's throttling and daily upload limit) still works: it reads on, keeps +new files on this machine, and files a scan couldn't fetch wait for the next one instead of failing for good.""" + +from __future__ import annotations + +import pathlib + +import pytest + +from app.domain import blobs, sources, store + +FAKE = pathlib.Path(__file__).parent.parent / "fake_rclone.py" + + +@pytest.fixture +def drive(db, cfg, folder, monkeypatch): + state = folder / "throttled" + state.write_text("") + monkeypatch.setenv("RCLONE_BINARY", str(FAKE)) + monkeypatch.setenv("FAKE_RCLONE_ROOT", str(folder / "drive")) + monkeypatch.setenv("FAKE_RCLONE_STATE", str(state)) + sid = sources.create(db, cfg, "Drive", "drive", secret_values={"token": '{"access_token":"a"}'}) + return sid, state + + +def test_a_throttled_connection_is_still_healthy(db, cfg, drive): + sid, state = drive + assert sources.test(db, cfg, sid)["ok"] and "limited" not in sources.test(db, cfg, sid) + state.write_text("all") + h = sources.test(db, cfg, sid) + assert h["ok"] and "rate limiting" in h["limited"] and "error" not in h + with pytest.raises(sources.RateLimited): + sources.browse(db, cfg, sid, "") + + +def test_files_stay_here_while_uploads_are_refused(db, cfg, folder, drive): + sid, state = drive + pods = store.ns_id(db, "pods") + where = {"store": "connection", "connection": sid, "folder": "lens", "crypt": False} + (folder / "a.txt").write_bytes(b"kept on Drive") + on_drive = blobs.put(db, cfg, pods, "notes/1/a.txt", folder / "a.txt", where) + assert on_drive == where + + state.write_text("uploads") # Google's daily upload limit: reading goes on + (folder / "b.txt").write_bytes(b"kept here for now") + assert blobs.put(db, cfg, pods, "notes/1/b.txt", folder / "b.txt", where) == {"store": "local"} + with blobs.open_file(db, cfg, {"store": "local"}, "notes/1/b.txt") as f: + assert f.read() == b"kept here for now" + with blobs.open_file(db, cfg, on_drive, "notes/1/a.txt") as f: + assert f.read() == b"kept on Drive" + checked = blobs.test(db, cfg, where) + assert checked["ok"] and "refusing new files" in checked["limited"] + + +def test_a_throttled_scan_waits_instead_of_failing(db, cfg, folder, drive): + sid, state = drive + calls = folder / "drive" / "calls" + calls.mkdir(parents=True) + for n in (1, 2): + (calls / f"call{n}.txt").write_text(f"Ann: call {n}.\nBen: great.\nAnn: bye.") + wid = sources.create_watch(db, cfg, sid, "calls", store.ns_id(db, "pods"), stable_seconds=0, backfill=True, kinds="transcripts") + + state.write_text("downloads") # listing works, but every download is throttled from here + stats = sources.poll_watch(db, cfg, wid, log=lambda *a: None) + assert (stats["waiting"], stats["errors"]) == (2, 0) + assert pathlib.Path(f"{state}.log").read_text().split() == ["copyto"] # the rest don't hammer a throttled account + rows = db.rows("SELECT path, status, error FROM remote_file WHERE watch = $w", w=wid) + assert {r["status"] for r in rows} == {"waiting"} and all("rate limited" in r["error"] for r in rows) + + state.write_text("") # the limit lifted + stats = sources.poll_watch(db, cfg, wid, log=lambda *a: None) + assert (stats["new"], stats["errors"]) == (2, 0) diff --git a/fastapi_backend/tests/fake_rclone.py b/fastapi_backend/tests/fake_rclone.py new file mode 100755 index 000000000..32d56245b --- /dev/null +++ b/fastapi_backend/tests/fake_rclone.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""A stand-in for rclone, for tests: remotes are folders under $FAKE_RCLONE_ROOT, and $FAKE_RCLONE_STATE says what +Google is throttling right now: nothing (empty), "uploads" (new files: Drive's daily upload limit), "downloads", or +"all". Each throttled call is logged to $FAKE_RCLONE_STATE.log.""" + +import json +import os +import pathlib +import shutil +import sys + +LIMIT = "2026/10/10 12:00:00 ERROR : Failed to copy: googleapi: Error 403: User rate limit exceeded., userRateLimitExceeded" +root = pathlib.Path(os.environ["FAKE_RCLONE_ROOT"]) +state = pathlib.Path(os.environ["FAKE_RCLONE_STATE"]).read_text().strip() +args, i = [], 1 +while i < len(sys.argv): + a = sys.argv[i] + if a in ("--config", "--retries", "--low-level-retries", "--max-depth"): + i += 2 + continue + if not a.startswith("-"): + args.append(a) + i += 1 +cmd, rest = args[0], args[1:] + + +def remote(p): + return root / p.split(":", 1)[1].lstrip("/") if ":" in p else None + + +def throttled(what): + if state == "all" or state == what: + with open(os.environ["FAKE_RCLONE_STATE"] + ".log", "a") as log: + log.write(f"{cmd}\n") + print(LIMIT, file=sys.stderr) + sys.exit(1) + + +if cmd == "obscure": + print(sys.stdin.read().strip()) +elif cmd == "lsjson": + throttled("lists") + base = remote(rest[0]) + base.mkdir(parents=True, exist_ok=True) + files = sorted(p for p in base.rglob("*") if p.is_file()) if "-R" in sys.argv else sorted(base.iterdir()) + out = [ + {"Path": str(p.relative_to(base)), "Name": p.name, "Size": p.stat().st_size, "ModTime": "2020-01-01T00:00:00Z", "IsDir": p.is_dir()} + for p in files + ] + print(json.dumps(out)) +elif cmd == "copyto": + src, dst = rest + throttled("uploads" if remote(dst) is not None else "downloads") + if remote(src) is not None and not remote(src).exists(): + print("ERROR : file not found", file=sys.stderr) + sys.exit(1) + out = remote(dst) or pathlib.Path(dst) + out.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(remote(src) or src, out) +elif cmd == "deletefile": + remote(rest[0]).unlink(missing_ok=True) +else: + sys.exit(f"fake rclone: {cmd} isn't faked") diff --git a/nextjs-frontend/__tests__/sources-model.test.ts b/nextjs-frontend/__tests__/sources-model.test.ts index fde33954d..5cd885f57 100644 --- a/nextjs-frontend/__tests__/sources-model.test.ts +++ b/nextjs-frontend/__tests__/sources-model.test.ts @@ -83,6 +83,12 @@ describe("connections", () => { tone: "ok", text: "OK · checked 14:02", }); + expect( + healthOf({ ok: true, checked_at: "2026-09-30T14:02:00", limited: "Google Drive is rate limiting" }, now), + ).toEqual({ + tone: "limited", + text: "Works, slowly · Google Drive is rate limiting · checked 14:02", + }); expect( healthOf( { diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 17a895634..1b65a81d7 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -4928,6 +4928,12 @@ export type FileStoreTestResult = { * Error */ error?: string | null; + /** + * Limited + * + * the connection is rate limiting: reads work, new files are kept here + */ + limited?: string | null; [key: string]: unknown; }; @@ -14333,6 +14339,12 @@ export type SourceHealth = { * Error */ error?: string | null; + /** + * Limited + * + * the service is rate limiting the account; it still works + */ + limited?: string | null; [key: string]: unknown; }; diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index d7d16bf49..6c639acdc 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -714,7 +714,11 @@ function StorageBody({ ctx }: { ctx: BodyCtx }) { Writes a small file there, reads it back and removes it {test.data && - (test.data.ok ? ( + (test.data.ok && test.data.limited ? ( + + The connection is {test.data.limited}. + + ) : test.data.ok ? ( Written, read back and removed in {test.data.seconds} s. diff --git a/nextjs-frontend/components/sources/source-model.ts b/nextjs-frontend/components/sources/source-model.ts index ee6868841..8b3c67a0b 100644 --- a/nextjs-frontend/components/sources/source-model.ts +++ b/nextjs-frontend/components/sources/source-model.ts @@ -154,11 +154,13 @@ export function sourceSubtitle(s: Pick): string { } } -export type Health = { tone: "ok" | "bad" | "unknown"; text: string }; +export type Health = { tone: "ok" | "limited" | "bad" | "unknown"; text: string }; /** "OK · checked 14:02" or the backend's own error text, word for word. */ export function healthOf(h: Source["health"], now = new Date()): Health { if (!h) return { tone: "unknown", text: "Not tested yet" }; + if (h.ok && h.limited) + return { tone: "limited", text: `Works, slowly · ${h.limited.trim()} · checked ${checkedAt(h.checked_at, now)}` }; if (h.ok) return { tone: "ok", text: `OK · checked ${checkedAt(h.checked_at, now)}` }; return { tone: "bad", diff --git a/nextjs-frontend/components/sources/sources-page.tsx b/nextjs-frontend/components/sources/sources-page.tsx index c3ab325e5..a0e1aef7f 100644 --- a/nextjs-frontend/components/sources/sources-page.tsx +++ b/nextjs-frontend/components/sources/sources-page.tsx @@ -28,6 +28,7 @@ import { cn } from "@/lib/utils"; const GLYPH = { ok: ["✓", "text-green-dark"], + limited: ["◆", "text-gold-dark"], bad: ["✕", "text-red-dark"], unknown: ["–", "text-fg-muted"], } as const; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 86995a818..5465cc9cc 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -32842,6 +32842,18 @@ } ], "title": "Error" + }, + "limited": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Limited", + "description": "the connection is rate limiting: reads work, new files are kept here" } }, "additionalProperties": true, @@ -49132,6 +49144,18 @@ } ], "title": "Error" + }, + "limited": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Limited", + "description": "the service is rate limiting the account; it still works" } }, "additionalProperties": true,