Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/processing.md
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,9 @@ files, then Import (`POST /api/v1/import/source`). The listing marks files that
- Google Drive can also sign in as a service account: choose Service account and upload its JSON key file
(`service_account_credentials`). Share the folders Lens should read with the account's address, or give a shared
drive ID (`team_drive`); in Google Workspace, `impersonate` reads a user's own Drive through domain-wide delegation.
- A rate-limited connection (Google Drive throttling an account) still works: its check shows it working, slowly;
rclone backs off and retries (Drive gets 10 low-level retries); and a file a scan couldn't fetch waits for the next
scan, as do the rest of that scan's files, rather than failing for good.
- Folders on this machine can only be watched inside `sources.local_roots`, and the rclone binary can only be set in
archive.yaml: the web app can neither open up the server's disk nor choose what runs.

Expand Down
4 changes: 4 additions & 0 deletions docs/storage.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ sealed like any setting's secret: there's nothing to type or remember.
Each file records where it went, so changing these settings never strands what is already stored: older files are read
from wherever they were written, and new ones go to the new place.

When a connection refuses new files for a while (Google Drive's daily upload limit), new files are kept on this
machine instead, and each records that, so nothing fails; files already on the connection are still read from it. The
check then says it works but is rate limited (`limited`).

`POST /api/v1/settings/files/test` writes a small file there, reads it back and removes it: what is saved, or a
`{store, connection, folder, crypt}` given in the body, to check before saving. The assistant (for admins) has
`file_storage`, which says where files go and which connections could take them, and checks it with `check`; it
Expand Down
24 changes: 20 additions & 4 deletions fastapi_backend/app/domain/blobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,14 +125,15 @@ def _missing(err):


def _rclone(cfg, conf, *argv, timeout=TIMEOUT):
# --drive-stop-on-upload-limit: past Google's daily upload limit, fail at once rather than retry for hours
out = subprocess.run(
[sources._bin(cfg), "--config", conf, "--retries", "2", "--low-level-retries", "3", *argv],
[sources._bin(cfg), "--config", conf, "--retries", "2", "--low-level-retries", "3", "--drive-stop-on-upload-limit", *argv],
capture_output=True,
text=True,
timeout=timeout,
)
if out.returncode:
raise RuntimeError(sources._clean_err(out.stderr) or f"rclone exited with {out.returncode}")
raise sources.failure(out.stderr) if out.stderr.strip() else RuntimeError(f"rclone exited with {out.returncode}")
return out.stdout


Expand Down Expand Up @@ -193,7 +194,13 @@ def put(db, cfg, sid, key, path, where=None):
keyring.protect(db, cfg, sid, tmp)
else:
keyring.encrypt_file(db, cfg, sid, tmp, force=True) # it leaves the machine: only ever as ciphertext
_send(db, cfg, where, key, tmp)
try:
_send(db, cfg, where, key, tmp)
except sources.RateLimited:
# the connection refuses new files for now (Drive's daily upload limit): keep this one here instead, so
# nothing fails; it records where it went, and what's on the connection is still read from there
where = {"store": "local"}
_send(db, cfg, where, key, tmp) # already encrypted with the namespace's key
return where


Expand Down Expand Up @@ -230,7 +237,16 @@ def test(db, cfg, where=None):
store_, key = where["store"], f"lens-check/{secrets.token_hex(8)}"
with _scratch(cfg) as tmp, _scratch(cfg) as back:
pathlib.Path(tmp).write_bytes(probe)
_send(db, cfg, where, key, tmp)
try:
_send(db, cfg, where, key, tmp)
except sources.RateLimited as e:
return {
"ok": True,
"store": store_,
"seconds": round(time.monotonic() - t0, 2),
"limited": f"refusing new files for now ({e}); until that lifts, new files are kept on this "
"machine and files already there are still read from it"[:400],
}
_fetch(db, cfg, where, key, back)
delete(db, cfg, where, key)
if pathlib.Path(back).read_bytes() != probe:
Expand Down
42 changes: 37 additions & 5 deletions fastapi_backend/app/domain/sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,26 @@
}


# what Google Drive (and others) say when they throttle: the connection works, it is only slowed or refusing new files
RATE_LIMIT = re.compile(r"rate ?limit|quota exceeded|exceeded .*quota|too many requests|\b429\b|upload limit", re.I)


class RateLimited(RuntimeError):
"""The service is throttling this account: reading usually still works, more slowly; new files may be refused."""


def failure(text):
"""The error to raise for what rclone printed on failing: RateLimited when the service is only throttling."""
msg = _clean_err(text)
return RateLimited(msg) if RATE_LIMIT.search(text or "") else RuntimeError(msg)


def retries(src):
"""rclone's retry flags for a source. Drive throttles in bursts: rclone backs off between low-level retries, so
Drive gets its default 10 rather than giving up on the first burst."""
return ["--retries", "1", "--low-level-retries", "10" if src["type"] == "drive" else "2"]


def _bin(cfg):
b = os.environ.get("RCLONE_BINARY") or cfg["sources"].get("rclone") or shutil.which("rclone")
if not b:
Expand Down Expand Up @@ -147,14 +167,14 @@ def run(db, cfg, src, argv, timeout=300):
name, d, conf = _private_conf(cfg, src)
try:
out = subprocess.run(
[_bin(cfg), "--config", conf, "--retries", "1", "--low-level-retries", "2", *argv(name)],
[_bin(cfg), "--config", conf, *retries(src), *argv(name)],
capture_output=True,
text=True,
timeout=timeout,
)
_writeback(db, cfg, src, conf)
if out.returncode:
raise RuntimeError(_clean_err(out.stderr) or f"rclone exited with {out.returncode}")
raise failure(out.stderr) if out.stderr.strip() else RuntimeError(f"rclone exited with {out.returncode}")
return out.stdout
finally:
shutil.rmtree(d, ignore_errors=True)
Expand Down Expand Up @@ -253,6 +273,12 @@ def test(db, cfg, sid):
if not feeds.handles(src):
run(db, cfg, src, lambda n: ["lsjson", "--max-depth", "1", "--dirs-only", f"{n}:{target}"], timeout=60)
health = {"ok": True, "checked_at": store.now()}
except RateLimited as e: # it signed in: the service is only slowing it down, and rclone keeps retrying
health = {
"ok": True,
"checked_at": store.now(),
"limited": f"{BACKENDS[src['type']]['label']} is rate limiting this account: {e}"[:300],
}
except (RuntimeError, ValueError, subprocess.TimeoutExpired) as e:
health = {"ok": False, "checked_at": store.now(), "error": str(e)[:300]}
db.q("UPDATE $r SET health = $h", r=R("storage_source", sid), h=health)
Expand Down Expand Up @@ -630,7 +656,7 @@ def poll_watch(db, cfg, wid, log=print):
spaces = set(store.space_names(db)) if routes else set()
if routes:
stats["routed"] = 0
files, held = list_files(db, cfg, src, w["path"], w.get("cursor")), set()
files, held, limited = list_files(db, cfg, src, w["path"], w.get("cursor")), set(), None
for f in files:
kind = kind_of(cfg, w, f)
if not kind:
Expand All @@ -649,8 +675,8 @@ def poll_watch(db, cfg, wid, log=print):
if space is None or (first and not w.get("backfill")) or (src["id"], f["path"]) in gone[space]:
db.q("UPSERT $k CONTENT $d", k=key, d={**row, "status": "skipped"})
stats["skipped"] += 1
elif (now - _when(f["modified"])).total_seconds() < (w.get("stable_seconds") or 0):
db.q("UPSERT $k CONTENT $d", k=key, d={**row, "status": "waiting"})
elif limited or (now - _when(f["modified"])).total_seconds() < (w.get("stable_seconds") or 0):
db.q("UPSERT $k CONTENT $d", k=key, d=store.clean({**row, "status": "waiting", "error": limited}))
stats["waiting"] += 1
held.add(f["path"])
else:
Expand All @@ -660,6 +686,12 @@ def poll_watch(db, cfg, wid, log=print):
stats["new"] += 1
if space != w["space"]:
stats["routed"] += 1
except RateLimited as e: # throttled, not broken: this file and the rest wait for the next scan
limited = f"rate limited: {e}"[:300]
db.q("UPSERT $k CONTENT $d", k=key, d={**row, "status": "waiting", "error": limited})
stats["waiting"] += 1
held.add(f["path"])
log(f" {f['path']}: {limited}")
except Exception as e: # noqa: BLE001 - one bad file must not stop the scan
db.q("UPSERT $k CONTENT $d", k=key, d={**row, "status": "error", "error": f"{type(e).__name__}: {e}"[:300]})
stats["errors"] += 1
Expand Down
1 change: 1 addition & 0 deletions fastapi_backend/app/schemas/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,7 @@ class FileStoreTestResult(ResponseModel):
store: Literal["local", "connection"] | None = None
seconds: float | None = None
error: str | None = None
limited: str | None = Field(default=None, description="the connection is rate limiting: reads work, new files are kept here")


class AuditEntry(ResponseModel):
Expand Down
1 change: 1 addition & 0 deletions fastapi_backend/app/schemas/sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ class SourceHealth(ResponseModel):
ok: bool
checked_at: str | None = None
error: str | None = None
limited: str | None = Field(default=None, description="the service is rate limiting the account; it still works")


class SecretFlag(ResponseModel):
Expand Down
72 changes: 72 additions & 0 deletions fastapi_backend/tests/api/test_rate_limits.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""A connection that is rate limited (Google Drive's throttling and daily upload limit) still works: it reads on, keeps
new files on this machine, and files a scan couldn't fetch wait for the next one instead of failing for good."""

from __future__ import annotations

import pathlib

import pytest

from app.domain import blobs, sources, store

FAKE = pathlib.Path(__file__).parent.parent / "fake_rclone.py"


@pytest.fixture
def drive(db, cfg, folder, monkeypatch):
state = folder / "throttled"
state.write_text("")
monkeypatch.setenv("RCLONE_BINARY", str(FAKE))
monkeypatch.setenv("FAKE_RCLONE_ROOT", str(folder / "drive"))
monkeypatch.setenv("FAKE_RCLONE_STATE", str(state))
sid = sources.create(db, cfg, "Drive", "drive", secret_values={"token": '{"access_token":"a"}'})
return sid, state


def test_a_throttled_connection_is_still_healthy(db, cfg, drive):
sid, state = drive
assert sources.test(db, cfg, sid)["ok"] and "limited" not in sources.test(db, cfg, sid)
state.write_text("all")
h = sources.test(db, cfg, sid)
assert h["ok"] and "rate limiting" in h["limited"] and "error" not in h
with pytest.raises(sources.RateLimited):
sources.browse(db, cfg, sid, "")


def test_files_stay_here_while_uploads_are_refused(db, cfg, folder, drive):
sid, state = drive
pods = store.ns_id(db, "pods")
where = {"store": "connection", "connection": sid, "folder": "lens", "crypt": False}
(folder / "a.txt").write_bytes(b"kept on Drive")
on_drive = blobs.put(db, cfg, pods, "notes/1/a.txt", folder / "a.txt", where)
assert on_drive == where

state.write_text("uploads") # Google's daily upload limit: reading goes on
(folder / "b.txt").write_bytes(b"kept here for now")
assert blobs.put(db, cfg, pods, "notes/1/b.txt", folder / "b.txt", where) == {"store": "local"}
with blobs.open_file(db, cfg, {"store": "local"}, "notes/1/b.txt") as f:
assert f.read() == b"kept here for now"
with blobs.open_file(db, cfg, on_drive, "notes/1/a.txt") as f:
assert f.read() == b"kept on Drive"
checked = blobs.test(db, cfg, where)
assert checked["ok"] and "refusing new files" in checked["limited"]


def test_a_throttled_scan_waits_instead_of_failing(db, cfg, folder, drive):
sid, state = drive
calls = folder / "drive" / "calls"
calls.mkdir(parents=True)
for n in (1, 2):
(calls / f"call{n}.txt").write_text(f"Ann: call {n}.\nBen: great.\nAnn: bye.")
wid = sources.create_watch(db, cfg, sid, "calls", store.ns_id(db, "pods"), stable_seconds=0, backfill=True, kinds="transcripts")

state.write_text("downloads") # listing works, but every download is throttled from here
stats = sources.poll_watch(db, cfg, wid, log=lambda *a: None)
assert (stats["waiting"], stats["errors"]) == (2, 0)
assert pathlib.Path(f"{state}.log").read_text().split() == ["copyto"] # the rest don't hammer a throttled account
rows = db.rows("SELECT path, status, error FROM remote_file WHERE watch = $w", w=wid)
assert {r["status"] for r in rows} == {"waiting"} and all("rate limited" in r["error"] for r in rows)

state.write_text("") # the limit lifted
stats = sources.poll_watch(db, cfg, wid, log=lambda *a: None)
assert (stats["new"], stats["errors"]) == (2, 0)
63 changes: 63 additions & 0 deletions fastapi_backend/tests/fake_rclone.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#!/usr/bin/env python3
"""A stand-in for rclone, for tests: remotes are folders under $FAKE_RCLONE_ROOT, and $FAKE_RCLONE_STATE says what
Google is throttling right now: nothing (empty), "uploads" (new files: Drive's daily upload limit), "downloads", or
"all". Each throttled call is logged to $FAKE_RCLONE_STATE.log."""

import json
import os
import pathlib
import shutil
import sys

LIMIT = "2026/10/10 12:00:00 ERROR : Failed to copy: googleapi: Error 403: User rate limit exceeded., userRateLimitExceeded"
root = pathlib.Path(os.environ["FAKE_RCLONE_ROOT"])
state = pathlib.Path(os.environ["FAKE_RCLONE_STATE"]).read_text().strip()
args, i = [], 1
while i < len(sys.argv):
a = sys.argv[i]
if a in ("--config", "--retries", "--low-level-retries", "--max-depth"):
i += 2
continue
if not a.startswith("-"):
args.append(a)
i += 1
cmd, rest = args[0], args[1:]


def remote(p):
return root / p.split(":", 1)[1].lstrip("/") if ":" in p else None


def throttled(what):
if state == "all" or state == what:
with open(os.environ["FAKE_RCLONE_STATE"] + ".log", "a") as log:
log.write(f"{cmd}\n")
print(LIMIT, file=sys.stderr)
sys.exit(1)


if cmd == "obscure":
print(sys.stdin.read().strip())
elif cmd == "lsjson":
throttled("lists")
base = remote(rest[0])
base.mkdir(parents=True, exist_ok=True)
files = sorted(p for p in base.rglob("*") if p.is_file()) if "-R" in sys.argv else sorted(base.iterdir())
out = [
{"Path": str(p.relative_to(base)), "Name": p.name, "Size": p.stat().st_size, "ModTime": "2020-01-01T00:00:00Z", "IsDir": p.is_dir()}
for p in files
]
print(json.dumps(out))
elif cmd == "copyto":
src, dst = rest
throttled("uploads" if remote(dst) is not None else "downloads")
if remote(src) is not None and not remote(src).exists():
print("ERROR : file not found", file=sys.stderr)
sys.exit(1)
out = remote(dst) or pathlib.Path(dst)
out.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(remote(src) or src, out)
elif cmd == "deletefile":
remote(rest[0]).unlink(missing_ok=True)
else:
sys.exit(f"fake rclone: {cmd} isn't faked")
6 changes: 6 additions & 0 deletions nextjs-frontend/__tests__/sources-model.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,12 @@ describe("connections", () => {
tone: "ok",
text: "OK · checked 14:02",
});
expect(
healthOf({ ok: true, checked_at: "2026-09-30T14:02:00", limited: "Google Drive is rate limiting" }, now),
).toEqual({
tone: "limited",
text: "Works, slowly · Google Drive is rate limiting · checked 14:02",
});
expect(
healthOf(
{
Expand Down
12 changes: 12 additions & 0 deletions nextjs-frontend/app/openapi-client/types.gen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4928,6 +4928,12 @@ export type FileStoreTestResult = {
* Error
*/
error?: string | null;
/**
* Limited
*
* the connection is rate limiting: reads work, new files are kept here
*/
limited?: string | null;
[key: string]: unknown;
};

Expand Down Expand Up @@ -14333,6 +14339,12 @@ export type SourceHealth = {
* Error
*/
error?: string | null;
/**
* Limited
*
* the service is rate limiting the account; it still works
*/
limited?: string | null;
[key: string]: unknown;
};

Expand Down
6 changes: 5 additions & 1 deletion nextjs-frontend/components/settings/bodies.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -714,7 +714,11 @@ function StorageBody({ ctx }: { ctx: BodyCtx }) {
<span className="text-[12px] text-fg-muted">Writes a small file there, reads it back and removes it</span>
</div>
{test.data &&
(test.data.ok ? (
(test.data.ok && test.data.limited ? (
<Banner tone="warning" title="It works, but it’s rate limited.">
The connection is {test.data.limited}.
</Banner>
) : test.data.ok ? (
<Banner tone="success" title="It works.">
Written, read back and removed in {test.data.seconds} s.
</Banner>
Expand Down
4 changes: 3 additions & 1 deletion nextjs-frontend/components/sources/source-model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -154,11 +154,13 @@ export function sourceSubtitle(s: Pick<Source, "type" | "params">): string {
}
}

export type Health = { tone: "ok" | "bad" | "unknown"; text: string };
export type Health = { tone: "ok" | "limited" | "bad" | "unknown"; text: string };

/** "OK · checked 14:02" or the backend's own error text, word for word. */
export function healthOf(h: Source["health"], now = new Date()): Health {
if (!h) return { tone: "unknown", text: "Not tested yet" };
if (h.ok && h.limited)
return { tone: "limited", text: `Works, slowly · ${h.limited.trim()} · checked ${checkedAt(h.checked_at, now)}` };
if (h.ok) return { tone: "ok", text: `OK · checked ${checkedAt(h.checked_at, now)}` };
return {
tone: "bad",
Expand Down
1 change: 1 addition & 0 deletions nextjs-frontend/components/sources/sources-page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import { cn } from "@/lib/utils";

const GLYPH = {
ok: ["✓", "text-green-dark"],
limited: ["◆", "text-gold-dark"],
bad: ["✕", "text-red-dark"],
unknown: ["–", "text-fg-muted"],
} as const;
Expand Down
Loading
Loading