diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eb3fde89..2b79657d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Watched folders and routines work in Docker and the packages.** They were checked only by the API's own + background work, which Docker, Synology, QNAP, Cloudron and Proxmox all turn off in favour of a `lens worker` + process; so folders were never scanned and routines never ran. `lens worker` now does both (not a worker limited + with `--steps`, nor one started with `--no-schedule`), and a folder scan is claimed first, as routine runs are. - **MCP server: agents search, read and cite the archive.** Add `https:///mcp` to Claude, Cursor, VS Code or another MCP client, sign in on Lens's consent page (OAuth), and the agent sees what you see: your namespaces, the collections you were given a role on, and their graphs. Read-only tools: `search` (moments said, on screen or diff --git a/docs/processing.md b/docs/processing.md index 60bb6de17..7bfb33ee1 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -143,6 +143,10 @@ has left. lens worker --steps transcribe,diarize # e.g. on the Mac, with SURREAL_URL pointing at the server +A worker that runs every step also scans watched folders and runs routines when they are due, so Docker and the +packages need no other process for them. One limited with `--steps`, or started with `--no-schedule`, only runs jobs. +A folder scan and a routine run are each claimed first, so several processes doing this never repeat one. + ## Storage sources Admins add sources in the app (`/api/v1/sources`): S3 or S3-compatible, Dropbox, Google Drive, OneDrive, SFTP, SMB, @@ -262,7 +266,7 @@ namespaces, or all of them: The schedule is a five-field cron expression (minute hour day-of-month month day-of-week, e.g. `0 3 * * *`) or `@hourly`, `@daily`, `@weekly`, `@monthly`, in a time zone; without one a routine runs only when someone presses Run -now. The API process checks every 30 seconds (`lens watch` does too, for setups that run it separately); a routine is +now. Every 30 seconds, `lens worker` checks (and so does the API process when it runs background work, and `lens watch`); a routine is never started twice at once, and one started in two processes runs once. Each run keeps what every action did and a log. diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py index eb0231e01..e7be34d06 100644 --- a/fastapi_backend/app/cli.py +++ b/fastapi_backend/app/cli.py @@ -212,6 +212,11 @@ def platform_main(argv, config): w.add_argument("--name") w.add_argument("--steps", help="comma-separated steps this worker runs (default: all)") w.add_argument("--once", action="store_true", help="run what is queued, then exit") + w.add_argument( + "--no-schedule", + action="store_true", + help="don't scan watched folders or run routines (a worker limited with --steps never does)", + ) x = sub.add_parser("watch", help="scan watched folders on storage sources and run the routines that are due") x.add_argument("--once", action="store_true") a = ap.parse_args(argv) @@ -258,6 +263,8 @@ def password(): print(f"worker {wk.name} runs {', '.join(sorted(wk.can))}; Ctrl-C to stop") stop = threading.Event() notify.start(db, C, stop, name=wk.name, log=print) # sends notifications too (docs/notifications.md) + if not (a.steps or a.no_schedule): # and scans watched folders and runs routines, as `lens watch` does + routines.start(db, C, stop, log=print) _stop_on_term() try: wk.loop(stop) diff --git a/fastapi_backend/app/core/runtime.py b/fastapi_backend/app/core/runtime.py index 998c00096..faa4cc88c 100644 --- a/fastapi_backend/app/core/runtime.py +++ b/fastapi_backend/app/core/runtime.py @@ -9,7 +9,7 @@ from typing import Any from app.config import settings as env -from app.domain import auth, content_types, jobs, notify, routines, settings, setup, sources, store, templates +from app.domain import auth, content_types, jobs, notify, routines, settings, setup, store, templates log = logging.getLogger("lens") @@ -47,24 +47,8 @@ def start_background(self) -> None: w = jobs.Worker(self.db, self.current, name=f"api-{os.getpid()}-{i}", log=log.info) threading.Thread(target=w.loop, args=(self.stop,), daemon=True, name=f"worker-{i}").start() - def watcher() -> None: - while not self.stop.wait(max(5, self.current()["sources"]["check_seconds"])): - try: - sources.poll_due(self.db, self.current(), log=log.info) - except Exception: # noqa: BLE001 - keep polling - log.exception("watcher failed") - - threading.Thread(target=watcher, daemon=True, name="watcher").start() notify.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log=log.warning) - - def scheduler() -> None: - while not self.stop.wait(routines.CHECK_SECONDS): - try: - routines.run_due(self.db, self.current(), log=log.info) - except Exception: # noqa: BLE001 - keep scheduling - log.exception("routines failed") - - threading.Thread(target=scheduler, daemon=True, name="routines").start() + routines.start(self.db, self.current, self.stop, log=log.info) def close(self) -> None: self.stop.set() diff --git a/fastapi_backend/app/domain/routines.py b/fastapi_backend/app/domain/routines.py index f6893512b..07224e8e8 100644 --- a/fastapi_backend/app/domain/routines.py +++ b/fastapi_backend/app/domain/routines.py @@ -16,6 +16,7 @@ from __future__ import annotations import datetime as dt +import threading from . import jobs, organize, schedule, sources, store, telemetry, workflows @@ -31,6 +32,7 @@ MAX_ACTIONS, DEFAULT_LIMIT, MAX_LIMIT = 20, 500, 5000 STALE_MINUTES = 180 CHECK_SECONDS = 30 # how often the scheduler looks for routines that are due +MIN_WAIT_SECONDS = 5 # the shortest wait between rounds, whatever sources.check_seconds says LOG_MAX = 500 FIELDS = ( "record::id(id) AS id, name, description, enabled, schedule, timezone, namespaces, actions, next_run_at, last_run_at, " @@ -458,6 +460,30 @@ def run_due(db, cfg, log=print, now=None): return done +def start(db, cfg_fn, stop, log=None): + """The scheduling threads, until `stop` is set: watched folders scanned when due (every sources.check_seconds) and + routines run when due (every CHECK_SECONDS). The API runs them with its background work; `lens worker` runs them too, + since Docker and the packages run the API without it.""" + + def every(seconds, fn, what): + def loop(): + while not stop.wait(max(MIN_WAIT_SECONDS, seconds())): + try: + fn(db, cfg_fn(), log=log) + except Exception as e: # noqa: BLE001 - keep going + if log: + log(f"{what}: {type(e).__name__}: {e}") + + th = threading.Thread(target=loop, daemon=True, name=what) + th.start() + return th + + return [ + every(lambda: cfg_fn()["sources"]["check_seconds"], sources.poll_due, "watched folders"), + every(lambda: CHECK_SECONDS, run_due, "routines"), + ] + + # ---------- what a fresh archive starts with ---------- def seed(db): """The graph-organising workflow, and a routine that runs it every night (off until someone turns it on).""" diff --git a/fastapi_backend/app/domain/sources.py b/fastapi_backend/app/domain/sources.py index 3292c501a..a3fefd8dc 100644 --- a/fastapi_backend/app/domain/sources.py +++ b/fastapi_backend/app/domain/sources.py @@ -639,11 +639,19 @@ def poll_due(db, cfg, log=print): for w in db.rows( "SELECT record::id(id) AS id, poll_minutes, next_scan_at FROM watch_path WHERE enabled = true AND next_scan_at <= $n", n=store.now() ): + nxt = (dt.datetime.now(dt.timezone.utc) + dt.timedelta(minutes=w.get("poll_minutes") or 5)).isoformat(timespec="seconds") + # taken first, so a second process polling (the API and a worker) doesn't scan it too + if not db.rows( + "UPDATE $r SET next_scan_at = $n WHERE next_scan_at = $was RETURN AFTER", + r=R("watch_path", w["id"]), + n=nxt, + was=w["next_scan_at"], + ): + continue try: poll_watch(db, cfg, w["id"], log) done += 1 except Exception as e: # noqa: BLE001 - recorded on the watch, retried next time - nxt = (dt.datetime.now(dt.timezone.utc) + dt.timedelta(minutes=w.get("poll_minutes") or 5)).isoformat(timespec="seconds") db.q("UPDATE $r SET last_error = $e, next_scan_at = $n", r=R("watch_path", w["id"]), e=f"{type(e).__name__}: {e}"[:300], n=nxt) log(f" watch {w['id']}: {type(e).__name__}: {e}") return done diff --git a/fastapi_backend/tests/api/test_routines.py b/fastapi_backend/tests/api/test_routines.py index 1173d2600..f0b930dc2 100644 --- a/fastapi_backend/tests/api/test_routines.py +++ b/fastapi_backend/tests/api/test_routines.py @@ -339,3 +339,25 @@ def flaky(*a, **k): if a.get("merge"): entities.undo_merge(db, a["merge"]) # undone from the entity page first assert organize.undo_run(db, run_id) == (len(applied), 0) + + +def test_scheduling_threads_scan_folders_and_run_routines(db, cfg, monkeypatch): + """What the API's background work and `lens worker` start: both rounds, until told to stop.""" + import threading + + from app.domain import sources + + calls = {"watched folders": threading.Event(), "routines": threading.Event()} + monkeypatch.setattr(routines, "MIN_WAIT_SECONDS", 0.01) + monkeypatch.setattr(routines, "CHECK_SECONDS", 0.01) + monkeypatch.setattr(sources, "poll_due", lambda db, cfg, log=None: calls["watched folders"].set()) + monkeypatch.setattr(routines, "run_due", lambda db, cfg, log=None: calls["routines"].set()) + stop = threading.Event() + threads = routines.start(db, lambda: {**cfg, "sources": {**cfg["sources"], "check_seconds": 0}}, stop) + try: + assert all(e.wait(5) for e in calls.values()) + finally: + stop.set() + for t in threads: + t.join(5) + assert not any(t.is_alive() for t in threads) diff --git a/fastapi_backend/tests/api/test_sources.py b/fastapi_backend/tests/api/test_sources.py index b86948ed3..319a41858 100644 --- a/fastapi_backend/tests/api/test_sources.py +++ b/fastapi_backend/tests/api/test_sources.py @@ -12,7 +12,7 @@ import pytest -from app.domain import deletion, pipelines, sources, store +from app.domain import convert, deletion, pipelines, sources, store from tests.helpers import drain, login, make_user, write_wav R = store.R @@ -49,6 +49,7 @@ def test_sources_and_watched_folders(client, new_client, db, cfg, folder): ) assert r.status_code == 200, r.text assert sources.poll_due(db, cfg) == 1 + assert sources.poll_due(db, cfg) == 0 # scanned: not due again until poll_minutes have passed drain(db, cfg) assert db.one("SELECT status FROM recording WHERE title = 'call1'")["status"] == "analyzed" # S3, through rclone's own S3 server @@ -102,7 +103,7 @@ def test_sources_and_watched_folders(client, new_client, db, cfg, folder): @needs_rclone -def test_watch_preview_counts_files(client, db, folder): +def test_watch_preview_counts_files(client, db, cfg, folder): inbox = folder / "inbox" inbox.mkdir() (inbox / "a.txt").write_text("Ann: hi.\nBen: hello.") @@ -112,9 +113,11 @@ def test_watch_preview_counts_files(client, db, folder): h = login(client, "root@x.io", "root password 1") sid = client.post("/api/v1/sources", json={"name": "inbox", "type": "local"}, headers=h).json()["id"] r = client.post("/api/v1/watches/preview", json={"source": sid, "path": str(inbox)}, headers=h) - assert r.json() == {"files": 2, "audio": 0, "transcripts": 2, "documents": 0, "images": 0} + # a text file is a document where the server can make a PDF of it (LibreOffice), else a transcript + docs = 0 if convert.unavailable(cfg, "a.txt") else 1 + assert r.json() == {"files": 2, "audio": 0, "transcripts": 2 - docs, "documents": docs, "images": 0} r = client.post("/api/v1/watches/preview", json={"source": sid, "path": str(inbox), "exclude": ["*.srt"]}, headers=h) - assert r.json()["transcripts"] == 1 + assert r.json()["transcripts"] == 1 - docs def test_sources_and_watches_without_rclone(client, new_client, db, cfg, folder):