Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version

## Unreleased

- **Watched folders and routines work in Docker and the packages.** They were checked only by the API's own
background work, which Docker, Synology, QNAP, Cloudron and Proxmox all turn off in favour of a `lens worker`
process; so folders were never scanned and routines never ran. `lens worker` now does both (not a worker limited
with `--steps`, nor one started with `--no-schedule`), and a folder scan is claimed first, as routine runs are.
- **MCP server: agents search, read and cite the archive.** Add `https://<your Lens>/mcp` to Claude, Cursor, VS Code
or another MCP client, sign in on Lens's consent page (OAuth), and the agent sees what you see: your namespaces,
the collections you were given a role on, and their graphs. Read-only tools: `search` (moments said, on screen or
Expand Down
6 changes: 5 additions & 1 deletion docs/processing.md
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ has left.

lens worker --steps transcribe,diarize # e.g. on the Mac, with SURREAL_URL pointing at the server

A worker that runs every step also scans watched folders and runs routines when they are due, so Docker and the
packages need no other process for them. One limited with `--steps`, or started with `--no-schedule`, only runs jobs.
A folder scan and a routine run are each claimed first, so several processes doing this never repeat one.

## Storage sources

Admins add sources in the app (`/api/v1/sources`): S3 or S3-compatible, Dropbox, Google Drive, OneDrive, SFTP, SMB,
Expand Down Expand Up @@ -262,7 +266,7 @@ namespaces, or all of them:

The schedule is a five-field cron expression (minute hour day-of-month month day-of-week, e.g. `0 3 * * *`) or
`@hourly`, `@daily`, `@weekly`, `@monthly`, in a time zone; without one a routine runs only when someone presses Run
now. The API process checks every 30 seconds (`lens watch` does too, for setups that run it separately); a routine is
now. Every 30 seconds, `lens worker` checks (and so does the API process when it runs background work, and `lens watch`); a routine is
never started twice at once, and one started in two processes runs once. Each run keeps what every action did and a
log.

Expand Down
7 changes: 7 additions & 0 deletions fastapi_backend/app/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,11 @@ def platform_main(argv, config):
w.add_argument("--name")
w.add_argument("--steps", help="comma-separated steps this worker runs (default: all)")
w.add_argument("--once", action="store_true", help="run what is queued, then exit")
w.add_argument(
"--no-schedule",
action="store_true",
help="don't scan watched folders or run routines (a worker limited with --steps never does)",
)
x = sub.add_parser("watch", help="scan watched folders on storage sources and run the routines that are due")
x.add_argument("--once", action="store_true")
a = ap.parse_args(argv)
Expand Down Expand Up @@ -258,6 +263,8 @@ def password():
print(f"worker {wk.name} runs {', '.join(sorted(wk.can))}; Ctrl-C to stop")
stop = threading.Event()
notify.start(db, C, stop, name=wk.name, log=print) # sends notifications too (docs/notifications.md)
if not (a.steps or a.no_schedule): # and scans watched folders and runs routines, as `lens watch` does
routines.start(db, C, stop, log=print)
_stop_on_term()
try:
wk.loop(stop)
Expand Down
20 changes: 2 additions & 18 deletions fastapi_backend/app/core/runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from typing import Any

from app.config import settings as env
from app.domain import auth, content_types, jobs, notify, routines, settings, setup, sources, store, templates
from app.domain import auth, content_types, jobs, notify, routines, settings, setup, store, templates

log = logging.getLogger("lens")

Expand Down Expand Up @@ -47,24 +47,8 @@ def start_background(self) -> None:
w = jobs.Worker(self.db, self.current, name=f"api-{os.getpid()}-{i}", log=log.info)
threading.Thread(target=w.loop, args=(self.stop,), daemon=True, name=f"worker-{i}").start()

def watcher() -> None:
while not self.stop.wait(max(5, self.current()["sources"]["check_seconds"])):
try:
sources.poll_due(self.db, self.current(), log=log.info)
except Exception: # noqa: BLE001 - keep polling
log.exception("watcher failed")

threading.Thread(target=watcher, daemon=True, name="watcher").start()
notify.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log=log.warning)

def scheduler() -> None:
while not self.stop.wait(routines.CHECK_SECONDS):
try:
routines.run_due(self.db, self.current(), log=log.info)
except Exception: # noqa: BLE001 - keep scheduling
log.exception("routines failed")

threading.Thread(target=scheduler, daemon=True, name="routines").start()
routines.start(self.db, self.current, self.stop, log=log.info)

def close(self) -> None:
self.stop.set()
Expand Down
26 changes: 26 additions & 0 deletions fastapi_backend/app/domain/routines.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from __future__ import annotations

import datetime as dt
import threading

from . import jobs, organize, schedule, sources, store, telemetry, workflows

Expand All @@ -31,6 +32,7 @@
MAX_ACTIONS, DEFAULT_LIMIT, MAX_LIMIT = 20, 500, 5000
STALE_MINUTES = 180
CHECK_SECONDS = 30 # how often the scheduler looks for routines that are due
MIN_WAIT_SECONDS = 5 # the shortest wait between rounds, whatever sources.check_seconds says
LOG_MAX = 500
FIELDS = (
"record::id(id) AS id, name, description, enabled, schedule, timezone, namespaces, actions, next_run_at, last_run_at, "
Expand Down Expand Up @@ -458,6 +460,30 @@ def run_due(db, cfg, log=print, now=None):
return done


def start(db, cfg_fn, stop, log=None):
"""The scheduling threads, until `stop` is set: watched folders scanned when due (every sources.check_seconds) and
routines run when due (every CHECK_SECONDS). The API runs them with its background work; `lens worker` runs them too,
since Docker and the packages run the API without it."""

def every(seconds, fn, what):
def loop():
while not stop.wait(max(MIN_WAIT_SECONDS, seconds())):
try:
fn(db, cfg_fn(), log=log)
except Exception as e: # noqa: BLE001 - keep going
if log:
log(f"{what}: {type(e).__name__}: {e}")

th = threading.Thread(target=loop, daemon=True, name=what)
th.start()
return th

return [
every(lambda: cfg_fn()["sources"]["check_seconds"], sources.poll_due, "watched folders"),
every(lambda: CHECK_SECONDS, run_due, "routines"),
]


# ---------- what a fresh archive starts with ----------
def seed(db):
"""The graph-organising workflow, and a routine that runs it every night (off until someone turns it on)."""
Expand Down
10 changes: 9 additions & 1 deletion fastapi_backend/app/domain/sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -639,11 +639,19 @@ def poll_due(db, cfg, log=print):
for w in db.rows(
"SELECT record::id(id) AS id, poll_minutes, next_scan_at FROM watch_path WHERE enabled = true AND next_scan_at <= $n", n=store.now()
):
nxt = (dt.datetime.now(dt.timezone.utc) + dt.timedelta(minutes=w.get("poll_minutes") or 5)).isoformat(timespec="seconds")
# taken first, so a second process polling (the API and a worker) doesn't scan it too
if not db.rows(
"UPDATE $r SET next_scan_at = $n WHERE next_scan_at = $was RETURN AFTER",
r=R("watch_path", w["id"]),
n=nxt,
was=w["next_scan_at"],
):
continue
try:
poll_watch(db, cfg, w["id"], log)
done += 1
except Exception as e: # noqa: BLE001 - recorded on the watch, retried next time
nxt = (dt.datetime.now(dt.timezone.utc) + dt.timedelta(minutes=w.get("poll_minutes") or 5)).isoformat(timespec="seconds")
db.q("UPDATE $r SET last_error = $e, next_scan_at = $n", r=R("watch_path", w["id"]), e=f"{type(e).__name__}: {e}"[:300], n=nxt)
log(f" watch {w['id']}: {type(e).__name__}: {e}")
return done
22 changes: 22 additions & 0 deletions fastapi_backend/tests/api/test_routines.py
Original file line number Diff line number Diff line change
Expand Up @@ -339,3 +339,25 @@ def flaky(*a, **k):
if a.get("merge"):
entities.undo_merge(db, a["merge"]) # undone from the entity page first
assert organize.undo_run(db, run_id) == (len(applied), 0)


def test_scheduling_threads_scan_folders_and_run_routines(db, cfg, monkeypatch):
"""What the API's background work and `lens worker` start: both rounds, until told to stop."""
import threading

from app.domain import sources

calls = {"watched folders": threading.Event(), "routines": threading.Event()}
monkeypatch.setattr(routines, "MIN_WAIT_SECONDS", 0.01)
monkeypatch.setattr(routines, "CHECK_SECONDS", 0.01)
monkeypatch.setattr(sources, "poll_due", lambda db, cfg, log=None: calls["watched folders"].set())
monkeypatch.setattr(routines, "run_due", lambda db, cfg, log=None: calls["routines"].set())
stop = threading.Event()
threads = routines.start(db, lambda: {**cfg, "sources": {**cfg["sources"], "check_seconds": 0}}, stop)
try:
assert all(e.wait(5) for e in calls.values())
finally:
stop.set()
for t in threads:
t.join(5)
assert not any(t.is_alive() for t in threads)
11 changes: 7 additions & 4 deletions fastapi_backend/tests/api/test_sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

import pytest

from app.domain import deletion, pipelines, sources, store
from app.domain import convert, deletion, pipelines, sources, store
from tests.helpers import drain, login, make_user, write_wav

R = store.R
Expand Down Expand Up @@ -49,6 +49,7 @@ def test_sources_and_watched_folders(client, new_client, db, cfg, folder):
)
assert r.status_code == 200, r.text
assert sources.poll_due(db, cfg) == 1
assert sources.poll_due(db, cfg) == 0 # scanned: not due again until poll_minutes have passed
drain(db, cfg)
assert db.one("SELECT status FROM recording WHERE title = 'call1'")["status"] == "analyzed"
# S3, through rclone's own S3 server
Expand Down Expand Up @@ -102,7 +103,7 @@ def test_sources_and_watched_folders(client, new_client, db, cfg, folder):


@needs_rclone
def test_watch_preview_counts_files(client, db, folder):
def test_watch_preview_counts_files(client, db, cfg, folder):
inbox = folder / "inbox"
inbox.mkdir()
(inbox / "a.txt").write_text("Ann: hi.\nBen: hello.")
Expand All @@ -112,9 +113,11 @@ def test_watch_preview_counts_files(client, db, folder):
h = login(client, "root@x.io", "root password 1")
sid = client.post("/api/v1/sources", json={"name": "inbox", "type": "local"}, headers=h).json()["id"]
r = client.post("/api/v1/watches/preview", json={"source": sid, "path": str(inbox)}, headers=h)
assert r.json() == {"files": 2, "audio": 0, "transcripts": 2, "documents": 0, "images": 0}
# a text file is a document where the server can make a PDF of it (LibreOffice), else a transcript
docs = 0 if convert.unavailable(cfg, "a.txt") else 1
assert r.json() == {"files": 2, "audio": 0, "transcripts": 2 - docs, "documents": docs, "images": 0}
r = client.post("/api/v1/watches/preview", json={"source": sid, "path": str(inbox), "exclude": ["*.srt"]}, headers=h)
assert r.json()["transcripts"] == 1
assert r.json()["transcripts"] == 1 - docs


def test_sources_and_watches_without_rclone(client, new_client, db, cfg, folder):
Expand Down
Loading