From 241b486ec1e92220661611b29d7c651ec4de82d3 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 07:06:56 +0000 Subject: [PATCH] feat(workers): keep heavy steps to their hours and to so many at once Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QFUNzy1QZy5HhKAyJxTAYJ --- README.md | 3 +- changelog.d/quiet-processing.md | 4 ++ docs/architecture.md | 10 +++ docs/requirements.md | 1 + fastapi_backend/app/domain/jobs.py | 7 +- fastapi_backend/app/domain/quiet.py | 68 +++++++++++++++++++ fastapi_backend/app/domain/settings.py | 25 +++++++ fastapi_backend/app/domain/store.py | 5 ++ fastapi_backend/tests/domain/test_quiet.py | 61 +++++++++++++++++ .../components/settings/bodies.tsx | 9 +++ nextjs-frontend/components/settings/model.ts | 39 +++++++++++ 11 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 changelog.d/quiet-processing.md create mode 100644 fastapi_backend/app/domain/quiet.py create mode 100644 fastapi_backend/tests/domain/test_quiet.py diff --git a/README.md b/README.md index 5a6b417d1..43ca72ff7 100644 --- a/README.md +++ b/README.md @@ -306,7 +306,8 @@ swapped for a cloud provider instead. Setup asks only what has no default: the m Picking a cloud model provider fills in its address and a model, so only its key is left to paste. Every setting lives in the web app, which works on a phone too: settings are in named groups, a search finds any one by name, and the options most people leave as they are stay folded. A thin bar along the top of the web app shows a page or its data is -still loading. ([Components](docs/components.md), +still loading. Heavy steps can be kept to the night, or to so many at a time (off by default). +([Components](docs/components.md), [quiet processing](docs/architecture.md#background-work), [speech providers](docs/speech-providers.md), [local models](docs/local-models.md), [cloud model providers](docs/configuration.md#cloud-model-providers); [checks](docs/requirements.md#light-by-default)) diff --git a/changelog.d/quiet-processing.md b/changelog.d/quiet-processing.md new file mode 100644 index 000000000..b767ddfa0 --- /dev/null +++ b/changelog.d/quiet-processing.md @@ -0,0 +1,4 @@ +- **Quiet processing.** Settings › Workers › Quiet processing keeps the heavy steps (transcribing, reading and looking + through media) to a window such as 01:00-06:00, or to so many at a time, both off by default; the rest runs as before + ([Background work](docs/architecture.md#background-work)). + diff --git a/docs/architecture.md b/docs/architecture.md index b9c79dd2a..c8013f87e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -81,6 +81,16 @@ whose worker stops responding is retried up to `workers.max_attempts`. Admins ca jobs), drain it (it also hands its job back after the step it's on) and resume it; the flag lives on the worker's record, so it holds across restarts. `GET /api/v1/events` streams job progress as server-sent events. +**Quiet processing** (Settings › Workers, both off by default) keeps the heavy steps (`workers.heavy_steps`: by +default transcribe, read, diarize, shots, ocr, faces, objects and nsfw, the ones that work through media on this +machine) to when it suits a small machine at home: + +- `workers.heavy_hours`, a window like `01:00-06:00` in `workers.timezone` (UTC by default; it may pass midnight). + Outside it no job starts a heavy step: a job that reaches one goes back on the queue, its log saying why, and waits + for the window. Light steps (analysis, summaries, reports) still run. +- `workers.heavy_at_once`, how many jobs may be in a heavy step at a time across every worker. A step already running + finishes; two workers claiming in the same instant can briefly make it one more. + In development the API can run workers in-process (`RUN_BACKGROUND=true` or `workers.inline > 0`); the Docker setups run a separate `worker` service. diff --git a/docs/requirements.md b/docs/requirements.md index 5cc40e247..ed17c4f9f 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -387,4 +387,5 @@ request, to another check that proves the same thing; loosening a requirement or | Settings are in named groups, a search finds any one by name, and rarely changed options stay folded | [settings-find.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/settings-find.test.tsx) `puts every section in exactly one group`, `finds a field by its label, and opens the section on it`; [settings-save.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/settings-save.test.tsx) `are folded under a line saying what's in them, the main one left in view` | | A thin bar along the top of the web app runs while a page or its first data loads, not for a quick answer or a background refresh | [top-progress.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/top-progress.test.tsx) `runs along the top while a page's first data loads`, `doesn't flash for a quick answer, or show for refreshing what's on screen`, `runs from a page change until Next writes the new address`, `finishes a page change that lands back on the page showing, as a redirect can` | | The web app works on a phone: every journey is surveyed on one, and none may take more than its baseline | [clicks.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/clicks.yml) `Survey on a phone`; [baseline-phone.json](https://github.com/adeelahmad/lense/blob/main/ci/clicks/baseline-phone.json); [phone-layout.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/phone-layout.test.tsx) | +| Heavy steps kept to their hours and to so many at a time, off by default | [test_quiet.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_quiet.py) `test_the_window_and_its_checks`, `test_heavy_steps_wait_for_their_hours_and_their_turn` | | The default install runs on a Raspberry Pi (arm64) with a cloud model | No check yet ([#486](https://github.com/adeelahmad/lense/issues/486)) | diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index 3d6b142bc..aaf09d80e 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -14,7 +14,7 @@ import threading import time -from . import activity, analyze, ingest, keyring, pipelines, render, speakers as spk, store, telemetry +from . import activity, analyze, ingest, keyring, pipelines, quiet, render, speakers as spk, store, telemetry R = store.R PIPELINE = [ @@ -800,6 +800,10 @@ def beat(): say(f"handing {step} to a worker that can run it") save("status = 'queued', step_index = $i, next_step = $s, worker = NONE", i=i, s=step) return "handed-off" + if step in quiet.held(db, cfg_fn(), job=jid): # Settings › Workers › Quiet processing + say(f"{step} waits: {quiet.why(cfg_fn())}") + save("status = 'queued', step_index = $i, next_step = $s, worker = NONE", i=i, s=step) + return "handed-off" t0 = started(i) save("step_index = $i, next_step = $s, heartbeat_at = $t", i=i, s=step) if spec.get("when") and not pipelines.condition_ok(db, rid, spec["when"]): @@ -1199,6 +1203,7 @@ def run_once(self): return False # steps whose engine or model is still being fetched wait for it can = self.can - self.keeper.blocked() if self.keeper else self.can + can = can - quiet.held(self.db, self.cfg_fn()) # heavy steps outside their hours, or at their limit job = claim(self.db, self.name, can) if can else None if not job: return False diff --git a/fastapi_backend/app/domain/quiet.py b/fastapi_backend/app/domain/quiet.py new file mode 100644 index 000000000..6d30105c9 --- /dev/null +++ b/fastapi_backend/app/domain/quiet.py @@ -0,0 +1,68 @@ +"""Quiet processing: hours for the heavy steps, and how many of them run at once (Settings › Workers). + +On a small machine at home, transcribing and reading video keep it busy and its fan loud. Two settings, both off by +default, keep that to when it suits: +- `workers.heavy_hours`: a window like "01:00-06:00" (in `workers.timezone`, UTC by default; it may pass midnight). + Outside it, no job starts a heavy step; one that reaches one goes back on the queue and waits for the window. +- `workers.heavy_at_once`: at most this many jobs in a heavy step at a time, across every worker (a step a worker is + already running finishes; two workers claiming in the same instant can briefly make it one more). + +The heavy steps are `workers.heavy_steps`: by default the ones that work through media on this machine (speech, +voices, shots, text on screen, faces, objects, NSFW, reading documents). Everything else runs as before. +""" + +from __future__ import annotations + +import datetime as dt +import re + +from . import schedule, store + +HEAVY = ["transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw"] +WINDOW = re.compile(r"^([01]\d|2[0-3]):([0-5]\d)-([01]\d|2[0-3]):([0-5]\d)$") + + +def window(text): + """'01:00-06:00' as (start, end) minutes of the day; ValueError when it isn't one.""" + m = WINDOW.match(str(text or "").strip()) + if not m: + raise ValueError("workers.heavy_hours is a window like 01:00-06:00 (24-hour times), or empty for any time") + a, b = int(m[1]) * 60 + int(m[2]), int(m[3]) * 60 + int(m[4]) + if a == b: + raise ValueError("workers.heavy_hours starts and ends at different times") + return a, b + + +def in_window(text, tz="UTC", now=None): + """Whether `now` (default: this moment) is inside the window, read in time zone `tz`.""" + a, b = window(text) + t = (now or dt.datetime.now(dt.timezone.utc)).astimezone(schedule.zone(tz)) + m = t.hour * 60 + t.minute + return a <= m < b if a < b else (m >= a or m < b) + + +def held(db, cfg, now=None, job=None): + """The heavy steps no job may start now: all of them outside the hours, or with as many running as allowed (not + counting `job`, the one asking); an empty set when neither setting is on.""" + w = cfg.get("workers") or {} + heavy = set(w.get("heavy_steps") or HEAVY) + if w.get("heavy_hours") and not in_window(w["heavy_hours"], w.get("timezone") or "UTC", now): + return heavy + cap = w.get("heavy_at_once") + if cap: + n = db.one( + "SELECT count() AS n FROM job WHERE status = 'running' AND next_step IN $h AND id != $j GROUP ALL", + h=sorted(heavy), + j=store.R("job", job if job is not None else -1), + ) + if (n or {}).get("n", 0) >= int(cap): + return heavy + return set() + + +def why(cfg): + """Why a heavy step is waiting, for the job's log.""" + w = cfg.get("workers") or {} + if w.get("heavy_hours") and not in_window(w["heavy_hours"], w.get("timezone") or "UTC"): + return f"heavy steps run {w['heavy_hours']} ({w.get('timezone') or 'UTC'})" + return f"{w.get('heavy_at_once')} heavy step(s) at a time are running" diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index 76ae0b4ed..2f3553be4 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -463,6 +463,29 @@ def secrets_view(db, base): return {"providers": secret_providers.providers(eff), "references": refs, "fields": fields} +def _quiet_setting(key, value): + """Quiet processing (quiet.py): the heavy steps' hours, how many at once, which steps, and the hours' time zone.""" + from . import jobs, quiet, schedule + + if key == "heavy_hours": + if value is None or (isinstance(value, str) and not value.strip()): + return None + quiet.window(value) # ValueError when it isn't a window + return value.strip() + if key == "heavy_at_once": + if value is not None and not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 64): + raise ValueError("workers.heavy_at_once is a whole number from 1 to 64, or empty for no limit") + return value + if key == "heavy_steps": + if not (isinstance(value, list) and all(isinstance(s, str) and s in jobs.STEPS for s in value)): + raise ValueError(f"workers.heavy_steps is a list of steps: {', '.join(jobs.STEPS)}") + return value + if not isinstance(value, str): + raise ValueError("workers.timezone is a time zone, like Europe/London") + schedule.zone(value.strip() or "UTC") + return value.strip() or "UTC" + + def _updates_setting(key, value): if key == "check_hours": if not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 24 * 30): @@ -687,6 +710,8 @@ def _check(section, key, value, default): if not isinstance(value, str) or len(value) > 200: raise ValueError("files.folder is a folder (a bucket and path for S3), up to 200 characters") return blobs._folder(value) + if section == "workers" and key in ("heavy_hours", "heavy_at_once", "heavy_steps", "timezone"): + return _quiet_setting(key, value) if (section, key) == ("video", "object_min_score"): if not (isinstance(value, (int, float)) and not isinstance(value, bool) and 0.05 <= value <= 0.95): raise ValueError("video.object_min_score is a number from 0.05 to 0.95") diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 166285009..970e62478 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -289,6 +289,11 @@ def labels(): "poll_seconds": 2, "stale_minutes": 15, "max_attempts": 3, + # quiet processing (quiet.py): a window like "01:00-06:00" for the heavy steps, and how many run at once; off + "heavy_hours": None, + "heavy_at_once": None, + "heavy_steps": ["transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw"], + "timezone": "UTC", "steps": [ "enhance", "clean_audio", diff --git a/fastapi_backend/tests/domain/test_quiet.py b/fastapi_backend/tests/domain/test_quiet.py new file mode 100644 index 000000000..a72a74cf5 --- /dev/null +++ b/fastapi_backend/tests/domain/test_quiet.py @@ -0,0 +1,61 @@ +"""Quiet processing (quiet.py): heavy steps only in their hours, and only so many at once.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from app.domain import ingest, jobs, quiet, settings, store +from tests import helpers +from tests.helpers import drain + + +def at(h, m=0): + return dt.datetime(2026, 10, 18, h, m, tzinfo=dt.UTC) + + +def test_the_window_and_its_checks(): + assert quiet.in_window("01:00-06:00", now=at(3)) and not quiet.in_window("01:00-06:00", now=at(6)) + assert quiet.in_window("23:00-06:00", now=at(23, 30)) and quiet.in_window("23:00-06:00", now=at(2)) + assert not quiet.in_window("23:00-06:00", now=at(12)) + assert quiet.in_window("01:00-06:00", "Europe/London", now=at(1, 30)) # 02:30 there in summer time + assert not quiet.in_window("01:00-06:00", "Europe/London", now=at(5, 30)) + for bad in ("1:00-6:00", "01:00-01:00", "25:00-06:00", "night"): + with pytest.raises(ValueError): + settings._check("workers", "heavy_hours", bad, None) + assert settings._check("workers", "heavy_hours", " ", None) is None # empty: any time + with pytest.raises(ValueError): + settings._check("workers", "heavy_at_once", 0, None) + with pytest.raises(ValueError): + settings._check("workers", "heavy_steps", ["transcribe", "nap"], None) + with pytest.raises(ValueError): + settings._check("workers", "timezone", "Mars/Olympus", None) + + +def test_heavy_steps_wait_for_their_hours_and_their_turn(cfg, db, folder): + tr = folder / "call.txt" + tr.write_text("[00:00] Mara: Here is the plan.") + rid = ingest.import_transcript(db, cfg, "pods", tr, log=helpers.quiet) + jid = jobs.enqueue(db, rid, ["transcribe"], by="test") # it says why there is nothing to do + assert quiet.held(db, cfg) == set() # off by default: nothing waits + + now = dt.datetime.now(dt.UTC) + later = now + dt.timedelta(hours=2) + cfg["workers"]["heavy_hours"] = f"{later:%H:%M}-{later + dt.timedelta(hours=1):%H:%M}" # not now + assert "transcribe" in quiet.held(db, cfg) and "analyze" not in quiet.held(db, cfg) + drain(db, cfg) + job = db.one("SELECT status, next_step FROM $j", j=store.R("job", jid)) + assert (job["status"], job["next_step"]) == ("queued", "transcribe") # still waiting for its hours + + cfg["workers"]["heavy_hours"] = f"{now - dt.timedelta(hours=1):%H:%M}-{now + dt.timedelta(hours=1):%H:%M}" + cfg["workers"]["heavy_at_once"] = 1 + db.q("CREATE job:9000 SET status = 'running', next_step = 'ocr', recording = $r, steps = ['ocr']", r=rid) + assert quiet.held(db, cfg) == set(quiet.HEAVY) # one heavy step running already: the limit + assert quiet.held(db, cfg, job=9000) == set() # not counting the job asking + drain(db, cfg) + assert db.one("SELECT status FROM $j", j=store.R("job", jid))["status"] == "queued" + + db.q("UPDATE job:9000 SET status = 'succeeded', next_step = NONE") + drain(db, cfg) + assert db.one("SELECT status FROM $j", j=store.R("job", jid))["status"] == "succeeded" diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index 615b1321b..5885fa1ea 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -492,6 +492,15 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) { + Quiet processing +
+ + + +
+ + +

The number of built-in workers and their steps take effect when the server restarts. See which workers are alive in{" "} diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index d6744e81f..27e3b618c 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -2902,6 +2902,45 @@ export const FIELDS: FieldSpec[] = [ min: 1, max: 20, }, + { + section: "workers", + key: "heavy_hours", + label: "Heavy steps run", + kind: "text", + nullable: true, + placeholder: "Any time", + hint: "A window like 01:00-06:00, for transcribing and reading media. Empty: any time", + }, + { + section: "workers", + key: "timezone", + label: "Time zone", + kind: "text", + placeholder: "UTC", + hint: "For those hours, like Europe/London", + }, + { + section: "workers", + key: "heavy_at_once", + label: "Heavy steps at once", + kind: "int", + min: 1, + max: 64, + nullable: true, + placeholder: "No limit", + }, + { + section: "workers", + key: "heavy_steps", + label: "Heavy steps", + kind: "checks", + options: ["transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "embed"].map( + (s) => ({ + value: s, + label: s, + }), + ), + }, { section: "workers", key: "steps",