From 6bf9c8bd570c80d51bb1e10b31c4debf22387247 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 02:55:38 +0000 Subject: [PATCH 1/3] feat(scaling): core of auto scaling (work in progress) Providers with sealed credentials, scaling groups with target, step and schedule rules, approvals only a web session gives, budgets as Lens budgets, a scaler loop with a lease, REST routes, schemas, settings and tables. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ALjNbLobvPe7VwvDL8j7V3 --- fastapi_backend/app/api/v1/router.py | 2 + fastapi_backend/app/api/v1/routes/scaling.py | 253 ++++ fastapi_backend/app/cli.py | 2 + fastapi_backend/app/core/runtime.py | 2 + fastapi_backend/app/domain/activity.py | 2 + fastapi_backend/app/domain/budgets.py | 8 +- fastapi_backend/app/domain/scaling.py | 1258 +++++++++++++++++ fastapi_backend/app/domain/scaling_drivers.py | 270 ++++ fastapi_backend/app/domain/settings.py | 24 + fastapi_backend/app/domain/store.py | 13 + fastapi_backend/app/schemas/scaling.py | 221 +++ 11 files changed, 2052 insertions(+), 3 deletions(-) create mode 100644 fastapi_backend/app/api/v1/routes/scaling.py create mode 100644 fastapi_backend/app/domain/scaling.py create mode 100644 fastapi_backend/app/domain/scaling_drivers.py create mode 100644 fastapi_backend/app/schemas/scaling.py diff --git a/fastapi_backend/app/api/v1/router.py b/fastapi_backend/app/api/v1/router.py index 701471d63..b6952159c 100644 --- a/fastapi_backend/app/api/v1/router.py +++ b/fastapi_backend/app/api/v1/router.py @@ -56,6 +56,7 @@ recordings, requests, routines, + scaling, search, searches, sensors, @@ -151,5 +152,6 @@ vaults, podcasts, specs, + scaling, ): api_router.include_router(module.router) diff --git a/fastapi_backend/app/api/v1/routes/scaling.py b/fastapi_backend/app/api/v1/routes/scaling.py new file mode 100644 index 000000000..bd9b2a45e --- /dev/null +++ b/fastapi_backend/app/api/v1/routes/scaling.py @@ -0,0 +1,253 @@ +"""Auto scaling (app/domain/scaling.py, docs/scaling.md; admins only): providers, scaling groups with their rules, +the units they launch, and what the scaler is doing. + +Anything an admin's API key, an app or an agent may do, they do here too: add and change providers and groups, set +the desired count within a group's bounds, pause, terminate a unit. Approving a group's terms is the one pick that +needs a person signed in to the web app (`POST /scaling/groups/{id}/approve` answers 403 to a token or an app), and a +group made or changed from the web app carries that approval with it. +""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, HTTPException, Query + +from app.api.deps import AdminReader, AdminWriter, Cfg, Db, Principal, domain_errors +from app.domain import auth, scaling +from app.domain import scaling_drivers as drivers +from app.schemas.common import Ok +from app.schemas.scaling import ( + Desired, + DriverSpec, + Group, + GroupChange, + GroupIn, + Metrics, + Overview, + Paused, + Protected, + Provider, + ProviderChange, + ProviderIn, + Terminate, + Unit, +) + +router = APIRouter(prefix="/scaling", tags=["scaling"]) + + +def _via(user: Principal) -> str: + return "web" if user.via == "access" else user.via + + +def _errors(): + """The domain's errors as answers: a provider's refusal is 502, a pick that isn't the caller's 403.""" + + class _Ctx: + def __enter__(self): + return self + + def __exit__(self, et, e, tb): + if et is None: + return False + if issubclass(et, scaling.Unapproved): + raise HTTPException(403, str(e)) from None + if issubclass(et, drivers.DriverError): + raise HTTPException(502, str(e)) from None + if issubclass(et, ValueError): + raise HTTPException(400, str(e)) from None + if issubclass(et, KeyError): + raise HTTPException(404, "not found") from None + return False + + return _Ctx() + + +@router.get("") +def overview(user: AdminReader, db: Db, cfg: Cfg) -> Overview: + """Everything at a glance: the switch, providers, groups with their units, the drivers on offer, the metrics now.""" + return Overview(**scaling.overview(db, cfg)) + + +@router.get("/drivers") +def list_drivers(user: AdminReader) -> list[DriverSpec]: + """The kinds of provider Lens can launch on, with the settings each asks for.""" + return [DriverSpec(**d) for d in scaling.driver_specs()] + + +@router.get("/metrics") +def metrics(user: AdminReader, db: Db, cfg: Cfg, steps: str | None = Query(None, description="comma-separated steps for the queue metrics")) -> Metrics: + """The values rules can watch, now.""" + want = [s.strip() for s in steps.split(",") if s.strip()] if steps else None + return Metrics(metrics=scaling.metrics(db, cfg, want), about=scaling.METRICS) + + +# ---------- providers ---------- +@router.get("/providers") +def list_providers(user: AdminReader, db: Db) -> list[Provider]: + return [Provider(**p) for p in scaling.providers(db)] + + +@router.post("/providers", status_code=201) +def add_provider(body: ProviderIn, user: AdminWriter, db: Db, cfg: Cfg) -> Provider: + """Add a provider. Its credentials are sealed (never shown again, only whether they're set) and checked with a + read-only call; its library is fetched the first time.""" + with _errors(): + p = scaling.add_provider(db, cfg, body.name, body.driver, body.settings, body.secrets, body.price, body.note, user.email, body.check) + auth.audit(db, user.as_audit(), "scaling.provider.add", f"scale_provider:{p['id']}", {"driver": body.driver}) + return Provider(**p) + + +@router.get("/providers/{pid}") +def get_provider(pid: int, user: AdminReader, db: Db) -> Provider: + with _errors(): + return Provider(**scaling.provider_view(scaling._provider(db, pid))) + + +@router.patch("/providers/{pid}") +def change_provider(pid: int, body: ProviderChange, user: AdminWriter, db: Db, cfg: Cfg) -> Provider: + changes = body.model_dump(exclude_unset=True, exclude={"check"}) + with _errors(): + p = scaling.update_provider(db, cfg, pid, changes, user.email, body.check) + auth.audit(db, user.as_audit(), "scaling.provider.change", f"scale_provider:{pid}", sorted(changes)) + return Provider(**p) + + +@router.post("/providers/{pid}/check") +def check_provider(pid: int, user: AdminReader, db: Db, cfg: Cfg) -> dict[str, Any]: + """Prove the provider's credentials with a read-only call: {ok, error?}.""" + with _errors(): + return scaling.check_provider(db, cfg, pid) + + +@router.delete("/providers/{pid}") +def remove_provider(pid: int, user: AdminWriter, db: Db, cfg: Cfg) -> Ok: + with _errors(): + scaling.remove_provider(db, cfg, pid, user.email) + auth.audit(db, user.as_audit(), "scaling.provider.remove", f"scale_provider:{pid}") + return Ok() + + +# ---------- groups ---------- +@router.get("/groups") +def list_groups(user: AdminReader, db: Db, cfg: Cfg) -> list[Group]: + gs = scaling.groups(db) + for g in gs: + g["waiting_for"] = scaling.waiting_for(g, cfg) + return [Group(**g) for g in gs] + + +@router.post("/groups", status_code=201) +def create_group(body: GroupIn, user: AdminWriter, db: Db, cfg: Cfg) -> Group: + """Make a group. From the web app it's approved as it's made (that is your pick); from a token or an app it + waits for a person's approval before it launches anything.""" + spec = body.model_dump(exclude_unset=True) + with _errors(): + g = scaling.create_group(db, cfg, spec, user.email, _via(user)) + auth.audit(db, user.as_audit(), "scaling.group.create", f"scale_group:{g['id']}", {"max": g["max"], "via": _via(user)}) + g["waiting_for"] = scaling.waiting_for(g, cfg) + return Group(**g) + + +@router.get("/groups/{gid}") +def get_group(gid: int, user: AdminReader, db: Db, cfg: Cfg) -> Group: + with _errors(): + g = scaling.get_group(db, gid) + g["waiting_for"] = scaling.waiting_for(g, cfg) + return Group(**g) + + +@router.patch("/groups/{gid}") +def change_group(gid: int, body: GroupChange, user: AdminWriter, db: Db, cfg: Cfg) -> Group: + """Change a group. A change that widens its approved terms (another provider or unit, a higher max or budget) + takes the approval away until a person gives it again, unless it's made from the web app.""" + changes = body.model_dump(exclude_unset=True) + with _errors(): + g = scaling.update_group(db, cfg, gid, changes, user.email, _via(user)) + auth.audit(db, user.as_audit(), "scaling.group.change", f"scale_group:{gid}", sorted(changes)) + g["waiting_for"] = scaling.waiting_for(g, cfg) + return Group(**g) + + +@router.post("/groups/{gid}/approve") +def approve_group(gid: int, user: AdminWriter, db: Db, cfg: Cfg) -> Group: + """Approve the group's terms as they stand: a person's pick, from a signed-in session only (403 otherwise).""" + with _errors(): + g = scaling.approve_group(db, cfg, gid, user.email, _via(user)) + auth.audit(db, user.as_audit(), "scaling.group.approve", f"scale_group:{gid}", g["terms"]) + g["waiting_for"] = scaling.waiting_for(g, cfg) + return Group(**g) + + +@router.post("/groups/{gid}/desired") +def set_desired(gid: int, body: Desired, user: AdminWriter, db: Db, cfg: Cfg) -> Group: + """Ask for a number of units now, within the group's min and max.""" + with _errors(): + g = scaling.set_desired(db, cfg, gid, body.desired, user.email, _via(user)) + auth.audit(db, user.as_audit(), "scaling.group.desired", f"scale_group:{gid}", {"desired": body.desired}) + g["waiting_for"] = scaling.waiting_for(g, cfg) + return Group(**g) + + +@router.post("/groups/{gid}/pause") +def pause_group(gid: int, body: Paused, user: AdminWriter, db: Db, cfg: Cfg) -> Group: + """Pause a group (nothing new launches; what runs stays) or resume it.""" + with _errors(): + g = scaling.pause_group(db, cfg, gid, body.paused, body.why, user.email) + auth.audit(db, user.as_audit(), "scaling.group.pause" if body.paused else "scaling.group.resume", f"scale_group:{gid}") + g["waiting_for"] = scaling.waiting_for(g, cfg) + return Group(**g) + + +@router.delete("/groups/{gid}") +def remove_group(gid: int, user: AdminWriter, db: Db, cfg: Cfg) -> Ok: + """Remove a group: its units are terminated first.""" + with _errors(): + scaling.remove_group(db, cfg, gid, user.email) + auth.audit(db, user.as_audit(), "scaling.group.remove", f"scale_group:{gid}") + return Ok() + + +@router.get("/groups/{gid}/history") +def group_history( + gid: int, user: AdminReader, db: Db, limit: int = Query(50, ge=1, le=200), before: int | None = Query(None) +) -> list[dict[str, Any]]: + """What happened to the group: launches, joins, terminations, changes and the cost it accrued (the ledger).""" + with _errors(): + scaling._group(db, gid) + return scaling.history(db, gid, limit, before) + + +# ---------- units ---------- +@router.get("/units") +def list_units( + user: AdminReader, db: Db, group: int | None = Query(None), alive: bool = Query(False, description="only units not yet gone") +) -> list[Unit]: + return [Unit(**n) for n in scaling.nodes(db, group, alive)] + + +@router.post("/units/{nid}/protect") +def protect_unit(nid: int, body: Protected, user: AdminWriter, db: Db, cfg: Cfg) -> Unit: + """Keep a unit out of scale-in (or let it go again).""" + with _errors(): + n = scaling.protect(db, cfg, nid, body.protected, user.email) + auth.audit(db, user.as_audit(), "scaling.unit.protect", f"scale_node:{nid}", {"protected": body.protected}) + return Unit(**n) + + +@router.post("/units/{nid}/terminate") +def terminate_unit(nid: int, body: Terminate, user: AdminWriter, db: Db, cfg: Cfg) -> Unit: + """Let a unit go now: a running worker drains first, so its job is handed back.""" + with _errors(): + n = scaling.terminate_node(db, cfg, nid, user.email, body.why) + auth.audit(db, user.as_audit(), "scaling.unit.terminate", f"scale_node:{nid}", {"why": body.why}) + return Unit(**n) + + +@router.post("/tick") +def tick(user: AdminWriter, db: Db, cfg: Cfg) -> dict[str, Any]: + """Run one pass of the scaler now, in this process (the scheduled one carries on): what it did.""" + with domain_errors(): + out = scaling.Scaler(db, lambda: cfg, f"api-tick-{user.id}").tick(lease=False) + return out or {} diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py index ef5ad9456..f023a604d 100644 --- a/fastapi_backend/app/cli.py +++ b/fastapi_backend/app/cli.py @@ -283,6 +283,7 @@ def platform_main(argv, config): notify, passkeys, routines, + scaling, sensors, settings, shares, @@ -407,6 +408,7 @@ def password(): tunnel.start(db, C, stop, name=wk.name, log=say) # and the Cloudflare tunnel, while it's on https.start(db, C, stop, name=wk.name, log=say) # and HTTPS, where the ports are (LENS_HTTPS_LISTEN) local_llm.start(db, C, stop, name=wk.name, log=say) # and the local chat model, while it's on + scaling.start(db, C, stop, name=wk.name, log_=say) # and auto scaling (docs/scaling.md), while it's on nfs_share.start(db, C, stop, name=wk.name, log=say) # and the file tree over NFS, while it's on updates.start(db, C, stop, log=say) # and the daily check for a new release (docs/updates.md) _stop_on_term() diff --git a/fastapi_backend/app/core/runtime.py b/fastapi_backend/app/core/runtime.py index e67a5668e..a97b66d48 100644 --- a/fastapi_backend/app/core/runtime.py +++ b/fastapi_backend/app/core/runtime.py @@ -22,6 +22,7 @@ local_llm, notify, routines, + scaling, sensors, settings, setup, @@ -102,6 +103,7 @@ def start_background(self) -> None: tunnel.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log=log.info) https.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log=log.info) local_llm.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log=log.info) + scaling.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log_=log.info) updates.start(self.db, self.current, self.stop, log=log.info) def close(self) -> None: diff --git a/fastapi_backend/app/domain/activity.py b/fastapi_backend/app/domain/activity.py index 89f91c460..7c0ab64f8 100644 --- a/fastapi_backend/app/domain/activity.py +++ b/fastapi_backend/app/domain/activity.py @@ -151,6 +151,8 @@ def _table(parts, i): return "note_page" if parts[i] == "runs" and i and parts[i - 1] == "automations": return "automation_run" + if i and parts[i - 1] == "scaling" and parts[i] in ("providers", "groups", "units"): # auto scaling (scaling.py) + return {"providers": "scale_provider", "groups": "scale_group", "units": "scale_node"}[parts[i]] return TABLES.get(parts[i]) diff --git a/fastapi_backend/app/domain/budgets.py b/fastapi_backend/app/domain/budgets.py index aed4fc047..455d7d2f6 100644 --- a/fastapi_backend/app/domain/budgets.py +++ b/fastapi_backend/app/domain/budgets.py @@ -32,8 +32,8 @@ R = store.R PERIODS = ("run", "day", "week", "month") ON_OVER = ("ask", "skip", "assistant") -TABLES = ("routine", "automation", "pipeline", "workflow", "space") -REF = re.compile(r"^(?:(routine|automation|pipeline|workflow|space):(\d+)|all)$") +TABLES = ("routine", "automation", "pipeline", "workflow", "space", "scale_group") +REF = re.compile(r"^(?:(routine|automation|pipeline|workflow|space|scale_group):(\d+)|all)$") ALL = "all" # everything Lens spends (set in setup); the meter measures all of Lens against it HARD_SECONDS = 30 # how long hard_stop trusts what it last found FIELDS = "resource, usd, tokens, period, on_over, warn_at, note, by, updated_at, alert" @@ -48,7 +48,9 @@ def _key(resource): def check_ref(resource): if not isinstance(resource, str) or not REF.match(resource): - raise ValueError("a budget is for all, routine:, automation:, pipeline:, workflow: or space:") + raise ValueError( + "a budget is for all, routine:, automation:, pipeline:, workflow:, space: or scale_group:" + ) return resource diff --git a/fastapi_backend/app/domain/scaling.py b/fastapi_backend/app/domain/scaling.py new file mode 100644 index 000000000..cc3296085 --- /dev/null +++ b/fastapi_backend/app/domain/scaling.py @@ -0,0 +1,1258 @@ +"""Auto scaling (docs/scaling.md): Lens launches more capacity when work waits, and lets it go when it doesn't. + +Two kinds of capacity, each a **scaling group** on one **provider** (scaling_drivers.py): + +* `node`: a machine that runs `lens worker` (more workers on this machine, or over the mesh a machine at a cloud, a + Proxmox host or any Linux box over SSH), so queued jobs run at once instead of one after another; +* `model`: an OpenAI-compatible model endpoint on a GPU (RunPod, Lambda, SageMaker, Bedrock, Hugging Face). + +A group has a floor and a ceiling (`min`, `max`), a desired count, and **rules** as AWS Auto Scaling has them: + +* target: keep a metric at a value (`desired = ceil(units × metric / target)`); +* step: when a metric is above (or below) a threshold for N minutes, add (or remove) units; +* schedule: at a cron time, set min, max or desired. + +Rules propose; Lens takes the largest proposal (out when any rule says so, in only when all agree), clamps it to +[min, max], then waits out the group's cooldown after a change and its warm-up while a unit is still joining. Metrics +come from what Lens already keeps (the job queue, workers' heartbeats, the activity ledger): nothing new is collected. + +**The person decides.** Scaling is off until an admin turns it on (`scaling.enabled`), and a group launches nothing +until a person, signed in to the web app, approves its terms (the provider, the unit, `max`, the monthly budget and +what happens at the cap). API keys, apps, MCP and the assistant create, change and propose groups, set the desired +count within the approved bounds, pause, and terminate units, but never approve: a change that widens the terms +(another provider or unit, a higher max or budget) takes the approval away until a person gives it again; running +units stay. A priced provider's group needs a budget, kept as a Lens budget (`budget:scale_group_`, budgets.py) so +the Costs page shows it; at its cap, or the overall budget's hard limit, the group drains and terminates its units +(`at_cap` stop) or keeps `min` of them. Each running unit's price is accrued into the activity ledger as it runs, +labelled an estimate, so what scaling costs shows next to what the model calls cost. + +One process runs the scaler (a lease on `app_service:scaling`, as the tunnel and the local model do), in `lens +worker` and the API: every `scaling.tick_seconds` it accrues cost, enforces budgets, reconciles each unit with its +provider (one that never joined within `join_minutes` is terminated, so a broken image costs one attempt; three +failures in a row pause the group), evaluates the rules, and launches or drains and terminates. Draining hands a +worker's job back (jobs.control), so nothing is lost. Every launch, termination and change is in the activity ledger +with who or what caused it, next to the audit log. +""" + +from __future__ import annotations + +import datetime as dt +import logging +import math +import re +import threading +import time + +from . import activity, budgets, jobs, schedule, settings, store +from . import scaling_drivers as drivers +from .store import R + +log = logging.getLogger("lens.scaling") +ROW = R("app_service", "scaling") +LEASE_SECONDS = 90 +DEFAULTS = {"enabled": False, "tick_seconds": 30, "join_minutes": 15, "accrue_minutes": 5, "tag": "lens"} +KINDS = ("node", "model") +STATES = ("launching", "running", "draining", "terminating", "terminated", "failed") +ALIVE = ("launching", "running", "draining", "terminating") +RULE_TYPES = ("target", "step", "schedule") +TERMINATE = ("oldest", "newest") +AT_CAP = ("stop", "min") +MAX_UNITS = 100 +MAX_RULES = 20 +MAX_FAILURES = 3 # launches that failed in a row before the group pauses itself +WORKER_ALIVE_SECONDS = 120 # a worker whose heartbeat is older isn't counted (they register every 30 seconds) +WORKER_LOST_SECONDS = 600 # a unit's worker silent this long is lost: terminated and replaced +MODEL_WINDOW_SECONDS = 300 # model metrics look at the last five minutes +NAME_RX = re.compile(r"^[^\x00-\x1f]{1,80}$") +TAG_RX = re.compile(r"^[a-z][a-z0-9-]{0,15}$") +METRICS = { + "jobs_waiting": "jobs waiting in the queue (those the group's steps run, when it names steps)", + "oldest_wait_minutes": "how many minutes the oldest waiting job has waited", + "workers_busy_share": "the share of live workers that are busy, 0 to 1", + "worker_load": "live workers' average load per CPU (1.0: every CPU busy)", + "model_calls_per_minute": "model calls per minute over the last five minutes", + "model_seconds": "how long a model call took on average over the last five minutes, in seconds", +} +PROVIDER_FIELDS = "record::id(id) AS id, name, driver, settings, sealed, price, note, created_at, updated_at, updated_by, checked_at, check" +GROUP_FIELDS = ( + "record::id(id) AS id, name, kind, provider, template, min, max, desired, cooldown_seconds, warmup_seconds, terminate, rules, " + "rule_state, at_cap, enabled, approval, paused, paused_why, capped, created_at, updated_at, updated_by, last_out_at, last_in_at, " + "failures, error, last_eval" +) +NODE_FIELDS = ( + "record::id(id) AS id, group, provider, driver, name, external_id, state, address, endpoint, price, launched_at, ready_at, " + "ended_at, by, why, protected, error, accrued_at" +) + + +class Unapproved(PermissionError): + """A pick only a person signed in to the web app makes.""" + + +def conf(cfg): + return {**DEFAULTS, **((cfg or {}).get("scaling") or {})} + + +def enabled(cfg): + return bool(conf(cfg).get("enabled")) + + +def _now(): + return dt.datetime.now(dt.timezone.utc) + + +def _iso(t): + return t.astimezone(dt.timezone.utc).isoformat(timespec="seconds") + + +def _parse(s): + if not s: + return None + try: + return dt.datetime.fromisoformat(str(s).replace("Z", "+00:00")) + except ValueError: + return None + + +def _age(s, now): + """Seconds since an ISO time, or None.""" + t = _parse(s) + return (now - t).total_seconds() if t else None + + +def _ref(table, key): + return activity.ref(table, key) + + +def _slug(name): + s = re.sub(r"[^a-z0-9]+", "-", (name or "").lower()).strip("-")[:24] + return s or "group" + + +def _int(v, what, lo, hi): + if isinstance(v, bool) or not isinstance(v, int) or not lo <= v <= hi: + raise ValueError(f"{what} is a whole number from {lo} to {hi}") + return v + + +def _num(v, what, lo=0.0, hi=1e9): + if isinstance(v, bool) or not isinstance(v, int | float) or not lo <= v <= hi: + raise ValueError(f"{what} is a number from {lo:g} to {hi:g}") + return float(v) + + +# ---------- providers ---------- +def driver_specs(): + return [d.spec() for d in drivers.DRIVERS.values()] + + +def _seal(cfg, pid, key, value): + return settings.seal(cfg, value, f"scale_provider:{pid}.{key}") + + +def _unseal(cfg, pid, key, sealed): + return settings.unseal(cfg, sealed, f"scale_provider:{pid}.{key}") + + +def providers(db): + return [provider_view(p) for p in db.rows(f"SELECT {PROVIDER_FIELDS} FROM scale_provider ORDER BY id")] + + +def _provider(db, pid): + row = db.one(f"SELECT {PROVIDER_FIELDS} FROM $r", r=R("scale_provider", int(pid))) + if not row: + raise KeyError(f"there's no provider {pid}") + return row + + +def provider_view(p): + """A provider as the API shows it: which credentials are set, never their values.""" + drv = drivers.DRIVERS.get(p.get("driver")) + return { + "id": p["id"], + "name": p.get("name"), + "driver": p.get("driver"), + "kind": drv.kind if drv else None, + "label": drv.label if drv else p.get("driver"), + "priced": drv.priced if drv else True, + "settings": p.get("settings") or {}, + "secrets": {k: True for k in (p.get("sealed") or {})}, + "price": p.get("price"), + "note": p.get("note"), + "checked_at": p.get("checked_at"), + "check": p.get("check"), + "created_at": p.get("created_at"), + "updated_at": p.get("updated_at"), + "updated_by": p.get("updated_by"), + } + + +def opened(db, cfg, pid): + """A provider with its credentials opened, for a driver's call only: never shown.""" + p = _provider(db, pid) + secrets = {} + for k, sealed in (p.get("sealed") or {}).items(): + try: + secrets[k] = _unseal(cfg, p["id"], k, sealed) + except Exception as e: # noqa: BLE001 - the key changed: the credential has to be given again + raise drivers.DriverError(f"the provider's {k} can't be read any more ({type(e).__name__}): save it again") from None + return {**p, "secrets": secrets} + + +def _check_provider_body(drv, settings_, secrets, price, note): + settings_ = drivers.check_settings(drv, settings_) + secrets = dict(secrets or {}) + unknown = sorted(set(secrets) - set(drv.secrets)) + if unknown: + raise ValueError(f"a {drv.label} provider has no secret {', '.join(unknown)}; it takes {', '.join(drv.secrets) or 'none'}") + for k, v in secrets.items(): + if v is not None and (not isinstance(v, str) or len(v) > 10_000): + raise ValueError(f"{k} is text") + if price is not None: + price = _num(price, "price (USD per unit-hour)", 0, 10_000) + if note is not None and (not isinstance(note, str) or len(note) > 300): + raise ValueError("note is up to 300 characters") + return settings_, secrets, price, note + + +def _check_call(db, cfg, p, drv): + """Prove the credentials with the driver's read-only call; kept on the provider either way.""" + try: + with activity.call(f"scaling.{drv.id}.check", cfg, refs=[_ref("scale_provider", p["id"])]): + out = drv.check(opened(db, cfg, p["id"])) + result = {"ok": True, **(out if isinstance(out, dict) else {})} + except Exception as e: # noqa: BLE001 - what the provider said, for the person + result = {"ok": False, "error": f"{type(e).__name__}: {e}"[:500] if not isinstance(e, drivers.DriverError) else str(e)[:500]} + db.q("UPDATE $r SET checked_at = $t, check = $c", r=R("scale_provider", p["id"]), t=store.now(), c=result) + return result + + +def _fetch(cfg, drv, say=None): + """The driver's library, fetched the first time a provider of its kind is saved (components.pip_install).""" + if drv.ready(): + return + from . import components + + components.pip_install(cfg, drv.extra, say or (lambda m: log.info("scaling: %s", m))) + if not drv.ready(): + raise ValueError(f"{drv.label} needs the {drv.extra} packages, which couldn't be installed here") + + +def add_provider(db, cfg, name, driver, settings_=None, secrets=None, price=None, note=None, by=None, check=True): + """Save a provider: its settings, its credentials sealed (bound to the provider's id and the field), and the + driver's read-only check. The view (never the credentials).""" + drv = drivers.get(driver) + if not isinstance(name, str) or not NAME_RX.match(name.strip()): + raise ValueError("a provider has a name of up to 80 characters") + settings_, secrets, price, note = _check_provider_body(drv, settings_, secrets, price, note) + missing = [k for k in drv.secrets if drv.fields[k].get("required") and not secrets.get(k)] + if missing: + raise ValueError(f"a {drv.label} provider needs {', '.join(missing)}") + if drv.extra: + _fetch(cfg, drv) + pid = db.next_id("scale_provider") + db.q( + "CREATE $r CONTENT $d", + r=R("scale_provider", pid), + d=store.clean( + { + "name": name.strip(), + "driver": drv.id, + "settings": settings_, + "sealed": {k: _seal(cfg, pid, k, v) for k, v in secrets.items() if v}, + "price": price, + "note": note, + "created_at": store.now(), + "updated_at": store.now(), + "updated_by": by, + } + ), + ) + activity.record("run", "scaling.provider.add", [_ref("scale_provider", pid)], cfg, db, actor=by, detail={"driver": drv.id}) + p = _provider(db, pid) + if check: + _check_provider(db, cfg, p, drv) + p = _provider(db, pid) + return provider_view(p) + + +def _check_provider(db, cfg, p, drv): + return _check_call(db, cfg, p, drv) + + +def update_provider(db, cfg, pid, changes, by=None, check=True): + """Change a provider's name, settings, credentials (a secret given as null is removed; one left out is kept), + price or note.""" + p = _provider(db, pid) + drv = drivers.get(p["driver"]) + changes = dict(changes or {}) + unknown = sorted(set(changes) - {"name", "settings", "secrets", "price", "note"}) + if unknown: + raise ValueError(f"a provider has no {', '.join(unknown)}") + name = changes.get("name", p.get("name")) + if not isinstance(name, str) or not NAME_RX.match(name.strip()): + raise ValueError("a provider has a name of up to 80 characters") + settings_ = {**(p.get("settings") or {}), **(changes.get("settings") or {})} if "settings" in changes else p.get("settings") or {} + settings_, secrets, price, note = _check_provider_body( + drv, settings_, changes.get("secrets"), changes.get("price", p.get("price")), changes.get("note", p.get("note")) + ) + sealed = dict(p.get("sealed") or {}) + for k, v in secrets.items(): + if v is None: + sealed.pop(k, None) + elif v: + sealed[k] = _seal(cfg, p["id"], k, v) + missing = [k for k in drv.secrets if drv.fields[k].get("required") and k not in sealed] + if missing: + raise ValueError(f"a {drv.label} provider needs {', '.join(missing)}") + db.q( + "UPDATE $r MERGE $d", + r=R("scale_provider", p["id"]), + d={ + "name": name.strip(), + "settings": settings_, + "sealed": sealed, + "price": price, + "note": note, + "updated_at": store.now(), + "updated_by": by, + }, + ) + activity.record( + "run", "scaling.provider.change", [_ref("scale_provider", p["id"])], cfg, db, actor=by, detail={"fields": sorted(changes)} + ) + out = _provider(db, p["id"]) + if check: + _check_provider(db, cfg, out, drv) + out = _provider(db, p["id"]) + return provider_view(out) + + +def check_provider(db, cfg, pid): + p = _provider(db, pid) + return _check_provider(db, cfg, p, drivers.get(p["driver"])) + + +def remove_provider(db, cfg, pid, by=None): + p = _provider(db, pid) + used = db.values("SELECT VALUE record::id(id) FROM scale_group WHERE provider = $p", p=p["id"]) + if used: + raise ValueError(f"groups {', '.join(map(str, used))} use this provider: remove them first") + db.q("DELETE $r", r=R("scale_provider", p["id"])) + activity.record("run", "scaling.provider.remove", [_ref("scale_provider", p["id"])], cfg, db, actor=by) + + +# ---------- groups ---------- +def _budget_ref(gid): + return f"scale_group:{int(gid)}" + + +def check_rule(rule): + """A rule as it's kept, or ValueError.""" + if not isinstance(rule, dict): + raise ValueError("a rule is an object") + t = rule.get("type") + if t not in RULE_TYPES: + raise ValueError(f"a rule's type is one of {', '.join(RULE_TYPES)}") + out = {"type": t} + if t in ("target", "step"): + m = rule.get("metric") + if m not in METRICS: + raise ValueError(f"a rule's metric is one of {', '.join(METRICS)}") + out["metric"] = m + if t == "target": + out["target"] = _num(rule.get("target"), "target", 0.000001) + elif t == "step": + above, below = rule.get("above"), rule.get("below") + if (above is None) == (below is None): + raise ValueError("a step rule has above or below: the threshold") + if above is not None: + out["above"] = _num(above, "above") + else: + out["below"] = _num(below, "below") + out["minutes"] = _int(rule.get("minutes", 1), "minutes", 0, 1440) + add, remove = rule.get("add"), rule.get("remove") + if (add is None) == (remove is None): + raise ValueError("a step rule adds or removes units: add or remove, a count or a percentage like \"50%\"") + for k, v in (("add", add), ("remove", remove)): + if v is None: + continue + if isinstance(v, str) and v.strip().endswith("%") and v.strip()[:-1].isdigit() and 0 < int(v.strip()[:-1]) <= 1000: + out[k] = v.strip() + else: + out[k] = _int(v, k, 1, MAX_UNITS) + else: + cron, tz = rule.get("cron"), rule.get("timezone") or "UTC" + if not isinstance(cron, str): + raise ValueError("a schedule rule has cron: when it fires, like 0 9 * * 1-5") + try: + schedule.check(cron.strip(), tz) + except ValueError as e: + raise ValueError(f"a schedule rule's cron: {e}") from None + out["cron"], out["timezone"] = cron.strip(), tz + if all(rule.get(k) is None for k in ("min", "max", "desired")): + raise ValueError("a schedule rule sets min, max or desired") + for k in ("min", "max", "desired"): + if rule.get(k) is not None: + out[k] = _int(rule[k], k, 0, MAX_UNITS) + if rule.get("name") is not None: + if not isinstance(rule["name"], str) or len(rule["name"]) > 80: + raise ValueError("a rule's name is up to 80 characters") + out["name"] = rule["name"] + return out + + +def describe_rule(r): + if r["type"] == "target": + return f"keep {r['metric']} at {r['target']:g}" + if r["type"] == "step": + cond = f"{r['metric']} {'above' if 'above' in r else 'below'} {r.get('above', r.get('below')):g} for {r['minutes']} min" + act = f"add {r['add']}" if "add" in r else f"remove {r['remove']}" + return f"{act} when {cond}" + parts = [f"{k} {r[k]}" for k in ("min", "max", "desired") if k in r] + return f"at {schedule.describe(r['cron'])} ({r['timezone']}): {', '.join(parts)}" + + +def terms(g): + """What an approval binds: what's launched, where, how many at most, and what it may cost.""" + return { + "kind": g.get("kind"), + "provider": g.get("provider"), + "template": g.get("template") or {}, + "max": g.get("max"), + "budget": (g.get("budget") or {}).get("usd") if isinstance(g.get("budget"), dict) else g.get("budget"), + "at_cap": g.get("at_cap") or "stop", + } + + +def widens(old, new): + """Whether new terms let more be spent than the old ones a person approved.""" + if old.get("kind") != new.get("kind") or old.get("provider") != new.get("provider") or old.get("template") != new.get("template"): + return True + if (new.get("max") or 0) > (old.get("max") or 0): + return True + ob, nb = old.get("budget"), new.get("budget") + if ob is not None and (nb is None or nb > ob): + return True + return old.get("at_cap", "stop") == "stop" and new.get("at_cap") == "min" + + +def _check_group_body(db, body, current=None): + """A group's fields checked: the row's content (budget apart) and the budget cap.""" + body = dict(body or {}) + unknown = sorted( + set(body) + - { + "name", + "kind", + "provider", + "template", + "min", + "max", + "desired", + "cooldown_seconds", + "warmup_seconds", + "terminate", + "rules", + "at_cap", + "enabled", + "budget", + } + ) + if unknown: + raise ValueError(f"a group has no {', '.join(unknown)}") + cur = current or {} + name = body.get("name", cur.get("name")) + if not isinstance(name, str) or not NAME_RX.match(name.strip()): + raise ValueError("a group has a name of up to 80 characters") + pid = body.get("provider", cur.get("provider")) + if pid is None: + raise ValueError("a group has a provider") + p = _provider(db, pid) + drv = drivers.get(p["driver"]) + kind = body.get("kind", cur.get("kind") or drv.kind) + if kind != drv.kind: + raise ValueError(f"a {drv.label} provider launches {drv.kind} units, not {kind}") + template = drivers.check_template(drv, body.get("template", cur.get("template"))) + lo = _int(body.get("min", cur.get("min", 0)), "min", 0, MAX_UNITS) + hi = _int(body.get("max", cur.get("max", 1)), "max", 0, MAX_UNITS) + if hi < lo: + raise ValueError("max is at least min") + desired = body.get("desired", cur.get("desired")) + desired = lo if desired is None else min(hi, max(lo, _int(desired, "desired", 0, MAX_UNITS))) + rules = [check_rule(r) for r in (body.get("rules", cur.get("rules")) or [])] + if len(rules) > MAX_RULES: + raise ValueError(f"a group has up to {MAX_RULES} rules") + terminate = body.get("terminate", cur.get("terminate") or "oldest") + if terminate not in TERMINATE: + raise ValueError(f"terminate is one of {', '.join(TERMINATE)}") + at_cap = body.get("at_cap", cur.get("at_cap") or "stop") + if at_cap not in AT_CAP: + raise ValueError(f"at_cap is one of {', '.join(AT_CAP)}: stop every unit at the budget's cap, or keep min") + budget = body.get("budget", cur.get("budget")) + if isinstance(budget, dict): + budget = budget.get("usd") + if budget is not None: + budget = _num(budget, "budget (USD per month)", 0.01, 1e7) + if drv.priced and budget is None: + raise ValueError(f"a group on {drv.label} needs a budget: USD per month") + row = { + "name": name.strip(), + "kind": kind, + "provider": p["id"], + "template": template, + "min": lo, + "max": hi, + "desired": desired, + "cooldown_seconds": _int(body.get("cooldown_seconds", cur.get("cooldown_seconds", 300)), "cooldown_seconds", 0, 86400), + "warmup_seconds": _int(body.get("warmup_seconds", cur.get("warmup_seconds", 300)), "warmup_seconds", 0, 7200), + "terminate": terminate, + "rules": rules, + "at_cap": at_cap, + "enabled": bool(body.get("enabled", cur.get("enabled", True))), + } + return row, budget, p, drv + + +def groups(db): + return [group_view(db, g) for g in db.rows(f"SELECT {GROUP_FIELDS} FROM scale_group ORDER BY id")] + + +def _group(db, gid): + row = db.one(f"SELECT {GROUP_FIELDS} FROM $r", r=R("scale_group", int(gid))) + if not row: + raise KeyError(f"there's no scaling group {gid}") + return row + + +def get_group(db, gid): + return group_view(db, _group(db, gid)) + + +def _alive(db, gid): + return db.rows(f"SELECT {NODE_FIELDS} FROM scale_node WHERE group = $g AND state IN $s ORDER BY id", g=int(gid), s=list(ALIVE)) + + +def group_view(db, g, now=None): + """A group as the API shows it: its rules in words, its budget's standing, its units, and whether it may launch.""" + now = now or _now() + units = _alive(db, g["id"]) + b = budgets.get(db, _budget_ref(g["id"])) + st = budgets.status(db, _budget_ref(g["id"]), b, now) if b else None + approval = g.get("approval") + out = { + "id": g["id"], + "name": g.get("name"), + "kind": g.get("kind"), + "provider": g.get("provider"), + "template": g.get("template") or {}, + "min": g.get("min") or 0, + "max": g.get("max") or 0, + "desired": g.get("desired") or 0, + "cooldown_seconds": g.get("cooldown_seconds"), + "warmup_seconds": g.get("warmup_seconds"), + "terminate": g.get("terminate") or "oldest", + "rules": [{**r, "says": describe_rule(r)} for r in (g.get("rules") or [])], + "at_cap": g.get("at_cap") or "stop", + "enabled": bool(g.get("enabled", True)), + "paused": bool(g.get("paused")), + "paused_why": g.get("paused_why"), + "capped": g.get("capped"), + "approval": approval, + "approved": bool(approval) and not widens(approval.get("terms") or {}, terms(g)), + "terms": terms(g), + "budget": {"usd": b.get("usd"), "period": b.get("period"), "state": st["state"], "spent": st["spent"], "left": st["left"]} if b else None, + "units": { + "launching": sum(u.get("state") == "launching" for u in units), + "running": sum(u.get("state") == "running" for u in units), + "draining": sum(u.get("state") in ("draining", "terminating") for u in units), + }, + "last_out_at": g.get("last_out_at"), + "last_in_at": g.get("last_in_at"), + "last_eval": g.get("last_eval"), + "failures": g.get("failures") or 0, + "error": g.get("error"), + "created_at": g.get("created_at"), + "updated_at": g.get("updated_at"), + "updated_by": g.get("updated_by"), + } + out["waiting_for"] = waiting_for(out) + return out + + +def waiting_for(view, cfg=None): + """Why a group launches nothing now, in words, or None.""" + if cfg is not None and not enabled(cfg): + return "scaling is off: turn it on in Settings › Scaling" + if not view.get("enabled"): + return "the group is off" + if view.get("paused"): + return f"paused: {view.get('paused_why') or 'by hand'}" + if not view.get("approved"): + return "a person's approval of its terms, in Settings › Scaling" + if view.get("capped"): + return f"its budget: {view['capped']}" + return None + + +def _set_budget(db, gid, usd, by): + ref = _budget_ref(gid) + if usd is None: + if budgets.get(db, ref): + budgets.remove(db, ref) + return + old = budgets.get(db, ref) or {} + budgets.put(db, ref, usd=usd, period="month", on_over="skip", warn_at=old.get("warn_at") or 0.8, note=old.get("note"), by=by) + + +def create_group(db, cfg, body, by=None, via="token"): + """Save a group. It launches nothing until a person approves its terms (approve_group): a group made by a person + signed in to the web app (`via` web) is approved as it's made, since that is their pick.""" + row, budget, p, drv = _check_group_body(db, body) + gid = db.next_id("scale_group") + row = {**row, "rule_state": {}, "paused": False, "failures": 0, "created_at": store.now(), "updated_at": store.now(), "updated_by": by} + db.q("CREATE $r CONTENT $d", r=R("scale_group", gid), d=store.clean(row)) + _set_budget(db, gid, budget, by) + activity.record( + "run", + "scaling.group.create", + [_ref("scale_group", gid), _ref("scale_provider", p["id"])], + cfg, + db, + actor=by, + detail={"max": row["max"], "min": row["min"], "budget": budget, "via": via}, + ) + if via == "web": + approve_group(db, cfg, gid, by, via) + return get_group(db, gid) + + +def update_group(db, cfg, gid, changes, by=None, via="token"): + """Change a group. A change that widens its approved terms takes the approval away (a person gives it again); + one made by a person signed in to the web app carries their approval.""" + g = _group(db, gid) + row, budget, p, drv = _check_group_body(db, changes, g) + before, after = terms({**g, "budget": (budgets.get(db, _budget_ref(g["id"])) or {}).get("usd")}), terms({**row, "budget": budget}) + approval = g.get("approval") + if approval and widens(approval.get("terms") or {}, after): + approval = None + row = {**row, "approval": approval, "updated_at": store.now(), "updated_by": by} + if "rules" in changes: + row["rule_state"] = {} + db.q("UPDATE $r MERGE $d", r=R("scale_group", g["id"]), d=row) + if approval is None and g.get("approval"): # MERGE keeps a field given as None: take it off + db.q("UPDATE $r SET approval = NONE", r=R("scale_group", g["id"])) + _set_budget(db, g["id"], budget, by) + activity.record( + "run", + "scaling.group.change", + [_ref("scale_group", g["id"]), _ref("scale_provider", p["id"])], + cfg, + db, + actor=by, + detail={"fields": sorted(changes or {}), "widens": widens(before, after), "via": via}, + ) + if via == "web" and approval is None: + approve_group(db, cfg, g["id"], by, via) + return get_group(db, g["id"]) + + +def approve_group(db, cfg, gid, by, via="web"): + """A person's approval of a group's terms as they stand now: only from a signed-in session in the web app. + Agents (API keys, apps, MCP, the assistant) never approve.""" + if via != "web": + raise Unapproved("approving a scaling group's terms is the person's own pick, signed in to Lens") + g = _group(db, gid) + t = terms({**g, "budget": (budgets.get(db, _budget_ref(g["id"])) or {}).get("usd")}) + drv = drivers.get(_provider(db, g["provider"])["driver"]) + if drv.priced and t["budget"] is None: + raise ValueError("give the group a budget first") + approval = {"by": by, "at": store.now(), "terms": t} + db.q("UPDATE $r SET approval = $a, updated_at = $t, updated_by = $u", r=R("scale_group", g["id"]), a=approval, t=store.now(), u=by) + activity.record("run", "scaling.group.approve", [_ref("scale_group", g["id"])], cfg, db, actor=by, detail=t) + return get_group(db, g["id"]) + + +def set_desired(db, cfg, gid, n, by=None, via="token"): + """Ask for a number of units now, within [min, max]: anyone who may change the group, agents included, since the + bounds are what the person approved. The rules take over again at their next proposal.""" + g = _group(db, gid) + n = _int(n, "desired", 0, MAX_UNITS) + if not g.get("min", 0) <= n <= g.get("max", 0): + raise ValueError(f"desired is from {g.get('min', 0)} to {g.get('max', 0)} for this group") + db.q("UPDATE $r SET desired = $n, updated_at = $t, updated_by = $u", r=R("scale_group", g["id"]), n=n, t=store.now(), u=by) + activity.record("run", "scaling.group.desired", [_ref("scale_group", g["id"])], cfg, db, actor=by, detail={"desired": n, "via": via}) + return get_group(db, g["id"]) + + +def pause_group(db, cfg, gid, paused=True, why=None, by=None): + g = _group(db, gid) + db.q( + "UPDATE $r SET paused = $p, paused_why = $w, failures = 0, updated_at = $t, updated_by = $u", + r=R("scale_group", g["id"]), + p=bool(paused), + w=(why or ("by hand" if by else None)) if paused else None, + t=store.now(), + u=by, + ) + activity.record("run", "scaling.group.pause" if paused else "scaling.group.resume", [_ref("scale_group", g["id"])], cfg, db, actor=by) + return get_group(db, g["id"]) + + +def remove_group(db, cfg, gid, by=None): + """Remove a group: its live units are terminated first (a draining one at once), then its rows and budget go.""" + g = _group(db, gid) + p = opened(db, cfg, g["provider"]) + drv = drivers.get(p["driver"]) + for n in _alive(db, g["id"]): + _terminate(db, cfg, drv, p, n, "the group was removed", by) + db.q("DELETE scale_node WHERE group = $g", g=g["id"]) + db.q("DELETE $r", r=R("scale_group", g["id"])) + _set_budget(db, g["id"], None, by) + activity.record("run", "scaling.group.remove", [_ref("scale_group", g["id"])], cfg, db, actor=by) + + +def history(db, gid, limit=50, before=None): + return activity.history(db, _budget_ref(gid), limit, before) + + +# ---------- units ---------- +def nodes(db, gid=None, alive=False): + where, p = [], {} + if gid is not None: + where.append("group = $g") + p["g"] = int(gid) + if alive: + where.append("state IN $s") + p["s"] = list(ALIVE) + q = f"SELECT {NODE_FIELDS} FROM scale_node" + (f" WHERE {' AND '.join(where)}" if where else "") + " ORDER BY id DESC LIMIT 500" + return [node_view(n) for n in db.rows(q, **p)] + + +def _node(db, nid): + row = db.one(f"SELECT {NODE_FIELDS} FROM $r", r=R("scale_node", int(nid))) + if not row: + raise KeyError(f"there's no unit {nid}") + return row + + +def node_view(n): + endpoint = n.get("endpoint") + return { + **{k: n.get(k) for k in NODE_FIELDS.replace("record::id(id) AS id", "id").split(", ")}, + "endpoint": {k: v for k, v in endpoint.items() if k != "api_key"} | ({"api_key": True} if endpoint.get("api_key") else {}) + if isinstance(endpoint, dict) + else None, + } + + +def protect(db, cfg, nid, on=True, by=None): + n = _node(db, nid) + db.q("UPDATE $r SET protected = $p", r=R("scale_node", n["id"]), p=bool(on)) + activity.record("run", "scaling.unit.protect", [_ref("scale_node", n["id"]), _ref("scale_group", n["group"])], cfg, db, actor=by, detail={"on": bool(on)}) + return node_view(_node(db, nid)) + + +def terminate_node(db, cfg, nid, by=None, why=None): + """Let a unit go now: a running worker drains first (its job is handed back), anything else is terminated.""" + n = _node(db, nid) + if n.get("state") not in ALIVE: + raise ValueError(f"unit {nid} is {n.get('state')}") + if n.get("state") == "running" and n.get("driver") and drivers.get(n["driver"]).kind == "node": + _drain(db, cfg, n, why or "asked to", by) + elif n.get("state") != "terminating": + p = opened(db, cfg, n["provider"]) + _terminate(db, cfg, drivers.get(p["driver"]), p, n, why or "asked to", by) + return node_view(_node(db, nid)) + + +def _drain(db, cfg, n, why, by=None): + try: + jobs.control(db, n["name"], "drain", by=by or "scaling") + except KeyError: + pass # never registered as a worker: nothing to hand back + db.q("UPDATE $r SET state = 'draining', why = $w", r=R("scale_node", n["id"]), w=why) + activity.record("run", "scaling.unit.drain", [_ref("scale_node", n["id"]), _ref("scale_group", n["group"])], cfg, db, actor=by, detail={"why": why}) + + +def _terminate(db, cfg, drv, p, n, why, by=None, state="terminated", now=None): + """Terminate a unit at its provider and close its row (accruing its last cost); a provider that fails to is tried + again next tick (the row stays terminating).""" + now = now or _now() + db.q("UPDATE $r SET state = 'terminating', why = $w", r=R("scale_node", n["id"]), w=why) + if n.get("external_id"): + try: + with activity.call(f"scaling.{drv.id}.terminate", cfg, refs=[_ref("scale_node", n["id"]), _ref("scale_group", n["group"])]): + drv.terminate(p, n) + except Exception as e: # noqa: BLE001 - tried again next tick + err = f"{type(e).__name__}: {e}"[:300] if not isinstance(e, drivers.DriverError) else str(e)[:300] + db.q("UPDATE $r SET error = $e", r=R("scale_node", n["id"]), e=f"couldn't terminate: {err}") + log.warning("scaling: couldn't terminate unit %s: %s", n["id"], err) + return False + accrue(db, cfg, n, now, final=True) + db.q( + "UPDATE $r SET state = $s, ended_at = $t, accrued_at = $t", + r=R("scale_node", n["id"]), + s=state, + t=_iso(now), + ) + db.q("DELETE $w", w=R("worker", n["name"])) + activity.record( + "run", + "scaling.terminate" if state == "terminated" else "scaling.fail", + [_ref("scale_node", n["id"]), _ref("scale_group", n["group"]), _ref("scale_provider", n["provider"])], + cfg, + db, + actor=by, + ok=state == "terminated", + detail={"why": why, "unit": n.get("name")}, + ) + return True + + +def accrue(db, cfg, n, now, final=False): + """What a unit cost since its last accrual, as a ledger row counted for it, its group and its provider: its price + per hour for the hours since, always an estimate (a list price, not a bill). Every accrue_minutes, and when it + ends.""" + price = n.get("price") + since = _parse(n.get("accrued_at") or n.get("launched_at")) + if not price or since is None: + return 0.0 + hours = max(0.0, (now - since).total_seconds() / 3600) + if hours <= 0 or (not final and hours * 60 < conf(cfg)["accrue_minutes"]): + return 0.0 + cost = round(price * hours, 6) + activity.record( + "out", + "scaling.unit", + [_ref("scale_node", n["id"]), _ref("scale_group", n["group"]), _ref("scale_provider", n["provider"])], + cfg, + db, + cost_usd=cost, + ms=int(hours * 3_600_000), + detail={"estimate": True, "price_per_hour": price, "unit": n.get("name")}, + ) + db.q("UPDATE $r SET accrued_at = $t", r=R("scale_node", n["id"]), t=_iso(now)) + return cost + + +# ---------- metrics ---------- +def _live_workers(db, now): + out = [] + for w in jobs.workers(db, now): + age = _age(w.get("heartbeat_at"), now) + if age is not None and age <= WORKER_ALIVE_SECONDS: + out.append(w) + return out + + +def metrics(db, cfg, steps=None, now=None): + """Every metric's value now: {name: value}. `steps` narrows the queue metrics to jobs those steps run.""" + now = now or _now() + q, p = "SELECT created_at FROM job WHERE status = 'queued'", {} + if steps: + q += " AND next_step IN $steps" + p["steps"] = sorted(steps) + waiting = db.rows(q, **p) + oldest = None + for j in waiting: + age = _age(j.get("created_at"), now) + if age is not None and (oldest is None or age > oldest): + oldest = age + live = _live_workers(db, now) + busy = [w for w in live if w.get("current") is not None] + loads = [w["load"] for w in live if isinstance(w.get("load"), int | float)] + since = _iso(now - dt.timedelta(seconds=MODEL_WINDOW_SECONDS)) + row = ( + db.one( + "SELECT count() AS n, math::sum(ms ?? 0) AS ms FROM activity WHERE kind = 'out' AND at >= $s AND string::starts_with(action, 'model.') GROUP ALL", + s=since, + ) + or {} + ) + calls = int(row.get("n") or 0) + return { + "jobs_waiting": len(waiting), + "oldest_wait_minutes": round(oldest / 60, 1) if oldest is not None else 0.0, + "workers_busy_share": round(len(busy) / len(live), 3) if live else 0.0, + "worker_load": round(sum(loads) / len(loads), 2) if loads else 0.0, + "model_calls_per_minute": round(calls / (MODEL_WINDOW_SECONDS / 60), 2), + "model_seconds": round(float(row.get("ms") or 0) / calls / 1000, 2) if calls else 0.0, + "workers_live": len(live), + } + + +# ---------- rules ---------- +def _count(v, units): + if isinstance(v, str): + return max(1, math.ceil(units * int(v[:-1]) / 100)) + return int(v) + + +def evaluate(g, m, units, now, state=None): + """What the rules propose for a group with `units` running: (desired or None when no rule speaks, the new rule + state, [what each rule said]). Step rules fire once their condition has held for their minutes; schedule rules + once per time they name.""" + state = dict(state or {}) + proposals, said = [], [] + new_min, new_max = None, None + for i, r in enumerate(g.get("rules") or []): + key, st = str(i), dict(state.get(str(i)) or {}) + if r["type"] == "target": + v = m.get(r["metric"]) or 0 + want = math.ceil(max(units, 1) * v / r["target"]) if v > 0 else 0 + proposals.append(want) + said.append(f"{describe_rule(r)}: {r['metric']} is {v:g}, so {want}") + elif r["type"] == "step": + v = m.get(r["metric"]) or 0 + breach = v > r["above"] if "above" in r else v < r["below"] + if not breach: + st.pop("since", None) + else: + since = _parse(st.get("since")) + if since is None: + since = now + st["since"] = _iso(now) + if (now - since).total_seconds() >= r["minutes"] * 60: + want = units + _count(r["add"], units) if "add" in r else units - _count(r["remove"], units) + proposals.append(max(0, want)) + said.append(f"{describe_rule(r)}: {r['metric']} is {v:g}, so {max(0, want)}") + else: + last = _parse(st.get("fired_at")) or _parse(g.get("updated_at")) or _parse(g.get("created_at")) or now + try: + nxt = schedule.next_after(r["cron"], r.get("timezone") or "UTC", last) + except ValueError: + nxt = None + if nxt is not None and nxt <= now: + st["fired_at"] = _iso(now) + if r.get("min") is not None: + new_min = r["min"] + if r.get("max") is not None: + new_max = r["max"] + if r.get("desired") is not None: + proposals.append(r["desired"]) + said.append(f"{describe_rule(r)}: fired") + state[key] = st + want = max(proposals) if proposals else None + return want, state, said, new_min, new_max + + +# ---------- the scaler ---------- +def mesh_module(): + """The Nebula mesh (mesh.py, docs/network.md) when this Lens has it, else None.""" + try: + from . import mesh + except ImportError: + return None + return mesh + + +def bootstrap(db, cfg, g, name): + """How a unit joins Lens: a worker started here gets its name and inherits the rest; a machine elsewhere gets the + mesh's install line (`LENS_JOIN=… LENS_JOIN_TOKEN=… sh install.sh`) with a one-time join token good for an hour, + named for the unit, so the provider's user-data carries no long-lived secret: the database credential travels + inside the sealed join answer. {env, line, token_id} (line and token_id None without the mesh).""" + boot = {"env": {"LENS_SCALING_UNIT": name}, "line": None, "token_id": None} + mesh = mesh_module() + if mesh is None or not (getattr(mesh, "conf", lambda c: {})(cfg) or {}).get("enabled"): + return boot + make, line = getattr(mesh, "make_token", None), getattr(mesh, "join_line", None) + if not (make and line): + return boot + row, token = make(db, cfg, by="scaling", role="worker", name=name, hours=1, uses=1) + boot["line"], boot["token_id"] = line(cfg, token), (row or {}).get("id") + return boot + + +class Scaler: + """Holds the lease and runs each group's loop.""" + + def __init__(self, db, cfg_fn, name, log_=None): + self.db, self.cfg_fn, self.name, self.log = db, cfg_fn, name, log_ or (lambda *a: None) + self.lock = threading.Lock() + + def _lease(self, now): + return bool( + self.db.rows( + "UPSERT $r MERGE {owner: $me, at: $t} WHERE owner = NONE OR owner = $me OR at < $stale RETURN AFTER", + r=ROW, + me=self.name, + t=_iso(now), + stale=_iso(now - dt.timedelta(seconds=LEASE_SECONDS)), + ) + ) + + def tick(self, now=None, lease=True): + """One pass over every group. Returns what it did: {launched, terminated, groups}.""" + now = now or _now() + cfg = self.cfg_fn() + if lease and not self._lease(now): + return None + did = {"launched": 0, "terminated": 0, "groups": 0} + for g in self.db.rows(f"SELECT {GROUP_FIELDS} FROM scale_group ORDER BY id"): + try: + self._group(cfg, g, now, did) + did["groups"] += 1 + except Exception as e: # noqa: BLE001 - one group's trouble doesn't stop the rest; it shows on the group + err = f"{type(e).__name__}: {e}"[:300] + self.db.q("UPDATE $r SET error = $e", r=R("scale_group", g["id"]), e=err) + self.log(f"scaling: group {g['id']}: {err}") + self.db.q("UPDATE $r MERGE {ticked_at: $t, running: true} WHERE owner = $me", r=ROW, t=_iso(now), me=self.name) + return did + + def _group(self, cfg, g, now, did): + db = self.db + p = opened(db, cfg, g["provider"]) + drv = drivers.get(p["driver"]) + units = _alive(db, g["id"]) + # 1. cost, 2. the budget + for n in units: + if n.get("state") in ("running", "draining") and n.get("price"): + accrue(db, cfg, n, now) + capped = budgets.used_up(db, [_budget_ref(g["id"])], now) if units or g.get("desired") else None + if (capped or None) != (g.get("capped") or None): + db.q("UPDATE $r SET capped = $c", r=R("scale_group", g["id"]), c=capped) + if capped: + activity.record("run", "scaling.group.capped", [_ref("scale_group", g["id"])], cfg, db, detail={"why": capped}) + # 3. each unit against its provider and Lens + units = [n for n in units if self._reconcile(cfg, g, drv, p, n, now, did)] + # 4. the rules, then act + running = [n for n in units if n.get("state") == "running"] + launching = [n for n in units if n.get("state") == "launching"] + view = group_view(db, g, now) + may_launch = enabled(cfg) and view["enabled"] and not view["paused"] and view["approved"] and not capped + lo, hi, desired = g.get("min") or 0, g.get("max") or 0, g.get("desired") or 0 + if may_launch and g.get("rules"): + want, state, said, new_min, new_max = evaluate(g, metrics(db, cfg, (g.get("template") or {}).get("steps"), now), len(running), now, g.get("rule_state")) + sets = {"rule_state": state, "last_eval": {"at": _iso(now), "said": said}} + if new_min is not None or new_max is not None: + lo = new_min if new_min is not None else lo + hi = new_max if new_max is not None else hi + if hi < lo: + hi = lo + sets |= {"min": lo, "max": hi} + if want is not None: + desired = want + desired = min(hi, max(lo, desired)) + sets["desired"] = desired + db.q("UPDATE $r MERGE $d", r=R("scale_group", g["id"]), d=sets) + target = desired + if not enabled(cfg) or not view["enabled"]: + target = 0 + elif capped: + target = lo if g.get("at_cap") == "min" else 0 + elif not view["approved"] or view["paused"]: + target = min(target, len(running) + len(launching)) # nothing new; what runs stays + have = len(running) + len(launching) + if target > have: + if any((_age(n.get("launched_at"), now) or 0) < (g.get("warmup_seconds") or 0) for n in launching): + return # warming up: wait for it to join before judging again + cool = _age(g.get("last_out_at"), now) + if cool is not None and cool < (g.get("cooldown_seconds") or 0): + return + for _ in range(target - have): + if self._launch(cfg, g, drv, p, now, f"desired {target}, {have} running or joining"): + did["launched"] += 1 + have += 1 + else: + break + db.q("UPDATE $r SET last_out_at = $t", r=R("scale_group", g["id"]), t=_iso(now)) + elif target < have: + cool = _age(g.get("last_in_at"), now) + if cool is not None and cool < (g.get("cooldown_seconds") or 0) and target > 0: + return + gone = 0 + for n in self._for_removal(g, launching + running, have - target, now): + why = f"desired {target}, {have} running" + if n.get("state") == "running" and drv.kind == "node": + _drain(db, cfg, n, why) + else: + _terminate(db, cfg, drv, p, n, why, now=now) + gone += 1 + did["terminated"] += gone + if gone: + db.q("UPDATE $r SET last_in_at = $t", r=R("scale_group", g["id"]), t=_iso(now)) + + def _for_removal(self, g, units, n, now): + """Which units go when scaling in: never a protected one; idle before busy; then the oldest (or newest).""" + workers = {w["name"]: w for w in jobs.workers(self.db, now)} + free = [u for u in units if not u.get("protected")] + busy = lambda u: 1 if (workers.get(u["name"]) or {}).get("current") is not None else 0 # noqa: E731 + free.sort(key=lambda u: (busy(u), u.get("launched_at") or "" if g.get("terminate") != "newest" else "")) + if g.get("terminate") == "newest": + free.sort(key=lambda u: (busy(u), -(_parse(u.get("launched_at")) or now).timestamp())) + return free[:n] + + def _launch(self, cfg, g, drv, p, now, why): + db = self.db + nid = db.next_id("scale_node") + name = f"{conf(cfg)['tag']}-{_slug(g.get('name'))}-{nid}" + price = p.get("price") + if price is None: + try: + price = drv.price(p, g.get("template") or {}) + except Exception: # noqa: BLE001 - no price: nothing accrues + price = None + db.q( + "CREATE $r CONTENT $d", + r=R("scale_node", nid), + d=store.clean( + { + "group": g["id"], + "provider": p["id"], + "driver": drv.id, + "name": name, + "state": "launching", + "price": price, + "launched_at": _iso(now), + "accrued_at": _iso(now), + "by": "rule" if g.get("rules") else "desired", + "why": why, + "protected": False, + } + ), + ) + refs = [_ref("scale_node", nid), _ref("scale_group", g["id"]), _ref("scale_provider", p["id"])] + try: + with activity.call(f"scaling.{drv.id}.launch", cfg, refs=refs): + got = drv.launch(p, g.get("template") or {}, name, bootstrap(db, cfg, g, name)) or {} + except Exception as e: # noqa: BLE001 - the unit failed; three in a row pause the group + err = f"{type(e).__name__}: {e}"[:300] if not isinstance(e, drivers.DriverError) else str(e)[:300] + db.q("UPDATE $r SET state = 'failed', error = $e, ended_at = $t", r=R("scale_node", nid), e=err, t=_iso(now)) + fails = (g.get("failures") or 0) + 1 + db.q("UPDATE $r SET failures = $f, error = $e", r=R("scale_group", g["id"]), f=fails, e=err) + activity.record("run", "scaling.fail", refs, cfg, db, ok=False, error=type(e).__name__, detail={"why": err, "unit": name}) + self.log(f"scaling: couldn't launch {name}: {err}") + if fails >= MAX_FAILURES: + pause_group(db, cfg, g["id"], True, f"{fails} launches failed in a row; the last: {err}") + return False + db.q( + "UPDATE $r MERGE $d", + r=R("scale_node", nid), + d=store.clean( + { + "external_id": str(got.get("id")) if got.get("id") is not None else None, + "address": got.get("address"), + "endpoint": got.get("endpoint"), + "price": got.get("price", price), + } + ), + ) + db.q("UPDATE $r SET failures = 0, error = NONE", r=R("scale_group", g["id"])) + activity.record("run", "scaling.launch", refs, cfg, db, detail={"why": why, "unit": name, "price_per_hour": got.get("price", price)}) + self.log(f"scaling: launched {name} on {p.get('name')} ({why})") + return True + + def _reconcile(self, cfg, g, drv, p, n, now, did): + """One unit against its provider and Lens: True while it's still alive.""" + db = self.db + state = n.get("state") + if state == "terminating": + _terminate(db, cfg, drv, p, n, n.get("why") or "terminating", now=now) + did["terminated"] += 1 + return False + try: + with activity.call(f"scaling.{drv.id}.describe", cfg, refs=[_ref("scale_node", n["id"])]): + d = drv.describe(p, n) or {} + except Exception as e: # noqa: BLE001 - the provider didn't answer: judged again next tick + db.q("UPDATE $r SET error = $e", r=R("scale_node", n["id"]), e=f"{type(e).__name__}: {e}"[:300]) + return True + if d.get("state") == "gone": + why = "stopped at the provider" if state != "launching" else "never came up at the provider" + _terminate(db, cfg, drv, p, {**n, "external_id": None}, why, state="failed" if state == "launching" else "terminated", now=now) + did["terminated"] += 1 + return False + sets = {k: d[k] for k in ("address", "endpoint") if d.get(k) and d.get(k) != n.get(k)} + worker = db.one("SELECT heartbeat_at, current FROM $w", w=R("worker", n["name"])) if drv.kind == "node" else None + beat = _age((worker or {}).get("heartbeat_at"), now) if worker else None + if state == "launching": + joined = (beat is not None and beat <= WORKER_ALIVE_SECONDS) if drv.kind == "node" else d.get("state") == "running" + if joined: + sets |= {"state": "running", "ready_at": _iso(now), "error": None} + activity.record( + "run", "scaling.join", [_ref("scale_node", n["id"]), _ref("scale_group", g["id"])], cfg, db, detail={"unit": n["name"]} + ) + self.log(f"scaling: {n['name']} joined") + elif (_age(n.get("launched_at"), now) or 0) > conf(cfg)["join_minutes"] * 60: + _terminate(db, cfg, drv, p, n, f"didn't join within {conf(cfg)['join_minutes']} minutes", state="failed", now=now) + fails = (g.get("failures") or 0) + 1 + db.q("UPDATE $r SET failures = $f", r=R("scale_group", g["id"]), f=fails) + if fails >= MAX_FAILURES: + pause_group(db, cfg, g["id"], True, f"{fails} units in a row failed to join") + did["terminated"] += 1 + return False + elif state == "running" and drv.kind == "node" and beat is not None and beat > WORKER_LOST_SECONDS: + _terminate(db, cfg, drv, p, n, f"its worker has been silent for {int(beat // 60)} minutes", state="failed", now=now) + did["terminated"] += 1 + return False + elif state == "draining" and (worker is None or worker.get("current") is None or beat is None or beat > WORKER_ALIVE_SECONDS): + _terminate(db, cfg, drv, p, n, n.get("why") or "drained", now=now) + did["terminated"] += 1 + return False + if sets: + db.q("UPDATE $r MERGE $d", r=R("scale_node", n["id"]), d=store.clean(sets)) + if sets.get("state") == "running": + db.q("UPDATE $r SET error = NONE", r=R("scale_node", n["id"])) + n.update(sets) + return True + + def loop(self, stop): + while not stop.is_set(): + try: + self.tick() + except Exception as e: # noqa: BLE001 - keep the scaler alive + self.log(f"scaling: {type(e).__name__}: {e}") + stop.wait(max(5, int(conf(self.cfg_fn()).get("tick_seconds") or 30))) + self.db.q("UPDATE $r SET owner = NONE, running = false WHERE owner = $me", r=ROW, me=self.name) + + +def start(db, cfg_fn, stop, name, log_=None): + s = Scaler(db, cfg_fn, name, log_) + threading.Thread(target=s.loop, args=(stop,), daemon=True, name="scaler").start() + return s + + +def state(db): + """Which process scales, and when it last looked.""" + return db.one("SELECT owner, at, ticked_at, running FROM $r", r=ROW) or {} + + +def overview(db, cfg, now=None): + """Everything at a glance: the switch, the providers, the groups with their units, the drivers on offer.""" + now = now or _now() + gs = [group_view(db, g, now) for g in db.rows(f"SELECT {GROUP_FIELDS} FROM scale_group ORDER BY id")] + for g in gs: + g["waiting_for"] = waiting_for(g, cfg) + return { + "enabled": enabled(cfg), + "settings": conf(cfg), + "scaler": state(db), + "providers": providers(db), + "groups": gs, + "units": nodes(db, alive=True), + "drivers": driver_specs(), + "metrics": metrics(db, cfg, None, now), + } + + +# ---------- the assistant and MCP ---------- +def apply_tool(db, cfg, tool, args, user, via="agent"): + """Carry out a scaling tool's change, for the assistant (ops_tools.apply) once approved, or at once for what + agents may do within the approved terms. `via` web is the person's own yes in the web app, which also approves + a group's terms.""" + by = user.get("email") if isinstance(user, dict) else user + if tool == "propose_scaling_group": + return {"group": create_group(db, cfg, args.get("group") or {}, by, via)} + if tool == "change_scaling_group": + return {"group": update_group(db, cfg, int(args["group_id"]), args.get("changes") or {}, by, via)} + if tool == "set_scaling_desired": + return {"group": set_desired(db, cfg, int(args["group_id"]), int(args["desired"]), by, via)} + if tool == "pause_scaling_group": + return {"group": pause_group(db, cfg, int(args["group_id"]), bool(args.get("paused", True)), args.get("why"), by)} + if tool == "terminate_scaling_unit": + return {"unit": terminate_node(db, cfg, int(args["unit_id"]), by, args.get("why"))} + raise ValueError(f"no tool {tool}") + + +def time_now(): + return time.time() diff --git a/fastapi_backend/app/domain/scaling_drivers.py b/fastapi_backend/app/domain/scaling_drivers.py new file mode 100644 index 000000000..65209530d --- /dev/null +++ b/fastapi_backend/app/domain/scaling_drivers.py @@ -0,0 +1,270 @@ +"""The providers auto scaling launches units on (scaling.py, docs/scaling.md): what each one asks for, and the calls +that launch, look at and terminate a unit there. + +A driver is the code for one kind of provider. It names the settings an admin fills in for a provider of its kind +(`fields`), which of them are credentials (`secrets`, sealed by scaling.py and handed to the driver opened, only for +a call), and what a scaling group picks for each unit (`template`: a size, an image, the steps a worker runs). A +driver that charges money (`priced`) needs a budget on every group that uses it. + +The calls get the provider as `p` ({id, name, region, settings, secrets}) and a unit as `node` (its scale_node row): + +* check(p): a read-only call that proves the credentials work (listing regions, say); raises DriverError. +* price(p, template): the list price in USD per unit-hour, when the driver knows it; None otherwise. +* launch(p, template, name, boot): start one unit named `name`. `boot` is how a node joins Lens (scaling.bootstrap: + a script and its environment). Returns {"id": the provider's id for it, "address"?, "endpoint"?, "price"?}. +* describe(p, node): where it stands: {"state": "pending" | "running" | "gone", "address"?, "endpoint"?}. +* terminate(p, node): stop it, and delete what was made for it. Terminating one that's already gone is no error. +* listing(p, tag): the units the provider has that carry Lens's tag, [{"id", "name"}], to find ones Lens lost. + +Drivers that need a library name it as `extra` (an optional dependency in pyproject.toml): it's fetched the first +time a provider of that kind is saved (components.pip_install), so the default install carries none of them. +""" + +from __future__ import annotations + +import os +import signal +import socket +import subprocess +import sys +import threading + +from . import jobs + + +class DriverError(RuntimeError): + """What a provider said when a call failed, for the person to read.""" + + +FIELD_TYPES = ("string", "integer", "number", "boolean", "list") + + +def _field(label, type_="string", required=False, default=None, help=None, options=None, secret=False): + return { + "label": label, + "type": type_, + "required": required, + "default": default, + "help": help, + "options": options, + "secret": secret, + } + + +class Driver: + id = "" + kind = "node" # node: a machine that runs `lens worker`; model: an OpenAI-compatible model endpoint + label = "" + about = "" + priced = True # it charges money: its groups need a budget + extra = None # the optional dependency (pyproject.toml) it needs, fetched when a provider of it is saved + modules = () # what that extra installs, to see whether it's here + license = None + fields: dict = {} # the provider's settings: {name: _field(...)}; secret ones are sealed + template: dict = {} # what a group picks for each unit + + @property + def secrets(self): + return tuple(k for k, f in self.fields.items() if f.get("secret")) + + def ready(self): + from . import components + + return not self.modules or components.importable(*self.modules) + + def spec(self): + return { + "id": self.id, + "kind": self.kind, + "label": self.label, + "about": self.about, + "priced": self.priced, + "extra": self.extra, + "license": self.license, + "ready": self.ready(), + "fields": self.fields, + "template": self.template, + } + + def check(self, p): + return {"ok": True} + + def price(self, p, template): + return None + + def launch(self, p, template, name, boot): + raise NotImplementedError + + def describe(self, p, node): + raise NotImplementedError + + def terminate(self, p, node): + raise NotImplementedError + + def listing(self, p, tag): + return [] + + +def _values(fields, given, what): + """`given` checked against `fields` (types, required, options), with defaults filled in: the settings a driver + keeps. ValueError naming the first problem.""" + given = dict(given or {}) + unknown = sorted(set(given) - set(fields)) + if unknown: + raise ValueError(f"{what} has no {', '.join(unknown)}; it takes {', '.join(fields) or 'nothing'}") + out = {} + for k, f in fields.items(): + if f.get("secret"): + continue + v = given.get(k, f.get("default")) + if v in (None, ""): + if f.get("required"): + raise ValueError(f"{what}: {f['label']} ({k}) is needed") + continue + t = f["type"] + if t == "string": + if not isinstance(v, str) or len(v) > 2000: + raise ValueError(f"{what}: {k} is text") + v = v.strip() + elif t == "integer": + if isinstance(v, bool) or not isinstance(v, int): + raise ValueError(f"{what}: {k} is a whole number") + elif t == "number": + if isinstance(v, bool) or not isinstance(v, int | float): + raise ValueError(f"{what}: {k} is a number") + elif t == "boolean": + if not isinstance(v, bool): + raise ValueError(f"{what}: {k} is true or false") + elif t == "list": + if not isinstance(v, list) or not all(isinstance(x, str) and x.strip() for x in v) or len(v) > 100: + raise ValueError(f"{what}: {k} is a list of names") + v = [x.strip() for x in v] + if f.get("options") and (v if isinstance(v, list) else [v]) and not set(v if isinstance(v, list) else [v]) <= set(f["options"]): + raise ValueError(f"{what}: {k} is one of {', '.join(map(str, f['options']))}") + out[k] = v + return out + + +def check_settings(drv, given): + return _values(drv.fields, given, f"a {drv.label} provider") + + +def check_template(drv, given): + return _values(drv.template, given, f"a {drv.label} unit") + + +# ---------- this machine ---------- +def _die_with_parent(): + """In the child, before it runs: be stopped when the process that started it ends (Linux), so a worker this + machine's scaler started never outlives it.""" + try: + import ctypes + + ctypes.CDLL("libc.so.6", use_errno=True).prctl(1, signal.SIGTERM) # PR_SET_PDEATHSIG + except (OSError, AttributeError): + pass + + +class Local(Driver): + """More `lens worker` processes on the machine the scaler runs on: when a big machine's one worker can't keep up, + more of them take jobs at once, and they stop again when the queue is empty. Nothing to sign in to and no cost. + Each one stops with the process that started it; the scaler that takes over starts them again where it runs.""" + + id = "local" + label = "This machine" + about = "Runs more Lens workers on the machine Lens runs on, so more jobs run at once. Costs nothing." + priced = False + fields = {} + template = { + "steps": _field( + "Steps", + "list", + help="the steps these workers run (transcribe, analyze, embed...); none: every step", + options=sorted(jobs.STEPS), + ) + } + + def __init__(self): + self.procs: dict[str, subprocess.Popen] = {} + self.lock = threading.Lock() + + def host(self): + return socket.gethostname() + + def command(self, name, template): + cmd = [sys.executable, "-c", "import sys; from app.cli import main; sys.exit(main())", "worker", "--name", name, "--no-schedule"] + if template.get("steps"): + cmd += ["--steps", ",".join(template["steps"])] + return cmd + + def launch(self, p, template, name, boot): + env = {**os.environ, **(boot.get("env") or {})} + try: + proc = subprocess.Popen( + self.command(name, template), + env=env, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + preexec_fn=_die_with_parent if sys.platform.startswith("linux") else None, + ) + except OSError as e: + raise DriverError(f"couldn't start a worker here: {e}") from None + with self.lock: + self.procs[name] = proc + return {"id": f"{self.host()}:{proc.pid}", "price": 0.0} + + def _proc(self, node): + with self.lock: + return self.procs.get(node.get("name")) + + def describe(self, p, node): + host, _, pid = (node.get("external_id") or "").rpartition(":") + if host != self.host(): + return {"state": "gone"} # started by a scaler on another machine, which stopped it when it ended + proc = self._proc(node) + if proc is not None: + return {"state": "running" if proc.poll() is None else "gone"} + try: + os.kill(int(pid), 0) + except (OSError, ValueError): + return {"state": "gone"} + return {"state": "running"} + + def terminate(self, p, node): + host, _, pid = (node.get("external_id") or "").rpartition(":") + proc = self._proc(node) + if proc is not None: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(15) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(5) + with self.lock: + self.procs.pop(node.get("name"), None) + return + if host == self.host() and pid.isdigit(): + try: + os.kill(int(pid), signal.SIGTERM) + except OSError: + pass + + +DRIVERS: dict[str, Driver] = {} + + +def register(drv): + DRIVERS[drv.id] = drv + return drv + + +def get(driver_id): + try: + return DRIVERS[driver_id] + except KeyError: + raise ValueError(f"there's no provider kind {driver_id}; there are {', '.join(sorted(DRIVERS))}") from None + + +register(Local()) diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index bba6483eb..0616471b7 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -60,6 +60,7 @@ "events": None, "automations": None, "backups": None, + "scaling": None, "encryption": None, "fedora": None, "tunnel": None, @@ -495,6 +496,8 @@ def _check(section, key, value, default): return _automations_setting(key, value) if section == "backups": return _backup_setting(key, value) + if section == "scaling": + return _scaling_setting(key, value) if section == "fedora": return _fedora_setting(key, value) if section == "shares": @@ -1122,6 +1125,27 @@ def _backup_setting(key, value): raise ValueError(f"unknown setting backups.{key}") +SCALING_RANGES = {"tick_seconds": (10, 600), "join_minutes": (1, 120), "accrue_minutes": (1, 60)} +SCALING_TAG_RX = re.compile(r"^[a-z][a-z0-9-]{0,15}$") + + +def _scaling_setting(key, value): + if key == "enabled": + if not isinstance(value, bool): + raise ValueError("scaling.enabled is true or false") + return value + if key in SCALING_RANGES: + lo, hi = SCALING_RANGES[key] + if not (isinstance(value, int) and not isinstance(value, bool) and lo <= value <= hi): + raise ValueError(f"scaling.{key} is a whole number from {lo} to {hi}") + return value + if key == "tag": + if not isinstance(value, str) or not SCALING_TAG_RX.match(value.strip()): + raise ValueError("scaling.tag is up to 16 lowercase letters, digits and dashes, starting with a letter") + return value.strip() + raise ValueError(f"unknown setting scaling.{key}") + + EMBED_RANGES = {"passage_chars": (200, 4000), "batch_size": (1, 256), "neighbours": (5, 500), "timeout": (5, 600)} diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 5d74ee50c..824e70e8b 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -690,6 +690,11 @@ def labels(): # for none; the newest `keep` scheduled backups are kept. connection: a storage connection (sources.py) every new # backup is also copied to, under folder, encrypted with a key from secret.key first (none: kept here only) "backups": {"schedule": None, "timezone": "UTC", "keep": 7, "connection": None, "folder": "lens-backups"}, + # auto scaling (domain/scaling.py, docs/scaling.md): off by default, and a group launches nothing until a person + # approves its terms in the web app. tick_seconds: how often the scaler looks; join_minutes: how long a launched + # unit may take to join before it's given up on; accrue_minutes: how often a running unit's price goes into the + # ledger; tag: the prefix of units' names (and the tag they carry at the provider) + "scaling": {"enabled": False, "tick_seconds": 30, "join_minutes": 15, "accrue_minutes": 5, "tag": "lens"}, "telemetry": { "enabled": False, "endpoint": None, @@ -1304,6 +1309,14 @@ def close(self): "DEFINE INDEX IF NOT EXISTS activity_resources ON activity FIELDS resources", # caps on what a routine, pipeline, workflow or namespace may cost (domain/budgets.py): budget:_ "DEFINE TABLE IF NOT EXISTS budget SCHEMALESS", + # auto scaling (domain/scaling.py): providers with their credentials sealed, groups with their rules and approval, + # and each unit launched (a worker machine or a model endpoint), alive or gone + "DEFINE TABLE IF NOT EXISTS scale_provider SCHEMALESS", + "DEFINE TABLE IF NOT EXISTS scale_group SCHEMALESS", + "DEFINE INDEX IF NOT EXISTS scale_group_provider ON scale_group FIELDS provider", + "DEFINE TABLE IF NOT EXISTS scale_node SCHEMALESS", + "DEFINE INDEX IF NOT EXISTS scale_node_group ON scale_node FIELDS group", + "DEFINE INDEX IF NOT EXISTS scale_node_state ON scale_node FIELDS state", # background work "DEFINE TABLE IF NOT EXISTS job SCHEMALESS", "DEFINE INDEX IF NOT EXISTS job_status ON job FIELDS status", diff --git a/fastapi_backend/app/schemas/scaling.py b/fastapi_backend/app/schemas/scaling.py new file mode 100644 index 000000000..96af44f38 --- /dev/null +++ b/fastapi_backend/app/schemas/scaling.py @@ -0,0 +1,221 @@ +"""Auto scaling (docs/scaling.md): providers, scaling groups with their rules, and the units they launch.""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import Field + +from app.schemas.common import RequestModel, ResponseModel + + +class DriverField(ResponseModel): + label: str + type: Literal["string", "integer", "number", "boolean", "list"] + required: bool = False + default: Any = None + help: str | None = None + options: list[Any] | None = None + secret: bool = Field(False, description="a credential: sealed when saved, shown only as set") + + +class DriverSpec(ResponseModel): + id: str + kind: Literal["node", "model"] = Field(description="node: machines that run `lens worker`; model: a model endpoint") + label: str + about: str + priced: bool = Field(description="it charges money, so its groups need a budget") + extra: str | None = Field(None, description="the optional packages it needs, fetched when a provider of it is saved") + license: str | None = None + ready: bool = Field(description="its packages are installed") + fields: dict[str, DriverField] = Field(description="the provider's settings") + template: dict[str, DriverField] = Field(description="what a group picks for each unit") + + +class ProviderIn(RequestModel): + name: str = Field(max_length=80) + driver: str = Field(description="the kind of provider (see /scaling/drivers)") + settings: dict[str, Any] = Field(default_factory=dict, description="the driver's non-secret settings") + secrets: dict[str, str | None] = Field(default_factory=dict, description="its credentials: sealed, never shown again") + price: float | None = Field(None, ge=0, description="USD per unit-hour, when the driver doesn't know it; an estimate") + note: str | None = Field(None, max_length=300) + check: bool = Field(True, description="prove the credentials with a read-only call before answering") + + +class ProviderChange(RequestModel): + name: str | None = Field(None, max_length=80) + settings: dict[str, Any] | None = None + secrets: dict[str, str | None] | None = Field(None, description="a secret given as null is removed; one left out is kept") + price: float | None = Field(None, ge=0) + note: str | None = Field(None, max_length=300) + check: bool = True + + +class Provider(ResponseModel): + id: int + name: str + driver: str + kind: str | None = None + label: str | None = None + priced: bool = True + settings: dict[str, Any] + secrets: dict[str, bool] = Field(description="which credentials are set") + price: float | None = None + note: str | None = None + checked_at: str | None = None + check: dict[str, Any] | None = Field(None, description="the last check: {ok, error?}") + created_at: str | None = None + updated_at: str | None = None + updated_by: str | None = None + + +class Rule(RequestModel): + type: Literal["target", "step", "schedule"] + name: str | None = Field(None, max_length=80) + metric: str | None = Field(None, description="target and step rules: see /scaling/metrics") + target: float | None = Field(None, description="target rules: the value to keep the metric at") + above: float | None = Field(None, description="step rules: fires while the metric is above this") + below: float | None = Field(None, description="step rules: fires while the metric is below this") + minutes: int | None = Field(None, ge=0, le=1440, description="step rules: for how long before it fires (default 1)") + add: int | str | None = Field(None, description="step rules: units to add, a count or a percentage like \"50%\"") + remove: int | str | None = Field(None, description="step rules: units to remove, a count or a percentage") + cron: str | None = Field(None, description="schedule rules: when, as a cron expression") + timezone: str | None = None + min: int | None = Field(None, ge=0, le=100, description="schedule rules: the new min") + max: int | None = Field(None, ge=0, le=100, description="schedule rules: the new max") + desired: int | None = Field(None, ge=0, le=100, description="schedule rules: the new desired count") + + +class GroupIn(RequestModel): + name: str = Field(max_length=80) + provider: int = Field(description="the provider's id") + kind: Literal["node", "model"] | None = Field(None, description="the provider's kind; filled in when left out") + template: dict[str, Any] = Field(default_factory=dict, description="what each unit is (the driver's template fields)") + min: int = Field(0, ge=0, le=100) + max: int = Field(1, ge=0, le=100) + desired: int | None = Field(None, ge=0, le=100, description="units wanted now (default min)") + cooldown_seconds: int = Field(300, ge=0, le=86400, description="how long after a change before the next") + warmup_seconds: int = Field(300, ge=0, le=7200, description="how long a new unit may take to join before more are judged") + terminate: Literal["oldest", "newest"] = Field("oldest", description="which unit goes first when scaling in") + rules: list[Rule] = Field(default_factory=list) + at_cap: Literal["stop", "min"] = Field("stop", description="at the budget's cap: stop every unit, or keep min") + enabled: bool = True + budget: float | None = Field(None, gt=0, description="USD per month; needed on a priced provider") + + +class GroupChange(RequestModel): + name: str | None = Field(None, max_length=80) + provider: int | None = None + template: dict[str, Any] | None = None + min: int | None = Field(None, ge=0, le=100) + max: int | None = Field(None, ge=0, le=100) + desired: int | None = Field(None, ge=0, le=100) + cooldown_seconds: int | None = Field(None, ge=0, le=86400) + warmup_seconds: int | None = Field(None, ge=0, le=7200) + terminate: Literal["oldest", "newest"] | None = None + rules: list[Rule] | None = None + at_cap: Literal["stop", "min"] | None = None + enabled: bool | None = None + budget: float | None = Field(None, gt=0) + + +class Desired(RequestModel): + desired: int = Field(ge=0, le=100) + + +class Paused(RequestModel): + paused: bool = True + why: str | None = Field(None, max_length=200) + + +class Protected(RequestModel): + protected: bool = True + + +class Terminate(RequestModel): + why: str | None = Field(None, max_length=200) + + +class UnitCounts(ResponseModel): + launching: int + running: int + draining: int + + +class GroupBudget(ResponseModel): + usd: float | None = None + period: str + state: str + spent: dict[str, Any] + left: dict[str, Any] + + +class Group(ResponseModel): + id: int + name: str + kind: str + provider: int + template: dict[str, Any] + min: int + max: int + desired: int + cooldown_seconds: int | None = None + warmup_seconds: int | None = None + terminate: str + rules: list[dict[str, Any]] = Field(description="each with `says`, the rule in words") + at_cap: str + enabled: bool + paused: bool + paused_why: str | None = None + capped: str | None = Field(None, description="why its budget stops it now, when it does") + approval: dict[str, Any] | None = Field(None, description="a person's approval: {by, at, terms}") + approved: bool = Field(description="the approval covers the terms as they stand") + terms: dict[str, Any] = Field(description="what an approval binds: kind, provider, template, max, budget, at_cap") + budget: GroupBudget | None = None + units: UnitCounts + waiting_for: str | None = Field(None, description="why it launches nothing now, when it doesn't") + last_out_at: str | None = None + last_in_at: str | None = None + last_eval: dict[str, Any] | None = Field(None, description="what the rules said last: {at, said}") + failures: int = 0 + error: str | None = None + created_at: str | None = None + updated_at: str | None = None + updated_by: str | None = None + + +class Unit(ResponseModel): + id: int + group: int + provider: int + driver: str | None = None + name: str + external_id: str | None = Field(None, description="the provider's id for it") + state: Literal["launching", "running", "draining", "terminating", "terminated", "failed"] + address: str | None = None + endpoint: dict[str, Any] | None = Field(None, description="a model unit's endpoint: base_url, model; api_key true when set") + price: float | None = Field(None, description="USD per hour, an estimate") + launched_at: str | None = None + ready_at: str | None = None + ended_at: str | None = None + by: str | None = None + why: str | None = None + protected: bool = False + error: str | None = None + accrued_at: str | None = None + + +class Overview(ResponseModel): + enabled: bool + settings: dict[str, Any] + scaler: dict[str, Any] = Field(description="which process scales and when it last looked") + providers: list[Provider] + groups: list[Group] + units: list[Unit] + drivers: list[DriverSpec] + metrics: dict[str, float | int] + + +class Metrics(ResponseModel): + metrics: dict[str, float | int] + about: dict[str, str] = Field(description="what each metric measures") From 2d55f0c18fbc603dad897b0b9a6708bc1e441c7a Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 03:16:24 +0000 Subject: [PATCH 2/3] feat(scaling): auto scaling groups, rules and approvals, with This machine as the first provider Scaling groups launch units on a provider between min and max, by target, step and schedule rules, within a monthly budget, and launch nothing until a person signed in to the web app approves their terms. Agents propose and manage groups over the API, MCP and the assistant, and never approve. The scaler holds a lease, accrues each unit's cost into the activity ledger as an estimate, stops a group at its budget's cap, drains busy workers before letting them go, and pauses a group after three failures. This machine runs more lens worker processes at no cost. Threat model: P-SCALING. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ALjNbLobvPe7VwvDL8j7V3 --- README.md | 11 +- docs/budgets.md | 2 +- docs/requirements.md | 13 + docs/scaling.md | 145 + fastapi_backend/app/api/mcp_tools.py | 118 + fastapi_backend/app/api/v1/routes/scaling.py | 8 +- fastapi_backend/app/domain/ai_tools.py | 10 +- fastapi_backend/app/domain/budgets.py | 4 +- fastapi_backend/app/domain/ops_tools.py | 143 +- fastapi_backend/app/domain/scaling.py | 64 +- fastapi_backend/app/schemas/scaling.py | 2 +- fastapi_backend/tests/api/test_mcp.py | 5 + fastapi_backend/tests/api/test_scaling.py | 356 ++ .../tests/api/test_scaling_agents.py | 126 + .../tests/domain/test_scaling_rules.py | 80 + mkdocs.yml | 1 + nextjs-frontend/app/openapi-client/index.ts | 119 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 392 ++ .../app/openapi-client/types.gen.ts | 1502 +++++++ nextjs-frontend/openapi.json | 3929 ++++++++++++++--- threat-model.json | 19 +- threat-model.md | 2 + threat-model.yaml | 20 +- 23 files changed, 6362 insertions(+), 709 deletions(-) create mode 100644 docs/scaling.md create mode 100644 fastapi_backend/tests/api/test_scaling.py create mode 100644 fastapi_backend/tests/api/test_scaling_agents.py create mode 100644 fastapi_backend/tests/domain/test_scaling_rules.py diff --git a/README.md b/README.md index 0237905d6..0386e1a8b 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,12 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p what cost most. Routine choices go to small decision models, and the LLM only when it adds value. ([Activity and costs](docs/activity.md), [budgets](docs/budgets.md#one-budget-for-everything), [the meter](docs/budgets.md#the-meter-and-the-costs-page); [checks](docs/requirements.md#see-what-it-costs)) +- **Scale out.** Scaling groups add workers when work waits and let them go when it doesn't, by target, step and + schedule rules, within min and max and a monthly budget, with every launch, join and hour of cost in the activity + ledger. A group launches nothing until you approve its terms signed in to the web app; agents propose and manage + groups over the API, MCP and the assistant, and never approve one. It runs more workers on this machine today; + cloud, Proxmox, Linux and model hosts are planned. ([Auto scaling](docs/scaling.md); + [checks](docs/requirements.md#scale-out)) - **Keep it up to date and backed up.** Lens checks for a new release once a day and tells admins by email or a notification. One click in Settings › Updates (or one API call) upgrades it through a small agent on the machine, showing each step, and rolls back when the new version doesn't start; nothing installs unless an admin says so. The @@ -241,6 +247,9 @@ Next, and not in a pull request yet; each links to where it is tracked: topics, recordings and the other things that have pages ([notes](docs/notes.md)). - **Local models:** GPU builds of llama.cpp fetched by Lens, and local embedding models ([local models](docs/local-models.md)). +- **Auto scaling:** providers for AWS, DigitalOcean, Hetzner, Proxmox and Linux hosts over SSH, joining over the + Nebula mesh, model hosts (RunPod, Lambda, Hugging Face, SageMaker, Bedrock), and the Settings › Scaling page + ([auto scaling](docs/scaling.md)). - **Speech providers:** their summaries, chapters and entities as optional imports, and per-minute prices in the activity ledger ([speech providers](docs/speech-providers.md)). - **Storage:** uploads and assets' media in the file store, and a storage key to download @@ -280,7 +289,7 @@ The guides are on the [docs site](https://adeelahmad.github.io/lense/) and in th [the graph as files](docs/file-tree.md), [NFS share](docs/file-shares.md), [SMB share](docs/smb.md), [linked data (RDF)](docs/rdf.md), [IIIF](docs/iiif.md), [Fedora](docs/fedora.md) - **Models and cost:** [local models](docs/local-models.md), [speech providers](docs/speech-providers.md), - [activity and costs](docs/activity.md), [budgets](docs/budgets.md) + [activity and costs](docs/activity.md), [budgets](docs/budgets.md), [auto scaling](docs/scaling.md) - **Operate:** [encryption](docs/encryption.md), [storage](docs/storage.md), [database](docs/database.md), [logging](docs/logging.md), [telemetry](docs/telemetry.md), [Prometheus](docs/prometheus.md), [usage analytics](docs/analytics.md) diff --git a/docs/budgets.md b/docs/budgets.md index ae6e7a97a..08912953f 100644 --- a/docs/budgets.md +++ b/docs/budgets.md @@ -1,6 +1,6 @@ # Budgets -A budget caps what a routine, automation, pipeline, workflow or namespace may cost. Budgets are off unless an admin sets one. +A budget caps what a routine, automation, pipeline, workflow, namespace or [scaling group](scaling.md) may cost. Budgets are off unless an admin sets one. What has been spent comes from the [activity ledger](activity.md), so a budget counts everything done for that resource: a namespace's budget counts every job in it, and a routine's counts its runs and the jobs it queued. diff --git a/docs/requirements.md b/docs/requirements.md index f1bb4da07..9f9e61357 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -201,6 +201,19 @@ request, to another check that proves the same thing; loosening a requirement or | The Costs page's charts | No check yet ([#486](https://github.com/adeelahmad/lense/issues/486)) | | Routine choices go to small decision models, the LLM only when it adds value | [test_decisions.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_decisions.py) `test_the_decision_model_answers_when_it_has_a_key`, `test_the_language_model_decides_without_one_and_when_it_fails` | +## Scale out + +| Requirement | Checks | +|---|---| +| Off until an admin turns it on; off, every unit is let go and the scaler writes nothing | [test_scaling.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling.py) `test_scaling_is_off_and_lists_its_drivers`, `test_the_settings_section`, `test_this_machine_runs_more_workers_at_no_cost` | +| Providers keep their credentials sealed, shown only as set, and prove them with a read-only call | [test_scaling.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling.py) `test_a_providers_credentials_are_sealed_and_checked` | +| A group launches nothing until a person signed in to the web app approves its terms; agents never approve, and widening the terms takes the approval away | [test_scaling.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling.py) `test_only_a_person_in_the_web_app_approves_a_groups_terms`; [test_scaling_rules.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_scaling_rules.py) `test_what_widens_an_approval` | +| Target, step and schedule rules pick the number of units, within min and max | [test_scaling_rules.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_scaling_rules.py) `test_a_target_rule_keeps_its_metric_at_the_target`, `test_a_step_rule_fires_once_its_condition_has_held`, `test_a_schedule_rule_sets_the_bounds_when_its_time_comes`, `test_the_largest_proposal_wins`; [test_scaling.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling.py) `test_rules_scale_on_waiting_jobs_and_on_a_schedule` | +| The scaler launches units, sees them join, drains busy workers before letting them go, and fails and pauses on units that don't come up | [test_scaling.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling.py) `test_the_scaler_launches_joins_drains_and_terminates`, `test_a_unit_that_never_joins_fails_and_failures_pause_the_group`, `test_protected_and_terminated_units` | +| Each unit's cost goes into the activity ledger as an estimate, and a group stops at its budget's cap | [test_scaling.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling.py) `test_the_scaler_launches_joins_drains_and_terminates`, `test_at_its_budgets_cap_a_group_stops` | +| This machine runs more workers at no cost | [test_scaling.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling.py) `test_this_machine_runs_more_workers_at_no_cost` | +| Agents manage scaling in the assistant and over MCP: proposals wait for the person, actions within the terms happen at once | [test_scaling_agents.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling_agents.py) `test_the_assistant_proposes_and_the_person_approves_in_the_web_app`, `test_a_setup_conversation_makes_a_group_that_still_waits_for_the_person`, `test_only_admins_get_the_scaling_tools`, `test_mcp_clients_manage_scaling_but_never_approve` | + ## Keep it up to date and backed up | Requirement | Checks | diff --git a/docs/scaling.md b/docs/scaling.md new file mode 100644 index 000000000..7c3576778 --- /dev/null +++ b/docs/scaling.md @@ -0,0 +1,145 @@ +# Auto scaling + +Auto scaling adds workers when there's work waiting and lets them go when there isn't, the way a cloud's auto +scaling groups do, within a budget you set. A **provider** is where units launch, a **scaling group** says what each +unit is, how many there may be and the rules that pick the number, and a **unit** is one machine or process the group +launched. Everything it does is in the [activity ledger](activity.md), and its cost counts against the group's +[budget](budgets.md). + +Status: in progress. Built: groups, rules, approvals, budgets, the scaler, the API, MCP and assistant tools, and +**This machine**, a provider that runs more `lens worker` processes where Lens runs. Planned: providers for AWS, +DigitalOcean, Hetzner, Proxmox and any Linux host over SSH, joining Lens over its Nebula mesh network; model +hosts (RunPod, Lambda, Hugging Face, SageMaker, Bedrock) for models; and the Settings › Scaling page. + +Auto scaling is **off** until an admin turns it on, and a group launches nothing until a person approves its terms +signed in to the web app. Agents can propose and manage groups, but never approve one. + +## Turn it on + +```http +PUT /api/v1/settings/scaling +{"enabled": true} +``` + +| Setting | Default | | +| --- | --- | --- | +| `enabled` | `false` | the master switch: off, every unit is let go and nothing launches | +| `tick_seconds` | `30` | how often the scaler looks (10 to 600) | +| `join_minutes` | `15` | how long a new unit may take to join before it counts as failed (1 to 120) | +| `accrue_minutes` | `5` | how often a running unit's cost goes into the ledger (1 to 60) | +| `tag` | `lens` | the prefix of every unit's name, so they can be told apart at the provider | + +One process runs the scaler at a time: the API and every `lens worker` (except those started with `--steps` or +`--no-schedule`, as scaled workers are) take turns holding its lease. While scaling is off and nothing is left to let +go, the scaler doesn't even take the lease. + +## Providers + +```http +POST /api/v1/scaling/providers +{"name": "Here", "driver": "local"} +``` + +`GET /api/v1/scaling/drivers` lists the kinds of provider, with the settings each asks for (`fields`) and what a group +picks for each unit (`template`). A provider's credentials (`secrets`) are sealed when saved, bound to that provider +and field, and opened only for a call to it; the API shows only whether each one is set. Saving a provider proves its +credentials with a read-only call (`check`), kept on the provider; `POST /scaling/providers/{id}/check` runs it again. +A driver that needs a library fetches it the first time a provider of its kind is saved, so the default install +carries none of them. + +| Driver | Units | Costs | Template | +| --- | --- | --- | --- | +| `local` (This machine) | more `lens worker` processes where the scaler runs | nothing | `steps`: the steps they run (none: every step) | + +A worker on This machine stops with the process that started it, and the scaler that takes over starts them again where +it runs. + +## Groups + +```http +POST /api/v1/scaling/groups +{"name": "Transcribers", "provider": 1, "template": {"steps": ["transcribe"]}, "min": 0, "max": 4, + "rules": [{"type": "step", "metric": "jobs_waiting", "above": 10, "minutes": 2, "add": 2}, + {"type": "step", "metric": "jobs_waiting", "below": 1, "minutes": 10, "remove": "50%"}]} +``` + +| Field | Default | | +| --- | --- | --- | +| `name`, `provider` | | | +| `template` | the driver's defaults | what each unit is | +| `min`, `max` | `0`, `1` | the bounds, up to 100 | +| `desired` | `min` | units wanted now; the rules change it | +| `cooldown_seconds` | `300` | after scaling out or in, how long before the next change the same way | +| `warmup_seconds` | `300` | how long a launching unit may take to join before more are judged | +| `terminate` | `oldest` | which goes first when scaling in: `oldest` or `newest`. Idle units go before busy ones, protected ones never | +| `rules` | none | up to 20, below | +| `budget` | none | USD a month; needed on a provider that charges | +| `at_cap` | `stop` | at the budget's cap: `stop` every unit, or keep `min` | +| `enabled` | `true` | off, its units are let go | + +`PATCH /scaling/groups/{id}` changes one, `POST /scaling/groups/{id}/desired {"desired": 3}` asks for a number now +(within min and max), `POST /scaling/groups/{id}/pause {"paused": true, "why": "…"}` stops new launches while what runs +stays, and `DELETE /scaling/groups/{id}` lets its units go and removes it. `GET /scaling/groups/{id}/history` is what +happened to it, from the ledger. Each group says `waiting_for`: why it launches nothing now, in words. + +### Rules + +| Type | Fields | Proposes | +| --- | --- | --- | +| `target` | `metric`, `target` | units × metric ÷ target, rounded up: keeps the metric near the target | +| `step` | `metric`, `above` or `below`, `minutes` (default 1), `add` or `remove` (a count or a percentage like `"50%"`) | once the condition has held for `minutes`, that many more or fewer | +| `schedule` | `cron`, `timezone`, and any of `min`, `max`, `desired` | at each time the cron names, the new bounds and count | + +When several rules speak, the largest number wins; the result stays within min and max. Each rule comes back with +`says`, the rule in words, and the group's `last_eval` keeps what each said last time. + +| Metric | | +| --- | --- | +| `jobs_waiting` | jobs queued (for a group's `steps`, those that run them) | +| `oldest_wait_minutes` | how long the oldest queued job has waited | +| `workers_busy_share` | the share of live workers running a job | +| `worker_load` | the live workers' average load | +| `model_calls_per_minute` | model calls over the last 5 minutes | +| `model_seconds` | how long a model call took on average over the last 5 minutes | + +`GET /api/v1/scaling/metrics` gives them now. + +## The person approves + +A group launches nothing until a person approves its **terms**: the provider, what each unit is, the max, the budget +and what happens at its cap. `POST /scaling/groups/{id}/approve` approves them as they stand, and only from a session +signed in to the web app: an API key, an app or an agent gets 403. A group made or changed by a person in the web app +is approved as it's saved, since that is their pick. + +Within the approved terms anyone who may change the group, agents included, acts at once: asking for a number of units, +lowering the max or the budget, pausing, letting a unit go. A change that widens the terms (another provider or unit, +a higher max or budget, keeping min at the cap) takes the approval away until a person gives it again. + +## The scaler + +Every `tick_seconds`, for each group, the scaler: + +1. puts each running unit's cost since last time into the ledger (`scaling.unit`, an estimate from its price per hour); +2. checks the group's budget and the `all` budget: at the cap the group is `capped` and lets its units go (or keeps + min, with `at_cap: "min"`); +3. looks at each unit at its provider and in Lens. A worker unit is running once its worker beats; one that hasn't + joined within `join_minutes`, or whose worker has been silent for 10 minutes, fails and is let go; +4. runs the rules and launches or lets go units to reach the desired count. A busy worker drains first: its job goes + back to the queue after the step it's on, and the unit goes once it's idle. + +Three launches or joins that fail in a row pause the group, with the reason. Every launch, join, drain, termination, +failure and change is a ledger row linked to the unit, its group and its provider. + +## Agents + +The same in the assistant (admins, in the server tools) and over MCP: `scaling` reads everything, +`propose_scaling_group` and `change_scaling_group` make and change groups, `set_scaling_desired`, +`pause_scaling_group` and `terminate_scaling_unit` act within the approved terms. In the assistant, proposing or +changing a group waits for the person's yes; a yes on the Approvals page in the web app also approves the group's +terms, while a group the assistant makes in a setup conversation, or an agent over MCP, still waits for the person. +Turning scaling on (`change_settings scaling`) always asks. + +## Refine later + +Providers beyond This machine; spot and preemptible instances; warm pools; predictive scaling; health checks beyond +joining; prices fetched from providers' pricing APIs. diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index 8df689404..9425e876b 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -65,6 +65,8 @@ render, research, rss, + scaling, + scaling_drivers, services, settings, sources, @@ -2542,6 +2544,122 @@ def set_file_storage( return {"saved": changes, "check": checked} +# ---------- auto scaling (admins) ---------- +def _scaler(ctx: Context, write: bool = False) -> None: + if not ctx.user.admin: + raise ToolError("only admins see or change auto scaling") + if write and not ctx.user.can_write: + raise ToolError("this token is read-only: ask for a token with the write scope to change auto scaling") + + +def _scaling_do(ctx: Context, tool: str, args: dict[str, Any]) -> dict[str, Any]: + """A scaling tool's change, as an agent: never an approval of a group's terms (that's the person's, in the web app).""" + _scaler(ctx, write=True) + try: + out = scaling.apply_tool(ctx.db, ctx.cfg, tool, args, ctx.user.email, via="agent") + except scaling_drivers.DriverError as e: + raise ToolError(f"the provider refused: {e}") from None + what = out.get("group") or out.get("unit") or {} + table = "scale_group" if "group" in out else "scale_node" + auth.audit(ctx.db, ctx.user.as_audit(), f"scaling.{tool}", f"{table}:{what.get('id')}", ["mcp"]) + if "group" in out and out["group"].get("waiting_for"): + out["approve"] = f"{ctx.web}/settings/scaling" + return out + + +_GROUP_HOW = ( + "A group: name, provider (its id), template (what each unit is, as the provider's driver asks), min, max, desired, " + "cooldown_seconds, warmup_seconds, terminate (oldest|newest), rules, at_cap (stop|min), enabled, budget (USD a " + "month, needed on a provider that charges). Rules: {type: target, metric, target}, {type: step, metric, " + 'above|below, minutes, add|remove (a count or "50%")}, {type: schedule, cron, timezone, min, max, desired}' +) + + +@tool( + "scaling", + "Auto scaling", + "Auto scaling: whether it's on, the providers (where units launch: this machine, clouds, Proxmox, a Linux host, " + "or model hosts; credentials show only as set), the scaling groups with their rules in words, their budgets and " + "why one launches nothing now, the units, the drivers on offer and the metrics rules can watch. Admins only.", + admin=True, +) +def scaling_overview(ctx: Context) -> dict[str, Any]: + _scaler(ctx) + return {**scaling.overview(ctx.db, ctx.cfg), "about_metrics": scaling.METRICS, "settings": f"{ctx.web}/settings/scaling"} + + +@tool( + "propose_scaling_group", + "Propose a scaling group", + "Make a scaling group: units launched on a provider between min and max, by its rules, within its budget. It " + "launches nothing until the person approves its terms, signed in to the web app (the answer links there); an agent " + f"can't approve them. {_GROUP_HOW}. Admins only, with a token that has the write scope.", + Arg("group", "object", "the group", required=True), + writes=True, + admin=True, +) +def propose_scaling_group(ctx: Context, group: dict[str, Any]) -> dict[str, Any]: + return _scaling_do(ctx, "propose_scaling_group", {"group": group}) + + +@tool( + "change_scaling_group", + "Change a scaling group", + "Change a scaling group: only the fields given change. A change that widens what the person approved (another " + "provider or unit, a higher max or budget, keeping min at the cap) waits for their approval again, in the web app. " + f"{_GROUP_HOW}. Admins only, with a token that has the write scope.", + Arg("group_id", "integer", "the group (from scaling)", required=True), + Arg("changes", "object", "the fields to change", required=True), + writes=True, + admin=True, +) +def change_scaling_group(ctx: Context, group_id: int, changes: dict[str, Any]) -> dict[str, Any]: + return _scaling_do(ctx, "change_scaling_group", {"group_id": group_id, "changes": changes}) + + +@tool( + "set_scaling_desired", + "Ask a scaling group for units", + "Ask a scaling group for a number of units now, within its min and max (what the person approved); its rules take " + "over again at their next change. Admins only, with a token that has the write scope.", + Arg("group_id", "integer", "the group (from scaling)", required=True), + Arg("desired", "integer", "units wanted now", required=True, minimum=0, maximum=scaling.MAX_UNITS), + writes=True, + admin=True, +) +def set_scaling_desired(ctx: Context, group_id: int, desired: int) -> dict[str, Any]: + return _scaling_do(ctx, "set_scaling_desired", {"group_id": group_id, "desired": desired}) + + +@tool( + "pause_scaling_group", + "Pause a scaling group", + "Pause a scaling group (nothing new launches; what runs stays) or resume it (paused false). Admins only, with a " + "token that has the write scope.", + Arg("group_id", "integer", "the group (from scaling)", required=True), + Arg("paused", "boolean", "pause (true) or resume (false)", default=True), + Arg("why", "string", "why, shown on the group", max_length=200), + writes=True, + admin=True, +) +def pause_scaling_group(ctx: Context, group_id: int, paused: bool = True, why: str | None = None) -> dict[str, Any]: + return _scaling_do(ctx, "pause_scaling_group", {"group_id": group_id, "paused": paused, "why": why}) + + +@tool( + "terminate_scaling_unit", + "Let a scaling unit go", + "Let one unit of a scaling group go now: a worker finishes or hands back its job first. Admins only, with a token " + "that has the write scope.", + Arg("unit_id", "integer", "the unit (from scaling)", required=True), + Arg("why", "string", "why, kept with the unit", max_length=200), + writes=True, + admin=True, +) +def terminate_scaling_unit(ctx: Context, unit_id: int, why: str | None = None) -> dict[str, Any]: + return _scaling_do(ctx, "terminate_scaling_unit", {"unit_id": unit_id, "why": why}) + + # ---------- monitoring: Prometheus and Grafana next to Lens (admins) ---------- @tool( "monitoring", diff --git a/fastapi_backend/app/api/v1/routes/scaling.py b/fastapi_backend/app/api/v1/routes/scaling.py index bd9b2a45e..a64a4d357 100644 --- a/fastapi_backend/app/api/v1/routes/scaling.py +++ b/fastapi_backend/app/api/v1/routes/scaling.py @@ -77,7 +77,9 @@ def list_drivers(user: AdminReader) -> list[DriverSpec]: @router.get("/metrics") -def metrics(user: AdminReader, db: Db, cfg: Cfg, steps: str | None = Query(None, description="comma-separated steps for the queue metrics")) -> Metrics: +def metrics( + user: AdminReader, db: Db, cfg: Cfg, steps: str | None = Query(None, description="comma-separated steps for the queue metrics") +) -> Metrics: """The values rules can watch, now.""" want = [s.strip() for s in steps.split(",") if s.strip()] if steps else None return Metrics(metrics=scaling.metrics(db, cfg, want), about=scaling.METRICS) @@ -94,7 +96,9 @@ def add_provider(body: ProviderIn, user: AdminWriter, db: Db, cfg: Cfg) -> Provi """Add a provider. Its credentials are sealed (never shown again, only whether they're set) and checked with a read-only call; its library is fetched the first time.""" with _errors(): - p = scaling.add_provider(db, cfg, body.name, body.driver, body.settings, body.secrets, body.price, body.note, user.email, body.check) + p = scaling.add_provider( + db, cfg, body.name, body.driver, body.settings, body.secrets, body.price, body.note, user.email, body.check + ) auth.audit(db, user.as_audit(), "scaling.provider.add", f"scale_provider:{p['id']}", {"driver": body.driver}) return Provider(**p) diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index bba22e734..a8e2ef600 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -564,6 +564,7 @@ def builtin_names(): "use_skill", "mark_activities", "decline_approval", + "scaling", } | { t[0] for t in TOOLS @@ -2213,7 +2214,7 @@ def approve(db, cfg, aid, user, editable, decision="approve", base=None, admin=F ) return {"status": "declined"} with graph_history.acting(actor=user["email"], via="assistant", approval=a["id"]): - result = _carry_out(db, cfg, a, user, editable, decision, base, admin, readable) + result = _carry_out(db, cfg, a, user, editable, decision, base, admin, readable, via) db.q( "UPDATE $r SET status = 'done', decided_at = $t, decided_by = $u, result = $res", r=R("approval", a["id"]), @@ -2224,7 +2225,7 @@ def approve(db, cfg, aid, user, editable, decision="approve", base=None, admin=F return {"status": "done", **result} -def _carry_out(db, cfg, a, user, editable, decision, base, admin, readable): +def _carry_out(db, cfg, a, user, editable, decision, base, admin, readable, via="web"): args = a["args"] if a["tool"] == "extension": box = Toolbox(db, cfg, user, readable if readable is not None else editable, editable, None, a["chat"], base, admin) @@ -2251,8 +2252,9 @@ def _carry_out(db, cfg, a, user, editable, decision, base, admin, readable): "change_person", "subscribe_feed", "unsubscribe_feed", - ): - result = ops_tools.apply(db, cfg, base or cfg, a["tool"], args, user, editable, admin) + *ops_tools.scaling.TOOLS, + ): # a scaling group's terms are approved only by a yes in the web app (via web) + result = ops_tools.apply(db, cfg, base or cfg, a["tool"], args, user, editable, admin, via) elif a["tool"] == "decline_approval": # the person's yes to declining another of their approvals from . import approvals diff --git a/fastapi_backend/app/domain/budgets.py b/fastapi_backend/app/domain/budgets.py index 455d7d2f6..ac919d0e2 100644 --- a/fastapi_backend/app/domain/budgets.py +++ b/fastapi_backend/app/domain/budgets.py @@ -48,9 +48,7 @@ def _key(resource): def check_ref(resource): if not isinstance(resource, str) or not REF.match(resource): - raise ValueError( - "a budget is for all, routine:, automation:, pipeline:, workflow:, space: or scale_group:" - ) + raise ValueError("a budget is for all, routine:, automation:, pipeline:, workflow:, space: or scale_group:") return resource diff --git a/fastapi_backend/app/domain/ops_tools.py b/fastapi_backend/app/domain/ops_tools.py index 9b8e2f500..655e50f09 100644 --- a/fastapi_backend/app/domain/ops_tools.py +++ b/fastapi_backend/app/domain/ops_tools.py @@ -11,7 +11,7 @@ from __future__ import annotations -from . import auth, blobs, convert, decide, feeds, jobs, links, llm, metadata, roles, rss, settings, setup, sources, store, uploads +from . import auth, blobs, convert, decide, feeds, jobs, links, llm, metadata, roles, rss, scaling, settings, setup, sources, store, uploads R = store.R _S = {"type": "string"} @@ -47,11 +47,70 @@ "sensors", "shares", "web", + "scaling", ) # telemetry, analytics and web search send data elsewhere; sensors and shares open ports on the network; files moves -# where new files go -ALWAYS_ASK = {"telemetry", "analytics", "sensors", "shares", "files", "web"} +# where new files go; scaling turns on what can launch paid machines +ALWAYS_ASK = {"telemetry", "analytics", "sensors", "shares", "files", "web", "scaling"} ASK_TOOLS = {"invite_person", "change_person"} # who gets in always waits for the admin's approval +# within a scaling group's approved terms (its min, max and budget), so made at once unless the answer read outside text +WITHIN_BOUNDS = {"set_scaling_desired", "pause_scaling_group", "terminate_scaling_unit"} +_GROUP = { + "type": "object", + "description": "name, provider (its id), template (what each unit is, as the provider's driver asks), min, max, " + "desired, cooldown_seconds, warmup_seconds, terminate (oldest|newest), rules, at_cap (stop|min), enabled, budget " + "(USD a month, needed on a provider that charges). Rules: {type: target, metric, target}, {type: step, metric, " + 'above|below, minutes, add|remove (a count or "50%")}, {type: schedule, cron, timezone, min, max, desired}.', +} +SCALING_TOOLS = [ + ( + "scaling", + "Auto scaling: whether it's on, the providers (where units launch: this machine, clouds, Proxmox, a Linux host, " + "or model hosts), the scaling groups with their rules in words, budgets and why one launches nothing now, the " + "units running, and the metrics rules can watch (" + ", ".join(scaling.METRICS) + ").", + {}, + [], + False, + ), + ( + "propose_scaling_group", + "Propose a scaling group: units launched on a provider between min and max, by its rules, within its budget. " + "It launches nothing until the person approves its terms in the web app (an approval of this proposal there " + "does that); you can't approve them. Read scaling first for the providers and metrics.", + {"group": _GROUP}, + ["group"], + True, + ), + ( + "change_scaling_group", + "Change a scaling group (its id from scaling): only the fields given change. A change that widens what the " + "person approved (another provider or unit, a higher max or budget) waits for their approval again.", + {"group_id": {"type": "integer"}, "changes": _GROUP}, + ["group_id", "changes"], + True, + ), + ( + "set_scaling_desired", + "Ask a scaling group for a number of units now, within its min and max; its rules take over again at their next change.", + {"group_id": {"type": "integer"}, "desired": {"type": "integer", "minimum": 0}}, + ["group_id", "desired"], + True, + ), + ( + "pause_scaling_group", + "Pause a scaling group (nothing new launches; what runs stays) or resume it (paused false).", + {"group_id": {"type": "integer"}, "paused": {"type": "boolean"}, "why": _S}, + ["group_id"], + True, + ), + ( + "terminate_scaling_unit", + "Let one unit of a scaling group go now, by its id from scaling: a worker finishes or hands back its job first.", + {"unit_id": {"type": "integer"}, "why": _S}, + ["unit_id"], + True, + ), +] ADMIN_TOOLS = [ ( @@ -159,6 +218,8 @@ True, ), ] +ADMIN_TOOLS += SCALING_TOOLS + FILE_TOOLS = [ ( "import_files", @@ -230,8 +291,18 @@ def status(db, cfg, base): } -def apply(db, cfg, base, tool, args, user, editable, admin): - """Carry out a change (approved, or in a conversation that lets the assistant act). Returns the result.""" +def apply(db, cfg, base, tool, args, user, editable, admin, via="agent"): + """Carry out a change (approved, or in a conversation that lets the assistant act). Returns the result. `via` is + where it was approved (web, chat, or agent when the assistant acts on its own): only web approves a scaling + group's terms.""" + if tool in scaling.TOOLS: + if not admin: + raise PermissionError("only admins manage scaling") + out = scaling.apply_tool(db, cfg, tool, args, user, via) + what = out.get("group") or out.get("unit") or {} + table = "scale_group" if "group" in out else "scale_node" + auth.audit(db, user, f"scaling.{tool}", f"{table}:{what.get('id')}", {"via": via, "assistant": True}) + return out if tool == "change_settings": if not admin: raise PermissionError("only admins change settings") @@ -392,7 +463,7 @@ class OpsTools: def _change(self, tool, args, summary): if ( - (self.act or self._acts_on(tool, args)) + (self.act or self._acts_on(tool, args) or tool in WITHIN_BOUNDS) and tool not in ASK_TOOLS and not (tool == "change_settings" and args.get("section") in ALWAYS_ASK) ): @@ -413,6 +484,66 @@ def _acts_on(self, tool, args): return False # a new namespace waits for approval, as create_namespace does return True + def t_scaling(self): + v = scaling.overview(self.db, self.cfg) + out = { + "enabled": v["enabled"], + "settings": v["settings"], + "providers": [{k: p.get(k) for k in ("id", "name", "driver", "kind", "priced", "price", "check")} for p in v["providers"]], + "groups": v["groups"], + "units": v["units"], + "metrics": v["metrics"], + "drivers": [{k: d[k] for k in ("id", "kind", "label", "priced", "ready")} for d in v["drivers"]], + } + if not v["enabled"]: + out["note"] = "scaling is off: an admin turns it on in Settings › Scaling (change_settings scaling {enabled: true})" + running = sum(u["state"] == "running" for u in v["units"]) + return out, f"Read auto scaling: {len(v['groups'])} groups, {running} units running" + + def _scaling_group(self, gid): + try: + return scaling.get_group(self.db, int(gid)) + except (KeyError, TypeError, ValueError): + raise ValueError("there's no scaling group with that id: scaling lists them") from None + + def _terms_said(self, row, budget): + p = scaling._provider(self.db, row["provider"]) + cost = f", up to ${budget:g} a month" if budget is not None else "" + return f"on {p['name']}: {row['min']} to {row['max']} {row['kind']} units{cost}" + + def t_propose_scaling_group(self, group): + row, budget, _p, _d = scaling._check_group_body(self.db, group) # said now, not after it waits for approval + summary = f"Make the scaling group {row['name']} {self._terms_said(row, budget)}" + return self._change("propose_scaling_group", {"group": group}, summary) + + def t_change_scaling_group(self, group_id, changes): + g = self._scaling_group(group_id) + cur = {**g, "budget": (g.get("budget") or {}).get("usd")} + row, budget, _p, _d = scaling._check_group_body(self.db, changes, cur) + what = ", ".join(sorted(changes)) or "nothing" + summary = f"Change the scaling group {g['name']} ({what}): {self._terms_said(row, budget)}" + return self._change("change_scaling_group", {"group_id": g["id"], "changes": changes}, summary) + + def t_set_scaling_desired(self, group_id, desired): + g = self._scaling_group(group_id) + n = int(desired) + if not g["min"] <= n <= g["max"]: + raise ValueError(f"desired is from {g['min']} to {g['max']} for {g['name']}") + return self._change("set_scaling_desired", {"group_id": g["id"], "desired": n}, f"Ask {g['name']} for {n} units") + + def t_pause_scaling_group(self, group_id, paused=True, why=None): + g = self._scaling_group(group_id) + args = store.clean({"group_id": g["id"], "paused": bool(paused), "why": why}) + return self._change("pause_scaling_group", args, f"{'Pause' if paused else 'Resume'} the scaling group {g['name']}") + + def t_terminate_scaling_unit(self, unit_id, why=None): + try: + n = scaling._node(self.db, int(unit_id)) + except (KeyError, TypeError, ValueError): + raise ValueError("there's no scaling unit with that id: scaling lists them") from None + args = store.clean({"unit_id": n["id"], "why": why}) + return self._change("terminate_scaling_unit", args, f"Let the unit {n['name']} go") + def t_server_status(self): return status(self.db, self.cfg, self.base), "Checked what's set up" diff --git a/fastapi_backend/app/domain/scaling.py b/fastapi_backend/app/domain/scaling.py index cc3296085..439b5badb 100644 --- a/fastapi_backend/app/domain/scaling.py +++ b/fastapi_backend/app/domain/scaling.py @@ -305,18 +305,16 @@ def update_provider(db, cfg, pid, changes, by=None, check=True): missing = [k for k in drv.secrets if drv.fields[k].get("required") and k not in sealed] if missing: raise ValueError(f"a {drv.label} provider needs {', '.join(missing)}") - db.q( - "UPDATE $r MERGE $d", + db.q( # SET, not MERGE: a merge would keep a setting or a credential taken away + "UPDATE $r SET name = $n, settings = $s, sealed = $k, price = $p, note = $o, updated_at = $t, updated_by = $u", r=R("scale_provider", p["id"]), - d={ - "name": name.strip(), - "settings": settings_, - "sealed": sealed, - "price": price, - "note": note, - "updated_at": store.now(), - "updated_by": by, - }, + n=name.strip(), + s=settings_, + k=sealed, + p=price, + o=note, + t=store.now(), + u=by, ) activity.record( "run", "scaling.provider.change", [_ref("scale_provider", p["id"])], cfg, db, actor=by, detail={"fields": sorted(changes)} @@ -373,7 +371,7 @@ def check_rule(rule): out["minutes"] = _int(rule.get("minutes", 1), "minutes", 0, 1440) add, remove = rule.get("add"), rule.get("remove") if (add is None) == (remove is None): - raise ValueError("a step rule adds or removes units: add or remove, a count or a percentage like \"50%\"") + raise ValueError('a step rule adds or removes units: add or remove, a count or a percentage like "50%"') for k, v in (("add", add), ("remove", remove)): if v is None: continue @@ -540,6 +538,7 @@ def group_view(db, g, now=None): b = budgets.get(db, _budget_ref(g["id"])) st = budgets.status(db, _budget_ref(g["id"]), b, now) if b else None approval = g.get("approval") + t = terms({**g, "budget": b.get("usd") if b else None}) # the budget lives in the budget table, not on the row out = { "id": g["id"], "name": g.get("name"), @@ -559,9 +558,11 @@ def group_view(db, g, now=None): "paused_why": g.get("paused_why"), "capped": g.get("capped"), "approval": approval, - "approved": bool(approval) and not widens(approval.get("terms") or {}, terms(g)), - "terms": terms(g), - "budget": {"usd": b.get("usd"), "period": b.get("period"), "state": st["state"], "spent": st["spent"], "left": st["left"]} if b else None, + "approved": bool(approval) and not widens(approval.get("terms") or {}, t), + "terms": t, + "budget": {"usd": b.get("usd"), "period": b.get("period"), "state": st["state"], "spent": st["spent"], "left": st["left"]} + if b + else None, "units": { "launching": sum(u.get("state") == "launching" for u in units), "running": sum(u.get("state") == "running" for u in units), @@ -631,8 +632,9 @@ def update_group(db, cfg, gid, changes, by=None, via="token"): """Change a group. A change that widens its approved terms takes the approval away (a person gives it again); one made by a person signed in to the web app carries their approval.""" g = _group(db, gid) + g = {**g, "budget": (budgets.get(db, _budget_ref(g["id"])) or {}).get("usd")} # a budget left out is kept row, budget, p, drv = _check_group_body(db, changes, g) - before, after = terms({**g, "budget": (budgets.get(db, _budget_ref(g["id"])) or {}).get("usd")}), terms({**row, "budget": budget}) + before, after = terms(g), terms({**row, "budget": budget}) approval = g.get("approval") if approval and widens(approval.get("terms") or {}, after): approval = None @@ -749,7 +751,15 @@ def node_view(n): def protect(db, cfg, nid, on=True, by=None): n = _node(db, nid) db.q("UPDATE $r SET protected = $p", r=R("scale_node", n["id"]), p=bool(on)) - activity.record("run", "scaling.unit.protect", [_ref("scale_node", n["id"]), _ref("scale_group", n["group"])], cfg, db, actor=by, detail={"on": bool(on)}) + activity.record( + "run", + "scaling.unit.protect", + [_ref("scale_node", n["id"]), _ref("scale_group", n["group"])], + cfg, + db, + actor=by, + detail={"on": bool(on)}, + ) return node_view(_node(db, nid)) @@ -772,7 +782,9 @@ def _drain(db, cfg, n, why, by=None): except KeyError: pass # never registered as a worker: nothing to hand back db.q("UPDATE $r SET state = 'draining', why = $w", r=R("scale_node", n["id"]), w=why) - activity.record("run", "scaling.unit.drain", [_ref("scale_node", n["id"]), _ref("scale_group", n["group"])], cfg, db, actor=by, detail={"why": why}) + activity.record( + "run", "scaling.unit.drain", [_ref("scale_node", n["id"]), _ref("scale_group", n["group"])], cfg, db, actor=by, detail={"why": why} + ) def _terminate(db, cfg, drv, p, n, why, by=None, state="terminated", now=None): @@ -891,7 +903,7 @@ def _count(v, units): def evaluate(g, m, units, now, state=None): """What the rules propose for a group with `units` running: (desired or None when no rule speaks, the new rule - state, [what each rule said]). Step rules fire once their condition has held for their minutes; schedule rules + state, [what each rule said], the new min, the new max: None unless a schedule rule set them). Step rules fire once their condition has held for their minutes; schedule rules once per time they name.""" state = dict(state or {}) proposals, said = [], [] @@ -986,6 +998,8 @@ def tick(self, now=None, lease=True): """One pass over every group. Returns what it did: {launched, terminated, groups}.""" now = now or _now() cfg = self.cfg_fn() + if not enabled(cfg) and not self.db.values("SELECT VALUE id FROM scale_node WHERE state IN $s LIMIT 1", s=list(ALIVE)): + return None # off, with nothing left to let go: not even the lease is written if lease and not self._lease(now): return None did = {"launched": 0, "terminated": 0, "groups": 0} @@ -1023,7 +1037,9 @@ def _group(self, cfg, g, now, did): may_launch = enabled(cfg) and view["enabled"] and not view["paused"] and view["approved"] and not capped lo, hi, desired = g.get("min") or 0, g.get("max") or 0, g.get("desired") or 0 if may_launch and g.get("rules"): - want, state, said, new_min, new_max = evaluate(g, metrics(db, cfg, (g.get("template") or {}).get("steps"), now), len(running), now, g.get("rule_state")) + want, state, said, new_min, new_max = evaluate( + g, metrics(db, cfg, (g.get("template") or {}).get("steps"), now), len(running), now, g.get("rule_state") + ) sets = {"rule_state": state, "last_eval": {"at": _iso(now), "said": said}} if new_min is not None or new_max is not None: lo = new_min if new_min is not None else lo @@ -1139,7 +1155,9 @@ def _launch(self, cfg, g, drv, p, now, why): ), ) db.q("UPDATE $r SET failures = 0, error = NONE", r=R("scale_group", g["id"])) - activity.record("run", "scaling.launch", refs, cfg, db, detail={"why": why, "unit": name, "price_per_hour": got.get("price", price)}) + activity.record( + "run", "scaling.launch", refs, cfg, db, detail={"why": why, "unit": name, "price_per_hour": got.get("price", price)} + ) self.log(f"scaling: launched {name} on {p.get('name')} ({why})") return True @@ -1236,6 +1254,10 @@ def overview(db, cfg, now=None): # ---------- the assistant and MCP ---------- +# the assistant's and MCP's tools that change scaling (ops_tools.SCALING_TOOLS, mcp_tools); `scaling` only reads +TOOLS = ("propose_scaling_group", "change_scaling_group", "set_scaling_desired", "pause_scaling_group", "terminate_scaling_unit") + + def apply_tool(db, cfg, tool, args, user, via="agent"): """Carry out a scaling tool's change, for the assistant (ops_tools.apply) once approved, or at once for what agents may do within the approved terms. `via` web is the person's own yes in the web app, which also approves diff --git a/fastapi_backend/app/schemas/scaling.py b/fastapi_backend/app/schemas/scaling.py index 96af44f38..db31ce956 100644 --- a/fastapi_backend/app/schemas/scaling.py +++ b/fastapi_backend/app/schemas/scaling.py @@ -77,7 +77,7 @@ class Rule(RequestModel): above: float | None = Field(None, description="step rules: fires while the metric is above this") below: float | None = Field(None, description="step rules: fires while the metric is below this") minutes: int | None = Field(None, ge=0, le=1440, description="step rules: for how long before it fires (default 1)") - add: int | str | None = Field(None, description="step rules: units to add, a count or a percentage like \"50%\"") + add: int | str | None = Field(None, description='step rules: units to add, a count or a percentage like "50%"') remove: int | str | None = Field(None, description="step rules: units to remove, a count or a percentage") cron: str | None = Field(None, description="schedule rules: when, as a cron expression") timezone: str | None = None diff --git a/fastapi_backend/tests/api/test_mcp.py b/fastapi_backend/tests/api/test_mcp.py index fd0027c52..485cf8feb 100644 --- a/fastapi_backend/tests/api/test_mcp.py +++ b/fastapi_backend/tests/api/test_mcp.py @@ -92,6 +92,11 @@ def test_clients_are_sent_to_sign_in(client, app, env): "set_namespace_integrations", "set_picture_descriptions", "set_file_storage", + "propose_scaling_group", + "change_scaling_group", + "set_scaling_desired", + "pause_scaling_group", + "terminate_scaling_unit", "set_monitoring", "set_service", "set_sandbox_gateway", diff --git a/fastapi_backend/tests/api/test_scaling.py b/fastapi_backend/tests/api/test_scaling.py new file mode 100644 index 000000000..eafdc8f31 --- /dev/null +++ b/fastapi_backend/tests/api/test_scaling.py @@ -0,0 +1,356 @@ +"""Auto scaling (docs/scaling.md): providers with sealed credentials, groups that launch nothing until a person +approves their terms in the web app, rules, budgets, and the scaler launching, joining, draining and terminating units +through a fake provider.""" + +from __future__ import annotations + +import datetime as dt +import sys + +import pytest + +from app.domain import activity, budgets, scaling, settings, store +from app.domain import scaling_drivers as drivers +from tests.helpers import login, make_user + +R = store.R +NOW = dt.datetime(2026, 10, 18, 12, 0, tzinfo=dt.UTC) + + +class Fake(drivers.Driver): + """A cloud in memory: machines by name, a key that has to be "good", 50 cents an hour.""" + + id = "fake" + kind = "node" + label = "Fake cloud" + about = "Machines in a dict." + priced = True + fields = { + "region": drivers._field("Region", required=True), + "api_key": drivers._field("API key", required=True, secret=True), + } + template = {"size": drivers._field("Size", default="small", options=["small", "large"])} + + def __init__(self): + self.machines, self.boots, self.seen_keys, self.fail = {}, {}, [], None + + def check(self, p): + self.seen_keys.append(p["secrets"].get("api_key")) + if p["secrets"].get("api_key") != "good": + raise drivers.DriverError("the key was refused") + return {"regions": 3} + + def price(self, p, template): + return 0.5 + + def launch(self, p, template, name, boot): + if self.fail: + raise drivers.DriverError(self.fail) + self.machines[name] = "pending" + self.boots[name] = boot + return {"id": name, "address": "10.0.0.9"} + + def describe(self, p, node): + return {"state": self.machines.get(node["external_id"], "gone")} + + def terminate(self, p, node): + self.machines.pop(node["external_id"], None) + + +@pytest.fixture +def fake(): + drv = Fake() + drivers.register(drv) + yield drv + drivers.DRIVERS.pop("fake", None) + + +@pytest.fixture +def h(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +@pytest.fixture +def key(client, h): + """An admin's API key: an agent, which may do everything but approve.""" + tok = client.post("/api/v1/tokens", headers=h, json={"name": "agent", "scope": "write"}).json()["token"] + return {"Authorization": f"Bearer {tok}"} + + +def on(db, cfg, **more): + settings.save(db, cfg, "scaling", {"enabled": True, **more}, "root@x.io") + return settings.effective(db, cfg) + + +def provider(client, h, **over): + body = {"name": "Fake", "driver": "fake", "settings": {"region": "eu"}, "secrets": {"api_key": "good"}, **over} + r = client.post("/api/v1/scaling/providers", headers=h, json=body) + assert r.status_code == 201, r.text + return r.json() + + +def group(client, h, pid, **over): + body = {"name": "Workers", "provider": pid, "min": 0, "max": 3, "budget": 20, **over} + return client.post("/api/v1/scaling/groups", headers=h, json=body) + + +def join(db, name, current=None, at=NOW): + """The unit's `lens worker` has started and beats.""" + db.q("UPSERT $w MERGE {name: $n, heartbeat_at: $t, current: $c}", w=R("worker", name), n=name, t=at.isoformat(), c=current) + + +def tick(db, cfg, now=NOW): + return scaling.Scaler(db, lambda: cfg, "test").tick(now=now, lease=False) + + +def unit_states(db, gid): + return sorted((n["name"], n["state"]) for n in scaling.nodes(db, gid)) + + +def test_scaling_is_off_and_lists_its_drivers(client, h, fake): + v = client.get("/api/v1/scaling", headers=h).json() + assert v["enabled"] is False and v["groups"] == [] and v["settings"]["tick_seconds"] == 30 + ids = {d["id"]: d for d in v["drivers"]} + assert ids["local"]["priced"] is False and ids["fake"]["fields"]["api_key"]["secret"] is True + assert set(v["metrics"]) >= set(scaling.METRICS) + assert client.get("/api/v1/scaling/metrics", headers=h).json()["about"]["jobs_waiting"] + + +def test_a_providers_credentials_are_sealed_and_checked(client, db, cfg, h, fake): + p = provider(client, h) + assert p["secrets"] == {"api_key": True} and p["check"] == {"ok": True, "regions": 3} + raw = db.one("SELECT sealed FROM $r", r=R("scale_provider", p["id"])) + assert "good" not in str(raw) and fake.seen_keys == ["good"] # kept sealed, opened only for the driver's call + assert "good" not in client.get("/api/v1/scaling/providers", headers=h).text + + bad = provider(client, h, name="Bad", secrets={"api_key": "nope"}) + assert bad["check"] == {"ok": False, "error": "the key was refused"} + for body in ( + {"name": "x", "driver": "fake", "settings": {"region": "eu"}}, # no key + {"name": "x", "driver": "fake", "settings": {}, "secrets": {"api_key": "good"}}, # no region + {"name": "x", "driver": "fake", "settings": {"region": "eu", "zone": 1}, "secrets": {"api_key": "good"}}, + {"name": "x", "driver": "nimbus", "settings": {}}, + ): + assert client.post("/api/v1/scaling/providers", headers=h, json=body).status_code in (400, 404, 422), body + + r = client.patch(f"/api/v1/scaling/providers/{bad['id']}", headers=h, json={"secrets": {"api_key": "good"}}) + assert r.json()["check"]["ok"] is True + assert client.post(f"/api/v1/scaling/providers/{p['id']}/check", headers=h).json()["ok"] is True + assert client.delete(f"/api/v1/scaling/providers/{bad['id']}", headers=h).status_code == 200 + assert [a["action"] for a in activity.history(db, f"scale_provider:{p['id']}")][-1] == "scaling.provider.add" + + +def test_only_a_person_in_the_web_app_approves_a_groups_terms(client, db, cfg, h, key, fake): + pid = provider(client, h)["id"] + assert group(client, key, pid, budget=None).status_code == 400 # a priced provider needs a budget + + # an agent proposes: saved, but it launches nothing until a person approves + g = group(client, key, pid).json() + assert (g["approved"], g["approval"]) == (False, None) and g["budget"]["usd"] == 20 + assert g["waiting_for"] == "scaling is off: turn it on in Settings › Scaling" + gid = g["id"] + assert client.post(f"/api/v1/scaling/groups/{gid}/approve", headers=key).status_code == 403 + cfg = on(db, cfg) + assert tick(db, cfg)["launched"] == 0 and scaling.nodes(db, gid) == [] + assert "approval" in client.get(f"/api/v1/scaling/groups/{gid}", headers=h).json()["waiting_for"] + + # the person, signed in, approves the terms as they stand + g = client.post(f"/api/v1/scaling/groups/{gid}/approve", headers=h).json() + assert g["approved"] is True and g["approval"]["by"] == "root@x.io" and g["terms"]["budget"] == 20 and g["waiting_for"] is None + + # within the terms an agent acts: fewer units, a lower budget, the desired count + g = client.patch(f"/api/v1/scaling/groups/{gid}", headers=key, json={"max": 2, "budget": 10}).json() + assert g["approved"] is True + assert client.post(f"/api/v1/scaling/groups/{gid}/desired", headers=key, json={"desired": 2}).json()["desired"] == 2 + assert client.post(f"/api/v1/scaling/groups/{gid}/desired", headers=key, json={"desired": 5}).status_code == 400 + # widening them takes the approval away, until the person gives it again + g = client.patch(f"/api/v1/scaling/groups/{gid}", headers=key, json={"max": 4}).json() + assert g["approved"] is False and g["approval"] is None + g = client.patch(f"/api/v1/scaling/groups/{gid}", headers=key, json={"max": 2, "budget": 50}).json() + assert g["approved"] is False + # a change the person makes in the web app carries their approval + g = client.patch(f"/api/v1/scaling/groups/{gid}", headers=h, json={"max": 4}).json() + assert g["approved"] is True and g["terms"]["max"] == 4 and g["terms"]["budget"] == 50 + # and a group the person makes there is approved as it's made + assert group(client, h, pid, name="Mine").json()["approved"] is True + acts = [a["action"] for a in activity.history(db, f"scale_group:{gid}", 50)] + assert {"scaling.group.create", "scaling.group.approve", "scaling.group.change", "scaling.group.desired"} <= set(acts) + + +def test_the_scaler_launches_joins_drains_and_terminates(client, db, cfg, h, fake): + cfg = on(db, cfg) + pid = provider(client, h)["id"] + gid = group(client, h, pid, min=1, max=3, cooldown_seconds=0, warmup_seconds=0).json()["id"] + + assert tick(db, cfg)["launched"] == 1 # min 1 + ((name, state),) = unit_states(db, gid) + assert state == "launching" and fake.machines == {name: "pending"} and name.startswith("lens-workers-") + assert fake.boots[name]["env"] == {"LENS_SCALING_UNIT": name} # no long-lived secret in what the provider gets + + join(db, name) + tick(db, cfg, NOW + dt.timedelta(seconds=30)) + assert unit_states(db, gid) == [(name, "running")] + assert client.get(f"/api/v1/scaling/groups/{gid}", headers=h).json()["units"] == {"launching": 0, "running": 1, "draining": 0} + + # asked for more: two more launch + client.post(f"/api/v1/scaling/groups/{gid}/desired", headers=h, json={"desired": 3}) + assert tick(db, cfg, NOW + dt.timedelta(minutes=1))["launched"] == 2 + for n in scaling.nodes(db, gid, alive=True): + join(db, n["name"], at=NOW + dt.timedelta(minutes=2)) + tick(db, cfg, NOW + dt.timedelta(minutes=2)) + assert [s for _, s in unit_states(db, gid)] == ["running"] * 3 + + # back to one: a busy worker drains (its job is handed back) and goes once it's idle; idle ones go first + busy = scaling.nodes(db, gid, alive=True)[0]["name"] + join(db, busy, current=7, at=NOW + dt.timedelta(minutes=3)) + client.post(f"/api/v1/scaling/groups/{gid}/desired", headers=h, json={"desired": 1}) + assert tick(db, cfg, NOW + dt.timedelta(minutes=3))["terminated"] == 2 + left = dict(unit_states(db, gid)) + assert left[busy] == "running" and sorted(left.values()) == ["draining", "draining", "running"] + for n, s in left.items(): + if s == "draining": + join(db, n, current=None, at=NOW + dt.timedelta(minutes=4)) # finished, idle + tick(db, cfg, NOW + dt.timedelta(minutes=4)) + assert sorted(dict(unit_states(db, gid)).values()) == ["running", "terminated", "terminated"] + assert list(fake.machines) == [busy] + + # the ledger has each step, and the hours each unit ran, as an estimate + acts = activity.history(db, f"scale_group:{gid}", 100) + names = [a["action"] for a in acts] + assert {"scaling.launch", "scaling.join", "scaling.unit.drain", "scaling.terminate", "scaling.unit"} <= set(names) + cost = [a for a in acts if a["action"] == "scaling.unit"] + assert all(a["detail"]["estimate"] for a in cost) and sum(a["cost_usd"] for a in cost) > 0 + assert budgets.status(db, f"scale_group:{gid}")["spent"]["usd"] > 0 + + # turning scaling off lets every unit go + settings.save(db, cfg, "scaling", {"enabled": False}, "root@x.io") + cfg = settings.effective(db, cfg) + tick(db, cfg, NOW + dt.timedelta(minutes=5)) + join(db, busy, current=None, at=NOW + dt.timedelta(minutes=6)) + tick(db, cfg, NOW + dt.timedelta(minutes=6)) + assert scaling.nodes(db, gid, alive=True) == [] and fake.machines == {} + + +def test_a_unit_that_never_joins_fails_and_failures_pause_the_group(client, db, cfg, h, fake): + cfg = on(db, cfg, join_minutes=5) + pid = provider(client, h)["id"] + gid = group(client, h, pid, min=1, max=1, cooldown_seconds=0, warmup_seconds=0).json()["id"] + tick(db, cfg) + first = scaling.nodes(db, gid)[0]["name"] + tick(db, cfg, NOW + dt.timedelta(minutes=6)) # it never joined: terminated as failed, and replaced + assert dict(unit_states(db, gid))[first] == "failed" and first not in fake.machines + assert sorted(dict(unit_states(db, gid)).values()) == ["failed", "launching"] + fake.fail = "out of capacity" + fake.machines.clear() # the provider lost the new one too, and launches no more + for i in range(3): + tick(db, cfg, NOW + dt.timedelta(minutes=10 + i)) + g = client.get(f"/api/v1/scaling/groups/{gid}", headers=h).json() + assert g["paused"] is True and "out of capacity" in g["paused_why"] and g["waiting_for"].startswith("paused") + assert tick(db, cfg, NOW + dt.timedelta(minutes=20))["launched"] == 0 + g = client.post(f"/api/v1/scaling/groups/{gid}/pause", headers=h, json={"paused": False}).json() + assert g["paused"] is False and g["failures"] == 0 + + +def test_at_its_budgets_cap_a_group_stops(client, db, cfg, h, fake): + cfg = on(db, cfg) + pid = provider(client, h)["id"] + gid = group(client, h, pid, min=2, max=2, budget=1, cooldown_seconds=0, warmup_seconds=0).json()["id"] + tick(db, cfg) + for n in scaling.nodes(db, gid, alive=True): + join(db, n["name"], at=NOW + dt.timedelta(hours=1)) + tick(db, cfg, NOW + dt.timedelta(hours=1)) # two units, an hour at 50 cents: the dollar is spent + tick(db, cfg, NOW + dt.timedelta(hours=1, minutes=1)) + g = client.get(f"/api/v1/scaling/groups/{gid}", headers=h).json() + assert g["capped"] and g["budget"]["state"] == "over" and "budget" in g["waiting_for"] + assert [s for _, s in unit_states(db, gid)] == ["draining", "draining"] # at_cap stop: every unit goes + assert tick(db, cfg, NOW + dt.timedelta(hours=1, minutes=2))["launched"] == 0 + + +def test_rules_scale_on_waiting_jobs_and_on_a_schedule(client, db, cfg, h, fake): + cfg = on(db, cfg) + pid = provider(client, h)["id"] + rules = [ + {"type": "step", "metric": "jobs_waiting", "above": 4, "minutes": 2, "add": 2}, + {"type": "schedule", "cron": "0 18 * * *", "timezone": "UTC", "max": 1, "desired": 0}, + ] + g = group(client, h, pid, max=5, rules=rules, cooldown_seconds=0, warmup_seconds=0).json() + assert [r["says"] for r in g["rules"]] == [ + "add 2 when jobs_waiting above 4 for 2 min", + g["rules"][1]["says"], + ] and "max 1" in g["rules"][1]["says"] + gid = g["id"] + for _ in range(6): + db.q("CREATE job CONTENT {status: 'queued', next_step: 'transcribe', created_at: $t}", t=NOW.isoformat()) + assert tick(db, cfg)["launched"] == 0 # the condition has to hold for 2 minutes + assert tick(db, cfg, NOW + dt.timedelta(minutes=2))["launched"] == 2 + said = client.get(f"/api/v1/scaling/groups/{gid}", headers=h).json()["last_eval"]["said"] + assert said == ["add 2 when jobs_waiting above 4 for 2 min: jobs_waiting is 6, so 2"] + # at 18:00 the schedule brings max down to 1 + tick(db, cfg, NOW.replace(hour=18, minute=1)) + g = client.get(f"/api/v1/scaling/groups/{gid}", headers=h).json() + assert (g["max"], g["desired"]) == (1, 1) + + for bad in ( + {"type": "step", "metric": "jobs_waiting", "above": 4, "add": 1, "remove": 1}, + {"type": "target", "metric": "mood", "target": 1}, + {"type": "schedule", "cron": "whenever", "desired": 1}, + {"type": "schedule", "cron": "0 9 * * *"}, + ): + assert client.patch(f"/api/v1/scaling/groups/{gid}", headers=h, json={"rules": [bad]}).status_code in (400, 422), bad + + +def test_protected_and_terminated_units(client, db, cfg, h, key, fake): + cfg = on(db, cfg) + pid = provider(client, h)["id"] + gid = group(client, h, pid, min=2, max=2, cooldown_seconds=0, warmup_seconds=0).json()["id"] + tick(db, cfg) + a, b = scaling.nodes(db, gid, alive=True) + assert client.post(f"/api/v1/scaling/units/{a['id']}/protect", headers=key, json={"protected": True}).json()["protected"] + out = client.post(f"/api/v1/scaling/units/{b['id']}/terminate", headers=key, json={"why": "testing"}).json() + assert out["state"] == "terminated" and out["why"] == "testing" and b["name"] not in fake.machines + assert client.post(f"/api/v1/scaling/units/{b['id']}/terminate", headers=key, json={}).status_code == 400 + assert client.delete(f"/api/v1/scaling/providers/{pid}", headers=h).status_code == 400 # a group uses it + assert client.delete(f"/api/v1/scaling/groups/{gid}", headers=h).status_code == 200 + assert fake.machines == {} and scaling.nodes(db, gid, alive=True) == [] + + +def test_the_settings_section(client, db, cfg, h): + assert settings.effective(db, cfg)["scaling"]["enabled"] is False + for bad in ({"tick_seconds": 1}, {"join_minutes": 500}, {"tag": "Not A Tag"}): + with pytest.raises(ValueError): + settings.save(db, cfg, "scaling", bad, "root@x.io") + settings.save(db, cfg, "scaling", {"tick_seconds": 60, "tag": "acme"}, "root@x.io") + s = settings.effective(db, cfg)["scaling"] + assert (s["tick_seconds"], s["tag"]) == (60, "acme") + + +def test_only_admins_see_scaling(client, db, h): + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + e = login(client, "ed@x.io", "editor password 1") + assert client.get("/api/v1/scaling", headers=e).status_code == 403 + assert client.post("/api/v1/scaling/groups", headers=e, json={"name": "x", "provider": 1}).status_code == 403 + + +def test_this_machine_runs_more_workers_at_no_cost(client, db, cfg, h, monkeypatch): + here = drivers.get("local") + monkeypatch.setattr(here, "command", lambda name, template: [sys.executable, "-c", "import time; time.sleep(60)"]) + assert tick(db, cfg) is None # off, with nothing running: the scaler doesn't even take its lease + cfg = on(db, cfg) + pid = client.post("/api/v1/scaling/providers", headers=h, json={"name": "Here", "driver": "local"}).json()["id"] + g = group(client, h, pid, budget=None, min=1, max=2, template={"steps": ["transcribe"]}, cooldown_seconds=0, warmup_seconds=0) + g = g.json() + assert g["approved"] is True and g["budget"] is None # it costs nothing, so it needs no budget + assert tick(db, cfg)["launched"] == 1 + (n,) = scaling.nodes(db, g["id"]) + assert n["price"] == 0.0 and n["external_id"].startswith(here.host() + ":") + assert here.describe(None, n)["state"] == "running" # a process on this machine + join(db, n["name"]) + tick(db, cfg, NOW + dt.timedelta(seconds=30)) + out = client.post(f"/api/v1/scaling/units/{n['id']}/terminate", headers=h, json={}).json() + assert out["state"] == "draining" + tick(db, cfg, NOW + dt.timedelta(minutes=1)) # idle: it goes, and its process stops + assert scaling.nodes(db, g["id"])[0]["state"] in ("terminated", "launching") + assert here.describe(None, n)["state"] == "gone" + for u in scaling.nodes(db, g["id"], alive=True): # min 1 started another; stop it too + here.terminate(None, u) diff --git a/fastapi_backend/tests/api/test_scaling_agents.py b/fastapi_backend/tests/api/test_scaling_agents.py new file mode 100644 index 000000000..082b74ec7 --- /dev/null +++ b/fastapi_backend/tests/api/test_scaling_agents.py @@ -0,0 +1,126 @@ +"""Auto scaling through agents (docs/scaling.md, "Agents"): the assistant proposes groups that wait for the person's +yes in the web app and acts at once within approved terms; MCP clients do the same, and never approve.""" + +from __future__ import annotations + +import json + +import pytest + +from app.domain import scaling, settings +from tests import fake_llm +from tests.api._assist import Assist, sse, start_llm +from tests.api.test_mcp import tool, tool_error +from tests.api.test_oauth import _bearer, _grant +from tests.api.test_scaling import fake # noqa: F401 - the fake provider +from tests.helpers import login, make_user + +GROUP = {"name": "Workers", "min": 0, "max": 3, "budget": 20} + + +@pytest.fixture +def llm(cfg): + srv = start_llm(cfg) + yield fake_llm.Handler + srv.shutdown() + + +@pytest.fixture +def app(cfg, db, llm): + from app.main import create_app + + return create_app(cfg, db, background=False) + + +def call(i, name, args): + return {"id": f"c{i}", "type": "function", "function": {"name": name, "arguments": json.dumps(args)}} + + +def provider(c, h): + body = {"name": "Fake", "driver": "fake", "settings": {"region": "eu"}, "secrets": {"api_key": "good"}} + return c.post("/api/v1/scaling/providers", headers=h, json=body).json()["id"] + + +def test_the_assistant_proposes_and_the_person_approves_in_the_web_app(app, db, cfg, folder, new_client, llm, fake): # noqa: F811 + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["admin"] + pid = provider(c, h) + llm.tool_script = [ + {"content": "", "tool_calls": [call(1, "scaling", {})]}, + {"content": "", "tool_calls": [call(2, "propose_scaling_group", {"group": {**GROUP, "provider": pid}})]}, + {"content": "I proposed a group of up to 3 workers."}, + ] + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "scale workers on the fake cloud"}).text) + seen = llm.seen[-2]["messages"][3]["content"] + read = json.loads(seen[seen.index("{") : seen.rindex("}") + 1]) + assert read["enabled"] is False and read["providers"][0]["name"] == "Fake" and "good" not in seen + appr = ev["approval"][0] + assert appr["summary"] == "Make the scaling group Workers on Fake: 0 to 3 node units, up to $20 a month" + assert scaling.groups(db) == [] + out = c.post(f"/api/v1/approvals/{appr['id']}", headers=h, json={"decision": "approve"}).json() + assert out["status"] == "done" and out["group"]["approved"] is True # the person's yes in the web app approves its terms + gid = out["group"]["id"] + + # within the approved terms the assistant acts at once; widening them asks again + llm.tool_script = [ + {"content": "", "tool_calls": [call(3, "set_scaling_desired", {"group_id": gid, "desired": 2})]}, + {"content": "", "tool_calls": [call(4, "change_scaling_group", {"group_id": gid, "changes": {"max": 10}})]}, + {"content": "Asked for 2 now; raising the max waits for you."}, + ] + ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "two now, and allow ten"}).text) + assert scaling.get_group(db, gid)["desired"] == 2 + assert [a["summary"] for a in ev["approval"]] == [ + "Change the scaling group Workers (max): on Fake: 0 to 10 node units, up to $20 a month" + ] + assert scaling.get_group(db, gid)["max"] == 3 + + +def test_a_setup_conversation_makes_a_group_that_still_waits_for_the_person(app, db, cfg, folder, new_client, llm, fake): # noqa: F811 + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["admin"] + pid = provider(c, h) + llm.tool_script = [ + {"content": "", "tool_calls": [call(1, "propose_scaling_group", {"group": {**GROUP, "provider": pid}})]}, + {"content": "", "tool_calls": [call(2, "change_settings", {"section": "scaling", "changes": {"enabled": True}})]}, + {"content": "Made it; approve its terms in Settings."}, + ] + cid = c.post("/api/v1/chats", headers=h, json={"kind": "setup"}).json()["id"] + ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "set up scaling"}).text) + (g,) = scaling.groups(db) + assert g["approved"] is False and "approval" in g["waiting_for"] # made at once, as the assistant, so not approved + assert [a["summary"] for a in ev["approval"]] == ["Set scaling: enabled = True"] # turning scaling on always asks + assert settings.effective(db, cfg)["scaling"]["enabled"] is False + + +def test_only_admins_get_the_scaling_tools(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + llm.tool_script = [{"content": "Hi."}] + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "hi"}).text) + offered = {t["function"]["name"] for t in llm.seen[-1].get("tools") or []} + assert not offered & {"scaling", *scaling.TOOLS} + + +def test_mcp_clients_manage_scaling_but_never_approve(client, db, cfg, fake): # noqa: F811 + make_user(db, "root@x.io", "root password 1", admin=True) + h = login(client, "root@x.io", "root password 1") + pid = provider(client, h) + _, tokens = _grant(client, h, scope="read write") + m = _bearer(tokens) + assert tool(client, m, "scaling")["providers"][0]["secrets"] == {"api_key": True} + out = tool(client, m, "propose_scaling_group", group={**GROUP, "provider": pid}) + g = out["group"] + assert g["approved"] is False and out["approve"].endswith("/settings/scaling") + assert "budget" in tool_error(client, m, "propose_scaling_group", group={**GROUP, "provider": pid, "budget": None}) + # the person approves in the web app; then the agent works within the terms + assert client.post(f"/api/v1/scaling/groups/{g['id']}/approve", headers=h).json()["approved"] is True + assert tool(client, m, "set_scaling_desired", group_id=g["id"], desired=1)["group"]["desired"] == 1 + assert "from 0 to 3" in tool_error(client, m, "set_scaling_desired", group_id=g["id"], desired=9) + assert tool(client, m, "pause_scaling_group", group_id=g["id"], why="night")["group"]["paused_why"] == "night" + assert tool(client, m, "change_scaling_group", group_id=g["id"], changes={"max": 5})["group"]["approved"] is False + + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + _, et = _grant(client, login(client, "ed@x.io", "editor password 1"), scope="read write") + assert "only admins" in tool_error(client, _bearer(et), "scaling") diff --git a/fastapi_backend/tests/domain/test_scaling_rules.py b/fastapi_backend/tests/domain/test_scaling_rules.py new file mode 100644 index 000000000..69337f0f1 --- /dev/null +++ b/fastapi_backend/tests/domain/test_scaling_rules.py @@ -0,0 +1,80 @@ +"""Auto scaling's rules and terms (app/domain/scaling.py): what each rule proposes, and which changes widen what a +person approved.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from app.domain import scaling + +NOW = dt.datetime(2026, 10, 18, 12, 0, tzinfo=dt.UTC) + + +def rules(*rs): + return {"rules": [scaling.check_rule(r) for r in rs], "created_at": NOW.isoformat()} + + +def test_a_target_rule_keeps_its_metric_at_the_target(): + g = rules({"type": "target", "metric": "workers_busy_share", "target": 0.5}) + want, _, said, *_ = scaling.evaluate(g, {"workers_busy_share": 1.0}, 2, NOW) + assert want == 4 and said == ["keep workers_busy_share at 0.5: workers_busy_share is 1, so 4"] + assert scaling.evaluate(g, {"workers_busy_share": 0.25}, 4, NOW)[0] == 2 + assert scaling.evaluate(g, {"workers_busy_share": 0}, 3, NOW)[0] == 0 + + +def test_a_step_rule_fires_once_its_condition_has_held(): + g = rules({"type": "step", "metric": "oldest_wait_minutes", "above": 10, "minutes": 5, "add": "50%"}) + want, state, *_ = scaling.evaluate(g, {"oldest_wait_minutes": 12}, 4, NOW) + assert want is None and state["0"]["since"] + want, state, *_ = scaling.evaluate(g, {"oldest_wait_minutes": 12}, 4, NOW + dt.timedelta(minutes=5), state) + assert want == 6 # 50% of 4 more + want, state, *_ = scaling.evaluate(g, {"oldest_wait_minutes": 3}, 6, NOW + dt.timedelta(minutes=6), state) + assert want is None and "since" not in state["0"] # the condition broke: it starts over + down = rules({"type": "step", "metric": "jobs_waiting", "below": 1, "minutes": 0, "remove": 1}) + assert scaling.evaluate(down, {"jobs_waiting": 0}, 2, NOW)[0] == 1 + + +def test_a_schedule_rule_sets_the_bounds_when_its_time_comes(): + g = { + **rules({"type": "schedule", "cron": "0 9 * * *", "timezone": "UTC", "min": 2, "desired": 2}), + "created_at": NOW.replace(hour=7).isoformat(), + } + assert scaling.evaluate(g, {}, 0, NOW.replace(hour=8))[0] is None + want, state, said, lo, hi = scaling.evaluate(g, {}, 0, NOW) # 9:00 has passed since it was made + assert (want, lo, hi) == (2, 2, None) and said[0].endswith("fired") + assert scaling.evaluate(g, {}, 2, NOW + dt.timedelta(minutes=1), state)[0] is None # once per time it names + + +def test_the_largest_proposal_wins(): + g = rules( + {"type": "target", "metric": "jobs_waiting", "target": 10}, + {"type": "step", "metric": "jobs_waiting", "above": 5, "minutes": 0, "add": 3}, + ) + assert scaling.evaluate(g, {"jobs_waiting": 20}, 1, NOW)[0] == 4 + + +@pytest.mark.parametrize( + "rule", + [ + {"type": "nudge"}, + {"type": "target", "metric": "jobs_waiting"}, + {"type": "target", "metric": "jobs_waiting", "target": 0}, + {"type": "step", "metric": "jobs_waiting", "above": 1}, + {"type": "step", "metric": "jobs_waiting", "above": 1, "below": 0, "add": 1}, + {"type": "step", "metric": "jobs_waiting", "above": 1, "add": "5000%"}, + {"type": "schedule", "cron": "0 9 * * *", "timezone": "Mars/Olympus", "min": 1}, + ], +) +def test_rules_that_dont_say_enough_are_refused(rule): + with pytest.raises(ValueError): + scaling.check_rule(rule) + + +def test_what_widens_an_approval(): + t = {"kind": "node", "provider": 1, "template": {"size": "small"}, "max": 3, "budget": 20.0, "at_cap": "stop"} + assert not scaling.widens(t, t) + assert not scaling.widens(t, {**t, "max": 2, "budget": 10.0}) + for wider in ({"max": 4}, {"budget": 21.0}, {"budget": None}, {"provider": 2}, {"template": {"size": "large"}}, {"at_cap": "min"}): + assert scaling.widens(t, {**t, **wider}), wider diff --git a/mkdocs.yml b/mkdocs.yml index 6fd7e1338..cbfc531fb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -106,6 +106,7 @@ nav: - Speech providers: speech-providers.md - Activity and costs: activity.md - Budgets: budgets.md + - Auto scaling: scaling.md - Operate: - Encryption: encryption.md - Storage: storage.md diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 123eda165..39ffafab2 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -40,6 +40,7 @@ export { Research as Research2, Resources, Routines, + Scaling, Search, Searches, Sensors, @@ -112,6 +113,11 @@ export type { AddPasskeyOptionsResponses, AddPasskeyResponse, AddPasskeyResponses, + AddProviderData, + AddProviderError, + AddProviderErrors, + AddProviderResponse, + AddProviderResponses, AddRecordingPermissionData, AddRecordingPermissionError, AddRecordingPermissionErrors, @@ -175,6 +181,11 @@ export type { ApproveAccessRequestErrors, ApproveAccessRequestResponse, ApproveAccessRequestResponses, + ApproveGroupData, + ApproveGroupError, + ApproveGroupErrors, + ApproveGroupResponse, + ApproveGroupResponses, AskGraphData, AskGraphError, AskGraphErrors, @@ -295,6 +306,11 @@ export type { CancelUploadResponse, CancelUploadResponses, ChangeAsk, + ChangeGroupData, + ChangeGroupError, + ChangeGroupErrors, + ChangeGroupResponse, + ChangeGroupResponses, ChangeLoginProviderData, ChangeLoginProviderError, ChangeLoginProviderErrors, @@ -305,6 +321,11 @@ export type { ChangePasswordErrors, ChangePasswordResponse, ChangePasswordResponses, + ChangeProviderData, + ChangeProviderError, + ChangeProviderErrors, + ChangeProviderResponse, + ChangeProviderResponses, Chat, ChatCapabilities, ChatCapabilitiesData, @@ -338,6 +359,11 @@ export type { CheckMessageErrors, CheckMessageResponse, CheckMessageResponses, + CheckProviderData, + CheckProviderError, + CheckProviderErrors, + CheckProviderResponse, + CheckProviderResponses, CheckUpdatesData, CheckUpdatesResponse, CheckUpdatesResponses, @@ -518,6 +544,11 @@ export type { CreateFieldErrors, CreateFieldResponse, CreateFieldResponses, + CreateGroupData, + CreateGroupError, + CreateGroupErrors, + CreateGroupResponse, + CreateGroupResponses, CreateHighlightData, CreateHighlightError, CreateHighlightErrors, @@ -892,6 +923,7 @@ export type { DeleteWatchResponses, Described, Description, + Desired, DetectLlmData, DetectLlmResponse, DetectLlmResponses, @@ -940,6 +972,8 @@ export type { DraftAutomationErrors, DraftAutomationResponse, DraftAutomationResponses, + DriverField, + DriverSpec, DropPasskeysData, DropPasskeysError, DropPasskeysErrors, @@ -1273,6 +1307,11 @@ export type { GetGraphErrors, GetGraphResponse, GetGraphResponses, + GetGroupData, + GetGroupError, + GetGroupErrors, + GetGroupResponse, + GetGroupResponses, GetHealthData, GetHealthResponse, GetHealthResponses, @@ -1376,6 +1415,11 @@ export type { GetPodcastErrors, GetPodcastResponse, GetPodcastResponses, + GetProviderData, + GetProviderError, + GetProviderErrors, + GetProviderResponse, + GetProviderResponses, GetPublicCollectionData, GetPublicCollectionError, GetPublicCollectionErrors, @@ -1583,6 +1627,15 @@ export type { GraphVerifyFixResponses, GraphVerifyResponse, GraphVerifyResponses, + Group, + GroupBudget, + GroupChange, + GroupHistoryData, + GroupHistoryError, + GroupHistoryErrors, + GroupHistoryResponse, + GroupHistoryResponses, + GroupIn, Handling, HandlingChange, HeadUpdate, @@ -1841,6 +1894,9 @@ export type { ListDatesErrors, ListDatesResponse, ListDatesResponses, + ListDriversData, + ListDriversResponse, + ListDriversResponses, ListEntitiesData, ListEntitiesError, ListEntitiesErrors, @@ -1907,6 +1963,9 @@ export type { ListGraphChangesErrors, ListGraphChangesResponse, ListGraphChangesResponses, + ListGroupsData, + ListGroupsResponse, + ListGroupsResponses, ListHighlightsData, ListHighlightsError, ListHighlightsErrors, @@ -2021,6 +2080,9 @@ export type { ListPodcastsErrors, ListPodcastsResponse, ListPodcastsResponses, + ListProvidersData, + ListProvidersResponse, + ListProvidersResponses, ListReadingsData, ListReadingsError, ListReadingsErrors, @@ -2128,6 +2190,11 @@ export type { ListTopicsErrors, ListTopicsResponse, ListTopicsResponses, + ListUnitsData, + ListUnitsError, + ListUnitsErrors, + ListUnitsResponse, + ListUnitsResponses, ListUploadsData, ListUploadsResponse, ListUploadsResponses, @@ -2291,6 +2358,12 @@ export type { MetadataBulk, MetadataBulkResult, MetadataEdit, + Metrics, + MetricsData, + MetricsError, + MetricsErrors, + MetricsResponse, + MetricsResponses, MeUpdate, ModelSpend, Monitoring, @@ -2407,6 +2480,10 @@ export type { OpenRecordingToIpGroupResponses, OriginCount, Output, + Overview, + OverviewData, + OverviewResponse, + OverviewResponses, Page, PageAboutData, PageAboutError, @@ -2470,6 +2547,12 @@ export type { PasswordChange, Pattern, PatternUpdate, + Paused, + PauseGroupData, + PauseGroupError, + PauseGroupErrors, + PauseGroupResponse, + PauseGroupResponses, Permission, PermissionAdd, PhotoHit, @@ -2561,6 +2644,15 @@ export type { ProposeGraphChangeErrors, ProposeGraphChangeResponse, ProposeGraphChangeResponses, + Protected, + ProtectUnitData, + ProtectUnitError, + ProtectUnitErrors, + ProtectUnitResponse, + ProtectUnitResponses, + Provider, + ProviderChange, + ProviderIn, PublicCard, PublicChapter, PublicCollection, @@ -2749,6 +2841,11 @@ export type { RemoveFeedItemErrors, RemoveFeedItemResponse, RemoveFeedItemResponses, + RemoveGroupData, + RemoveGroupError, + RemoveGroupErrors, + RemoveGroupResponse, + RemoveGroupResponses, RemoveLocalModelData, RemoveLocalModelError, RemoveLocalModelErrors, @@ -2764,6 +2861,11 @@ export type { RemovePasskeyErrors, RemovePasskeyResponse, RemovePasskeyResponses, + RemoveProviderData, + RemoveProviderError, + RemoveProviderErrors, + RemoveProviderResponse, + RemoveProviderResponses, RemoveRecordingPermissionData, RemoveRecordingPermissionError, RemoveRecordingPermissionErrors, @@ -2982,6 +3084,7 @@ export type { RoutineRun, RoutineRunRequest, RoutineUpdate, + Rule, RunAutomationData, RunAutomationError, RunAutomationErrors, @@ -3156,6 +3259,11 @@ export type { SetCollectionMemberErrors, SetCollectionMemberResponse, SetCollectionMemberResponses, + SetDesiredData, + SetDesiredError, + SetDesiredErrors, + SetDesiredResponse, + SetDesiredResponses, SetInstallData, SetInstallError, SetInstallErrors, @@ -3415,6 +3523,12 @@ export type { TemplateUpdate, TemplateVersionCreate, TemplateVersionInfo, + Terminate, + TerminateUnitData, + TerminateUnitError, + TerminateUnitErrors, + TerminateUnitResponse, + TerminateUnitResponses, TermSuggestion, TestAnalyticsData, TestAnalyticsResponse, @@ -3497,7 +3611,10 @@ export type { TestTikaData, TestTikaResponse, TestTikaResponses, + TickData, TicketRequest, + TickResponse, + TickResponses, TikaTestResult, TitleChatData, TitleChatError, @@ -3593,6 +3710,8 @@ export type { UndoTopicChangeErrors, UndoTopicChangeResponse, UndoTopicChangeResponses, + Unit, + UnitCounts, UnlinkEntityData, UnlinkEntityError, UnlinkEntityErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index e361bb209..93274f102 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -34,6 +34,9 @@ import type { AddPasskeyOptionsData, AddPasskeyOptionsResponses, AddPasskeyResponses, + AddProviderData, + AddProviderErrors, + AddProviderResponses, AddRecordingPermissionData, AddRecordingPermissionErrors, AddRecordingPermissionResponses, @@ -59,6 +62,9 @@ import type { ApproveAccessRequestData, ApproveAccessRequestErrors, ApproveAccessRequestResponses, + ApproveGroupData, + ApproveGroupErrors, + ApproveGroupResponses, AskGraphData, AskGraphErrors, AskGraphResponses, @@ -94,12 +100,18 @@ import type { CancelUploadData, CancelUploadErrors, CancelUploadResponses, + ChangeGroupData, + ChangeGroupErrors, + ChangeGroupResponses, ChangeLoginProviderData, ChangeLoginProviderErrors, ChangeLoginProviderResponses, ChangePasswordData, ChangePasswordErrors, ChangePasswordResponses, + ChangeProviderData, + ChangeProviderErrors, + ChangeProviderResponses, ChatCapabilitiesData, ChatCapabilitiesResponses, ChatToolsData, @@ -113,6 +125,9 @@ import type { CheckMessageData, CheckMessageErrors, CheckMessageResponses, + CheckProviderData, + CheckProviderErrors, + CheckProviderResponses, CheckUpdatesData, CheckUpdatesResponses, ClassifyData, @@ -195,6 +210,9 @@ import type { CreateFieldData, CreateFieldErrors, CreateFieldResponses, + CreateGroupData, + CreateGroupErrors, + CreateGroupResponses, CreateHighlightData, CreateHighlightErrors, CreateHighlightResponses, @@ -594,6 +612,9 @@ import type { GetGraphData, GetGraphErrors, GetGraphResponses, + GetGroupData, + GetGroupErrors, + GetGroupResponses, GetHealthData, GetHealthResponses, GetInstallData, @@ -657,6 +678,9 @@ import type { GetPodcastData, GetPodcastErrors, GetPodcastResponses, + GetProviderData, + GetProviderErrors, + GetProviderResponses, GetPublicCollectionData, GetPublicCollectionErrors, GetPublicCollectionResponses, @@ -778,6 +802,9 @@ import type { GraphVerifyFixData, GraphVerifyFixResponses, GraphVerifyResponses, + GroupHistoryData, + GroupHistoryErrors, + GroupHistoryResponses, HideEntityData, HideEntityErrors, HideEntityResponses, @@ -896,6 +923,8 @@ import type { ListDatesData, ListDatesErrors, ListDatesResponses, + ListDriversData, + ListDriversResponses, ListEntitiesData, ListEntitiesErrors, ListEntitiesResponses, @@ -936,6 +965,8 @@ import type { ListGraphChangesData, ListGraphChangesErrors, ListGraphChangesResponses, + ListGroupsData, + ListGroupsResponses, ListHighlightsData, ListHighlightsErrors, ListHighlightsResponses, @@ -1006,6 +1037,8 @@ import type { ListPodcastsData, ListPodcastsErrors, ListPodcastsResponses, + ListProvidersData, + ListProvidersResponses, ListReadingsData, ListReadingsErrors, ListReadingsResponses, @@ -1072,6 +1105,9 @@ import type { ListTopicsData, ListTopicsErrors, ListTopicsResponses, + ListUnitsData, + ListUnitsErrors, + ListUnitsResponses, ListUploadsData, ListUploadsResponses, ListUsersData, @@ -1142,6 +1178,9 @@ import type { MergeTopicsData, MergeTopicsErrors, MergeTopicsResponses, + MetricsData, + MetricsErrors, + MetricsResponses, MoveMentionData, MoveMentionErrors, MoveMentionResponses, @@ -1168,6 +1207,8 @@ import type { OpenRecordingToIpGroupData, OpenRecordingToIpGroupErrors, OpenRecordingToIpGroupResponses, + OverviewData, + OverviewResponses, PageAboutData, PageAboutErrors, PageAboutResponses, @@ -1199,6 +1240,9 @@ import type { PasskeySetupOptionsErrors, PasskeySetupOptionsResponses, PasskeySetupResponses, + PauseGroupData, + PauseGroupErrors, + PauseGroupResponses, PinPageData, PinPageErrors, PinPageResponses, @@ -1235,6 +1279,9 @@ import type { ProposeGraphChangeData, ProposeGraphChangeErrors, ProposeGraphChangeResponses, + ProtectUnitData, + ProtectUnitErrors, + ProtectUnitResponses, PublishToFeedData, PublishToFeedErrors, PublishToFeedResponses, @@ -1316,6 +1363,9 @@ import type { RemoveFeedItemData, RemoveFeedItemErrors, RemoveFeedItemResponses, + RemoveGroupData, + RemoveGroupErrors, + RemoveGroupResponses, RemoveLocalModelData, RemoveLocalModelErrors, RemoveLocalModelResponses, @@ -1325,6 +1375,9 @@ import type { RemovePasskeyData, RemovePasskeyErrors, RemovePasskeyResponses, + RemoveProviderData, + RemoveProviderErrors, + RemoveProviderResponses, RemoveRecordingPermissionData, RemoveRecordingPermissionErrors, RemoveRecordingPermissionResponses, @@ -1517,6 +1570,9 @@ import type { SetCollectionMemberData, SetCollectionMemberErrors, SetCollectionMemberResponses, + SetDesiredData, + SetDesiredErrors, + SetDesiredResponses, SetInstallData, SetInstallErrors, SetInstallResponses, @@ -1620,6 +1676,9 @@ import type { TemplatesNoteData, TemplatesNoteErrors, TemplatesNoteResponses, + TerminateUnitData, + TerminateUnitErrors, + TerminateUnitResponses, TestAnalyticsData, TestAnalyticsResponses, TestAutomationData, @@ -1671,6 +1730,8 @@ import type { TestTelemetryResponses, TestTikaData, TestTikaResponses, + TickData, + TickResponses, TitleChatData, TitleChatErrors, TitleChatResponses, @@ -12612,3 +12673,334 @@ export class Specs { }); } } + +export class Scaling { + /** + * Overview + * + * Everything at a glance: the switch, providers, groups with their units, the drivers on offer, the metrics now. + */ + public static overview( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/scaling", + ...options, + }); + } + + /** + * List Drivers + * + * The kinds of provider Lens can launch on, with the settings each asks for. + */ + public static listDrivers( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/scaling/drivers", + ...options, + }); + } + + /** + * Metrics + * + * The values rules can watch, now. + */ + public static metrics( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/scaling/metrics", + ...options, + }); + } + + /** + * List Providers + */ + public static listProviders( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/scaling/providers", + ...options, + }); + } + + /** + * Add Provider + * + * Add a provider. Its credentials are sealed (never shown again, only whether they're set) and checked with a + * read-only call; its library is fetched the first time. + */ + public static addProvider( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/scaling/providers", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Remove Provider + */ + public static removeProvider( + options: Options, + ): RequestResult { + return (options.client ?? client).delete({ + url: "/api/v1/scaling/providers/{pid}", + ...options, + }); + } + + /** + * Get Provider + */ + public static getProvider( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/scaling/providers/{pid}", + ...options, + }); + } + + /** + * Change Provider + */ + public static changeProvider( + options: Options, + ): RequestResult { + return (options.client ?? client).patch({ + url: "/api/v1/scaling/providers/{pid}", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Check Provider + * + * Prove the provider's credentials with a read-only call: {ok, error?}. + */ + public static checkProvider( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/scaling/providers/{pid}/check", + ...options, + }); + } + + /** + * List Groups + */ + public static listGroups( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/scaling/groups", + ...options, + }); + } + + /** + * Create Group + * + * Make a group. From the web app it's approved as it's made (that is your pick); from a token or an app it + * waits for a person's approval before it launches anything. + */ + public static createGroup( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/scaling/groups", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Remove Group + * + * Remove a group: its units are terminated first. + */ + public static removeGroup( + options: Options, + ): RequestResult { + return (options.client ?? client).delete({ + url: "/api/v1/scaling/groups/{gid}", + ...options, + }); + } + + /** + * Get Group + */ + public static getGroup( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/scaling/groups/{gid}", + ...options, + }); + } + + /** + * Change Group + * + * Change a group. A change that widens its approved terms (another provider or unit, a higher max or budget) + * takes the approval away until a person gives it again, unless it's made from the web app. + */ + public static changeGroup( + options: Options, + ): RequestResult { + return (options.client ?? client).patch({ + url: "/api/v1/scaling/groups/{gid}", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Approve Group + * + * Approve the group's terms as they stand: a person's pick, from a signed-in session only (403 otherwise). + */ + public static approveGroup( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/scaling/groups/{gid}/approve", + ...options, + }); + } + + /** + * Set Desired + * + * Ask for a number of units now, within the group's min and max. + */ + public static setDesired( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/scaling/groups/{gid}/desired", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Pause Group + * + * Pause a group (nothing new launches; what runs stays) or resume it. + */ + public static pauseGroup( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/scaling/groups/{gid}/pause", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Group History + * + * What happened to the group: launches, joins, terminations, changes and the cost it accrued (the ledger). + */ + public static groupHistory( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/scaling/groups/{gid}/history", + ...options, + }); + } + + /** + * List Units + */ + public static listUnits( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/scaling/units", + ...options, + }); + } + + /** + * Protect Unit + * + * Keep a unit out of scale-in (or let it go again). + */ + public static protectUnit( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/scaling/units/{nid}/protect", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Terminate Unit + * + * Let a unit go now: a running worker drains first, so its job is handed back. + */ + public static terminateUnit( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/scaling/units/{nid}/terminate", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Tick + * + * Run one pass of the scaler now, in this process (the scheduled one carries on): what it did. + */ + public static tick( + options?: Options, + ): RequestResult { + return (options?.client ?? client).post({ + url: "/api/v1/scaling/tick", + ...options, + }); + } +} diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index db087a598..38a442c89 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -5871,6 +5871,116 @@ export type Description = { [key: string]: unknown; }; +/** + * Desired + */ +export type Desired = { + /** + * Desired + */ + desired: number; +}; + +/** + * DriverField + */ +export type DriverField = { + /** + * Label + */ + label: string; + /** + * Type + */ + type: "string" | "integer" | "number" | "boolean" | "list"; + /** + * Required + */ + required?: boolean; + /** + * Default + */ + default?: unknown; + /** + * Help + */ + help?: string | null; + /** + * Options + */ + options?: Array | null; + /** + * Secret + * + * a credential: sealed when saved, shown only as set + */ + secret?: boolean; + [key: string]: unknown; +}; + +/** + * DriverSpec + */ +export type DriverSpec = { + /** + * Id + */ + id: string; + /** + * Kind + * + * node: machines that run `lens worker`; model: a model endpoint + */ + kind: "node" | "model"; + /** + * Label + */ + label: string; + /** + * About + */ + about: string; + /** + * Priced + * + * it charges money, so its groups need a budget + */ + priced: boolean; + /** + * Extra + * + * the optional packages it needs, fetched when a provider of it is saved + */ + extra?: string | null; + /** + * License + */ + license?: string | null; + /** + * Ready + * + * its packages are installed + */ + ready: boolean; + /** + * Fields + * + * the provider's settings + */ + fields: { + [key: string]: DriverField; + }; + /** + * Template + * + * what a group picks for each unit + */ + template: { + [key: string]: DriverField; + }; + [key: string]: unknown; +}; + /** * EmailInfo * @@ -8947,6 +9057,328 @@ export type GraphQuestion = { as_of?: string | null; }; +/** + * Group + */ +export type Group = { + /** + * Id + */ + id: number; + /** + * Name + */ + name: string; + /** + * Kind + */ + kind: string; + /** + * Provider + */ + provider: number; + /** + * Template + */ + template: { + [key: string]: unknown; + }; + /** + * Min + */ + min: number; + /** + * Max + */ + max: number; + /** + * Desired + */ + desired: number; + /** + * Cooldown Seconds + */ + cooldown_seconds?: number | null; + /** + * Warmup Seconds + */ + warmup_seconds?: number | null; + /** + * Terminate + */ + terminate: string; + /** + * Rules + * + * each with `says`, the rule in words + */ + rules: Array<{ + [key: string]: unknown; + }>; + /** + * At Cap + */ + at_cap: string; + /** + * Enabled + */ + enabled: boolean; + /** + * Paused + */ + paused: boolean; + /** + * Paused Why + */ + paused_why?: string | null; + /** + * Capped + * + * why its budget stops it now, when it does + */ + capped?: string | null; + /** + * Approval + * + * a person's approval: {by, at, terms} + */ + approval?: { + [key: string]: unknown; + } | null; + /** + * Approved + * + * the approval covers the terms as they stand + */ + approved: boolean; + /** + * Terms + * + * what an approval binds: kind, provider, template, max, budget, at_cap + */ + terms: { + [key: string]: unknown; + }; + budget?: GroupBudget | null; + units: UnitCounts; + /** + * Waiting For + * + * why it launches nothing now, when it doesn't + */ + waiting_for?: string | null; + /** + * Last Out At + */ + last_out_at?: string | null; + /** + * Last In At + */ + last_in_at?: string | null; + /** + * Last Eval + * + * what the rules said last: {at, said} + */ + last_eval?: { + [key: string]: unknown; + } | null; + /** + * Failures + */ + failures?: number; + /** + * Error + */ + error?: string | null; + /** + * Created At + */ + created_at?: string | null; + /** + * Updated At + */ + updated_at?: string | null; + /** + * Updated By + */ + updated_by?: string | null; + [key: string]: unknown; +}; + +/** + * GroupBudget + */ +export type GroupBudget = { + /** + * Usd + */ + usd?: number | null; + /** + * Period + */ + period: string; + /** + * State + */ + state: string; + /** + * Spent + */ + spent: { + [key: string]: unknown; + }; + /** + * Left + */ + left: { + [key: string]: unknown; + }; + [key: string]: unknown; +}; + +/** + * GroupChange + */ +export type GroupChange = { + /** + * Name + */ + name?: string | null; + /** + * Provider + */ + provider?: number | null; + /** + * Template + */ + template?: { + [key: string]: unknown; + } | null; + /** + * Min + */ + min?: number | null; + /** + * Max + */ + max?: number | null; + /** + * Desired + */ + desired?: number | null; + /** + * Cooldown Seconds + */ + cooldown_seconds?: number | null; + /** + * Warmup Seconds + */ + warmup_seconds?: number | null; + /** + * Terminate + */ + terminate?: "oldest" | "newest" | null; + /** + * Rules + */ + rules?: Array | null; + /** + * At Cap + */ + at_cap?: "stop" | "min" | null; + /** + * Enabled + */ + enabled?: boolean | null; + /** + * Budget + */ + budget?: number | null; +}; + +/** + * GroupIn + */ +export type GroupIn = { + /** + * Name + */ + name: string; + /** + * Provider + * + * the provider's id + */ + provider: number; + /** + * Kind + * + * the provider's kind; filled in when left out + */ + kind?: "node" | "model" | null; + /** + * Template + * + * what each unit is (the driver's template fields) + */ + template?: { + [key: string]: unknown; + }; + /** + * Min + */ + min?: number; + /** + * Max + */ + max?: number; + /** + * Desired + * + * units wanted now (default min) + */ + desired?: number | null; + /** + * Cooldown Seconds + * + * how long after a change before the next + */ + cooldown_seconds?: number; + /** + * Warmup Seconds + * + * how long a new unit may take to join before more are judged + */ + warmup_seconds?: number; + /** + * Terminate + * + * which unit goes first when scaling in + */ + terminate?: "oldest" | "newest"; + /** + * Rules + */ + rules?: Array; + /** + * At Cap + * + * at the budget's cap: stop every unit, or keep min + */ + at_cap?: "stop" | "min"; + /** + * Enabled + */ + enabled?: boolean; + /** + * Budget + * + * USD per month; needed on a priced provider + */ + budget?: number | null; +}; + /** * HTTPValidationError */ @@ -12486,6 +12918,27 @@ export type MetadataEdit = { [key: string]: unknown; }; +/** + * Metrics + */ +export type Metrics = { + /** + * Metrics + */ + metrics: { + [key: string]: number | number; + }; + /** + * About + * + * what each metric measures + */ + about: { + [key: string]: string; + }; + [key: string]: unknown; +}; + /** * ModelSpend */ @@ -15008,6 +15461,53 @@ export type Output = { [key: string]: unknown; }; +/** + * Overview + */ +export type Overview = { + /** + * Enabled + */ + enabled: boolean; + /** + * Settings + */ + settings: { + [key: string]: unknown; + }; + /** + * Scaler + * + * which process scales and when it last looked + */ + scaler: { + [key: string]: unknown; + }; + /** + * Providers + */ + providers: Array; + /** + * Groups + */ + groups: Array; + /** + * Units + */ + units: Array; + /** + * Drivers + */ + drivers: Array; + /** + * Metrics + */ + metrics: { + [key: string]: number | number; + }; + [key: string]: unknown; +}; + /** * Page * @@ -15412,6 +15912,20 @@ export type PatternUpdate = { action?: "keep" | "drop" | null; }; +/** + * Paused + */ +export type Paused = { + /** + * Paused + */ + paused?: boolean; + /** + * Why + */ + why?: string | null; +}; + /** * Permission * @@ -16386,6 +16900,177 @@ export type PrimaryFile = { [key: string]: unknown; }; +/** + * Protected + */ +export type Protected = { + /** + * Protected + */ + protected?: boolean; +}; + +/** + * Provider + */ +export type Provider = { + /** + * Id + */ + id: number; + /** + * Name + */ + name: string; + /** + * Driver + */ + driver: string; + /** + * Kind + */ + kind?: string | null; + /** + * Label + */ + label?: string | null; + /** + * Priced + */ + priced?: boolean; + /** + * Settings + */ + settings: { + [key: string]: unknown; + }; + /** + * Secrets + * + * which credentials are set + */ + secrets: { + [key: string]: boolean; + }; + /** + * Price + */ + price?: number | null; + /** + * Note + */ + note?: string | null; + /** + * Checked At + */ + checked_at?: string | null; + /** + * Check + * + * the last check: {ok, error?} + */ + check?: { + [key: string]: unknown; + } | null; + /** + * Created At + */ + created_at?: string | null; + /** + * Updated At + */ + updated_at?: string | null; + /** + * Updated By + */ + updated_by?: string | null; + [key: string]: unknown; +}; + +/** + * ProviderChange + */ +export type ProviderChange = { + /** + * Name + */ + name?: string | null; + /** + * Settings + */ + settings?: { + [key: string]: unknown; + } | null; + /** + * Secrets + * + * a secret given as null is removed; one left out is kept + */ + secrets?: { + [key: string]: string | null; + } | null; + /** + * Price + */ + price?: number | null; + /** + * Note + */ + note?: string | null; + /** + * Check + */ + check?: boolean; +}; + +/** + * ProviderIn + */ +export type ProviderIn = { + /** + * Name + */ + name: string; + /** + * Driver + * + * the kind of provider (see /scaling/drivers) + */ + driver: string; + /** + * Settings + * + * the driver's non-secret settings + */ + settings?: { + [key: string]: unknown; + }; + /** + * Secrets + * + * its credentials: sealed, never shown again + */ + secrets?: { + [key: string]: string | null; + }; + /** + * Price + * + * USD per unit-hour, when the driver doesn't know it; an estimate + */ + price?: number | null; + /** + * Note + */ + note?: string | null; + /** + * Check + * + * prove the credentials with a read-only call before answering + */ + check?: boolean; +}; + /** * PublicCard * @@ -19460,6 +20145,90 @@ export type RoutineUpdate = { enabled?: boolean | null; }; +/** + * Rule + */ +export type Rule = { + /** + * Type + */ + type: "target" | "step" | "schedule"; + /** + * Name + */ + name?: string | null; + /** + * Metric + * + * target and step rules: see /scaling/metrics + */ + metric?: string | null; + /** + * Target + * + * target rules: the value to keep the metric at + */ + target?: number | null; + /** + * Above + * + * step rules: fires while the metric is above this + */ + above?: number | null; + /** + * Below + * + * step rules: fires while the metric is below this + */ + below?: number | null; + /** + * Minutes + * + * step rules: for how long before it fires (default 1) + */ + minutes?: number | null; + /** + * Add + * + * step rules: units to add, a count or a percentage like "50%" + */ + add?: number | string | null; + /** + * Remove + * + * step rules: units to remove, a count or a percentage + */ + remove?: number | string | null; + /** + * Cron + * + * schedule rules: when, as a cron expression + */ + cron?: string | null; + /** + * Timezone + */ + timezone?: string | null; + /** + * Min + * + * schedule rules: the new min + */ + min?: number | null; + /** + * Max + * + * schedule rules: the new max + */ + max?: number | null; + /** + * Desired + * + * schedule rules: the new desired count + */ + desired?: number | null; +}; + /** * S3Key */ @@ -23459,6 +24228,16 @@ export type TermSuggestion = { [key: string]: unknown; }; +/** + * Terminate + */ +export type Terminate = { + /** + * Why + */ + why?: string | null; +}; + /** * TicketRequest */ @@ -24229,6 +25008,112 @@ export type Undone = { [key: string]: unknown; }; +/** + * Unit + */ +export type Unit = { + /** + * Id + */ + id: number; + /** + * Group + */ + group: number; + /** + * Provider + */ + provider: number; + /** + * Driver + */ + driver?: string | null; + /** + * Name + */ + name: string; + /** + * External Id + * + * the provider's id for it + */ + external_id?: string | null; + /** + * State + */ + state: "launching" | "running" | "draining" | "terminating" | "terminated" | "failed"; + /** + * Address + */ + address?: string | null; + /** + * Endpoint + * + * a model unit's endpoint: base_url, model; api_key true when set + */ + endpoint?: { + [key: string]: unknown; + } | null; + /** + * Price + * + * USD per hour, an estimate + */ + price?: number | null; + /** + * Launched At + */ + launched_at?: string | null; + /** + * Ready At + */ + ready_at?: string | null; + /** + * Ended At + */ + ended_at?: string | null; + /** + * By + */ + by?: string | null; + /** + * Why + */ + why?: string | null; + /** + * Protected + */ + protected?: boolean; + /** + * Error + */ + error?: string | null; + /** + * Accrued At + */ + accrued_at?: string | null; + [key: string]: unknown; +}; + +/** + * UnitCounts + */ +export type UnitCounts = { + /** + * Launching + */ + launching: number; + /** + * Running + */ + running: number; + /** + * Draining + */ + draining: number; + [key: string]: unknown; +}; + /** * UpdateStatus */ @@ -45873,3 +46758,620 @@ export type WriteSpecResponses = { }; export type WriteSpecResponse = WriteSpecResponses[keyof WriteSpecResponses]; + +export type OverviewData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/scaling"; +}; + +export type OverviewResponses = { + /** + * Successful Response + */ + 200: Overview; +}; + +export type OverviewResponse = OverviewResponses[keyof OverviewResponses]; + +export type ListDriversData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/scaling/drivers"; +}; + +export type ListDriversResponses = { + /** + * Response Scaling-List Drivers + * + * Successful Response + */ + 200: Array; +}; + +export type ListDriversResponse = ListDriversResponses[keyof ListDriversResponses]; + +export type MetricsData = { + body?: never; + path?: never; + query?: { + /** + * Steps + * + * comma-separated steps for the queue metrics + */ + steps?: string | null; + }; + url: "/api/v1/scaling/metrics"; +}; + +export type MetricsErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type MetricsError = MetricsErrors[keyof MetricsErrors]; + +export type MetricsResponses = { + /** + * Successful Response + */ + 200: Metrics; +}; + +export type MetricsResponse = MetricsResponses[keyof MetricsResponses]; + +export type ListProvidersData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/scaling/providers"; +}; + +export type ListProvidersResponses = { + /** + * Response Scaling-List Providers + * + * Successful Response + */ + 200: Array; +}; + +export type ListProvidersResponse = ListProvidersResponses[keyof ListProvidersResponses]; + +export type AddProviderData = { + body: ProviderIn; + path?: never; + query?: never; + url: "/api/v1/scaling/providers"; +}; + +export type AddProviderErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type AddProviderError = AddProviderErrors[keyof AddProviderErrors]; + +export type AddProviderResponses = { + /** + * Successful Response + */ + 201: Provider; +}; + +export type AddProviderResponse = AddProviderResponses[keyof AddProviderResponses]; + +export type RemoveProviderData = { + body?: never; + path: { + /** + * Pid + */ + pid: number; + }; + query?: never; + url: "/api/v1/scaling/providers/{pid}"; +}; + +export type RemoveProviderErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type RemoveProviderError = RemoveProviderErrors[keyof RemoveProviderErrors]; + +export type RemoveProviderResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type RemoveProviderResponse = RemoveProviderResponses[keyof RemoveProviderResponses]; + +export type GetProviderData = { + body?: never; + path: { + /** + * Pid + */ + pid: number; + }; + query?: never; + url: "/api/v1/scaling/providers/{pid}"; +}; + +export type GetProviderErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GetProviderError = GetProviderErrors[keyof GetProviderErrors]; + +export type GetProviderResponses = { + /** + * Successful Response + */ + 200: Provider; +}; + +export type GetProviderResponse = GetProviderResponses[keyof GetProviderResponses]; + +export type ChangeProviderData = { + body: ProviderChange; + path: { + /** + * Pid + */ + pid: number; + }; + query?: never; + url: "/api/v1/scaling/providers/{pid}"; +}; + +export type ChangeProviderErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ChangeProviderError = ChangeProviderErrors[keyof ChangeProviderErrors]; + +export type ChangeProviderResponses = { + /** + * Successful Response + */ + 200: Provider; +}; + +export type ChangeProviderResponse = ChangeProviderResponses[keyof ChangeProviderResponses]; + +export type CheckProviderData = { + body?: never; + path: { + /** + * Pid + */ + pid: number; + }; + query?: never; + url: "/api/v1/scaling/providers/{pid}/check"; +}; + +export type CheckProviderErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type CheckProviderError = CheckProviderErrors[keyof CheckProviderErrors]; + +export type CheckProviderResponses = { + /** + * Response Scaling-Check Provider + * + * Successful Response + */ + 200: { + [key: string]: unknown; + }; +}; + +export type CheckProviderResponse = CheckProviderResponses[keyof CheckProviderResponses]; + +export type ListGroupsData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/scaling/groups"; +}; + +export type ListGroupsResponses = { + /** + * Response Scaling-List Groups + * + * Successful Response + */ + 200: Array; +}; + +export type ListGroupsResponse = ListGroupsResponses[keyof ListGroupsResponses]; + +export type CreateGroupData = { + body: GroupIn; + path?: never; + query?: never; + url: "/api/v1/scaling/groups"; +}; + +export type CreateGroupErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type CreateGroupError = CreateGroupErrors[keyof CreateGroupErrors]; + +export type CreateGroupResponses = { + /** + * Successful Response + */ + 201: Group; +}; + +export type CreateGroupResponse = CreateGroupResponses[keyof CreateGroupResponses]; + +export type RemoveGroupData = { + body?: never; + path: { + /** + * Gid + */ + gid: number; + }; + query?: never; + url: "/api/v1/scaling/groups/{gid}"; +}; + +export type RemoveGroupErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type RemoveGroupError = RemoveGroupErrors[keyof RemoveGroupErrors]; + +export type RemoveGroupResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type RemoveGroupResponse = RemoveGroupResponses[keyof RemoveGroupResponses]; + +export type GetGroupData = { + body?: never; + path: { + /** + * Gid + */ + gid: number; + }; + query?: never; + url: "/api/v1/scaling/groups/{gid}"; +}; + +export type GetGroupErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GetGroupError = GetGroupErrors[keyof GetGroupErrors]; + +export type GetGroupResponses = { + /** + * Successful Response + */ + 200: Group; +}; + +export type GetGroupResponse = GetGroupResponses[keyof GetGroupResponses]; + +export type ChangeGroupData = { + body: GroupChange; + path: { + /** + * Gid + */ + gid: number; + }; + query?: never; + url: "/api/v1/scaling/groups/{gid}"; +}; + +export type ChangeGroupErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ChangeGroupError = ChangeGroupErrors[keyof ChangeGroupErrors]; + +export type ChangeGroupResponses = { + /** + * Successful Response + */ + 200: Group; +}; + +export type ChangeGroupResponse = ChangeGroupResponses[keyof ChangeGroupResponses]; + +export type ApproveGroupData = { + body?: never; + path: { + /** + * Gid + */ + gid: number; + }; + query?: never; + url: "/api/v1/scaling/groups/{gid}/approve"; +}; + +export type ApproveGroupErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ApproveGroupError = ApproveGroupErrors[keyof ApproveGroupErrors]; + +export type ApproveGroupResponses = { + /** + * Successful Response + */ + 200: Group; +}; + +export type ApproveGroupResponse = ApproveGroupResponses[keyof ApproveGroupResponses]; + +export type SetDesiredData = { + body: Desired; + path: { + /** + * Gid + */ + gid: number; + }; + query?: never; + url: "/api/v1/scaling/groups/{gid}/desired"; +}; + +export type SetDesiredErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type SetDesiredError = SetDesiredErrors[keyof SetDesiredErrors]; + +export type SetDesiredResponses = { + /** + * Successful Response + */ + 200: Group; +}; + +export type SetDesiredResponse = SetDesiredResponses[keyof SetDesiredResponses]; + +export type PauseGroupData = { + body: Paused; + path: { + /** + * Gid + */ + gid: number; + }; + query?: never; + url: "/api/v1/scaling/groups/{gid}/pause"; +}; + +export type PauseGroupErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type PauseGroupError = PauseGroupErrors[keyof PauseGroupErrors]; + +export type PauseGroupResponses = { + /** + * Successful Response + */ + 200: Group; +}; + +export type PauseGroupResponse = PauseGroupResponses[keyof PauseGroupResponses]; + +export type GroupHistoryData = { + body?: never; + path: { + /** + * Gid + */ + gid: number; + }; + query?: { + /** + * Limit + */ + limit?: number; + /** + * Before + */ + before?: number | null; + }; + url: "/api/v1/scaling/groups/{gid}/history"; +}; + +export type GroupHistoryErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GroupHistoryError = GroupHistoryErrors[keyof GroupHistoryErrors]; + +export type GroupHistoryResponses = { + /** + * Response Scaling-Group History + * + * Successful Response + */ + 200: Array<{ + [key: string]: unknown; + }>; +}; + +export type GroupHistoryResponse = GroupHistoryResponses[keyof GroupHistoryResponses]; + +export type ListUnitsData = { + body?: never; + path?: never; + query?: { + /** + * Group + */ + group?: number | null; + /** + * Alive + * + * only units not yet gone + */ + alive?: boolean; + }; + url: "/api/v1/scaling/units"; +}; + +export type ListUnitsErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ListUnitsError = ListUnitsErrors[keyof ListUnitsErrors]; + +export type ListUnitsResponses = { + /** + * Response Scaling-List Units + * + * Successful Response + */ + 200: Array; +}; + +export type ListUnitsResponse = ListUnitsResponses[keyof ListUnitsResponses]; + +export type ProtectUnitData = { + body: Protected; + path: { + /** + * Nid + */ + nid: number; + }; + query?: never; + url: "/api/v1/scaling/units/{nid}/protect"; +}; + +export type ProtectUnitErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ProtectUnitError = ProtectUnitErrors[keyof ProtectUnitErrors]; + +export type ProtectUnitResponses = { + /** + * Successful Response + */ + 200: Unit; +}; + +export type ProtectUnitResponse = ProtectUnitResponses[keyof ProtectUnitResponses]; + +export type TerminateUnitData = { + body: Terminate; + path: { + /** + * Nid + */ + nid: number; + }; + query?: never; + url: "/api/v1/scaling/units/{nid}/terminate"; +}; + +export type TerminateUnitErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type TerminateUnitError = TerminateUnitErrors[keyof TerminateUnitErrors]; + +export type TerminateUnitResponses = { + /** + * Successful Response + */ + 200: Unit; +}; + +export type TerminateUnitResponse = TerminateUnitResponses[keyof TerminateUnitResponses]; + +export type TickData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/scaling/tick"; +}; + +export type TickResponses = { + /** + * Response Scaling-Tick + * + * Successful Response + */ + 200: { + [key: string]: unknown; + }; +}; + +export type TickResponse = TickResponses[keyof TickResponses]; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 13d46a9ca..325aec9a4 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -32523,6 +32523,967 @@ } } } + }, + "/api/v1/scaling": { + "get": { + "tags": [ + "scaling" + ], + "summary": "Overview", + "description": "Everything at a glance: the switch, providers, groups with their units, the drivers on offer, the metrics now.", + "operationId": "overview", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Overview" + } + } + } + } + } + } + }, + "/api/v1/scaling/drivers": { + "get": { + "tags": [ + "scaling" + ], + "summary": "List Drivers", + "description": "The kinds of provider Lens can launch on, with the settings each asks for.", + "operationId": "list_drivers", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/DriverSpec" + }, + "type": "array", + "title": "Response Scaling-List Drivers" + } + } + } + } + } + } + }, + "/api/v1/scaling/metrics": { + "get": { + "tags": [ + "scaling" + ], + "summary": "Metrics", + "description": "The values rules can watch, now.", + "operationId": "metrics", + "parameters": [ + { + "name": "steps", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "comma-separated steps for the queue metrics", + "title": "Steps" + }, + "description": "comma-separated steps for the queue metrics" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Metrics" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/scaling/providers": { + "get": { + "tags": [ + "scaling" + ], + "summary": "List Providers", + "operationId": "list_providers", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/Provider" + }, + "type": "array", + "title": "Response Scaling-List Providers" + } + } + } + } + } + }, + "post": { + "tags": [ + "scaling" + ], + "summary": "Add Provider", + "description": "Add a provider. Its credentials are sealed (never shown again, only whether they're set) and checked with a\nread-only call; its library is fetched the first time.", + "operationId": "add_provider", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProviderIn" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Provider" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/scaling/providers/{pid}": { + "get": { + "tags": [ + "scaling" + ], + "summary": "Get Provider", + "operationId": "get_provider", + "parameters": [ + { + "name": "pid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Pid" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Provider" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "patch": { + "tags": [ + "scaling" + ], + "summary": "Change Provider", + "operationId": "change_provider", + "parameters": [ + { + "name": "pid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Pid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProviderChange" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Provider" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "scaling" + ], + "summary": "Remove Provider", + "operationId": "remove_provider", + "parameters": [ + { + "name": "pid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Pid" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ok" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/scaling/providers/{pid}/check": { + "post": { + "tags": [ + "scaling" + ], + "summary": "Check Provider", + "description": "Prove the provider's credentials with a read-only call: {ok, error?}.", + "operationId": "check_provider", + "parameters": [ + { + "name": "pid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Pid" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Scaling-Check Provider" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/scaling/groups": { + "get": { + "tags": [ + "scaling" + ], + "summary": "List Groups", + "operationId": "list_groups", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/Group" + }, + "type": "array", + "title": "Response Scaling-List Groups" + } + } + } + } + } + }, + "post": { + "tags": [ + "scaling" + ], + "summary": "Create Group", + "description": "Make a group. From the web app it's approved as it's made (that is your pick); from a token or an app it\nwaits for a person's approval before it launches anything.", + "operationId": "create_group", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GroupIn" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Group" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/scaling/groups/{gid}": { + "get": { + "tags": [ + "scaling" + ], + "summary": "Get Group", + "operationId": "get_group", + "parameters": [ + { + "name": "gid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Gid" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Group" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "patch": { + "tags": [ + "scaling" + ], + "summary": "Change Group", + "description": "Change a group. A change that widens its approved terms (another provider or unit, a higher max or budget)\ntakes the approval away until a person gives it again, unless it's made from the web app.", + "operationId": "change_group", + "parameters": [ + { + "name": "gid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Gid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GroupChange" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Group" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "scaling" + ], + "summary": "Remove Group", + "description": "Remove a group: its units are terminated first.", + "operationId": "remove_group", + "parameters": [ + { + "name": "gid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Gid" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ok" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/scaling/groups/{gid}/approve": { + "post": { + "tags": [ + "scaling" + ], + "summary": "Approve Group", + "description": "Approve the group's terms as they stand: a person's pick, from a signed-in session only (403 otherwise).", + "operationId": "approve_group", + "parameters": [ + { + "name": "gid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Gid" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Group" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/scaling/groups/{gid}/desired": { + "post": { + "tags": [ + "scaling" + ], + "summary": "Set Desired", + "description": "Ask for a number of units now, within the group's min and max.", + "operationId": "set_desired", + "parameters": [ + { + "name": "gid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Gid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Desired" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Group" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/scaling/groups/{gid}/pause": { + "post": { + "tags": [ + "scaling" + ], + "summary": "Pause Group", + "description": "Pause a group (nothing new launches; what runs stays) or resume it.", + "operationId": "pause_group", + "parameters": [ + { + "name": "gid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Gid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Paused" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Group" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/scaling/groups/{gid}/history": { + "get": { + "tags": [ + "scaling" + ], + "summary": "Group History", + "description": "What happened to the group: launches, joins, terminations, changes and the cost it accrued (the ledger).", + "operationId": "group_history", + "parameters": [ + { + "name": "gid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Gid" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 200, + "minimum": 1, + "default": 50, + "title": "Limit" + } + }, + { + "name": "before", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Before" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true + }, + "title": "Response Scaling-Group History" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/scaling/units": { + "get": { + "tags": [ + "scaling" + ], + "summary": "List Units", + "operationId": "list_units", + "parameters": [ + { + "name": "group", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Group" + } + }, + { + "name": "alive", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "description": "only units not yet gone", + "default": false, + "title": "Alive" + }, + "description": "only units not yet gone" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Unit" + }, + "title": "Response Scaling-List Units" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/scaling/units/{nid}/protect": { + "post": { + "tags": [ + "scaling" + ], + "summary": "Protect Unit", + "description": "Keep a unit out of scale-in (or let it go again).", + "operationId": "protect_unit", + "parameters": [ + { + "name": "nid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Nid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Protected" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Unit" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/scaling/units/{nid}/terminate": { + "post": { + "tags": [ + "scaling" + ], + "summary": "Terminate Unit", + "description": "Let a unit go now: a running worker drains first, so its job is handed back.", + "operationId": "terminate_unit", + "parameters": [ + { + "name": "nid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Nid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Terminate" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Unit" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/scaling/tick": { + "post": { + "tags": [ + "scaling" + ], + "summary": "Tick", + "description": "Run one pass of the scaler now, in this process (the scheduled one carries on): what it did.", + "operationId": "tick", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Scaling-Tick" + } + } + } + } + } + } } }, "components": { @@ -42702,6 +43663,172 @@ "title": "Description", "description": "What a model that can see images said a page or a shot shows." }, + "Desired": { + "properties": { + "desired": { + "type": "integer", + "maximum": 100.0, + "minimum": 0.0, + "title": "Desired" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "desired" + ], + "title": "Desired" + }, + "DriverField": { + "properties": { + "label": { + "type": "string", + "title": "Label" + }, + "type": { + "type": "string", + "enum": [ + "string", + "integer", + "number", + "boolean", + "list" + ], + "title": "Type" + }, + "required": { + "type": "boolean", + "title": "Required", + "default": false + }, + "default": { + "title": "Default" + }, + "help": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Help" + }, + "options": { + "anyOf": [ + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Options" + }, + "secret": { + "type": "boolean", + "title": "Secret", + "description": "a credential: sealed when saved, shown only as set", + "default": false + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "label", + "type" + ], + "title": "DriverField" + }, + "DriverSpec": { + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "kind": { + "type": "string", + "enum": [ + "node", + "model" + ], + "title": "Kind", + "description": "node: machines that run `lens worker`; model: a model endpoint" + }, + "label": { + "type": "string", + "title": "Label" + }, + "about": { + "type": "string", + "title": "About" + }, + "priced": { + "type": "boolean", + "title": "Priced", + "description": "it charges money, so its groups need a budget" + }, + "extra": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Extra", + "description": "the optional packages it needs, fetched when a provider of it is saved" + }, + "license": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "License" + }, + "ready": { + "type": "boolean", + "title": "Ready", + "description": "its packages are installed" + }, + "fields": { + "additionalProperties": { + "$ref": "#/components/schemas/DriverField" + }, + "type": "object", + "title": "Fields", + "description": "the provider's settings" + }, + "template": { + "additionalProperties": { + "$ref": "#/components/schemas/DriverField" + }, + "type": "object", + "title": "Template", + "description": "what a group picks for each unit" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "kind", + "label", + "about", + "priced", + "ready", + "fields", + "template" + ], + "title": "DriverSpec" + }, "EmailInfo": { "properties": { "subject": { @@ -46988,7 +48115,413 @@ "type": "array", "title": "Tags" }, - "language": { + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language", + "description": "its language code, when known" + }, + "origin": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Origin", + "description": "where it came from: source: (a connected source), upload, paste, iiif, folder or file" + }, + "origin_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Origin Name", + "description": "what to call its origin: the source's name, or e.g. Uploaded" + }, + "role": { + "anyOf": [ + { + "type": "string", + "enum": [ + "viewer", + "editor", + "owner" + ] + }, + { + "type": "null" + } + ], + "title": "Role", + "description": "your role on it: through its namespace or its collection (an admin of the collection is an owner)" + }, + "nsfw": { + "anyOf": [ + { + "$ref": "#/components/schemas/NsfwState" + }, + { + "type": "null" + } + ], + "description": "NSFW classification, where it's on in its namespace and it was classified" + }, + "why": { + "items": { + "$ref": "#/components/schemas/FindReason" + }, + "type": "array", + "title": "Why", + "description": "why it matched" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "space", + "media_kind" + ], + "title": "FindItem" + }, + "FindMoment": { + "properties": { + "t0": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "T0", + "description": "ms from the start, on a timeline" + }, + "t1": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "T1", + "description": "ms, where it ends" + }, + "page": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Page", + "description": "in a document or an image: the page (from 0)" + }, + "last_page": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Last Page", + "description": "a face over several pages: the last of them" + }, + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Text", + "description": "the words there, when it has any" + }, + "speaker": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Speaker" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source", + "description": "words: said, screen, file, page, object or described (as search has it)" + } + }, + "additionalProperties": true, + "type": "object", + "title": "FindMoment" + }, + "FindNamed": { + "properties": { + "id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Id" + }, + "key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Namespace" + } + }, + "additionalProperties": true, + "type": "object", + "title": "FindNamed" + }, + "FindQuery": { + "properties": { + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Namespace", + "description": "one namespace (default: every namespace you can read)" + }, + "text": { + "anyOf": [ + { + "type": "string", + "maxLength": 300 + }, + { + "type": "null" + } + ], + "title": "Text", + "description": "words said or written in them, as search takes them (words, \"phrases\", OR); by meaning too when that is set up" + }, + "title": { + "anyOf": [ + { + "type": "string", + "maxLength": 200 + }, + { + "type": "null" + } + ], + "title": "Title", + "description": "words that must all appear in the title, the namespace or a person's name" + }, + "faces": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + }, + "type": "array", + "maxItems": 20 + }, + { + "type": "null" + } + ], + "title": "Faces", + "description": "people seen in them: face identities, by id or name" + }, + "speakers": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + }, + "type": "array", + "maxItems": 20 + }, + { + "type": "null" + } + ], + "title": "Speakers", + "description": "people heard in them: speakers, by id or name" + }, + "entities": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + }, + "type": "array", + "maxItems": 20 + }, + { + "type": "null" + } + ], + "title": "Entities", + "description": "entities they mention, by id or name" + }, + "topics": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + }, + "type": "array", + "maxItems": 20 + }, + { + "type": "null" + } + ], + "title": "Topics", + "description": "topics they were accepted as about, by id or label; a topic's narrower topics count too" + }, + "types": { + "anyOf": [ + { + "items": { + "type": "string", + "enum": [ + "audio", + "video", + "transcript", + "document", + "image" + ] + }, + "type": "array", + "maxItems": 5 + }, + { + "type": "null" + } + ], + "title": "Types", + "description": "their kind: audio, video, transcript, document or image" + }, + "content_types": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array", + "maxItems": 20 + }, + { + "type": "null" + } + ], + "title": "Content Types", + "description": "content types (podcast, interview, scan, photo, email \u2026, GET /content-types), by key or label" + }, + "date_from": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Date From", + "description": "made on or after this day, month or year (2026-09-30, 2026-09, 2026)" + }, + "date_to": { "anyOf": [ { "type": "string" @@ -46997,205 +48530,325 @@ "type": "null" } ], - "title": "Language", - "description": "its language code, when known" + "title": "Date To", + "description": "made on or before this day, month or year" }, - "origin": { + "min_duration": { "anyOf": [ { - "type": "string" + "type": "number", + "minimum": 0.0 }, { "type": "null" } ], - "title": "Origin", - "description": "where it came from: source: (a connected source), upload, paste, iiif, folder or file" + "title": "Min Duration", + "description": "at least this many seconds long" }, - "origin_name": { + "max_duration": { "anyOf": [ { - "type": "string" + "type": "number", + "minimum": 0.0 }, { "type": "null" } ], - "title": "Origin Name", - "description": "what to call its origin: the source's name, or e.g. Uploaded" + "title": "Max Duration", + "description": "shorter than this many seconds" }, - "role": { + "tags": { "anyOf": [ { - "type": "string", - "enum": [ - "viewer", - "editor", - "owner" - ] + "items": { + "type": "string" + }, + "type": "array", + "maxItems": 20 }, { "type": "null" } ], - "title": "Role", - "description": "your role on it: through its namespace or its collection (an admin of the collection is an owner)" + "title": "Tags", + "description": "tags (ignoring case)" }, - "nsfw": { + "languages": { "anyOf": [ { - "$ref": "#/components/schemas/NsfwState" + "items": { + "type": "string" + }, + "type": "array", + "maxItems": 20 }, { "type": "null" } ], - "description": "NSFW classification, where it's on in its namespace and it was classified" + "title": "Languages", + "description": "language codes; none for those whose language isn't known" }, - "why": { - "items": { - "$ref": "#/components/schemas/FindReason" - }, - "type": "array", - "title": "Why", - "description": "why it matched" - } - }, - "additionalProperties": true, - "type": "object", - "required": [ - "id", - "space", - "media_kind" - ], - "title": "FindItem" - }, - "FindMoment": { - "properties": { - "t0": { + "objects": { "anyOf": [ { - "type": "integer" + "items": { + "type": "string" + }, + "type": "array", + "maxItems": 20 }, { "type": "null" } ], - "title": "T0", - "description": "ms from the start, on a timeline" + "title": "Objects", + "description": "kinds of object seen in them (person, car, dog \u2026)" }, - "t1": { + "emotions": { "anyOf": [ { - "type": "integer" + "items": { + "type": "string" + }, + "type": "array", + "maxItems": 20 }, { "type": "null" } ], - "title": "T1", - "description": "ms, where it ends" + "title": "Emotions", + "description": "emotions heard in them (ignoring case)" }, - "page": { + "status": { "anyOf": [ { - "type": "integer" + "items": { + "type": "string", + "enum": [ + "new", + "transcribed", + "diarized", + "analyzed", + "error", + "processing", + "failed" + ] + }, + "type": "array", + "maxItems": 10 }, { "type": "null" } ], - "title": "Page", - "description": "in a document or an image: the page (from 0)" + "title": "Status", + "description": "statuses, or processing and failed" }, - "last_page": { + "origins": { "anyOf": [ { - "type": "integer" + "items": { + "type": "string" + }, + "type": "array", + "maxItems": 20 }, { "type": "null" } ], - "title": "Last Page", - "description": "a face over several pages: the last of them" + "title": "Origins", + "description": "where they came from: source:, upload, paste, iiif, folder or file" }, - "text": { + "collection": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "null" } ], - "title": "Text", - "description": "the words there, when it has any" + "title": "Collection", + "description": "a collection: what's in it and in the collections inside it" }, - "speaker": { + "sensitive": { "anyOf": [ { - "type": "string" + "type": "string", + "enum": [ + "include", + "only", + "exclude" + ] }, { "type": "null" } ], - "title": "Speaker" + "title": "Sensitive", + "description": "where NSFW classification is on: only those flagged, exclude them, or include them" }, - "source": { + "sort": { "anyOf": [ { - "type": "string" + "type": "string", + "enum": [ + "relevance", + "-date", + "date", + "-title", + "title", + "-duration", + "duration" + ] }, { "type": "null" } ], - "title": "Source", - "description": "words: said, screen, file, page, object or described (as search has it)" + "title": "Sort", + "description": "relevance (the default with text: best matching words first), else newest first" + }, + "limit": { + "type": "integer", + "maximum": 100.0, + "minimum": 1.0, + "title": "Limit", + "default": 20 + }, + "offset": { + "type": "integer", + "maximum": 10000.0, + "minimum": 0.0, + "title": "Offset", + "default": 0 } }, - "additionalProperties": true, + "additionalProperties": false, "type": "object", - "title": "FindMoment" + "title": "FindQuery", + "description": "Filters combine with AND, the values of one filter with OR. Faces, speakers, entities and topics are ids or names\n(ignoring case; an entity's other names count, and any of a topic's labels)." }, - "FindNamed": { + "FindReason": { "properties": { - "id": { + "by": { + "type": "string", + "enum": [ + "words", + "face", + "voice", + "entity", + "topic", + "content_type", + "object", + "emotion" + ], + "title": "By" + }, + "name": { "anyOf": [ { - "type": "integer" + "type": "string" }, { "type": "null" } ], - "title": "Id" + "title": "Name", + "description": "what matched: the words, the person's or the entity's name, the topic's label \u2026" }, - "key": { + "moments": { + "items": { + "$ref": "#/components/schemas/FindMoment" + }, + "type": "array", + "title": "Moments", + "description": "up to 3 moments that show it, to cite" + }, + "count": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "null" } ], - "title": "Key" + "title": "Count", + "description": "emotion: how many lines have it" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "by" + ], + "title": "FindReason" + }, + "FindResults": { + "properties": { + "total": { + "type": "integer", + "title": "Total", + "description": "how many assets match, on all pages" }, - "name": { + "items": { + "items": { + "$ref": "#/components/schemas/FindItem" + }, + "type": "array", + "title": "Items" + }, + "resolved": { + "additionalProperties": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/FindNamed" + }, + "type": "array" + }, + { + "type": "integer" + } + ] + }, + "type": "object", + "title": "Resolved", + "description": "what the names given matched, by filter (and how many narrower topics came with them)" + }, + "text": { "anyOf": [ { - "type": "string" + "$ref": "#/components/schemas/FindText" }, { "type": "null" } - ], - "title": "Name" + ] + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "total", + "items" + ], + "title": "FindResults" + }, + "FindText": { + "properties": { + "mode": { + "type": "string", + "title": "Mode", + "description": "how the words were matched: keyword, semantic or hybrid" }, - "namespace": { + "meaning": { "anyOf": [ { "type": "string" @@ -47204,340 +48857,503 @@ "type": "null" } ], - "title": "Namespace" + "title": "Meaning", + "description": "why not by meaning, when it was asked and couldn't be" + }, + "moments": { + "type": "integer", + "title": "Moments", + "description": "moments the words were found in" + }, + "capped": { + "type": "boolean", + "title": "Capped", + "description": "more moments matched than were looked through, so some assets may be missing" } }, "additionalProperties": true, "type": "object", - "title": "FindNamed" + "required": [ + "mode", + "moments", + "capped" + ], + "title": "FindText" }, - "FindQuery": { + "ForgotPasswordRequest": { "properties": { - "namespace": { + "email": { + "type": "string", + "title": "Email" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "email" + ], + "title": "ForgotPasswordRequest" + }, + "GoogleClient": { + "properties": { + "origin": { "anyOf": [ { - "type": "string" + "type": "string", + "enum": [ + "sign-in", + "connection" + ] }, { "type": "null" } ], - "title": "Namespace", - "description": "one namespace (default: every namespace you can read)" + "title": "Origin", + "description": "where the OAuth client for connecting a Google account comes from: the Google sign-in provider, an earlier Google connection, or none (enter one to connect)" }, - "text": { + "client_id": { "anyOf": [ { - "type": "string", - "maxLength": 300 + "type": "string" }, { "type": "null" } ], - "title": "Text", - "description": "words said or written in them, as search takes them (words, \"phrases\", OR); by meaning too when that is set up" + "title": "Client Id" }, - "title": { + "redirect_uri": { + "type": "string", + "title": "Redirect Uri", + "description": "the address to add to the OAuth client's authorized redirect URIs" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "origin", + "redirect_uri" + ], + "title": "GoogleClient" + }, + "GoogleConnect": { + "properties": { + "client_id": { "anyOf": [ { "type": "string", - "maxLength": 200 + "maxLength": 300 }, { "type": "null" } ], - "title": "Title", - "description": "words that must all appear in the title, the namespace or a person's name" + "title": "Client Id", + "description": "an OAuth client of your own (else the one Lens has)" }, - "faces": { + "client_secret": { "anyOf": [ { - "items": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "string" - } - ] - }, - "type": "array", - "maxItems": 20 + "type": "string", + "maxLength": 300 }, { "type": "null" } ], - "title": "Faces", - "description": "people seen in them: face identities, by id or name" + "title": "Client Secret" + } + }, + "additionalProperties": false, + "type": "object", + "title": "GoogleConnect" + }, + "GoogleConnectStart": { + "properties": { + "url": { + "type": "string", + "title": "Url", + "description": "Google's consent page; open it in this browser" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "url" + ], + "title": "GoogleConnectStart" + }, + "Gpu": { + "properties": { + "name": { + "type": "string", + "title": "Name" }, - "speakers": { + "memory_gb": { "anyOf": [ { - "items": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "string" - } - ] - }, - "type": "array", - "maxItems": 20 + "type": "number" }, { "type": "null" } ], - "title": "Speakers", - "description": "people heard in them: speakers, by id or name" + "title": "Memory Gb" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "name" + ], + "title": "Gpu" + }, + "Graph": { + "properties": { + "scope": { + "type": "string", + "title": "Scope" + }, + "namespaces": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Namespaces" + }, + "nodes": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array", + "title": "Nodes" + }, + "edges": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array", + "title": "Edges" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "scope", + "namespaces", + "nodes", + "edges" + ], + "title": "Graph" + }, + "GraphChange": { + "properties": { + "id": { + "type": "integer", + "title": "Id" + }, + "kind": { + "type": "string", + "enum": [ + "merge", + "link" + ], + "title": "Kind" + }, + "a": { + "additionalProperties": true, + "type": "object", + "title": "A" + }, + "b": { + "additionalProperties": true, + "type": "object", + "title": "B" + }, + "spaces": { + "items": { + "type": "integer" + }, + "type": "array", + "title": "Spaces" }, - "entities": { + "reason": { "anyOf": [ { - "items": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "string" - } - ] - }, - "type": "array", - "maxItems": 20 + "type": "string" }, { "type": "null" } ], - "title": "Entities", - "description": "entities they mention, by id or name" + "title": "Reason" }, - "topics": { + "confidence": { "anyOf": [ { - "items": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "string" - } - ] - }, - "type": "array", - "maxItems": 20 + "type": "number" }, { "type": "null" } ], - "title": "Topics", - "description": "topics they were accepted as about, by id or label; a topic's narrower topics count too" + "title": "Confidence" }, - "types": { + "verdict": { "anyOf": [ { - "items": { - "type": "string", - "enum": [ - "audio", - "video", - "transcript", - "document", - "image" - ] - }, - "type": "array", - "maxItems": 5 + "additionalProperties": true, + "type": "object" }, { "type": "null" } ], - "title": "Types", - "description": "their kind: audio, video, transcript, document or image" + "title": "Verdict" }, - "content_types": { + "keep": { "anyOf": [ { - "items": { - "type": "string" - }, - "type": "array", - "maxItems": 20 + "type": "integer" }, { "type": "null" } ], - "title": "Content Types", - "description": "content types (podcast, interview, scan, photo, email \u2026, GET /content-types), by key or label" + "title": "Keep" }, - "date_from": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } + "status": { + "type": "string", + "enum": [ + "proposed", + "applied", + "dismissed", + "undone" ], - "title": "Date From", - "description": "made on or after this day, month or year (2026-09-30, 2026-09, 2026)" + "title": "Status" }, - "date_to": { + "routine": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "null" } ], - "title": "Date To", - "description": "made on or before this day, month or year" + "title": "Routine" }, - "min_duration": { + "run": { "anyOf": [ { - "type": "number", - "minimum": 0.0 + "type": "integer" }, { "type": "null" } ], - "title": "Min Duration", - "description": "at least this many seconds long" + "title": "Run" }, - "max_duration": { + "workflow": { "anyOf": [ { - "type": "number", - "minimum": 0.0 + "type": "integer" }, { "type": "null" } ], - "title": "Max Duration", - "description": "shorter than this many seconds" + "title": "Workflow" }, - "tags": { + "created_at": { "anyOf": [ { - "items": { - "type": "string" - }, - "type": "array", - "maxItems": 20 + "type": "string" }, { "type": "null" } ], - "title": "Tags", - "description": "tags (ignoring case)" + "title": "Created At" }, - "languages": { + "decided_at": { "anyOf": [ { - "items": { - "type": "string" - }, - "type": "array", - "maxItems": 20 + "type": "string" }, { "type": "null" } ], - "title": "Languages", - "description": "language codes; none for those whose language isn't known" + "title": "Decided At" }, - "objects": { + "decided_by": { "anyOf": [ { - "items": { - "type": "string" - }, - "type": "array", - "maxItems": 20 + "type": "string" }, { "type": "null" } ], - "title": "Objects", - "description": "kinds of object seen in them (person, car, dog \u2026)" - }, - "emotions": { + "title": "Decided By" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "kind", + "a", + "b", + "spaces", + "status" + ], + "title": "GraphChange" + }, + "GraphChangeAccept": { + "properties": { + "keep": { "anyOf": [ { - "items": { - "type": "string" - }, - "type": "array", - "maxItems": 20 + "type": "integer" }, { "type": "null" } ], - "title": "Emotions", - "description": "emotions heard in them (ignoring case)" + "title": "Keep", + "description": "for a merge: the entity to keep (default: the one proposed)" + } + }, + "additionalProperties": false, + "type": "object", + "title": "GraphChangeAccept" + }, + "GraphQuery": { + "properties": { + "query": { + "type": "string", + "title": "Query", + "description": "read-only Cypher, e.g. MATCH (e:Person)<-[:MENTIONS]-(r:Recording) RETURN e.name, count(r)" }, - "status": { + "params": { + "additionalProperties": true, + "type": "object", + "title": "Params", + "description": "values for $parameters in the query" + }, + "scope": { + "type": "string", + "title": "Scope", + "default": "global" + }, + "limit": { + "type": "integer", + "maximum": 5000.0, + "minimum": 1.0, + "title": "Limit", + "description": "rows at most", + "default": 500 + }, + "as_of": { "anyOf": [ { - "items": { - "type": "string", - "enum": [ - "new", - "transcribed", - "diarized", - "analyzed", - "error", - "processing", - "failed" - ] - }, - "type": "array", - "maxItems": 10 + "type": "string" }, { "type": "null" } ], - "title": "Status", - "description": "statuses, or processing and failed" + "title": "As Of", + "description": "query the graph as of a version: a number or a version's name" + } + }, + "type": "object", + "required": [ + "query" + ], + "title": "GraphQuery" + }, + "GraphQuestion": { + "properties": { + "question": { + "type": "string", + "maxLength": 1000, + "minLength": 1, + "title": "Question", + "description": "a question in plain language" }, - "origins": { + "scope": { + "type": "string", + "title": "Scope", + "default": "global" + }, + "limit": { + "type": "integer", + "maximum": 2000.0, + "minimum": 1.0, + "title": "Limit", + "default": 200 + }, + "as_of": { "anyOf": [ { - "items": { - "type": "string" - }, - "type": "array", - "maxItems": 20 + "type": "string" }, { "type": "null" } ], - "title": "Origins", - "description": "where they came from: source:, upload, paste, iiif, folder or file" + "title": "As Of", + "description": "ask the graph as of a version: a number or a version's name" + } + }, + "type": "object", + "required": [ + "question" + ], + "title": "GraphQuestion" + }, + "Group": { + "properties": { + "id": { + "type": "integer", + "title": "Id" }, - "collection": { + "name": { + "type": "string", + "title": "Name" + }, + "kind": { + "type": "string", + "title": "Kind" + }, + "provider": { + "type": "integer", + "title": "Provider" + }, + "template": { + "additionalProperties": true, + "type": "object", + "title": "Template" + }, + "min": { + "type": "integer", + "title": "Min" + }, + "max": { + "type": "integer", + "title": "Max" + }, + "desired": { + "type": "integer", + "title": "Desired" + }, + "cooldown_seconds": { "anyOf": [ { "type": "integer" @@ -47546,84 +49362,56 @@ "type": "null" } ], - "title": "Collection", - "description": "a collection: what's in it and in the collections inside it" + "title": "Cooldown Seconds" }, - "sensitive": { + "warmup_seconds": { "anyOf": [ { - "type": "string", - "enum": [ - "include", - "only", - "exclude" - ] + "type": "integer" }, { "type": "null" } ], - "title": "Sensitive", - "description": "where NSFW classification is on: only those flagged, exclude them, or include them" + "title": "Warmup Seconds" }, - "sort": { + "terminate": { + "type": "string", + "title": "Terminate" + }, + "rules": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array", + "title": "Rules", + "description": "each with `says`, the rule in words" + }, + "at_cap": { + "type": "string", + "title": "At Cap" + }, + "enabled": { + "type": "boolean", + "title": "Enabled" + }, + "paused": { + "type": "boolean", + "title": "Paused" + }, + "paused_why": { "anyOf": [ { - "type": "string", - "enum": [ - "relevance", - "-date", - "date", - "-title", - "title", - "-duration", - "duration" - ] + "type": "string" }, { "type": "null" } ], - "title": "Sort", - "description": "relevance (the default with text: best matching words first), else newest first" - }, - "limit": { - "type": "integer", - "maximum": 100.0, - "minimum": 1.0, - "title": "Limit", - "default": 20 - }, - "offset": { - "type": "integer", - "maximum": 10000.0, - "minimum": 0.0, - "title": "Offset", - "default": 0 - } - }, - "additionalProperties": false, - "type": "object", - "title": "FindQuery", - "description": "Filters combine with AND, the values of one filter with OR. Faces, speakers, entities and topics are ids or names\n(ignoring case; an entity's other names count, and any of a topic's labels)." - }, - "FindReason": { - "properties": { - "by": { - "type": "string", - "enum": [ - "words", - "face", - "voice", - "entity", - "topic", - "content_type", - "object", - "emotion" - ], - "title": "By" + "title": "Paused Why" }, - "name": { + "capped": { "anyOf": [ { "type": "string" @@ -47632,96 +49420,47 @@ "type": "null" } ], - "title": "Name", - "description": "what matched: the words, the person's or the entity's name, the topic's label \u2026" - }, - "moments": { - "items": { - "$ref": "#/components/schemas/FindMoment" - }, - "type": "array", - "title": "Moments", - "description": "up to 3 moments that show it, to cite" + "title": "Capped", + "description": "why its budget stops it now, when it does" }, - "count": { + "approval": { "anyOf": [ { - "type": "integer" + "additionalProperties": true, + "type": "object" }, { "type": "null" } ], - "title": "Count", - "description": "emotion: how many lines have it" - } - }, - "additionalProperties": true, - "type": "object", - "required": [ - "by" - ], - "title": "FindReason" - }, - "FindResults": { - "properties": { - "total": { - "type": "integer", - "title": "Total", - "description": "how many assets match, on all pages" + "title": "Approval", + "description": "a person's approval: {by, at, terms}" }, - "items": { - "items": { - "$ref": "#/components/schemas/FindItem" - }, - "type": "array", - "title": "Items" + "approved": { + "type": "boolean", + "title": "Approved", + "description": "the approval covers the terms as they stand" }, - "resolved": { - "additionalProperties": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/FindNamed" - }, - "type": "array" - }, - { - "type": "integer" - } - ] - }, + "terms": { + "additionalProperties": true, "type": "object", - "title": "Resolved", - "description": "what the names given matched, by filter (and how many narrower topics came with them)" + "title": "Terms", + "description": "what an approval binds: kind, provider, template, max, budget, at_cap" }, - "text": { + "budget": { "anyOf": [ { - "$ref": "#/components/schemas/FindText" + "$ref": "#/components/schemas/GroupBudget" }, { "type": "null" } ] - } - }, - "additionalProperties": true, - "type": "object", - "required": [ - "total", - "items" - ], - "title": "FindResults" - }, - "FindText": { - "properties": { - "mode": { - "type": "string", - "title": "Mode", - "description": "how the words were matched: keyword, semantic or hybrid" }, - "meaning": { + "units": { + "$ref": "#/components/schemas/UnitCounts" + }, + "waiting_for": { "anyOf": [ { "type": "string" @@ -47730,62 +49469,50 @@ "type": "null" } ], - "title": "Meaning", - "description": "why not by meaning, when it was asked and couldn't be" - }, - "moments": { - "type": "integer", - "title": "Moments", - "description": "moments the words were found in" + "title": "Waiting For", + "description": "why it launches nothing now, when it doesn't" }, - "capped": { - "type": "boolean", - "title": "Capped", - "description": "more moments matched than were looked through, so some assets may be missing" - } - }, - "additionalProperties": true, - "type": "object", - "required": [ - "mode", - "moments", - "capped" - ], - "title": "FindText" - }, - "ForgotPasswordRequest": { - "properties": { - "email": { - "type": "string", - "title": "Email" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "email" - ], - "title": "ForgotPasswordRequest" - }, - "GoogleClient": { - "properties": { - "origin": { + "last_out_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Out At" + }, + "last_in_at": { "anyOf": [ { - "type": "string", - "enum": [ - "sign-in", - "connection" - ] + "type": "string" }, { "type": "null" } ], - "title": "Origin", - "description": "where the OAuth client for connecting a Google account comes from: the Google sign-in provider, an earlier Google connection, or none (enter one to connect)" + "title": "Last In At" }, - "client_id": { + "last_eval": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Last Eval", + "description": "what the rules said last: {at, said}" + }, + "failures": { + "type": "integer", + "title": "Failures", + "default": 0 + }, + "error": { "anyOf": [ { "type": "string" @@ -47794,188 +49521,132 @@ "type": "null" } ], - "title": "Client Id" + "title": "Error" }, - "redirect_uri": { - "type": "string", - "title": "Redirect Uri", - "description": "the address to add to the OAuth client's authorized redirect URIs" - } - }, - "additionalProperties": true, - "type": "object", - "required": [ - "origin", - "redirect_uri" - ], - "title": "GoogleClient" - }, - "GoogleConnect": { - "properties": { - "client_id": { + "created_at": { "anyOf": [ { - "type": "string", - "maxLength": 300 + "type": "string" }, { "type": "null" } ], - "title": "Client Id", - "description": "an OAuth client of your own (else the one Lens has)" + "title": "Created At" }, - "client_secret": { + "updated_at": { "anyOf": [ { - "type": "string", - "maxLength": 300 + "type": "string" }, { "type": "null" } ], - "title": "Client Secret" - } - }, - "additionalProperties": false, - "type": "object", - "title": "GoogleConnect" - }, - "GoogleConnectStart": { - "properties": { - "url": { - "type": "string", - "title": "Url", - "description": "Google's consent page; open it in this browser" - } - }, - "additionalProperties": true, - "type": "object", - "required": [ - "url" - ], - "title": "GoogleConnectStart" - }, - "Gpu": { - "properties": { - "name": { - "type": "string", - "title": "Name" + "title": "Updated At" }, - "memory_gb": { + "updated_by": { "anyOf": [ { - "type": "number" + "type": "string" }, { "type": "null" } ], - "title": "Memory Gb" + "title": "Updated By" } }, "additionalProperties": true, "type": "object", "required": [ - "name" + "id", + "name", + "kind", + "provider", + "template", + "min", + "max", + "desired", + "terminate", + "rules", + "at_cap", + "enabled", + "paused", + "approved", + "terms", + "units" ], - "title": "Gpu" + "title": "Group" }, - "Graph": { + "GroupBudget": { "properties": { - "scope": { + "usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Usd" + }, + "period": { "type": "string", - "title": "Scope" + "title": "Period" }, - "namespaces": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Namespaces" + "state": { + "type": "string", + "title": "State" }, - "nodes": { - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array", - "title": "Nodes" + "spent": { + "additionalProperties": true, + "type": "object", + "title": "Spent" }, - "edges": { - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array", - "title": "Edges" + "left": { + "additionalProperties": true, + "type": "object", + "title": "Left" } }, "additionalProperties": true, "type": "object", "required": [ - "scope", - "namespaces", - "nodes", - "edges" + "period", + "state", + "spent", + "left" ], - "title": "Graph" + "title": "GroupBudget" }, - "GraphChange": { + "GroupChange": { "properties": { - "id": { - "type": "integer", - "title": "Id" - }, - "kind": { - "type": "string", - "enum": [ - "merge", - "link" - ], - "title": "Kind" - }, - "a": { - "additionalProperties": true, - "type": "object", - "title": "A" - }, - "b": { - "additionalProperties": true, - "type": "object", - "title": "B" - }, - "spaces": { - "items": { - "type": "integer" - }, - "type": "array", - "title": "Spaces" - }, - "reason": { + "name": { "anyOf": [ { - "type": "string" + "type": "string", + "maxLength": 80 }, { "type": "null" } ], - "title": "Reason" + "title": "Name" }, - "confidence": { + "provider": { "anyOf": [ { - "type": "number" + "type": "integer" }, { "type": "null" } ], - "title": "Confidence" + "title": "Provider" }, - "verdict": { + "template": { "anyOf": [ { "additionalProperties": true, @@ -47985,211 +49656,276 @@ "type": "null" } ], - "title": "Verdict" + "title": "Template" }, - "keep": { + "min": { "anyOf": [ { - "type": "integer" + "type": "integer", + "maximum": 100.0, + "minimum": 0.0 }, { "type": "null" } ], - "title": "Keep" + "title": "Min" }, - "status": { - "type": "string", - "enum": [ - "proposed", - "applied", - "dismissed", - "undone" + "max": { + "anyOf": [ + { + "type": "integer", + "maximum": 100.0, + "minimum": 0.0 + }, + { + "type": "null" + } ], - "title": "Status" + "title": "Max" }, - "routine": { + "desired": { "anyOf": [ { - "type": "integer" + "type": "integer", + "maximum": 100.0, + "minimum": 0.0 }, { "type": "null" } ], - "title": "Routine" + "title": "Desired" }, - "run": { + "cooldown_seconds": { "anyOf": [ { - "type": "integer" + "type": "integer", + "maximum": 86400.0, + "minimum": 0.0 }, { "type": "null" } ], - "title": "Run" + "title": "Cooldown Seconds" }, - "workflow": { + "warmup_seconds": { "anyOf": [ { - "type": "integer" + "type": "integer", + "maximum": 7200.0, + "minimum": 0.0 }, { "type": "null" } ], - "title": "Workflow" + "title": "Warmup Seconds" }, - "created_at": { + "terminate": { "anyOf": [ { - "type": "string" + "type": "string", + "enum": [ + "oldest", + "newest" + ] }, { "type": "null" } ], - "title": "Created At" + "title": "Terminate" }, - "decided_at": { + "rules": { "anyOf": [ { - "type": "string" + "items": { + "$ref": "#/components/schemas/Rule" + }, + "type": "array" }, { "type": "null" } ], - "title": "Decided At" + "title": "Rules" }, - "decided_by": { + "at_cap": { "anyOf": [ { - "type": "string" + "type": "string", + "enum": [ + "stop", + "min" + ] }, { "type": "null" } ], - "title": "Decided By" - } - }, - "additionalProperties": true, - "type": "object", - "required": [ - "id", - "kind", - "a", - "b", - "spaces", - "status" - ], - "title": "GraphChange" - }, - "GraphChangeAccept": { - "properties": { - "keep": { + "title": "At Cap" + }, + "enabled": { "anyOf": [ { - "type": "integer" + "type": "boolean" }, { "type": "null" } ], - "title": "Keep", - "description": "for a merge: the entity to keep (default: the one proposed)" + "title": "Enabled" + }, + "budget": { + "anyOf": [ + { + "type": "number", + "exclusiveMinimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Budget" } }, "additionalProperties": false, "type": "object", - "title": "GraphChangeAccept" + "title": "GroupChange" }, - "GraphQuery": { + "GroupIn": { "properties": { - "query": { + "name": { "type": "string", - "title": "Query", - "description": "read-only Cypher, e.g. MATCH (e:Person)<-[:MENTIONS]-(r:Recording) RETURN e.name, count(r)" + "maxLength": 80, + "title": "Name" }, - "params": { + "provider": { + "type": "integer", + "title": "Provider", + "description": "the provider's id" + }, + "kind": { + "anyOf": [ + { + "type": "string", + "enum": [ + "node", + "model" + ] + }, + { + "type": "null" + } + ], + "title": "Kind", + "description": "the provider's kind; filled in when left out" + }, + "template": { "additionalProperties": true, "type": "object", - "title": "Params", - "description": "values for $parameters in the query" + "title": "Template", + "description": "what each unit is (the driver's template fields)" }, - "scope": { - "type": "string", - "title": "Scope", - "default": "global" + "min": { + "type": "integer", + "maximum": 100.0, + "minimum": 0.0, + "title": "Min", + "default": 0 }, - "limit": { + "max": { "type": "integer", - "maximum": 5000.0, - "minimum": 1.0, - "title": "Limit", - "description": "rows at most", - "default": 500 + "maximum": 100.0, + "minimum": 0.0, + "title": "Max", + "default": 1 }, - "as_of": { + "desired": { "anyOf": [ { - "type": "string" + "type": "integer", + "maximum": 100.0, + "minimum": 0.0 }, { "type": "null" } ], - "title": "As Of", - "description": "query the graph as of a version: a number or a version's name" - } - }, - "type": "object", - "required": [ - "query" - ], - "title": "GraphQuery" - }, - "GraphQuestion": { - "properties": { - "question": { + "title": "Desired", + "description": "units wanted now (default min)" + }, + "cooldown_seconds": { + "type": "integer", + "maximum": 86400.0, + "minimum": 0.0, + "title": "Cooldown Seconds", + "description": "how long after a change before the next", + "default": 300 + }, + "warmup_seconds": { + "type": "integer", + "maximum": 7200.0, + "minimum": 0.0, + "title": "Warmup Seconds", + "description": "how long a new unit may take to join before more are judged", + "default": 300 + }, + "terminate": { "type": "string", - "maxLength": 1000, - "minLength": 1, - "title": "Question", - "description": "a question in plain language" + "enum": [ + "oldest", + "newest" + ], + "title": "Terminate", + "description": "which unit goes first when scaling in", + "default": "oldest" }, - "scope": { + "rules": { + "items": { + "$ref": "#/components/schemas/Rule" + }, + "type": "array", + "title": "Rules" + }, + "at_cap": { "type": "string", - "title": "Scope", - "default": "global" + "enum": [ + "stop", + "min" + ], + "title": "At Cap", + "description": "at the budget's cap: stop every unit, or keep min", + "default": "stop" }, - "limit": { - "type": "integer", - "maximum": 2000.0, - "minimum": 1.0, - "title": "Limit", - "default": 200 + "enabled": { + "type": "boolean", + "title": "Enabled", + "default": true }, - "as_of": { + "budget": { "anyOf": [ { - "type": "string" + "type": "number", + "exclusiveMinimum": 0.0 }, { "type": "null" } ], - "title": "As Of", - "description": "ask the graph as of a version: a number or a version's name" + "title": "Budget", + "description": "USD per month; needed on a priced provider" } }, + "additionalProperties": false, "type": "object", "required": [ - "question" + "name", + "provider" ], - "title": "GraphQuestion" + "title": "GroupIn" }, "HTTPValidationError": { "properties": { @@ -54196,6 +55932,39 @@ ], "title": "MetadataEdit" }, + "Metrics": { + "properties": { + "metrics": { + "additionalProperties": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "integer" + } + ] + }, + "type": "object", + "title": "Metrics" + }, + "about": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "title": "About", + "description": "what each metric measures" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "metrics", + "about" + ], + "title": "Metrics" + }, "ModelSpend": { "properties": { "model": { @@ -58452,6 +60221,80 @@ ], "title": "Output" }, + "Overview": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled" + }, + "settings": { + "additionalProperties": true, + "type": "object", + "title": "Settings" + }, + "scaler": { + "additionalProperties": true, + "type": "object", + "title": "Scaler", + "description": "which process scales and when it last looked" + }, + "providers": { + "items": { + "$ref": "#/components/schemas/Provider" + }, + "type": "array", + "title": "Providers" + }, + "groups": { + "items": { + "$ref": "#/components/schemas/Group" + }, + "type": "array", + "title": "Groups" + }, + "units": { + "items": { + "$ref": "#/components/schemas/Unit" + }, + "type": "array", + "title": "Units" + }, + "drivers": { + "items": { + "$ref": "#/components/schemas/DriverSpec" + }, + "type": "array", + "title": "Drivers" + }, + "metrics": { + "additionalProperties": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "integer" + } + ] + }, + "type": "object", + "title": "Metrics" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "enabled", + "settings", + "scaler", + "providers", + "groups", + "units", + "drivers", + "metrics" + ], + "title": "Overview" + }, "Page": { "properties": { "idx": { @@ -59157,6 +61000,30 @@ "type": "object", "title": "PatternUpdate" }, + "Paused": { + "properties": { + "paused": { + "type": "boolean", + "title": "Paused", + "default": true + }, + "why": { + "anyOf": [ + { + "type": "string", + "maxLength": 200 + }, + { + "type": "null" + } + ], + "title": "Why" + } + }, + "additionalProperties": false, + "type": "object", + "title": "Paused" + }, "Permission": { "properties": { "account": { @@ -60838,6 +62705,318 @@ "title": "PrimaryFile", "description": "The audio, video, document or image the resource's pipeline runs on." }, + "Protected": { + "properties": { + "protected": { + "type": "boolean", + "title": "Protected", + "default": true + } + }, + "additionalProperties": false, + "type": "object", + "title": "Protected" + }, + "Provider": { + "properties": { + "id": { + "type": "integer", + "title": "Id" + }, + "name": { + "type": "string", + "title": "Name" + }, + "driver": { + "type": "string", + "title": "Driver" + }, + "kind": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Kind" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Label" + }, + "priced": { + "type": "boolean", + "title": "Priced", + "default": true + }, + "settings": { + "additionalProperties": true, + "type": "object", + "title": "Settings" + }, + "secrets": { + "additionalProperties": { + "type": "boolean" + }, + "type": "object", + "title": "Secrets", + "description": "which credentials are set" + }, + "price": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Price" + }, + "note": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Note" + }, + "checked_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Checked At" + }, + "check": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Check", + "description": "the last check: {ok, error?}" + }, + "created_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated By" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "name", + "driver", + "settings", + "secrets" + ], + "title": "Provider" + }, + "ProviderChange": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string", + "maxLength": 80 + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "settings": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Settings" + }, + "secrets": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Secrets", + "description": "a secret given as null is removed; one left out is kept" + }, + "price": { + "anyOf": [ + { + "type": "number", + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Price" + }, + "note": { + "anyOf": [ + { + "type": "string", + "maxLength": 300 + }, + { + "type": "null" + } + ], + "title": "Note" + }, + "check": { + "type": "boolean", + "title": "Check", + "default": true + } + }, + "additionalProperties": false, + "type": "object", + "title": "ProviderChange" + }, + "ProviderIn": { + "properties": { + "name": { + "type": "string", + "maxLength": 80, + "title": "Name" + }, + "driver": { + "type": "string", + "title": "Driver", + "description": "the kind of provider (see /scaling/drivers)" + }, + "settings": { + "additionalProperties": true, + "type": "object", + "title": "Settings", + "description": "the driver's non-secret settings" + }, + "secrets": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": "object", + "title": "Secrets", + "description": "its credentials: sealed, never shown again" + }, + "price": { + "anyOf": [ + { + "type": "number", + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Price", + "description": "USD per unit-hour, when the driver doesn't know it; an estimate" + }, + "note": { + "anyOf": [ + { + "type": "string", + "maxLength": 300 + }, + { + "type": "null" + } + ], + "title": "Note" + }, + "check": { + "type": "boolean", + "title": "Check", + "description": "prove the credentials with a read-only call before answering", + "default": true + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "name", + "driver" + ], + "title": "ProviderIn" + }, "PublicCard": { "properties": { "id": { @@ -66085,6 +68264,194 @@ "type": "object", "title": "RoutineUpdate" }, + "Rule": { + "properties": { + "type": { + "type": "string", + "enum": [ + "target", + "step", + "schedule" + ], + "title": "Type" + }, + "name": { + "anyOf": [ + { + "type": "string", + "maxLength": 80 + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "metric": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Metric", + "description": "target and step rules: see /scaling/metrics" + }, + "target": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Target", + "description": "target rules: the value to keep the metric at" + }, + "above": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Above", + "description": "step rules: fires while the metric is above this" + }, + "below": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Below", + "description": "step rules: fires while the metric is below this" + }, + "minutes": { + "anyOf": [ + { + "type": "integer", + "maximum": 1440.0, + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Minutes", + "description": "step rules: for how long before it fires (default 1)" + }, + "add": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Add", + "description": "step rules: units to add, a count or a percentage like \"50%\"" + }, + "remove": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Remove", + "description": "step rules: units to remove, a count or a percentage" + }, + "cron": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cron", + "description": "schedule rules: when, as a cron expression" + }, + "timezone": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Timezone" + }, + "min": { + "anyOf": [ + { + "type": "integer", + "maximum": 100.0, + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Min", + "description": "schedule rules: the new min" + }, + "max": { + "anyOf": [ + { + "type": "integer", + "maximum": 100.0, + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Max", + "description": "schedule rules: the new max" + }, + "desired": { + "anyOf": [ + { + "type": "integer", + "maximum": 100.0, + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Desired", + "description": "schedule rules: the new desired count" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type" + ], + "title": "Rule" + }, "S3Key": { "properties": { "id": { @@ -72956,6 +75323,25 @@ ], "title": "TermSuggestion" }, + "Terminate": { + "properties": { + "why": { + "anyOf": [ + { + "type": "string", + "maxLength": 200 + }, + { + "type": "null" + } + ], + "title": "Why" + } + }, + "additionalProperties": false, + "type": "object", + "title": "Terminate" + }, "TicketRequest": { "properties": { "ticket": { @@ -74236,6 +76622,213 @@ ], "title": "Undone" }, + "Unit": { + "properties": { + "id": { + "type": "integer", + "title": "Id" + }, + "group": { + "type": "integer", + "title": "Group" + }, + "provider": { + "type": "integer", + "title": "Provider" + }, + "driver": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Driver" + }, + "name": { + "type": "string", + "title": "Name" + }, + "external_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "External Id", + "description": "the provider's id for it" + }, + "state": { + "type": "string", + "enum": [ + "launching", + "running", + "draining", + "terminating", + "terminated", + "failed" + ], + "title": "State" + }, + "address": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Address" + }, + "endpoint": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Endpoint", + "description": "a model unit's endpoint: base_url, model; api_key true when set" + }, + "price": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Price", + "description": "USD per hour, an estimate" + }, + "launched_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Launched At" + }, + "ready_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Ready At" + }, + "ended_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Ended At" + }, + "by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "By" + }, + "why": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Why" + }, + "protected": { + "type": "boolean", + "title": "Protected", + "default": false + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "accrued_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Accrued At" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "group", + "provider", + "name", + "state" + ], + "title": "Unit" + }, + "UnitCounts": { + "properties": { + "launching": { + "type": "integer", + "title": "Launching" + }, + "running": { + "type": "integer", + "title": "Running" + }, + "draining": { + "type": "integer", + "title": "Draining" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "launching", + "running", + "draining" + ], + "title": "UnitCounts" + }, "UpdateStatus": { "properties": { "current": { diff --git a/threat-model.json b/threat-model.json index 8f5bce67f..14b0a7fe5 100644 --- a/threat-model.json +++ b/threat-model.json @@ -6,7 +6,7 @@ "scope_subpath": null, "description": "Lens is a self-hosted archive for documents, recordings, photos and messages: it reads, transcribes, indexes and publishes them, and people search and chat over them. One operator (the instance admin) runs an instance for their own people: a FastAPI API, a Next.js web app that proxies the API and holds the sign-in session, and background workers, around one SurrealDB database, with content split into namespaces and per-namespace roles. Exposing an instance to the internet behind HTTPS (reverse proxy or Cloudflare tunnel) is supported. Only the instance admin (equivalent to host code execution) and the host operator are trusted; everyone else is trusted only as far as their role.", "confidence": { - "documented": 255, + "documented": 257, "inferred": 99 }, "components": [ @@ -658,6 +658,15 @@ "provenance": "documented", "source": "docs/assistant.md \"Optional containers\"" }, + { + "entry_point": "admin auto scaling", + "parameter": "/api/v1/scaling (providers, groups, units, tick), PUT /api/v1/settings/scaling, MCP and assistant scaling tools: provider credentials, a group's terms and rules, a unit id", + "attacker_controllable": "no", + "component": "authz", + "caller_must_enforce": "admin is trusted; off until turned on; audited (scaling.*) and in the activity ledger; credentials sealed and shown only as set; a group launches nothing until a person signed in to the web app approves its terms (P-SCALING)", + "provenance": "documented", + "source": "docs/scaling.md \"The person approves\"" + }, { "entry_point": "entity and topic curation tools", "parameter": "MCP edit_entity, edit_topic, merge_topics, delete_topic, undo_topic_change, undo_graph_change; assistant propose_topic_change: entity and topic ids, names, labels, definitions", @@ -1026,6 +1035,14 @@ "provenance": "documented", "source": "docs/feeds.md \"Private by default\"" }, + { + "property": "P-SCALING", + "violation_symptom": "integrity-bypass", + "severity_tier": "security", + "conditions": "auto scaling launches nothing unless a person signed in to the web app approved the group's terms (its provider, what each unit is, its max, its budget, what happens at its cap); an API key, an app, MCP and the assistant acting on its own never approve, and a change that widens the terms takes the approval away; a provider's credentials are sealed per provider and field, opened only for a call to it and never shown; a unit is handed no long-lived Lens secret", + "provenance": "documented", + "source": "docs/scaling.md \"The person approves\"" + }, { "property": "P-AUTH-REQ", "violation_symptom": "integrity-bypass", diff --git a/threat-model.md b/threat-model.md index 9cb118871..e65911b79 100644 --- a/threat-model.md +++ b/threat-model.md @@ -320,6 +320,7 @@ classified by gate only *(inferred, Q8)*; unread modules are listed in Q34. | `.lens-control/inbox/upgrade` (host upgrade agent), `/metrics`, `.lens-control/prometheus-token` | `ref` (release tag or this install's branch; empty: the version running), with `https=`, `public=` (`0`/`1`) and `target=` (`full`/`lean`); `action=services` with `profile=` (`mcp`, `browser` or `monitoring`) and `turn=on`/`off` (or an older Lens's `monitoring=on`/`off`); the scrape token | the inbox is writable by the containers (and local users); `/metrics` by anyone reaching the web app | x-env-config, credential | the agent installs only a release tag or the install's branch, changes only the HTTPS ports, the web app's network reach and the edition (any other value is dropped) and only toggles the `mcp`, `browser` and `monitoring` profiles (any other name is refused), nothing else from the file, so a writer of the inbox can at most restart Lens with those choices or open it to the network; `/metrics` answers only the scrape token (constant-time compare); the token file is world-readable on the host and grants metrics only, which carry no personal content | *(documented, docs/updates.md, docs/prometheus.md "Privacy")* | | `PUT /api/v1/admin/install`, `POST /api/v1/settings/https/ports`, MCP `set_install_settings` | `public`, `target`, `open` | admin only (MCP: with the write scope) | x-env-config | admin is trusted; audited (`lens.install`, `https.ports`); the change reaches the machine only as an inbox request, which the agent checks again | *(documented, docs/updates.md "Setting the machine up again")* | | `PUT /api/v1/admin/services/{profile}`, MCP `set_service` | `profile` (`mcp`, `browser`, `monitoring`), `on` | admin only (MCP: with the write scope) | x-env-config | admin is trusted; audited (`lens.services`, `lens.monitoring`); an unknown profile is a 404; one change at a time; it reaches the machine only as an inbox request, which the agent checks again | *(documented, docs/assistant.md "Optional containers")* | +| `/api/v1/scaling` (providers, groups, units), `PUT /api/v1/settings/scaling`, MCP and assistant scaling tools | provider credentials, a group's terms and rules, a unit id | admin only (MCP: with the write scope) | x-credential, x-env-config | admin is trusted; off until turned on; audited (`scaling.*`) and in the activity ledger; a provider's credentials are sealed per provider and field and shown only as set; a group launches nothing until a person signed in to the web app approves its terms, and agents act only within them (P-SCALING) | *(documented, docs/scaling.md "The person approves")* | | `PUT /api/v1/admin/sandbox`, MCP `set_sandbox_gateway` | `on` | admin only (MCP: with the write scope) | x-env-config | admin is trusted; audited (`lens.sandbox`); refused when the machine can't run the gateway (preflight); reaches the machine only as an inbox request (`action=services`, `profile=sandbox`) the agent checks again; off deletes every sandbox first | *(documented, docs/assistant.md "The gateway")* | | MCP `edit_entity`, `edit_topic`, `merge_topics`, `delete_topic`, `undo_topic_change`, `undo_graph_change`; assistant `propose_topic_change` | entity and topic ids, names, labels, definitions | yes, an agent with an editor's write-scope token; indirectly through content the model reads | data, x-prompt-injection | the curate permission in each namespace touched and a write-capable principal (the same route functions as the web app); the assistant's changes wait for the person's approval; merges and deletes are kept to undo, and audited | *(documented, docs/mcp.md "curate entities and topics as an editor does")* | | `/api/v1/automations` (save, try, run now, decide, draft from a sentence), the MCP and assistant automation tools; the activities an automation runs on | triggers, `where` and If match trees, the event workflow graph (action params are Jinja templates), `policy`, `limits`, an activity id; the text of the activities themselves | yes: any editor, for automations in namespaces they edit; indirectly through activity text (mail, titles, chat) that reaches action params | callback/code (declarative), data, x-prompt-injection | none; it runs as its maker with their permissions at run time (P-AUTOMATION-ACTOR), actions reaching outside Lens ask first, outside text marks intents untrusted and breaks up links (P-INJECTION-EGRESS), and chains and bursts are bounded (P-AUTOMATION-LOOP) | *(documented, docs/automations.md "Running automations")* | @@ -528,6 +529,7 @@ Paths in the Voided by column are relative to `fastapi_backend/app/`. Searches c | P-MEDIA-LINK | Media, frames, documents and reports open without a session only through a link the server signed for that path, or a live share link. Text people or models write is never signed. | info-leak | security-critical | *(documented, docs/architecture.md "is never signed")* | Voided by: Knowing `ACCESS_SECRET_KEY` signs any path — `core/security.py:43`; dev compose key — `docker-compose.yml:24`. Lifetime `MEDIA_URL_EXPIRE_SECONDS` — `config.py:30`. Report pages sign every asset of the namespace — `api/pages.py:140-141`. Signing is limited to allow-listed fields — `api/media.py:25-30`. | | P-SHARE | A share link opens one asset, read-only. It expires and can be revoked. Only hashes are stored. A dead link answers the same 410 whatever went wrong. | integrity-bypass | security-critical | *(documented, docs/authentication.md "give read-only access to one asset's player and embed")* | Voided by: Liveness plus asset match only — `domain/auth.py:441-444`; `?s=` routes — `api/deps.py:252` (Q7). | | P-FEEDS | A published feed's address opens that feed alone, read-only, with no session: the feed's XML and the audio of items it lists. Only an editor of the namespace makes a feed, and what it lists is bounded by who already reads that namespace: an item's resource or activity is in the namespace, or in one every reader of the feed's namespace reads too, never a vault's, and a namespace made a vault stops its feeds. The token is found by its hash; rotating or deleting the feed stops the old address at once. Text in an item cannot add markup to a reader. | info-leak | security-critical | *(documented, docs/feeds.md "Private by default")* | Voided by: the token is sealed with the server key, not only hashed, so the namespace's readers (and whoever holds `secret.key`) see the address again — `domain/syndication.py` `token_of`; anyone given the address reads the feed until it is rotated; editors choose what goes in, and a resource's title and summary show as they are now, so an edit after publishing shows; the fit is by namespace roles (`domain/syndication.py` `Fit`), not per-asset grants, so an asset later closed by access level still shows while its namespace fits. | +| P-SCALING | Auto scaling launches nothing that costs money unless a person, signed in to the web app, approved the group's terms: its provider, what each unit is, its max, its budget and what happens at its cap. An API key, an app, MCP and the assistant acting on its own never approve, and a change that widens the terms takes the approval away. A provider's credentials are sealed per provider and field, opened only for a call to it, and never shown. A unit is handed no long-lived Lens secret to start. | integrity-bypass | security-critical | *(documented, docs/scaling.md "The person approves")* | Voided by: an admin's web session approves, and an admin is trusted; a yes on the Approvals page to the assistant's proposal approves the terms that proposal names; within the terms agents may raise the desired count up to max, spending up to the budget; costs are estimates from list prices (`domain/scaling.py` `accrue`), not bills, so a provider can charge more than the budget counts. | | P-AUTH-REQ | A request is authenticated only if its bearer verifies and the account and session are live. Sign-out and disabling take effect at once, for every token kind. Revoking an app grant or API token ends its tokens at once. | integrity-bypass | security-critical | *(documented, docs/authentication.md "Every request checks that the account is still active")* | Voided by: Liveness checked per request — `api/deps.py:92`; JWT algorithm from `ALGORITHM` — `config.py:27`; dev secrets — `docker-compose.yml:24`. No bypass flag: `grep -rniE "disable_auth\|skip_auth\|no_auth\|auth_bypass\|AUTH_DISABLED\|ALLOW_ANON\|DEV_MODE" fastapi_backend/app nextjs-frontend/lib nextjs-frontend/auth.ts nextjs-frontend/proxy.ts docker-compose*.yml install.sh` — 0 hits. | | P-REMOVE | Removing an account ends every way it signs in at once (sessions, passkeys, sign-in links, connected accounts, API tokens, app grants, IIIF tokens) and takes every role, collection role and recording permission. Its email can make a new account, which inherits nothing. Admins only; never yourself, never the Matterbridge account, never the last passkey into a vault unless the admin says its files may be lost. | integrity-bypass | security-critical | *(documented, docs/authentication.md "Removing someone")* | Voided by: the account row stays as a marker with an `.invalid` email — `domain/auth.py` `remove_account`; shared notes, views and comments keep its id. | | P-BEARER-ONLY | `/api/v1` takes bearer tokens only, never cookies. The IIIF authorization flow and the web app's NextAuth handlers are the cookie surfaces. | integrity-bypass | security-critical | *(documented, docs/architecture.md "so there is no CSRF surface")* | Voided by: IIIF auth cookies — `api/iiif.py:232`, `api/iiif.py:333`; external-login state cookie — `api/v1/routes/external.py:34`. | diff --git a/threat-model.yaml b/threat-model.yaml index 061ada6d9..b6bfc0979 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -8,7 +8,7 @@ project: lens prose_version: "threat-model.md@same-commit" model_status: under-review triage_policy: strict -confidence: {documented: 208, maintainer: 46, inferred: 91, assumption: 8} +confidence: {documented: 210, maintainer: 46, inferred: 91, assumption: 8} generation: model: "Claude Opus 5.5 (claude-opus-5-5)" @@ -602,6 +602,16 @@ entry_points: obligation_id: admin-is-trusted caller_must_enforce: "admin is trusted; audited (lens.services, lens.monitoring); unknown profiles are a 404; one change at a time; reaches the machine only as an inbox request the agent checks again" provenance: {kind: documented, source: 'docs/assistant.md "Optional containers"'} + - id: "admin auto scaling" + component: authz + parameters: + - name: "/api/v1/scaling (providers, groups, units, tick), PUT /api/v1/settings/scaling, MCP and assistant scaling tools: provider credentials, a group's terms and rules, a unit id" + attacker_controllable: false + x-condition: "admin only (MCP: with the write scope)" + control_kinds: [x-credential, x-env-config] + obligation_id: admin-is-trusted + caller_must_enforce: "admin is trusted; off until turned on; audited (scaling.*) and in the activity ledger; credentials sealed and shown only as set; a group launches nothing until a person signed in to the web app approves its terms (P-SCALING)" + provenance: {kind: documented, source: 'docs/scaling.md "The person approves"'} - id: "admin sandbox gateway" component: authz parameters: @@ -1478,6 +1488,14 @@ properties_claimed: violation_symptoms: [info-leak] x-voided-by: "the token is sealed with the server key, not only hashed, so the namespace's readers and whoever holds secret.key see the address again (domain/syndication.py token_of); anyone given the address reads the feed until it is rotated; editors choose what goes in, and a resource's title and summary show as they are now; the fit is by namespace roles (domain/syndication.py Fit), not per-asset grants" provenance: {kind: documented, source: 'docs/feeds.md "Private by default"'} + - id: P-SCALING + kind: integrity + components: [authz, optional] + tier: security-critical + conditions: "auto scaling launches nothing unless a person signed in to the web app approved the group's terms (its provider, what each unit is, its max, its budget, what happens at its cap); an API key, an app, MCP and the assistant acting on its own never approve, and a change that widens the terms takes the approval away; a provider's credentials are sealed per provider and field, opened only for a call to it and never shown; a unit is handed no long-lived Lens secret" + violation_symptoms: [integrity-bypass] + x-voided-by: "an admin's web session approves, and an admin is trusted; a yes on the Approvals page to the assistant's proposal approves the terms that proposal names; within the terms agents may raise the desired count up to max, spending up to the budget; costs are estimates from list prices (domain/scaling.py accrue), not bills, so a provider can charge more than the budget counts" + provenance: {kind: documented, source: 'docs/scaling.md "The person approves"'} - id: P-AUTH-REQ kind: authentication components: [web, identity] From 8bf2a2b69e17f43c4513cd2681eb2722558e3ff3 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 03:24:40 +0000 Subject: [PATCH 3/3] =?UTF-8?q?feat(scaling):=20Settings=20=E2=80=BA=20Aut?= =?UTF-8?q?o=20scaling,=20to=20turn=20it=20on=20and=20approve=20each=20gro?= =?UTF-8?q?up's=20terms?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The web app gets an Auto scaling section under System: the master switch, the scaler's timings, and each group with its terms in words, an Approve button for a group waiting on the person, and Pause or Resume. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ALjNbLobvPe7VwvDL8j7V3 --- README.md | 6 +- docs/requirements.md | 1 + docs/scaling.md | 16 ++- fastapi_backend/app/domain/ops_tools.py | 2 +- fastapi_backend/app/domain/scaling.py | 4 +- fastapi_backend/tests/api/test_scaling.py | 2 +- .../__tests__/settings-scaling.test.tsx | 96 ++++++++++++++ .../components/settings/bodies.tsx | 26 ++++ nextjs-frontend/components/settings/find.ts | 1 + nextjs-frontend/components/settings/model.ts | 53 ++++++++ .../components/settings/scaling-panel.tsx | 117 ++++++++++++++++++ .../components/settings/settings-shell.tsx | 2 + 12 files changed, 313 insertions(+), 13 deletions(-) create mode 100644 nextjs-frontend/__tests__/settings-scaling.test.tsx create mode 100644 nextjs-frontend/components/settings/scaling-panel.tsx diff --git a/README.md b/README.md index 0386e1a8b..701f98daf 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p [the meter](docs/budgets.md#the-meter-and-the-costs-page); [checks](docs/requirements.md#see-what-it-costs)) - **Scale out.** Scaling groups add workers when work waits and let them go when it doesn't, by target, step and schedule rules, within min and max and a monthly budget, with every launch, join and hour of cost in the activity - ledger. A group launches nothing until you approve its terms signed in to the web app; agents propose and manage + ledger. A group launches nothing until you approve its terms in Settings › Auto scaling; agents propose and manage groups over the API, MCP and the assistant, and never approve one. It runs more workers on this machine today; cloud, Proxmox, Linux and model hosts are planned. ([Auto scaling](docs/scaling.md); [checks](docs/requirements.md#scale-out)) @@ -248,8 +248,8 @@ Next, and not in a pull request yet; each links to where it is tracked: - **Local models:** GPU builds of llama.cpp fetched by Lens, and local embedding models ([local models](docs/local-models.md)). - **Auto scaling:** providers for AWS, DigitalOcean, Hetzner, Proxmox and Linux hosts over SSH, joining over the - Nebula mesh, model hosts (RunPod, Lambda, Hugging Face, SageMaker, Bedrock), and the Settings › Scaling page - ([auto scaling](docs/scaling.md)). + Nebula mesh, model hosts (RunPod, Lambda, Hugging Face, SageMaker, Bedrock), and making providers and groups in + Settings ([auto scaling](docs/scaling.md)). - **Speech providers:** their summaries, chapters and entities as optional imports, and per-minute prices in the activity ledger ([speech providers](docs/speech-providers.md)). - **Storage:** uploads and assets' media in the file store, and a storage key to download diff --git a/docs/requirements.md b/docs/requirements.md index 9f9e61357..3c24b1867 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -211,6 +211,7 @@ request, to another check that proves the same thing; loosening a requirement or | Target, step and schedule rules pick the number of units, within min and max | [test_scaling_rules.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_scaling_rules.py) `test_a_target_rule_keeps_its_metric_at_the_target`, `test_a_step_rule_fires_once_its_condition_has_held`, `test_a_schedule_rule_sets_the_bounds_when_its_time_comes`, `test_the_largest_proposal_wins`; [test_scaling.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling.py) `test_rules_scale_on_waiting_jobs_and_on_a_schedule` | | The scaler launches units, sees them join, drains busy workers before letting them go, and fails and pauses on units that don't come up | [test_scaling.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling.py) `test_the_scaler_launches_joins_drains_and_terminates`, `test_a_unit_that_never_joins_fails_and_failures_pause_the_group`, `test_protected_and_terminated_units` | | Each unit's cost goes into the activity ledger as an estimate, and a group stops at its budget's cap | [test_scaling.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling.py) `test_the_scaler_launches_joins_drains_and_terminates`, `test_at_its_budgets_cap_a_group_stops` | +| Settings › Auto scaling shows each group's terms in words and approves them in one click | [settings-scaling.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/settings-scaling.test.tsx) `shows each group's terms and approves them in one click`, `puts the terms and the state in words` | | This machine runs more workers at no cost | [test_scaling.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling.py) `test_this_machine_runs_more_workers_at_no_cost` | | Agents manage scaling in the assistant and over MCP: proposals wait for the person, actions within the terms happen at once | [test_scaling_agents.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling_agents.py) `test_the_assistant_proposes_and_the_person_approves_in_the_web_app`, `test_a_setup_conversation_makes_a_group_that_still_waits_for_the_person`, `test_only_admins_get_the_scaling_tools`, `test_mcp_clients_manage_scaling_but_never_approve` | diff --git a/docs/scaling.md b/docs/scaling.md index 7c3576778..4ada0de54 100644 --- a/docs/scaling.md +++ b/docs/scaling.md @@ -6,16 +6,19 @@ unit is, how many there may be and the rules that pick the number, and a **unit* launched. Everything it does is in the [activity ledger](activity.md), and its cost counts against the group's [budget](budgets.md). -Status: in progress. Built: groups, rules, approvals, budgets, the scaler, the API, MCP and assistant tools, and -**This machine**, a provider that runs more `lens worker` processes where Lens runs. Planned: providers for AWS, -DigitalOcean, Hetzner, Proxmox and any Linux host over SSH, joining Lens over its Nebula mesh network; model -hosts (RunPod, Lambda, Hugging Face, SageMaker, Bedrock) for models; and the Settings › Scaling page. +Status: in progress. Built: groups, rules, approvals, budgets, the scaler, the API, MCP and assistant tools, +Settings › Auto scaling (the switch, and each group's terms to approve or pause), and **This machine**, a provider that +runs more `lens worker` processes where Lens runs. Planned: providers for AWS, DigitalOcean, Hetzner, Proxmox and any +Linux host over SSH, joining Lens over its Nebula mesh network; model hosts (RunPod, Lambda, Hugging Face, SageMaker, +Bedrock) for models; and making providers and groups in Settings. Auto scaling is **off** until an admin turns it on, and a group launches nothing until a person approves its terms signed in to the web app. Agents can propose and manage groups, but never approve one. ## Turn it on +**Settings › Auto scaling** turns it on, or: + ```http PUT /api/v1/settings/scaling {"enabled": true} @@ -107,8 +110,9 @@ When several rules speak, the largest number wins; the result stays within min a ## The person approves A group launches nothing until a person approves its **terms**: the provider, what each unit is, the max, the budget -and what happens at its cap. `POST /scaling/groups/{id}/approve` approves them as they stand, and only from a session -signed in to the web app: an API key, an app or an agent gets 403. A group made or changed by a person in the web app +and what happens at its cap. Settings › Auto scaling lists each group with its terms in words and an **Approve** +button; `POST /scaling/groups/{id}/approve` approves them as they stand, and only from a session signed in to the web +app: an API key, an app or an agent gets 403. A group made or changed by a person in the web app is approved as it's saved, since that is their pick. Within the approved terms anyone who may change the group, agents included, acts at once: asking for a number of units, diff --git a/fastapi_backend/app/domain/ops_tools.py b/fastapi_backend/app/domain/ops_tools.py index 655e50f09..d2648d393 100644 --- a/fastapi_backend/app/domain/ops_tools.py +++ b/fastapi_backend/app/domain/ops_tools.py @@ -496,7 +496,7 @@ def t_scaling(self): "drivers": [{k: d[k] for k in ("id", "kind", "label", "priced", "ready")} for d in v["drivers"]], } if not v["enabled"]: - out["note"] = "scaling is off: an admin turns it on in Settings › Scaling (change_settings scaling {enabled: true})" + out["note"] = "scaling is off: an admin turns it on in Settings › Auto scaling (change_settings scaling {enabled: true})" running = sum(u["state"] == "running" for u in v["units"]) return out, f"Read auto scaling: {len(v['groups'])} groups, {running} units running" diff --git a/fastapi_backend/app/domain/scaling.py b/fastapi_backend/app/domain/scaling.py index 439b5badb..03aac825e 100644 --- a/fastapi_backend/app/domain/scaling.py +++ b/fastapi_backend/app/domain/scaling.py @@ -584,13 +584,13 @@ def group_view(db, g, now=None): def waiting_for(view, cfg=None): """Why a group launches nothing now, in words, or None.""" if cfg is not None and not enabled(cfg): - return "scaling is off: turn it on in Settings › Scaling" + return "scaling is off: turn it on in Settings › Auto scaling" if not view.get("enabled"): return "the group is off" if view.get("paused"): return f"paused: {view.get('paused_why') or 'by hand'}" if not view.get("approved"): - return "a person's approval of its terms, in Settings › Scaling" + return "a person's approval of its terms, in Settings › Auto scaling" if view.get("capped"): return f"its budget: {view['capped']}" return None diff --git a/fastapi_backend/tests/api/test_scaling.py b/fastapi_backend/tests/api/test_scaling.py index eafdc8f31..91300004d 100644 --- a/fastapi_backend/tests/api/test_scaling.py +++ b/fastapi_backend/tests/api/test_scaling.py @@ -148,7 +148,7 @@ def test_only_a_person_in_the_web_app_approves_a_groups_terms(client, db, cfg, h # an agent proposes: saved, but it launches nothing until a person approves g = group(client, key, pid).json() assert (g["approved"], g["approval"]) == (False, None) and g["budget"]["usd"] == 20 - assert g["waiting_for"] == "scaling is off: turn it on in Settings › Scaling" + assert g["waiting_for"] == "scaling is off: turn it on in Settings › Auto scaling" gid = g["id"] assert client.post(f"/api/v1/scaling/groups/{gid}/approve", headers=key).status_code == 403 cfg = on(db, cfg) diff --git a/nextjs-frontend/__tests__/settings-scaling.test.tsx b/nextjs-frontend/__tests__/settings-scaling.test.tsx new file mode 100644 index 000000000..87dcd1c1b --- /dev/null +++ b/nextjs-frontend/__tests__/settings-scaling.test.tsx @@ -0,0 +1,96 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import "@testing-library/jest-dom"; + +import { Scaling } from "@/app/openapi-client"; +import type { Group, Provider } from "@/app/openapi-client/types.gen"; +import { groupState, termsText } from "@/components/settings/scaling-panel"; +import { SettingsSection } from "@/components/settings/section"; +import type { SettingsView } from "@/components/settings/model"; +import { TooltipProvider } from "@/components/ui/tooltip"; + +jest.mock("@/app/openapi-client", () => ({ + Admin: { updateSettings: jest.fn() }, + Scaling: { listGroups: jest.fn(), listProviders: jest.fn(), approveGroup: jest.fn(), pauseGroup: jest.fn() }, + Metadata: {}, +})); +jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); +jest.mock("next/navigation", () => ({ useRouter: () => ({ push: jest.fn() }) })); +jest.mock("@/lib/hooks/session", () => ({ useArchive: () => ({ namespaces: [] }) })); +jest.mock("@/components/ui/toast", () => ({ useToast: () => jest.fn() })); + +const ok = (data: unknown) => Promise.resolve({ data, response: { ok: true, status: 200 } }); +const VIEW = { + scaling: { values: { enabled: false, tick_seconds: 30, join_minutes: 15, accrue_minutes: 5, tag: "lens" } }, +} as unknown as SettingsView; +const PROVIDERS = [ + { id: 1, name: "Here", driver: "local", priced: false, settings: {}, secrets: {} }, + { id: 2, name: "Cloud", driver: "fake", priced: true, settings: {}, secrets: { api_key: true } }, +] as Provider[]; +const GROUP = { + id: 7, + name: "Transcribers", + kind: "node", + provider: 2, + template: {}, + min: 0, + max: 3, + desired: 0, + terminate: "oldest", + rules: [], + at_cap: "stop", + enabled: true, + paused: false, + approved: false, + terms: {}, + budget: { usd: 20, period: "month", state: "ok", spent: {}, left: {} }, + units: { launching: 0, running: 0, draining: 0 }, + waiting_for: "a person's approval of its terms, in Settings › Auto scaling", + failures: 0, +} as Group; + +function show() { + const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( + + + undefined} /> + + , + ); +} + +beforeEach(() => { + (Scaling.listProviders as jest.Mock).mockImplementation(() => ok(PROVIDERS)); +}); + +describe("auto scaling settings", () => { + it("shows each group's terms and approves them in one click", async () => { + (Scaling.listGroups as jest.Mock).mockImplementation(() => ok([GROUP])); + (Scaling.approveGroup as jest.Mock).mockImplementation(() => ok({ ...GROUP, approved: true })); + show(); + expect(screen.getByText("Scale workers")).toBeInTheDocument(); + expect(await screen.findByText("Up to 3 units on Cloud, up to $20 a month")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: /Approve Transcribers/ })); + await waitFor(() => expect(Scaling.approveGroup).toHaveBeenCalled()); + expect((Scaling.approveGroup as jest.Mock).mock.calls[0][0].path).toEqual({ gid: 7 }); + }); + + it("says when there are no groups yet", async () => { + (Scaling.listGroups as jest.Mock).mockImplementation(() => ok([])); + show(); + expect(await screen.findByText(/No scaling groups yet/)).toBeInTheDocument(); + }); + + it("puts the terms and the state in words", () => { + expect(termsText({ ...GROUP, provider: 1, budget: null, max: 1 }, PROVIDERS)).toBe( + "Up to 1 unit on Here, at no cost", + ); + expect(termsText({ ...GROUP, at_cap: "min", min: 1 }, PROVIDERS)).toBe( + "Up to 3 units on Cloud, up to $20 a month (at the cap, 1 keep running)", + ); + expect(groupState({ ...GROUP, waiting_for: null, units: { launching: 1, running: 2, draining: 0 } })).toBe( + "2 of 3 running, 1 joining", + ); + }); +}); diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index 4087f0def..45c530953 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -25,6 +25,7 @@ import { type ModelProvider, } from "@/components/settings/model-providers"; import { SandboxGatewaySwitch } from "@/components/settings/sandbox-gateway"; +import { ScalingGroups } from "@/components/settings/scaling-panel"; import { SignInProviders } from "@/components/settings/sign-in-providers"; import { UpdatesPanel } from "@/components/settings/updates-panel"; import { VoicePicker } from "@/components/settings/voice-picker"; @@ -543,6 +544,31 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) {

); + case "scaling": + return ( + <> + + Groups + + +
+ + + + +
+
+

+ A group launches nothing until you approve its terms here; agents (the assistant, MCP and API keys) make and + manage groups but can’t approve them. Costs are estimates from list prices, counted against each group’s + budget. Today groups run more workers on this machine; cloud providers are planned. +

+ + ); case "encryption": return ; case "backups": diff --git a/nextjs-frontend/components/settings/find.ts b/nextjs-frontend/components/settings/find.ts index 0bfe230c3..68676c536 100644 --- a/nextjs-frontend/components/settings/find.ts +++ b/nextjs-frontend/components/settings/find.ts @@ -27,6 +27,7 @@ export const SETTINGS_GROUPS: { label: string; sections: SectionId[] }[] = [ "backups", "components", "workers", + "scaling", "automations", "logs", "telemetry", diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index e49da533c..6d86885e1 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -73,6 +73,7 @@ export type SectionId = | "video" | "nsfw" | "workers" + | "scaling" | "automations" | "components" | "access" @@ -364,6 +365,13 @@ export const SECTIONS: SectionSpec[] = [ description: "Audio and video people upload in the web app: which types, how large, and how long an unfinished upload waits.", }, + { + id: "scaling", + label: "Auto scaling", + backend: ["scaling"], + description: + "Off unless you turn it on. Scaling groups add workers when work waits and let them go when it doesn’t, within a budget; each launches nothing until you approve its terms here. Agents make and manage groups over the API, MCP and the assistant.", + }, { id: "backups", label: "Backups", @@ -2657,6 +2665,51 @@ export const FIELDS: FieldSpec[] = [ hint: "One per line, like 192.168.1.0/24. NFS has no login, so machines anywhere else are turned away", }, { section: "nfs", key: "port", label: "Port inside Lens", kind: "int", min: 1, max: 65535 }, + // Auto scaling (docs/scaling.md), off by default + { + section: "scaling", + key: "enabled", + label: "Scale workers", + kind: "switch", + hint: "Off: nothing launches, and units already running are let go", + }, + { + section: "scaling", + key: "tick_seconds", + label: "Look every", + kind: "int", + min: 10, + max: 600, + unit: "seconds", + }, + { + section: "scaling", + key: "join_minutes", + label: "A new unit has to join within", + kind: "int", + min: 1, + max: 120, + unit: "minutes", + hint: "One that hasn’t by then counts as failed and is let go", + }, + { + section: "scaling", + key: "accrue_minutes", + label: "Count a unit’s cost every", + kind: "int", + min: 1, + max: 60, + unit: "minutes", + }, + { + section: "scaling", + key: "tag", + label: "Units’ names start with", + kind: "text", + mono: true, + placeholder: "lens", + hint: "Lowercase letters, digits and -, so they can be told apart at the provider", + }, // Backups (docs/database.md#backups) { section: "backups", diff --git a/nextjs-frontend/components/settings/scaling-panel.tsx b/nextjs-frontend/components/settings/scaling-panel.tsx new file mode 100644 index 000000000..f164a4fca --- /dev/null +++ b/nextjs-frontend/components/settings/scaling-panel.tsx @@ -0,0 +1,117 @@ +"use client"; + +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + +import { Scaling } from "@/app/openapi-client"; +import type { Group, Provider } from "@/app/openapi-client/types.gen"; +import { Banner } from "@/components/ui/banner"; +import { Button } from "@/components/ui/button"; +import { SkeletonRows } from "@/components/ui/states"; +import { useToast } from "@/components/ui/toast"; +import { data, useApiClient } from "@/lib/api/browser"; + +/** One of the API's reasons as a sentence: capitalised, with a full stop. */ +function sentence(s: string): string { + const t = s.trim(); + return t ? `${t.charAt(0).toUpperCase()}${t.slice(1)}${/[.!?]$/.test(t) ? "" : "."}` : t; +} + +/** What approving a group binds, in words: how many units at most, where, and what they may cost. */ +export function termsText(g: Group, providers: Provider[]): string { + const p = providers.find((x) => x.id === g.provider); + const where = p ? ` on ${p.name}` : ""; + const usd = g.budget?.usd; + const cost = usd != null ? `, up to $${usd} a month` : p && !p.priced ? ", at no cost" : ""; + const cap = g.at_cap === "min" && usd != null ? ` (at the cap, ${g.min} keep running)` : ""; + return `Up to ${g.max} unit${g.max === 1 ? "" : "s"}${where}${cost}${cap}`; +} + +/** Where a group stands: its units now, and why it launches nothing when it doesn't. */ +export function groupState(g: Group): string { + const u = g.units; + const parts = [`${u.running} of ${g.max} running`]; + if (u.launching) parts.push(`${u.launching} joining`); + if (u.draining) parts.push(`${u.draining} leaving`); + return parts.join(", ") + (g.waiting_for ? `. Waiting for ${g.waiting_for}` : ""); +} + +/** The scaling groups: each one's terms and units, approving its terms (a person's pick, here), pausing it. */ +export function ScalingGroups() { + const client = useApiClient(); + const qc = useQueryClient(); + const toast = useToast(); + const groups = useQuery({ + queryKey: ["scaling-groups"], + queryFn: () => data(Scaling.listGroups({ client })), + refetchInterval: 15_000, + }); + const providers = useQuery({ + queryKey: ["scaling-providers"], + queryFn: () => data(Scaling.listProviders({ client })), + }); + const fail = (title: string) => (e: Error) => toast({ title, body: e.message, tone: "red" }); + const done = () => void qc.invalidateQueries({ queryKey: ["scaling-groups"] }); + const approve = useMutation({ + mutationFn: (gid: number) => data(Scaling.approveGroup({ client, path: { gid } })), + onSuccess: (g) => { + done(); + toast({ title: `Approved ${g.name}`, tone: "green" }); + }, + onError: fail("Couldn’t approve the group"), + }); + const pause = useMutation({ + mutationFn: ({ gid, paused }: { gid: number; paused: boolean }) => + data(Scaling.pauseGroup({ client, path: { gid }, body: { paused } })), + onSuccess: done, + onError: fail("Couldn’t change the group"), + }); + if (groups.isPending) return ; + if (groups.error) + return ( + + {groups.error.message} + + ); + const list = groups.data; + if (!list.length) + return ( +

+ No scaling groups yet. Ask the assistant to set one up, or make one over the API or MCP; it shows here for you + to approve. +

+ ); + return ( +
    + {list.map((g) => ( +
  • +
    +
    {g.name}
    +
    {termsText(g, providers.data ?? [])}
    +
    {sentence(groupState(g))}
    +
    +
    + {!g.approved && ( + + )} + +
    +
  • + ))} +
+ ); +} diff --git a/nextjs-frontend/components/settings/settings-shell.tsx b/nextjs-frontend/components/settings/settings-shell.tsx index 2c3f1884f..050c05d1c 100644 --- a/nextjs-frontend/components/settings/settings-shell.tsx +++ b/nextjs-frontend/components/settings/settings-shell.tsx @@ -23,6 +23,7 @@ import { FileLock, FileType, FolderTree, + Gauge, Fingerprint, Globe, ScanText, @@ -90,6 +91,7 @@ const ICON: Record = { video: Clapperboard, nsfw: EyeOff, workers: Cpu, + scaling: Gauge, automations: Workflow, components: PackageCheck, access: ShieldCheck,