From fee61a6b99121d23bfb397e44a6cd74065f9d750 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 4 Oct 2026 09:32:22 +0000 Subject: [PATCH 1/6] feat(activity): ledger of calls, changes and cost per resource Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AMq5CNuWmtn3Suj1EamYBT --- CHANGELOG.md | 8 + DEVLOG.md | 35 ++ docs/activity.md | 68 +++ fastapi_backend/app/api/v1/router.py | 2 + fastapi_backend/app/api/v1/routes/activity.py | 83 +++ fastapi_backend/app/cli.py | 3 +- fastapi_backend/app/core/runtime.py | 3 +- fastapi_backend/app/domain/activity.py | 453 ++++++++++++++ fastapi_backend/app/domain/decide.py | 20 +- fastapi_backend/app/domain/extensions.py | 5 +- fastapi_backend/app/domain/jobs.py | 37 +- fastapi_backend/app/domain/notify.py | 12 +- fastapi_backend/app/domain/routines.py | 23 +- fastapi_backend/app/domain/semantic.py | 23 +- fastapi_backend/app/domain/settings.py | 21 + fastapi_backend/app/domain/store.py | 9 + fastapi_backend/app/domain/telemetry.py | 27 +- fastapi_backend/app/domain/voice.py | 8 +- fastapi_backend/app/main.py | 4 +- fastapi_backend/app/schemas/activity.py | 65 ++ fastapi_backend/tests/api/test_activity.py | 157 +++++ mkdocs.yml | 1 + nextjs-frontend/app/openapi-client/index.ts | 21 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 54 ++ .../app/openapi-client/types.gen.ts | 329 ++++++++++ nextjs-frontend/openapi.json | 575 ++++++++++++++++++ 26 files changed, 2005 insertions(+), 41 deletions(-) create mode 100644 docs/activity.md create mode 100644 fastapi_backend/app/api/v1/routes/activity.py create mode 100644 fastapi_backend/app/domain/activity.py create mode 100644 fastapi_backend/app/schemas/activity.py create mode 100644 fastapi_backend/tests/api/test_activity.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fc1854751..0460427ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Every resource has an activity history, with what it cost.** Lens now keeps a ledger of every request that + changes something, every call it makes out (models, embeddings, the decision model, text to speech, webhooks, web + tools) and every job and routine run, each counted for the resources it touched: a model call in a job a routine + queued counts for the job, its recording, namespace, pipeline, workflow and the routine. Tokens and estimated cost + come from the prices in Settings. `GET /api/v1/activity?resource=routine:3` gives a resource's history (with its + audit log entries), `/activity/totals` what it cost this day, week or month, `/activity/top` what cost most. On by + default; `activity.keep_days` (365) keeps it in bounds. See docs/activity.md. + - **Query the graph in Cypher, and walk it.** The archive is now a property graph of namespaces, collections, recordings, speakers and entities. `POST /api/v1/graph/query` runs read-only Cypher (the language of Neo4j and ISO GQL) over the namespaces you can read; `/graph/related` gives a node's parents, children, ancestors, descendants or diff --git a/DEVLOG.md b/DEVLOG.md index e6e642a46..d0acf27db 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -2,6 +2,41 @@ Plans and progress for work in flight. Newest first. +## 2026-10-04 · Activity history and budgets + +Goal (Adeel): every change and every call coming in or going out is tracked, with its cost, down to the resource, so +each resource has an activity history. Pipelines, workflows and routines can be capped at a budget, and before a run +(or something larger) the assistant looks at where the budget stands and whether the run should go ahead. + +Model: + +- One ledger, `activity`: a row per call in (API requests that change something), call out (model, embeddings, + decision model, webhooks, web tools) and run (a job, a routine run). Each row names every resource it touched + (`routine:3`, `routine_run:12`, `job:40`, `recording:7`, `pipeline:2`, `workflow:4`, `chat:9`, `space:1`, + `account:1`), with tokens, estimated cost (the prices in Settings), duration and outcome; never prompt or reply text. + Who a call is for comes from the work it runs in (a scope), so a model call inside a job counts for the job, its + recording, pipeline, namespace and the routine that queued it. +- A resource's history is its ledger rows plus its audit log entries. On by default, nothing to set up; kept + `activity.keep_days` (365). Reads (GET) aren't logged unless `activity.reads` is on. +- Budgets are off unless set: a budget on a routine, pipeline, workflow or namespace caps cost (USD) and/or tokens per + run, day, week or month. Before a routine run or a job, Lens estimates the run from past runs and checks what is + left. Over budget, it asks you (the default: the run waits for your pick), skips, or lets the assistant weigh it (the + decision model; unsure means ask). Doing nothing changes nothing: a waiting run stays waiting. + +Todo: + +- [ ] Activity ledger: scopes, model/embedding/decision/webhook/web-tool calls, changing API requests, job and routine + run totals, per-resource history and cost summary API, retention sweep +- [ ] Activity in the web app: a history panel on routines, pipelines, workflows and recordings; Settings → Activity + and costs with totals by resource +- [ ] Budgets: set per routine, pipeline, workflow or namespace; spent and estimate; checked before routine runs and + jobs; over budget waits for your pick (run once, skip), or the assistant decides +- [ ] Budgets in the web app: set a budget, see where it stands, approve or skip a waiting run +- [ ] Periodic check: warn at 80% and 100% of a budget (notifications and the assistant) + +Refine later: compute time as a cost (CPU seconds × a rate for local models), per-person budgets, forecasting from +schedules (a routine's next runs this period), budget alerts in the weekly digest. + ## 2026-10-04 · The graph, end to end Goal (Adeel): make the graph the one focus and nail it: a human explorer canvas, agent queries with rights, questions diff --git a/docs/activity.md b/docs/activity.md new file mode 100644 index 000000000..13f9787c2 --- /dev/null +++ b/docs/activity.md @@ -0,0 +1,68 @@ +# Activity and costs + +Lens keeps a ledger of what happens to each resource and what it cost: every request that changes something, every +call it makes out (a language model, embeddings, the decision model, text to speech, a webhook, a web tool), and every +job or routine run that ends. It is on by default and needs nothing set up. + +## What a row holds + +Each row names every resource it counts for, as `table:id`: + +| Resource | Example | +| --- | --- | +| A routine and one of its runs | `routine:3`, `routine_run:12` | +| A job, its recording, namespace and pipeline | `job:40`, `recording:7`, `space:1`, `pipeline:2` | +| A workflow | `workflow:4` | +| A chat | `chat:9` | +| The person who asked | `account:1` | + +A model call inside a workflow step of a job that a routine queued counts for the workflow, the job, the recording, +its namespace, its pipeline and the routine. A request counts for the resources in its path (`PATCH +/api/v1/routines/3` is `routine:3`), for what a `POST` made (from the `id` in its reply), and for who sent it. + +Besides the resources, a row has its kind (`in`, `out` or `run`), what it was (`POST /api/v1/routines/{rid}`, +`model.chat`, `embeddings`, `decision`, `notify.webhook`, `tool.http`, `job.succeeded`, `routine.done`), the model, +tokens in and out, the estimated cost in USD, how long it took, and how it ended (an error's type or HTTP status, +never its message). It never holds prompt, reply, transcript or file content. + +Cost comes from the prices you set: Settings › Telemetry › prices (USD per million tokens, per model), or the price +of the assistant's model under Settings › AI assistant. A model with no price counts tokens but no cost; local models +usually have none. `decisions.price_per_call` gives the decision model's price per decision. + +A run row (a job or a routine run ending) repeats what its calls cost in all, and the job or run keeps it too +(`cost_usd`, `tokens`). Totals don't count run rows twice. + +## Reading it + +A resource's history merges its ledger rows with its audit log entries (kind `change`), newest first: + +```http +GET /api/v1/activity?resource=routine:3&limit=100 +GET /api/v1/activity?resource=routine:3&kind=out&before=2026-10-04T09:00:00.000+00:00 +``` + +What it cost this day, week (from Monday) or month (UTC), or all time: + +```http +GET /api/v1/activity/totals?resource=pipeline:2&period=month +GET /api/v1/activity/totals?period=day # everything (admins) +GET /api/v1/activity/top?table=routine&period=week # what cost most (admins) +``` + +Admins see every resource. Everyone else sees their own account's activity, their own chats', and the recordings and +namespaces they can read. + +## Settings + +| Setting | Default | | +| --- | --- | --- | +| `activity.enabled` | `true` | Off stops new rows (the audit log carries on). | +| `activity.reads` | `false` | Log API reads (GET) too. The web app reads often, so this grows the ledger fast. | +| `activity.keep_days` | `365` | Rows older than this are dropped, hourly, by the routines scheduler. | + +Sign-in requests (`/api/v1/auth/`, `/api/v1/passkeys/`) aren't logged here; the audit log records sign-ins. + +## Refine later + +Compute time as a cost for local models (CPU seconds at a rate), per-person totals in the web app, and calls the +ledger doesn't see yet (Fedora sync, Matterbridge messages). diff --git a/fastapi_backend/app/api/v1/router.py b/fastapi_backend/app/api/v1/router.py index 85fb24028..06a5dee5f 100644 --- a/fastapi_backend/app/api/v1/router.py +++ b/fastapi_backend/app/api/v1/router.py @@ -3,6 +3,7 @@ from fastapi import APIRouter from app.api.v1.routes import ( + activity, admin, auth, batches, @@ -62,6 +63,7 @@ for module in ( users, admin, + activity, fedora, setup, namespaces, diff --git a/fastapi_backend/app/api/v1/routes/activity.py b/fastapi_backend/app/api/v1/routes/activity.py new file mode 100644 index 000000000..3773157ee --- /dev/null +++ b/fastapi_backend/app/api/v1/routes/activity.py @@ -0,0 +1,83 @@ +"""The activity ledger (docs/activity.md): a resource's history (its calls in and out, runs and changes) and what it +cost. Admins see every resource's; others see their own account's, their chats', and the recordings and namespaces they +can read.""" + +from __future__ import annotations + +import re +from typing import Literal + +from fastapi import APIRouter, HTTPException, Query + +from app.api.deps import Acl, AdminReader, CurrentUser, Db +from app.domain import activity +from app.schemas.activity import ActivityEntry, ActivityTop, ActivityTotals + +router = APIRouter(tags=["activity"]) +Period = Literal["day", "week", "month", "all"] +RESOURCE = re.compile(r"^([a-z_]{1,40}):([A-Za-z0-9_.-]{1,80})$") + + +def _may_see(db: Db, user: CurrentUser, acl: Acl, resource: str) -> None: + m = RESOURCE.match(resource) + if not m: + raise HTTPException(422, "resource is table:id, like routine:3 or recording:12") + if user.admin: + return + table, key = m.groups() + if table == "account" and key == str(user.id): + return + if key.isdigit(): + if table == "recording": + acl.recording(int(key)) + return + if table == "space": + acl.need(int(key)) + return + if table == "chat" and db.one("SELECT id FROM $r WHERE account = $a", r=activity.R("chat", int(key)), a=user.id): + return + raise HTTPException(403, "only admins see this resource's activity") + + +@router.get("/activity") +def resource_history( + user: CurrentUser, + acl: Acl, + db: Db, + resource: str = Query(description="table:id, like routine:3, pipeline:2, workflow:4, recording:12, account:1"), + kind: Literal["in", "out", "run", "change"] | None = None, + before: str | None = Query(None, description="only entries before this time (the last `at` of the page before)"), + limit: int = Query(100, ge=1, le=1000), +) -> list[ActivityEntry]: + """A resource's history, newest first: API requests that changed it, calls made for it (models, embeddings, the + decision model, webhooks, web tools) with tokens and cost, runs that ended, and its audit log entries.""" + _may_see(db, user, acl, resource) + return activity.history(db, resource, limit, before, kind) + + +@router.get("/activity/totals") +def resource_totals( + user: CurrentUser, acl: Acl, db: Db, resource: str | None = Query(None, description="table:id; none for everything (admins)"), + period: Period = "month", +) -> ActivityTotals: + """What a resource's calls cost this day, week (from Monday), month (UTC) or all time: calls, tokens, USD, time.""" + if resource is None: + if not user.admin: + raise HTTPException(403, "only admins see the totals for everything") + else: + _may_see(db, user, acl, resource) + since = activity._since(period) + return ActivityTotals(resource=resource, period=period, since=since, **activity.totals(db, resource, since)) + + +@router.get("/activity/top") +def top_resources( + user: AdminReader, + db: Db, + table: str | None = Query(None, pattern=r"^[a-z_]{1,40}$", description="only this kind of resource: routine, pipeline, workflow, space"), + period: Period = "month", + limit: int = Query(20, ge=1, le=200), +) -> ActivityTop: + """The resources that cost most this period (admins).""" + since = activity._since(period) + return ActivityTop(period=period, since=since, resources=activity.top(db, table, since, limit)) diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py index 49d9ce3ab..ec94af6a3 100644 --- a/fastapi_backend/app/cli.py +++ b/fastapi_backend/app/cli.py @@ -8,7 +8,7 @@ import shutil import sys -from .domain import analyze, components, graph, ingest, render, store +from .domain import activity, analyze, components, graph, ingest, render, store from .domain import search as searchmod from .domain import speakers as spk @@ -249,6 +249,7 @@ def platform_main(argv, config): cfg = store.load_config(config) components.activate(cfg) # packages and models fetched into the data folder db = store.connect(cfg) + activity.bind(db) # calls made here reach the activity ledger (docs/activity.md) C = settings.Settings(db, cfg).current def password(): diff --git a/fastapi_backend/app/core/runtime.py b/fastapi_backend/app/core/runtime.py index 4587dcc21..f3a4dd572 100644 --- a/fastapi_backend/app/core/runtime.py +++ b/fastapi_backend/app/core/runtime.py @@ -10,7 +10,7 @@ from typing import Any from app.config import settings as env -from app.domain import auth, bridge, components, content_types, jobs, notify, routines, sensors, settings, setup, store, templates, tunnel +from app.domain import activity, auth, bridge, components, content_types, jobs, notify, routines, sensors, settings, setup, store, templates, tunnel log = logging.getLogger("lens") @@ -22,6 +22,7 @@ def __init__(self, cfg: dict[str, Any] | None = None, db: store.DB | None = None self.base = cfg or store.load_config(env.ARCHIVE_CONFIG) components.activate(self.base) # packages and models fetched into the data folder self.db = db or store.connect(self.base) + activity.bind(self.db) # calls made outside any job, run or request still reach the ledger self.owns_db = db is None self.settings = settings.Settings(self.db, self.base) self.stop = threading.Event() diff --git a/fastapi_backend/app/domain/activity.py b/fastapi_backend/app/domain/activity.py new file mode 100644 index 000000000..215cc5ec2 --- /dev/null +++ b/fastapi_backend/app/domain/activity.py @@ -0,0 +1,453 @@ +"""Activity: what happened to each resource, and what it cost (docs/activity.md). + +One ledger, the `activity` table, a row for each + +* call in: an API request that changes something (POST, PUT, PATCH, DELETE; reads too when `activity.reads` is on), + counted for the resources in its path, what it made (a POST's reply {"id"}) and who called; +* call out: a model call, embeddings, the decision model, text to speech, a webhook, a web tool; +* run: a job or a routine run ending, with what it cost in all. + +A row names every resource it touched as `table:id` strings (`routine:3`, `routine_run:12`, `job:40`, `recording:7`, +`pipeline:2`, `workflow:4`, `chat:9`, `space:1`, `account:1`), so a resource's history is the rows that name it, next +to its audit log entries. It carries tokens, the estimated cost in USD (from the prices in Settings), how long it took +and how it ended; never prompt, reply or file content. + +Who a call is for comes from the work it runs in: `with activity.scope(db, "job:40", "recording:7"):` around a job, a +routine run or a workflow, and the API request's own resources (from its path) and caller. Scopes nest, so a model +call inside a workflow step of a job queued by a routine counts for all of them. + +On by default (activity.enabled) and cheap: one small row per call. Kept activity.keep_days (365), then dropped by the +routines scheduler. Writing a row never breaks the work it describes. +""" + +from __future__ import annotations + +import contextlib +import contextvars +import datetime as dt +import json +import logging +import re +import time + +from . import store + +log = logging.getLogger("lens") +R = store.R + +KINDS = ("in", "out", "run") +TABLES = { # the API's path segments, and the table a number after one names + "recordings": "recording", + "resources": "recording", + "routines": "routine", + "runs": "routine_run", + "pipelines": "pipeline", + "workflows": "workflow", + "custom-nodes": "custom_node", + "chats": "chat", + "jobs": "job", + "namespaces": "space", + "spaces": "space", + "collections": "collection", + "extensions": "extension", + "sources": "watch_path", + "watches": "watch_path", + "sensors": "sensor", + "templates": "template", + "notes": "note", + "comments": "comment", + "entities": "entity", + "speakers": "speaker", + "users": "account", + "batches": "batch", + "budgets": "budget", +} +REF = re.compile(r"^[a-z_]{1,40}:[A-Za-z0-9_.-]{1,80}$") +CHANGING = {"POST", "PUT", "PATCH", "DELETE"} +SKIP_PATHS = ("/api/v1/auth/", "/api/v1/passkeys/") # sign-in traffic is in the audit log; its bodies are secrets +SWEEP_SECONDS = 3600 +CREATED_BYTES = 2048 # how much of a POST's reply is read for the id of what it made +FIELDS = "record::id(id) AS id, at, kind, action, resources, actor, model, tokens_in, tokens_out, cost_usd, ms, ok, error, detail" + +_scope: contextvars.ContextVar[dict | None] = contextvars.ContextVar("lens_activity", default=None) +_default = {"db": None} +_swept = {"at": 0.0} + + +def bind(db): + """The process's database, for calls made outside any scope (the API, `lens worker`).""" + _default["db"] = db + + +def _section(cfg): + return (cfg or {}).get("activity") or {} + + +def enabled(cfg): + return _section(cfg).get("enabled", True) is not False + + +def ref(table, key): + return f"{table}:{key}" + + +@contextlib.contextmanager +def scope(db, *refs, cfg=None): + """Calls made inside count for these resources too (and for the scopes around this one).""" + outer = _scope.get() + mine = [r for r in refs if r] + token = _scope.set( + { + "db": db if db is not None else (outer or {}).get("db"), + "refs": [*((outer or {}).get("refs") or []), *mine], + "cfg": cfg if cfg is not None else (outer or {}).get("cfg"), + "asgi": (outer or {}).get("asgi"), + } + ) + try: + yield + finally: + _scope.reset(token) + + +def current(): + """The resources calls made here count for, in order, without repeats.""" + s = _scope.get() or {} + out = list(s.get("refs") or []) + asgi = s.get("asgi") + if asgi is not None: + out += request_refs(asgi) + return list(dict.fromkeys(r for r in out if REF.match(r))) + + +def request_refs(asgi): + """An API request's resources: each number (or path parameter) after a known path segment, and who called.""" + out = [] + params = {str(v) for v in (asgi.get("path_params") or {}).values()} + parts = [p for p in (asgi.get("path") or "").split("/") if p] + for a, b in zip(parts, parts[1:]): + table = TABLES.get(a) + if table and (b.isdigit() or b in params) and b not in TABLES: + out.append(ref(table, b)) + p = (asgi.get("state") or {}).get("principal") + if p is not None and getattr(p, "id", None) is not None: + out.append(ref("account", p.id)) + return out + + +def _actor(asgi): + p = ((asgi or {}).get("state") or {}).get("principal") + return getattr(p, "id", None) + + +def record(kind, action, refs=(), cfg=None, db=None, **fields): + """Write one row. `fields`: model, tokens_in, tokens_out, cost_usd, ms, ok, error (a type name), detail (a small + dict of counts and ids). Returns the row's id, or None when it wasn't written.""" + s = _scope.get() or {} + cfg = cfg if cfg is not None else s.get("cfg") + if cfg is not None and not enabled(cfg): + return None + db = db or s.get("db") or _default["db"] + if db is None: + return None + resources = list(dict.fromkeys([*current(), *(r for r in refs if r and REF.match(r))])) + row = store.clean( + { + "at": dt.datetime.now(dt.timezone.utc).isoformat(timespec="milliseconds"), # in order within a second + "kind": kind, + "action": action[:200], + "resources": resources, + "actor": fields.pop("actor", None) or _actor(s.get("asgi")), + **{k: v for k, v in fields.items() if k in ("model", "tokens_in", "tokens_out", "cost_usd", "ms", "ok", "error", "detail")}, + } + ) + row.setdefault("ok", True) + try: + got = db.values("CREATE activity CONTENT $d RETURN VALUE record::id(id)", d=row) + return got[0] if got else None + except Exception as e: # noqa: BLE001 - the ledger never breaks the work it describes + log.debug("activity: couldn't write a row: %s", e) + return None + + +class Call: + """A call out, timed: `with activity.call("embeddings", model=m) as c: ...; c.usage(tin, tout)`. An exception that + leaves the block marks the row failed with its type.""" + + def __init__(self, action, cfg=None, model=None, refs=(), detail=None, price=None): + self.action, self.cfg, self.model, self.refs, self.detail, self.price = action, cfg, model, refs, detail, price + self.tin = self.tout = self.cost = None + self.t0 = time.monotonic() + self.done = False + + def usage(self, tokens_in=None, tokens_out=None, cost_usd=None): + if isinstance(tokens_in, int): + self.tin = tokens_in + if isinstance(tokens_out, int): + self.tout = tokens_out + if cost_usd is not None: + self.cost = cost_usd + return self + + def end(self, error=None): + if self.done: + return + self.done = True + cost = self.cost + if cost is None and self.price is not None: + cost = self.price(self.model, self.tin, self.tout) + record( + "out", + self.action, + self.refs, + self.cfg, + model=self.model, + tokens_in=self.tin, + tokens_out=self.tout, + cost_usd=cost, + ms=int((time.monotonic() - self.t0) * 1000), + ok=error is None, + error=type(error).__name__ if error is not None else None, + detail=self.detail, + ) + + def __enter__(self): + return self + + def __exit__(self, et, e, tb): + self.end(e) + return False + + +def call(action, cfg=None, model=None, refs=(), detail=None, price=None): + return Call(action, cfg, model, refs, detail, price) + + +def token_cost(cfg): + """A price function for Call: USD from the per-million-token prices in Settings (telemetry.prices).""" + from . import telemetry + + prices = telemetry.prices(cfg) + + def price(model, tin, tout): + if tin is None and tout is None: + return None + return telemetry.cost(model, tin or 0, tout or 0, prices) + + return price + + +# ---------- the API ---------- +class Middleware: + """A row for each API request that changes something (or every request, with activity.reads on), naming the + resources in its path and who called; calls the request makes count for those resources too.""" + + def __init__(self, app): + self.app = app + + async def __call__(self, scope_, receive, send): + if scope_["type"] != "http" or not scope_.get("path", "").startswith("/api/"): + await self.app(scope_, receive, send) + return + app = scope_.get("app") + db = getattr(getattr(app, "state", None), "db", None) + token = _scope.set({"db": db, "refs": [], "cfg": None, "asgi": scope_}) + status, head = [None], [b""] + + async def sending(message): + if message["type"] == "http.response.start": + status[0] = message["status"] + elif message["type"] == "http.response.body" and len(head[0]) < CREATED_BYTES and scope_.get("method") == "POST": + head[0] += message.get("body", b"")[:CREATED_BYTES] + await send(message) + + t0, error = time.monotonic(), None + try: + await self.app(scope_, receive, sending) + except Exception as e: + error = e + raise + finally: + try: + self._log(scope_, app, db, status[0], error, t0, head[0]) + finally: + _scope.reset(token) + + def _log(self, scope_, app, db, status, error, t0, head=b""): + method = scope_.get("method") or "GET" + path = scope_.get("path") or "" + if db is None or path.startswith(SKIP_PATHS): + return + try: + cfg = app.state.settings.current() + except Exception: # noqa: BLE001 + return + if not enabled(cfg) or (method not in CHANGING and not _section(cfg).get("reads")): + return + from . import telemetry + + code = status or 500 + record( + "in", + f"{method} {telemetry._route_template(scope_)}", + [created(path, head)] if method == "POST" and 200 <= code < 300 else (), + cfg=cfg, + db=db, + ms=int((time.monotonic() - t0) * 1000), + ok=error is None and code < 400, + error=type(error).__name__ if error is not None else (str(code) if code >= 400 else None), + detail={"status": code}, + ) + + +def created(path, head): + """What a POST to a collection made, from its reply ({"id": n}): `routine:7` for POST /api/v1/routines.""" + table = TABLES.get(path.rstrip("/").rsplit("/", 1)[-1]) + if not table or not head: + return None + try: + got = json.loads(head) + except ValueError: + return None + key = got.get("id") if isinstance(got, dict) else None + return ref(table, key) if isinstance(key, (int, str)) and not isinstance(key, bool) else None + + +# ---------- reading it ---------- +def _since(period, now=None): + """The start of the period now is in (UTC): day, week (from Monday), month; None for all time.""" + now = now or dt.datetime.now(dt.timezone.utc) + if period == "day": + start = now.replace(hour=0, minute=0, second=0, microsecond=0) + elif period == "week": + start = (now - dt.timedelta(days=now.weekday())).replace(hour=0, minute=0, second=0, microsecond=0) + elif period == "month": + start = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + else: + return None + return start.isoformat(timespec="seconds") + + +def _when(at): + """A row's time, for ordering ledger rows (milliseconds) and audit entries (seconds) together.""" + try: + t = dt.datetime.fromisoformat(str(at)) + return t if t.tzinfo else t.replace(tzinfo=dt.timezone.utc) + except ValueError: + return dt.datetime.min.replace(tzinfo=dt.timezone.utc) + + +def history(db, resource, limit=100, before=None, kind=None): + """A resource's activity, newest first: its ledger rows and its audit log entries, merged.""" + where, p = ["resources CONTAINS $res"], {"res": resource, "n": limit} + if before: + where.append("at < $before") + p["before"] = before + if kind: + where.append("kind = $kind") + p["kind"] = kind + rows = db.rows(f"SELECT {FIELDS} FROM activity WHERE {' AND '.join(where)} ORDER BY at DESC LIMIT $n", **p) + if kind in (None, "change"): + audit = db.rows( + "SELECT at, account, email, action, target, detail FROM audit_log WHERE target = $res" + + (" AND at < $before" if before else "") + + " ORDER BY at DESC LIMIT $n", + **p, + ) + rows += [ + { + "id": None, + "at": a["at"], + "kind": "change", + "action": a["action"], + "resources": [resource], + "actor": a.get("account"), + "email": a.get("email"), + "ok": True, + "detail": a.get("detail") if isinstance(a.get("detail"), dict) else ({"items": a["detail"]} if a.get("detail") else None), + } + for a in audit + ] + rows.sort(key=lambda r: _when(r.get("at")), reverse=True) + rows = rows[:limit] + return rows + + +def totals(db, resource=None, since=None, until=None): + """What a resource (or everything) cost: calls, tokens, USD and time, by kind of call.""" + where, p = [], {} + if resource: + where.append("resources CONTAINS $res") + p["res"] = resource + if since: + where.append("at >= $since") + p["since"] = since + if until: + where.append("at < $until") + p["until"] = until + rows = db.rows( + "SELECT kind, count() AS calls, math::sum(tokens_in ?? 0) AS tokens_in, math::sum(tokens_out ?? 0) AS tokens_out, " + "math::sum(cost_usd ?? 0) AS cost_usd, math::sum(ms ?? 0) AS ms, count(ok = false) AS failed FROM activity" + + (f" WHERE {' AND '.join(where)}" if where else "") + + " GROUP BY kind", + **p, + ) + out = {"calls": 0, "tokens_in": 0, "tokens_out": 0, "cost_usd": 0.0, "ms": 0, "failed": 0, "by_kind": {}} + for r in rows: + k = r.get("kind") or "?" + # a run row repeats what its calls cost (its cost is their sum): counted on its own, not twice + out["by_kind"][k] = {x: r.get(x) or 0 for x in ("calls", "tokens_in", "tokens_out", "cost_usd", "ms", "failed")} + out["calls"] += r.get("calls") or 0 + out["failed"] += r.get("failed") or 0 + if k != "run": + for x in ("tokens_in", "tokens_out", "cost_usd", "ms"): + out[x] += r.get(x) or 0 + out["cost_usd"] = round(out["cost_usd"], 6) + return out + + +def spent(db, resource, since=None): + """(USD, tokens) a resource's calls cost since a time (not counting run rows, which repeat them).""" + p = {"res": resource} + q = "SELECT math::sum(cost_usd ?? 0) AS usd, math::sum((tokens_in ?? 0) + (tokens_out ?? 0)) AS tokens FROM activity WHERE resources CONTAINS $res AND kind != 'run'" + if since: + q += " AND at >= $since" + p["since"] = since + row = db.one(q + " GROUP ALL", **p) or {} + return round(float(row.get("usd") or 0), 6), int(row.get("tokens") or 0) + + +def top(db, prefix=None, since=None, limit=20): + """The resources that cost most since a time, of one table (`routine`) or any: [{resource, cost_usd, tokens, calls}].""" + p = {"n": limit, "p": f"{prefix}:" if prefix else ""} + inner = "kind != 'run' AND array::len(resources ?? []) > 0" + if since: + inner += " AND at >= $since" + p["since"] = since + rows = db.rows( + "SELECT res AS resource, math::sum(cost_usd ?? 0) AS cost_usd, math::sum((tokens_in ?? 0) + (tokens_out ?? 0)) AS tokens, " + f"count() AS calls FROM (SELECT resources AS res, cost_usd, tokens_in, tokens_out FROM activity WHERE {inner} SPLIT res) " + "WHERE string::starts_with(res, $p) GROUP BY res ORDER BY cost_usd DESC, tokens DESC LIMIT $n", + **p, + ) + for r in rows: + r["cost_usd"] = round(float(r.get("cost_usd") or 0), 6) + return rows + + +def run_total(db, resource): + """What one run's calls cost: (USD or None, tokens or None), for its run row.""" + usd, tokens = spent(db, resource) + return (usd or None), (tokens or None) + + +def tidy(db, cfg, now=None): + """Drop rows older than activity.keep_days, at most once an hour per process; the cut-off, or 0 when it didn't look.""" + t = time.monotonic() + if now is None and t - _swept["at"] < SWEEP_SECONDS and _swept["at"]: + return 0 + _swept["at"] = t + days = int(_section(cfg).get("keep_days") or 365) + cut = ((now or dt.datetime.now(dt.timezone.utc)) - dt.timedelta(days=days)).isoformat(timespec="seconds") + db.q("DELETE activity WHERE at < $cut", cut=cut) + return cut diff --git a/fastapi_backend/app/domain/decide.py b/fastapi_backend/app/domain/decide.py index 7a0e8d6fb..7f3bc5356 100644 --- a/fastapi_backend/app/domain/decide.py +++ b/fastapi_backend/app/domain/decide.py @@ -14,7 +14,7 @@ import urllib.error import urllib.request -from . import llm +from . import activity, llm log = logging.getLogger(__name__) # Jev reads up to about 32k tokens; a state is cut well below that @@ -53,13 +53,17 @@ def _jev(cfg, questions, state): headers=headers, method="POST", ) - try: - with urllib.request.urlopen(req, timeout=d.get("timeout") or 10) as r: - return json.load(r)["answers"] - except urllib.error.HTTPError as e: - raise Undecided(f"{e.code} from the decision model: {e.read().decode('utf-8', 'replace')[:200]}") from None - except (urllib.error.URLError, TimeoutError, OSError, ValueError, KeyError) as e: - raise Undecided(f"can't reach the decision model: {e}") from None + with activity.call("decision", cfg, body["model"], detail={"questions": len(questions)}) as ledger: + try: + with urllib.request.urlopen(req, timeout=d.get("timeout") or 10) as r: + answers = json.load(r)["answers"] + except urllib.error.HTTPError as e: + raise Undecided(f"{e.code} from the decision model: {e.read().decode('utf-8', 'replace')[:200]}") from None + except (urllib.error.URLError, TimeoutError, OSError, ValueError, KeyError) as e: + raise Undecided(f"can't reach the decision model: {e}") from None + if d.get("price_per_call") is not None: + ledger.usage(cost_usd=float(d["price_per_call"])) + return answers def _ranked(probs): diff --git a/fastapi_backend/app/domain/extensions.py b/fastapi_backend/app/domain/extensions.py index 25466b2ac..38c9a2c22 100644 --- a/fastapi_backend/app/domain/extensions.py +++ b/fastapi_backend/app/domain/extensions.py @@ -32,7 +32,7 @@ import yaml -from . import auth, store +from . import activity, auth, store R = store.R KINDS = ("tool", "skill", "hook", "plugin") @@ -748,7 +748,8 @@ def http_call(cfg, run, args): data = json.dumps(fill_json(body, args)).encode("utf-8") headers.setdefault("Content-Type", "application/json") req = urllib.request.Request(url, data=data, method=run.get("method") or "GET", headers=headers) - with netguard.Guard(forward=True, networks=webcapture.networks(cfg), max_bytes=MAX_REPLY + 65536) as guard: + host = urllib.parse.urlsplit(url).hostname + with activity.call("tool.http", cfg, detail={"host": host, "method": req.get_method()}), netguard.Guard(forward=True, networks=webcapture.networks(cfg), max_bytes=MAX_REPLY + 65536) as guard: try: with feeds._opener(guard).open(req, timeout=30) as r: status, ctype, raw = r.status, r.headers.get("Content-Type", ""), r.read(MAX_REPLY) diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index af33d0fce..3aaa1b415 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -14,7 +14,7 @@ import threading import time -from . import analyze, ingest, keyring, pipelines, render, speakers as spk, store, telemetry +from . import activity, analyze, ingest, keyring, pipelines, render, speakers as spk, store, telemetry R = store.R PIPELINE = ["transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe", "analyze", "embed", "summarize", "report"] @@ -116,7 +116,8 @@ def _export(db, cfg, rid, say, spec=None): def _workflow(db, cfg, rid, say, spec=None): from . import workflows - done = workflows.run(db, cfg, rid, int(spec["workflow"]), spec.get("version"), say) + with activity.scope(db, f"workflow:{int(spec['workflow'])}"): + done = workflows.run(db, cfg, rid, int(spec["workflow"]), spec.get("version"), say) ran = sum(1 for v in done.values() if v == "done") - 1 # not counting the input say(f"workflow ran {ran} of {len(done) - 1} nodes") @@ -421,7 +422,39 @@ def eta(job, est, now=None): return round(left + sum(est[i + 1 :]), 1) +FINAL = ("succeeded", "failed", "cancelled") + + +def job_refs(job): + """The resources a job's calls count for: it, its recording, namespace and pipeline, and the routine that queued it.""" + pipe = job.get("pipeline") if isinstance(job.get("pipeline"), dict) else {} + by = job.get("created_by") + return [ + f"job:{job['id']}", + f"recording:{job['recording']}", + f"space:{job['space']}" if job.get("space") is not None else None, + f"pipeline:{pipe['id']}" if pipe.get("id") is not None else None, + by if isinstance(by, str) and by.startswith("routine:") else None, + ] + + def run_job(db, cfg_fn, job, worker, can, log=None): + """Run a job's steps from where it is, with its calls counted for it (activity.py); when it ends, a run row with + what it cost in all, kept on the job too (cost_usd, tokens).""" + refs = job_refs(job) + with activity.scope(db, *refs): + outcome = _run_job(db, cfg_fn, job, worker, can, log) + if outcome in FINAL: + usd, tokens = activity.run_total(db, refs[0]) + try: + db.q("UPDATE $j SET cost_usd = $u, tokens = $n", j=R("job", job["id"]), u=usd, n=tokens) + except Exception: # noqa: BLE001 - the ledger never breaks the work + pass + activity.record("run", f"job.{outcome}", refs, cfg_fn(), db, cost_usd=usd, tokens_in=tokens, ok=outcome != "failed") + return outcome + + +def _run_job(db, cfg_fn, job, worker, can, log=None): jid, rid, jr, wr = job["id"], job["recording"], R("job", job["id"]), R("worker", worker) steps, i = job["steps"], job.get("step_index") or 0 out = RunLog(db, jid, job.get("space"), job.get("log"), job.get("log_total")) diff --git a/fastapi_backend/app/domain/notify.py b/fastapi_backend/app/domain/notify.py index f6804622c..e3ae9a08b 100644 --- a/fastapi_backend/app/domain/notify.py +++ b/fastapi_backend/app/domain/notify.py @@ -39,7 +39,7 @@ import time import urllib.parse -from . import netguard, settings, store +from . import activity, netguard, settings, store R = store.R EVENTS = { @@ -381,6 +381,16 @@ def send(db, cfg, t, event, msg_id): error = f"couldn't reach it: {type(e).__name__}: {e}"[:300] last = store.clean({"at": store.now(), "ok": error is None, "code": code, "error": error, "event": event["type"]}) db.q("UPDATE $r SET last = $l", r=R("notify_target", int(t["id"])), l=last) + activity.record( + "out", + f"notify.{t.get('kind') or 'webhook'}", + [f"notify_target:{t['id']}", f"space:{t['space']}" if t.get("space") is not None else None], + cfg, + db, + ok=error is None, + error=None if error is None else (f"HTTP {code}" if code else "unreachable"), + detail=store.clean({"event": event["type"], "status": code}), + ) return {"ok": error is None, "code": code, "error": error} diff --git a/fastapi_backend/app/domain/routines.py b/fastapi_backend/app/domain/routines.py index b321fa24a..dbef575f8 100644 --- a/fastapi_backend/app/domain/routines.py +++ b/fastapi_backend/app/domain/routines.py @@ -22,7 +22,7 @@ import datetime as dt import threading -from . import fedora, jobs, organize, schedule, semantic, sensor_digests, sensor_patterns, sensors, sources, store, telemetry, workflows +from . import activity, fedora, jobs, organize, schedule, semantic, sensor_digests, sensor_patterns, sensors, sources, store, telemetry, workflows R = store.R ACTIONS = ("sync", "pipeline", "workflow", "sensors") @@ -335,7 +335,8 @@ def _action(db, cfg, routine, run_id, a, seen, propose_only, say): w = workflows.get(db, int(a["workflow"]), a.get("version")) if w["scope"] == "graph": origin = {"routine": routine["id"], "run": run_id} - done, stats = organize.run(db, cfg, w["id"], spaces, w["version"], say, origin, propose_only or a.get("propose_only", False)) + with activity.scope(db, f"workflow:{w['id']}"): + done, stats = organize.run(db, cfg, w["id"], spaces, w["version"], say, origin, propose_only or a.get("propose_only", False)) return {"workflow": w["name"], "version": w["version"], "nodes": sum(v == "done" for v in done.values()), **stats} rids = _recordings(db, spaces, a.get("recordings", "new"), seen, limit, cfg) step = {"type": "workflow", "workflow": w["id"], "version": w["version"]} @@ -363,10 +364,23 @@ def run(db, cfg, rid, trigger="manual", by=None, propose_only=False, log=None): def _run(db, cfg, rid, trigger, by, propose_only, log): + """A run, with its calls counted for the routine and the run (activity.py); its run row says what it cost.""" + run_id = db.next_id("routine_run") + refs = [f"routine:{int(rid)}", f"routine_run:{run_id}"] + with activity.scope(db, *refs, cfg=cfg): + status = _run_actions(db, cfg, rid, run_id, trigger, by, propose_only, log) + usd, tokens = activity.run_total(db, refs[1]) + db.q("UPDATE $r SET cost_usd = $u, tokens = $n", r=R("routine_run", run_id), u=usd, n=tokens) + activity.record("run", f"routine.{status}", refs, cfg, db, cost_usd=usd, tokens_in=tokens, ok=status != "error", + detail={"trigger": trigger}) + return run_id + + +def _run_actions(db, cfg, rid, run_id, trigger, by, propose_only, log): routine = get(db, rid) since = routine.get("seen_recording") or 0 seen = {"since": since, "marks": [], "failed": False} - run_id, lines = db.next_id("routine_run"), [] + lines = [] started = store.now() def say(msg): @@ -416,7 +430,7 @@ def say(msg): s=status, i=run_id, ) - return run_id + return status def _claim(db, r, now): @@ -474,6 +488,7 @@ def run_due(db, cfg, log=print, now=None): now = now or dt.datetime.now(dt.timezone.utc) stamp = _iso(now) sweep(db, now) + activity.tidy(db, cfg) # the activity ledger past activity.keep_days, once an hour due = db.rows( "SELECT record::id(id) AS id, schedule, timezone, next_run_at, run_now, namespaces, actions FROM routine " "WHERE run_now != NONE OR (enabled = true AND next_run_at != NONE AND next_run_at <= $n)", diff --git a/fastapi_backend/app/domain/semantic.py b/fastapi_backend/app/domain/semantic.py index dff804435..c8a087d0f 100644 --- a/fastapi_backend/app/domain/semantic.py +++ b/fastapi_backend/app/domain/semantic.py @@ -24,7 +24,7 @@ import urllib.request from collections import OrderedDict -from . import store +from . import activity, store R = store.R log = logging.getLogger("lens") @@ -125,15 +125,18 @@ def embed(cfg, texts, timeout=None): headers["Authorization"] = f"Bearer {key}" body = json.dumps({"model": model, "input": [t[:EMBED_CHARS] for t in texts]}).encode() req = urllib.request.Request(base + "/embeddings", data=body, headers=headers, method="POST") - try: - with urllib.request.urlopen(req, timeout=timeout or _section(cfg).get("timeout") or 60) as r: - j = json.load(r) - except urllib.error.HTTPError as e: - raise EmbedError(f"{e.code} from the embeddings server for {model}: {e.read().decode('utf-8', 'replace')[:300]}") from None - except (urllib.error.URLError, TimeoutError, OSError) as e: - raise EmbedError(f"can't reach the embeddings server at {base}: {e}") from None - except ValueError: - raise EmbedError("the embeddings server's reply wasn't JSON") from None + with activity.call("embeddings", cfg, model, detail={"texts": len(texts)}, price=activity.token_cost(cfg)) as ledger: + try: + with urllib.request.urlopen(req, timeout=timeout or _section(cfg).get("timeout") or 60) as r: + j = json.load(r) + except urllib.error.HTTPError as e: + raise EmbedError(f"{e.code} from the embeddings server for {model}: {e.read().decode('utf-8', 'replace')[:300]}") from None + except (urllib.error.URLError, TimeoutError, OSError) as e: + raise EmbedError(f"can't reach the embeddings server at {base}: {e}") from None + except ValueError: + raise EmbedError("the embeddings server's reply wasn't JSON") from None + u = j.get("usage") if isinstance(j, dict) and isinstance(j.get("usage"), dict) else {} + ledger.usage(u.get("prompt_tokens", u.get("total_tokens"))) try: data = sorted(j["data"], key=lambda d: d.get("index", 0)) vecs = [[float(x) for x in d["embedding"]] for d in data] diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index 44a249726..3f0c44a7c 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -42,6 +42,7 @@ "bridge": None, "notifications": None, "telemetry": None, + "activity": None, "encryption": None, "fedora": None, "tunnel": None, @@ -354,6 +355,8 @@ def _check(section, key, value, default): return _notify_setting(key, value) if section == "telemetry": return _telemetry_setting(key, value) + if section == "activity": + return _activity_setting(key, value) if section == "fedora": return _fedora_setting(key, value) if section == "sensors": @@ -513,9 +516,27 @@ def _decision_setting(key, value): if not (isinstance(value, (int, float)) and not isinstance(value, bool) and 1 <= value <= 120): raise ValueError("decisions.timeout is a number of seconds from 1 to 120") return value + if key == "price_per_call": + if value is None or value == "": + return None + if not (isinstance(value, (int, float)) and not isinstance(value, bool) and 0 <= value <= 100): + raise ValueError("decisions.price_per_call is what one decision costs, in USD, from 0 to 100") + return float(value) raise ValueError(f"unknown setting decisions.{key}") +def _activity_setting(key, value): + if key in ("enabled", "reads"): + if not isinstance(value, bool): + raise ValueError(f"activity.{key} is true or false") + return value + if key == "keep_days": + if not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 3650): + raise ValueError("activity.keep_days is a whole number of days from 1 to 3650") + return value + raise ValueError(f"unknown setting activity.{key}") + + EMBED_RANGES = {"passage_chars": (200, 4000), "batch_size": (1, 256), "neighbours": (5, 500), "timeout": (5, 600)} diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 2916aa33e..78de6bec6 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -275,6 +275,7 @@ def labels(): "api_key": None, "act_above": 0.8, "timeout": 10, + "price_per_call": None, # USD per decision, for the activity ledger's cost (none: not counted) }, "video": { "sample_seconds": 5, @@ -346,6 +347,9 @@ def labels(): # encryption at rest (docs/encryption.md): files Lens keeps under data_dir, encrypted with their namespace's key; # work_minutes: how long a plain working copy for ffmpeg and the other tools is kept after its last use "encryption": {"files": False, "work_minutes": 30, "vault_minutes": 60}, + # the activity ledger (docs/activity.md): every change and call in or out, with its cost, per resource. On by + # default; reads: log API reads (GET) too; keep_days: how long rows are kept + "activity": {"enabled": True, "reads": False, "keep_days": 365}, "telemetry": { "enabled": False, "endpoint": None, @@ -747,6 +751,11 @@ def close(self): "DEFINE INDEX IF NOT EXISTS share_embed_share ON share_embed FIELDS share", "DEFINE TABLE IF NOT EXISTS audit_log SCHEMALESS", "DEFINE INDEX IF NOT EXISTS audit_at ON audit_log FIELDS at", + "DEFINE INDEX IF NOT EXISTS audit_target ON audit_log FIELDS target", + # what happened to each resource and what it cost (domain/activity.py): a row per call in, call out and run + "DEFINE TABLE IF NOT EXISTS activity SCHEMALESS", + "DEFINE INDEX IF NOT EXISTS activity_at ON activity FIELDS at", + "DEFINE INDEX IF NOT EXISTS activity_resources ON activity FIELDS resources", # background work "DEFINE TABLE IF NOT EXISTS job SCHEMALESS", "DEFINE INDEX IF NOT EXISTS job_status ON job FIELDS status", diff --git a/fastapi_backend/app/domain/telemetry.py b/fastapi_backend/app/domain/telemetry.py index 3bcad2c3a..343370cbf 100644 --- a/fastapi_backend/app/domain/telemetry.py +++ b/fastapi_backend/app/domain/telemetry.py @@ -352,16 +352,20 @@ def cost(model, input_tokens, output_tokens, prices=None): class ModelCall: - """One call to the model server: a client span (gen_ai.* attributes), its duration, tokens and estimated cost. - Neither the prompt nor the reply is recorded.""" + """One call to the model server: a client span (gen_ai.* attributes), its duration, tokens and estimated cost, and + a row in the activity ledger (activity.py) whether or not telemetry is on. Neither the prompt nor the reply is + recorded.""" def __init__(self, cfg, payload, operation="chat", current=True): + from . import activity # the ledger counts every call, whether or not telemetry is on + + self.model = payload.get("model") + self.operation, self.t0, self.done, self.response_model, self.usage = operation, time.monotonic(), False, None, None + self.ledger = activity.call(f"model.{operation}", cfg, self.model, price=activity.token_cost(cfg)) self.on = active() if not self.on: return l = (cfg or {}).get("llm") or {} - self.model = payload.get("model") - self.operation, self.t0, self.done, self.response_model, self.usage = operation, time.monotonic(), False, None, None server = urllib.parse.urlsplit(l.get("base_url") or "") attrs = { "gen_ai.operation.name": operation, @@ -386,7 +390,7 @@ def __init__(self, cfg, payload, operation="chat", current=True): def reply(self, j): """Note the usage and model of a reply (a parsed JSON body or a streamed chunk); returns it.""" - if self.on and isinstance(j, dict): + if isinstance(j, dict): if isinstance(j.get("usage"), dict): self.usage = j["usage"] if isinstance(j.get("model"), str): @@ -394,9 +398,15 @@ def reply(self, j): return j def end(self, error=None): - if not self.on or self.done: + if self.done: return self.done = True + u = self.usage or {} + tin, tout = u.get("prompt_tokens", u.get("input_tokens")), u.get("completion_tokens", u.get("output_tokens")) + self.ledger.usage(tin, tout) + self.ledger.end(error) + if not self.on: + return seconds = time.monotonic() - self.t0 base = {"gen_ai.operation.name": self.operation, "gen_ai.provider.name": "openai", "gen_ai.request.model": self.model} if self.response_model: @@ -407,8 +417,6 @@ def end(self, error=None): record("gen_ai.client.operation.duration", seconds, {**base, "error.type": type(error).__name__}) else: record("gen_ai.client.operation.duration", seconds, base) - u = self.usage or {} - tin, tout = u.get("prompt_tokens", u.get("input_tokens")), u.get("completion_tokens", u.get("output_tokens")) if isinstance(tin, int): self.span.set_attribute("gen_ai.usage.input_tokens", tin) record("gen_ai.client.token.usage", tin, {**base, "gen_ai.token.type": "input"}) @@ -434,7 +442,8 @@ def __exit__(self, et, e, tb): def model_call(cfg, payload, operation="chat", current=True): - """`with model_call(cfg, payload) as call: ... call.reply(json)`: a measured model call (a no-op while off).""" + """`with model_call(cfg, payload) as call: ... call.reply(json)`: a measured model call (only the ledger's row while + telemetry is off).""" return ModelCall(cfg, payload, operation, current) diff --git a/fastapi_backend/app/domain/voice.py b/fastapi_backend/app/domain/voice.py index 997a50cec..4db63b1a6 100644 --- a/fastapi_backend/app/domain/voice.py +++ b/fastapi_backend/app/domain/voice.py @@ -16,7 +16,7 @@ import urllib.error import urllib.request -from . import ingest +from . import activity, ingest log = logging.getLogger(__name__) IDLE_SECONDS = 600 @@ -127,9 +127,13 @@ def speak(cfg, text): headers={"Content-Type": "application/json", **({"Authorization": f"Bearer {key}"} if key else {})}, method="POST", ) + ledger = activity.call("model.speech", cfg, v["tts_model"], detail={"chars": len(body["input"])}) try: with urllib.request.urlopen(req, timeout=60) as r: - return r.read(), r.headers.get_content_type() or "audio/mpeg" + out = r.read(), r.headers.get_content_type() or "audio/mpeg" + ledger.end() + return out except (urllib.error.URLError, TimeoutError, OSError) as e: + ledger.end(e) log.info("voice: text-to-speech failed, the browser reads it: %s", e) return None diff --git a/fastapi_backend/app/main.py b/fastapi_backend/app/main.py index bcad215a1..96e31f1df 100644 --- a/fastapi_backend/app/main.py +++ b/fastapi_backend/app/main.py @@ -26,7 +26,7 @@ from app.config import settings from app.core import middleware from app.core.runtime import Archive -from app.domain import __version__, store, telemetry +from app.domain import __version__, activity, store, telemetry from app.domain.render import WEB_DIR from app.utils import simple_generate_unique_route_id @@ -73,6 +73,8 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # innermost, so the router has picked the route it names spans by; a no-op while telemetry is off app.add_middleware(telemetry.Middleware) + # a row in the activity ledger for each request that changes something, naming its resources (docs/activity.md) + app.add_middleware(activity.Middleware) app.add_middleware( CORSMiddleware, allow_origins=sorted(settings.CORS_ORIGINS), diff --git a/fastapi_backend/app/schemas/activity.py b/fastapi_backend/app/schemas/activity.py new file mode 100644 index 000000000..8f80eb9d5 --- /dev/null +++ b/fastapi_backend/app/schemas/activity.py @@ -0,0 +1,65 @@ +"""The activity ledger: what happened to a resource and what it cost (docs/activity.md).""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import Field + +from app.schemas.common import ResponseModel + + +class ActivityEntry(ResponseModel): + id: str | None = Field(None, description="the ledger row's id; none for an audit log entry") + at: str + kind: Literal["in", "out", "run", "change"] = Field( + description="in: an API request; out: a call Lens made (a model, embeddings, the decision model, a webhook, a " + "web tool); run: a job or routine run ended; change: an audit log entry" + ) + action: str = Field(description="what it was: `POST /api/v1/routines/{rid}/run`, `model.chat`, `job.succeeded`, `routine.update`") + resources: list[str] = Field(default_factory=list, description="every resource it counts for, as table:id") + actor: int | None = Field(None, description="the account that made it, when a person did") + email: str | None = None + model: str | None = None + tokens_in: int | None = None + tokens_out: int | None = None + cost_usd: float | None = Field(None, description="estimated from the prices in Settings; none when the model has no price") + ms: int | None = None + ok: bool = True + error: str | None = Field(None, description="the error's type or HTTP status; never its message") + detail: Any = None + + +class KindTotals(ResponseModel): + calls: int = 0 + tokens_in: int = 0 + tokens_out: int = 0 + cost_usd: float = 0 + ms: int = 0 + failed: int = 0 + + +class ActivityTotals(ResponseModel): + resource: str | None = None + period: str + since: str | None = None + calls: int + failed: int + tokens_in: int + tokens_out: int + cost_usd: float = Field(description="what its calls cost (run rows repeat their calls' cost and aren't added again)") + ms: int + by_kind: dict[str, KindTotals] + + +class ResourceCost(ResponseModel): + resource: str + cost_usd: float + tokens: int + calls: int + + +class ActivityTop(ResponseModel): + period: str + since: str | None = None + resources: list[ResourceCost] diff --git a/fastapi_backend/tests/api/test_activity.py b/fastapi_backend/tests/api/test_activity.py new file mode 100644 index 000000000..afc249932 --- /dev/null +++ b/fastapi_backend/tests/api/test_activity.py @@ -0,0 +1,157 @@ +"""The activity ledger: every changing request and every call out, with its cost, counted for each resource it touched.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from app.domain import activity, routines, store +from tests import fake_llm +from tests.helpers import drain, login, make_user, seed + + +@pytest.fixture +def model(cfg): + srv, url = fake_llm.start() + cfg["llm"].update(base_url=url, model="fake") + cfg["telemetry"]["prices"] = {"fake": {"input": 1.0, "output": 2.0}} # USD per million tokens + cfg["ai"]["tools"] = False # chats get the plain, streamed answer + fake_llm.Handler.usage = {"prompt_tokens": 1000, "completion_tokens": 500, "total_tokens": 1500} + yield fake_llm.Handler + fake_llm.Handler.usage = None + srv.shutdown() + + +@pytest.fixture +def app(cfg, db, model): + from app.main import create_app + + return create_app(cfg, db, background=False) + + +@pytest.fixture +def h(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +def uid(db, email): + return db.one("SELECT record::id(id) AS id FROM account WHERE email = $e", e=email)["id"] + + +def rows(db, resource, kind=None): + return [r for r in activity.history(db, resource, 500) if kind is None or r["kind"] == kind] + + +def test_a_routines_calls_count_for_it_its_run_and_the_jobs_it_queued(client, db, cfg, folder, h): + seed(db, cfg, folder) + rid = client.post( + "/api/v1/routines", + headers=h, + json={"name": "Summaries", "actions": [{"type": "pipeline", "steps": ["summarize"], "recordings": "all"}]}, + ).json()["id"] + run_id = routines.run(db, cfg, rid, by="test") + assert drain(db, cfg) == 3 + + calls = rows(db, f"routine:{rid}", "out") + assert len(calls) == 3 and {c["action"] for c in calls} == {"model.chat"} + for c in calls: + assert (c["model"], c["tokens_in"], c["tokens_out"], c["ok"]) == ("fake", 1000, 500, True) + assert c["cost_usd"] == pytest.approx(0.002) + job = next(r for r in c["resources"] if r.startswith("job:")) + assert {f"routine:{rid}", "pipeline:standard"} - set(c["resources"]) <= {"pipeline:standard"} + assert any(r.startswith("recording:") for r in c["resources"]) and any(r.startswith("space:") for r in c["resources"]) + # each job's own total, on the job and as its run row + jid = int(job.split(":")[1]) + assert db.one("SELECT cost_usd, tokens FROM $j", j=store.R("job", jid)) == {"cost_usd": pytest.approx(0.002), "tokens": 1500} + assert [r["action"] for r in rows(db, job, "run")] == ["job.succeeded"] + + # the routine's history: the request that made it, its run, and the audit entry + hist = rows(db, f"routine:{rid}") + assert {"in", "out", "run", "change"} <= {r["kind"] for r in hist} + assert any(r["kind"] == "change" and r["action"] == "routine.create" for r in hist) + assert db.one("SELECT cost_usd FROM $r", r=store.R("routine_run", run_id))["cost_usd"] is None # queued, not run, in the run + assert [r["action"] for r in rows(db, f"routine_run:{run_id}", "run")] == ["routine.done"] + + t = client.get("/api/v1/activity/totals", headers=h, params={"resource": f"routine:{rid}", "period": "day"}).json() + assert (t["tokens_in"], t["tokens_out"]) == (3000, 1500) and t["cost_usd"] == pytest.approx(0.006) + assert t["by_kind"]["out"]["calls"] == 3 and t["by_kind"]["run"]["calls"] >= 4 # three jobs and the routine run + top = client.get("/api/v1/activity/top", headers=h, params={"table": "routine"}).json()["resources"] + assert top[0]["resource"] == f"routine:{rid}" and top[0]["cost_usd"] == pytest.approx(0.006) and top[0]["tokens"] == 4500 + everything = client.get("/api/v1/activity/totals", headers=h).json() + assert everything["cost_usd"] == pytest.approx(0.006) + + +def test_changing_requests_are_logged_with_their_resources_and_caller(client, db, h): + rid = client.post("/api/v1/routines", headers=h, json={"name": "Nightly", "actions": [{"type": "sync"}]}).json()["id"] + assert client.patch(f"/api/v1/routines/{rid}", headers=h, json={"name": "Nightly sync"}).status_code == 200 + assert client.get(f"/api/v1/routines/{rid}", headers=h).status_code == 200 # reads aren't logged + assert client.patch("/api/v1/routines/999", headers=h, json={"name": "x"}).status_code == 404 + me = uid(db, "root@x.io") + + got = rows(db, f"routine:{rid}", "in") + assert [r["action"] for r in got] == ["PATCH /api/v1/routines/{rid}", "POST /api/v1/routines"] # made, then changed + assert got[0]["actor"] == me and f"account:{me}" in got[0]["resources"] and got[0]["detail"] == {"status": 200} + failed = rows(db, "routine:999", "in") + assert (failed[0]["ok"], failed[0]["error"]) == (False, "404") + mine = [r["action"] for r in rows(db, f"account:{me}", "in")] + assert "POST /api/v1/routines" in mine + assert not any("/auth/" in a for a in mine) # signing in is in the audit log, never with its body + + # reads too, once asked + assert client.put("/api/v1/settings/activity", headers=h, json={"reads": True}).status_code == 200 + client.get(f"/api/v1/routines/{rid}", headers=h) + assert "GET /api/v1/routines/{rid}" in [r["action"] for r in rows(db, f"routine:{rid}", "in")] + + # off: nothing more is written + assert client.put("/api/v1/settings/activity", headers=h, json={"enabled": False, "reads": False}).status_code == 200 + ins, changes = len(rows(db, f"routine:{rid}", "in")), len(rows(db, f"routine:{rid}", "change")) + client.patch(f"/api/v1/routines/{rid}", headers=h, json={"name": "again"}) + assert len(rows(db, f"routine:{rid}", "in")) == ins and len(rows(db, f"routine:{rid}", "change")) == changes + 1 # audit only + + +def test_chats_and_people_see_only_their_own(client, db, cfg, folder, h, model): + seed(db, cfg, folder) + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + e = login(client, "ed@x.io", "editor password 1") + cid = client.post("/api/v1/chats", headers=e, json={"scope": {"namespaces": ["pods"]}}).json()["id"] + assert client.post(f"/api/v1/chats/{cid}/messages", headers=e, json={"content": "What was it about?"}).status_code == 200 + me = uid(db, "ed@x.io") + + got = client.get("/api/v1/activity", headers=e, params={"resource": f"chat:{cid}", "kind": "out"}).json() + assert [r["action"] for r in got] == ["model.chat"] and f"account:{me}" in got[0]["resources"] + assert got[0]["cost_usd"] == pytest.approx(0.002), [(r["tokens_in"], r["model"]) for r in got] + assert client.get("/api/v1/activity", headers=e, params={"resource": f"account:{me}"}).status_code == 200 + pods = store.ns_id(db, "pods") + assert client.get("/api/v1/activity", headers=e, params={"resource": f"space:{pods}"}).status_code == 200 + for other in ("routine:1", "account:1", f"space:{store.ns_id(db, 'calls')}"): + assert client.get("/api/v1/activity", headers=e, params={"resource": other}).status_code in (403, 404), other + assert client.get("/api/v1/activity", headers=e, params={"resource": "not a resource"}).status_code == 422 + assert client.get("/api/v1/activity/totals", headers=e).status_code == 403 + assert client.get("/api/v1/activity/top", headers=e).status_code == 403 + # an admin sees anyone's + assert client.get("/api/v1/activity", headers=h, params={"resource": f"chat:{cid}"}).status_code == 200 + + +def test_old_rows_go_after_keep_days(db, cfg): + old = (dt.datetime.now(dt.UTC) - dt.timedelta(days=400)).isoformat(timespec="seconds") + db.q("CREATE activity CONTENT $d", d={"at": old, "kind": "out", "action": "model.chat", "resources": ["routine:1"]}) + activity.record("out", "model.chat", ["routine:1"], cfg, db) + activity.tidy(db, cfg, now=dt.datetime.now(dt.UTC)) + assert [r["at"] > old for r in rows(db, "routine:1")] == [True] + + +def test_scopes_nest_and_writing_never_breaks_the_work(db, cfg): + with activity.scope(db, "routine:1", cfg=cfg): + with activity.scope(None, "job:2", None, "not a ref"): + assert activity.current() == ["routine:1", "job:2"] + activity.record("out", "x") + assert activity.current() == ["routine:1"] + assert rows(db, "job:2")[0]["resources"] == ["routine:1", "job:2"] + + class Broken: + def values(self, *a, **k): + raise RuntimeError("down") + + assert activity.record("out", "x", db=Broken()) is None diff --git a/mkdocs.yml b/mkdocs.yml index ebf3509dd..02571a2f5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -75,6 +75,7 @@ nav: - Chat rooms: chat-rooms.md - Sensors: sensors.md - Telemetry: telemetry.md + - Activity and costs: activity.md - IIIF: iiif.md - Graph and Cypher: graph.md - Linked data (RDF): rdf.md diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index d43175085..ce8042583 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -2,6 +2,7 @@ export { AccessRequests, + Activity, Admin, Auth, Batches, @@ -53,6 +54,9 @@ export type { AcceptGraphChangeResponses, AccessRequest, AccountToken, + ActivityEntry, + ActivityTop, + ActivityTotals, AddFileData, AddFileError, AddFileErrors, @@ -1070,6 +1074,7 @@ export type { JobQueued, JobsCreate, JobsQueued, + KindTotals, LanguageCount, LastSend, LinkEntityData, @@ -1725,8 +1730,19 @@ export type { ResetPasswordRequest, ResetPasswordResponse, ResetPasswordResponses, + ResourceCost, ResourceFile, ResourceFiles, + ResourceHistoryData, + ResourceHistoryError, + ResourceHistoryErrors, + ResourceHistoryResponse, + ResourceHistoryResponses, + ResourceTotalsData, + ResourceTotalsError, + ResourceTotalsErrors, + ResourceTotalsResponse, + ResourceTotalsResponses, RetagRecordingsData, RetagRecordingsError, RetagRecordingsErrors, @@ -2098,6 +2114,11 @@ export type { TokenResponse, TokenResponses, ToolStep, + TopResourcesData, + TopResourcesError, + TopResourcesErrors, + TopResourcesResponse, + TopResourcesResponses, TranscribeClipData, TranscribeClipResponse, TranscribeClipResponses, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 83b2dc969..fb765bfe5 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -847,6 +847,12 @@ import type { ResetPasswordData, ResetPasswordErrors, ResetPasswordResponses, + ResourceHistoryData, + ResourceHistoryErrors, + ResourceHistoryResponses, + ResourceTotalsData, + ResourceTotalsErrors, + ResourceTotalsResponses, RetagRecordingsData, RetagRecordingsErrors, RetagRecordingsResponses, @@ -1013,6 +1019,9 @@ import type { TokenLimitsData, TokenLimitsResponses, TokenResponses, + TopResourcesData, + TopResourcesErrors, + TopResourcesResponses, TranscribeClipData, TranscribeClipResponses, TryWorkflowData, @@ -2323,6 +2332,51 @@ export class Admin { } } +export class Activity { + /** + * Resource History + * + * A resource's history, newest first: API requests that changed it, calls made for it (models, embeddings, the + * decision model, webhooks, web tools) with tokens and cost, runs that ended, and its audit log entries. + */ + public static resourceHistory( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/activity", + ...options, + }); + } + + /** + * Resource Totals + * + * What a resource's calls cost this day, week (from Monday), month (UTC) or all time: calls, tokens, USD, time. + */ + public static resourceTotals( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/activity/totals", + ...options, + }); + } + + /** + * Top Resources + * + * The resources that cost most this period (admins). + */ + public static topResources( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/activity/top", + ...options, + }); + } +} + export class Fedora { /** * Get Fedora Status diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 217fdbd3d..74b85f134 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -102,6 +102,157 @@ export type AccountToken = { [key: string]: unknown; }; +/** + * ActivityEntry + */ +export type ActivityEntry = { + /** + * Id + * + * the ledger row's id; none for an audit log entry + */ + id?: string | null; + /** + * At + */ + at: string; + /** + * Kind + * + * in: an API request; out: a call Lens made (a model, embeddings, the decision model, a webhook, a web tool); run: a job or routine run ended; change: an audit log entry + */ + kind: "in" | "out" | "run" | "change"; + /** + * Action + * + * what it was: `POST /api/v1/routines/{rid}/run`, `model.chat`, `job.succeeded`, `routine.update` + */ + action: string; + /** + * Resources + * + * every resource it counts for, as table:id + */ + resources?: Array; + /** + * Actor + * + * the account that made it, when a person did + */ + actor?: number | null; + /** + * Email + */ + email?: string | null; + /** + * Model + */ + model?: string | null; + /** + * Tokens In + */ + tokens_in?: number | null; + /** + * Tokens Out + */ + tokens_out?: number | null; + /** + * Cost Usd + * + * estimated from the prices in Settings; none when the model has no price + */ + cost_usd?: number | null; + /** + * Ms + */ + ms?: number | null; + /** + * Ok + */ + ok?: boolean; + /** + * Error + * + * the error's type or HTTP status; never its message + */ + error?: string | null; + /** + * Detail + */ + detail?: unknown; + [key: string]: unknown; +}; + +/** + * ActivityTop + */ +export type ActivityTop = { + /** + * Period + */ + period: string; + /** + * Since + */ + since?: string | null; + /** + * Resources + */ + resources: Array; + [key: string]: unknown; +}; + +/** + * ActivityTotals + */ +export type ActivityTotals = { + /** + * Resource + */ + resource?: string | null; + /** + * Period + */ + period: string; + /** + * Since + */ + since?: string | null; + /** + * Calls + */ + calls: number; + /** + * Failed + */ + failed: number; + /** + * Tokens In + */ + tokens_in: number; + /** + * Tokens Out + */ + tokens_out: number; + /** + * Cost Usd + * + * what its calls cost (run rows repeat their calls' cost and aren't added again) + */ + cost_usd: number; + /** + * Ms + */ + ms: number; + /** + * By Kind + */ + by_kind: { + [key: string]: KindTotals; + }; + [key: string]: unknown; +}; + /** * AnswerCheck */ @@ -5540,6 +5691,37 @@ export type JobsQueued = { [key: string]: unknown; }; +/** + * KindTotals + */ +export type KindTotals = { + /** + * Calls + */ + calls?: number; + /** + * Tokens In + */ + tokens_in?: number; + /** + * Tokens Out + */ + tokens_out?: number; + /** + * Cost Usd + */ + cost_usd?: number; + /** + * Ms + */ + ms?: number; + /** + * Failed + */ + failed?: number; + [key: string]: unknown; +}; + /** * LanguageCount */ @@ -9474,6 +9656,29 @@ export type ResetPasswordRequest = { password: string; }; +/** + * ResourceCost + */ +export type ResourceCost = { + /** + * Resource + */ + resource: string; + /** + * Cost Usd + */ + cost_usd: number; + /** + * Tokens + */ + tokens: number; + /** + * Calls + */ + calls: number; + [key: string]: unknown; +}; + /** * ResourceFile */ @@ -15670,6 +15875,130 @@ export type ReindexSearchResponses = { export type ReindexSearchResponse = ReindexSearchResponses[keyof ReindexSearchResponses]; +export type ResourceHistoryData = { + body?: never; + path?: never; + query: { + /** + * Resource + * + * table:id, like routine:3, pipeline:2, workflow:4, recording:12, account:1 + */ + resource: string; + /** + * Kind + */ + kind?: "in" | "out" | "run" | "change" | null; + /** + * Before + * + * only entries before this time (the last `at` of the page before) + */ + before?: string | null; + /** + * Limit + */ + limit?: number; + }; + url: "/api/v1/activity"; +}; + +export type ResourceHistoryErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ResourceHistoryError = ResourceHistoryErrors[keyof ResourceHistoryErrors]; + +export type ResourceHistoryResponses = { + /** + * Response Activity-Resource History + * + * Successful Response + */ + 200: Array; +}; + +export type ResourceHistoryResponse = ResourceHistoryResponses[keyof ResourceHistoryResponses]; + +export type ResourceTotalsData = { + body?: never; + path?: never; + query?: { + /** + * Resource + * + * table:id; none for everything (admins) + */ + resource?: string | null; + /** + * Period + */ + period?: "day" | "week" | "month" | "all"; + }; + url: "/api/v1/activity/totals"; +}; + +export type ResourceTotalsErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ResourceTotalsError = ResourceTotalsErrors[keyof ResourceTotalsErrors]; + +export type ResourceTotalsResponses = { + /** + * Successful Response + */ + 200: ActivityTotals; +}; + +export type ResourceTotalsResponse = ResourceTotalsResponses[keyof ResourceTotalsResponses]; + +export type TopResourcesData = { + body?: never; + path?: never; + query?: { + /** + * Table + * + * only this kind of resource: routine, pipeline, workflow, space + */ + table?: string | null; + /** + * Period + */ + period?: "day" | "week" | "month" | "all"; + /** + * Limit + */ + limit?: number; + }; + url: "/api/v1/activity/top"; +}; + +export type TopResourcesErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type TopResourcesError = TopResourcesErrors[keyof TopResourcesErrors]; + +export type TopResourcesResponses = { + /** + * Successful Response + */ + 200: ActivityTop; +}; + +export type TopResourcesResponse = TopResourcesResponses[keyof TopResourcesResponses]; + export type GetFedoraStatusData = { body?: never; path?: never; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 5cb83de2b..982f797df 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -2626,6 +2626,256 @@ } } }, + "/api/v1/activity": { + "get": { + "tags": [ + "activity" + ], + "summary": "Resource History", + "description": "A resource's history, newest first: API requests that changed it, calls made for it (models, embeddings, the\ndecision model, webhooks, web tools) with tokens and cost, runs that ended, and its audit log entries.", + "operationId": "resource_history", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "table:id, like routine:3, pipeline:2, workflow:4, recording:12, account:1", + "title": "Resource" + }, + "description": "table:id, like routine:3, pipeline:2, workflow:4, recording:12, account:1" + }, + { + "name": "kind", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "enum": [ + "in", + "out", + "run", + "change" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Kind" + } + }, + { + "name": "before", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "only entries before this time (the last `at` of the page before)", + "title": "Before" + }, + "description": "only entries before this time (the last `at` of the page before)" + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 1000, + "minimum": 1, + "default": 100, + "title": "Limit" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ActivityEntry" + }, + "title": "Response Activity-Resource History" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/activity/totals": { + "get": { + "tags": [ + "activity" + ], + "summary": "Resource Totals", + "description": "What a resource's calls cost this day, week (from Monday), month (UTC) or all time: calls, tokens, USD, time.", + "operationId": "resource_totals", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "table:id; none for everything (admins)", + "title": "Resource" + }, + "description": "table:id; none for everything (admins)" + }, + { + "name": "period", + "in": "query", + "required": false, + "schema": { + "enum": [ + "day", + "week", + "month", + "all" + ], + "type": "string", + "default": "month", + "title": "Period" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ActivityTotals" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/activity/top": { + "get": { + "tags": [ + "activity" + ], + "summary": "Top Resources", + "description": "The resources that cost most this period (admins).", + "operationId": "top_resources", + "parameters": [ + { + "name": "table", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-z_]{1,40}$" + }, + { + "type": "null" + } + ], + "description": "only this kind of resource: routine, pipeline, workflow, space", + "title": "Table" + }, + "description": "only this kind of resource: routine, pipeline, workflow, space" + }, + { + "name": "period", + "in": "query", + "required": false, + "schema": { + "enum": [ + "day", + "week", + "month", + "all" + ], + "type": "string", + "default": "month", + "title": "Period" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 200, + "minimum": 1, + "default": 20, + "title": "Limit" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ActivityTop" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/admin/fedora": { "get": { "tags": [ @@ -19744,6 +19994,265 @@ "title": "AccountToken", "description": "Anyone's key, for admins." }, + "ActivityEntry": { + "properties": { + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id", + "description": "the ledger row's id; none for an audit log entry" + }, + "at": { + "type": "string", + "title": "At" + }, + "kind": { + "type": "string", + "enum": [ + "in", + "out", + "run", + "change" + ], + "title": "Kind", + "description": "in: an API request; out: a call Lens made (a model, embeddings, the decision model, a webhook, a web tool); run: a job or routine run ended; change: an audit log entry" + }, + "action": { + "type": "string", + "title": "Action", + "description": "what it was: `POST /api/v1/routines/{rid}/run`, `model.chat`, `job.succeeded`, `routine.update`" + }, + "resources": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Resources", + "description": "every resource it counts for, as table:id" + }, + "actor": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Actor", + "description": "the account that made it, when a person did" + }, + "email": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Email" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "tokens_in": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens In" + }, + "tokens_out": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens Out" + }, + "cost_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost Usd", + "description": "estimated from the prices in Settings; none when the model has no price" + }, + "ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Ms" + }, + "ok": { + "type": "boolean", + "title": "Ok", + "default": true + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error", + "description": "the error's type or HTTP status; never its message" + }, + "detail": { + "title": "Detail" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "at", + "kind", + "action" + ], + "title": "ActivityEntry" + }, + "ActivityTop": { + "properties": { + "period": { + "type": "string", + "title": "Period" + }, + "since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Since" + }, + "resources": { + "items": { + "$ref": "#/components/schemas/ResourceCost" + }, + "type": "array", + "title": "Resources" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "period", + "resources" + ], + "title": "ActivityTop" + }, + "ActivityTotals": { + "properties": { + "resource": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Resource" + }, + "period": { + "type": "string", + "title": "Period" + }, + "since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Since" + }, + "calls": { + "type": "integer", + "title": "Calls" + }, + "failed": { + "type": "integer", + "title": "Failed" + }, + "tokens_in": { + "type": "integer", + "title": "Tokens In" + }, + "tokens_out": { + "type": "integer", + "title": "Tokens Out" + }, + "cost_usd": { + "type": "number", + "title": "Cost Usd", + "description": "what its calls cost (run rows repeat their calls' cost and aren't added again)" + }, + "ms": { + "type": "integer", + "title": "Ms" + }, + "by_kind": { + "additionalProperties": { + "$ref": "#/components/schemas/KindTotals" + }, + "type": "object", + "title": "By Kind" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "period", + "calls", + "failed", + "tokens_in", + "tokens_out", + "cost_usd", + "ms", + "by_kind" + ], + "title": "ActivityTotals" + }, "AnswerCheck": { "properties": { "claims": { @@ -29171,6 +29680,43 @@ ], "title": "JobsQueued" }, + "KindTotals": { + "properties": { + "calls": { + "type": "integer", + "title": "Calls", + "default": 0 + }, + "tokens_in": { + "type": "integer", + "title": "Tokens In", + "default": 0 + }, + "tokens_out": { + "type": "integer", + "title": "Tokens Out", + "default": 0 + }, + "cost_usd": { + "type": "number", + "title": "Cost Usd", + "default": 0 + }, + "ms": { + "type": "integer", + "title": "Ms", + "default": 0 + }, + "failed": { + "type": "integer", + "title": "Failed", + "default": 0 + } + }, + "additionalProperties": true, + "type": "object", + "title": "KindTotals" + }, "LanguageCount": { "properties": { "language": { @@ -35869,6 +36415,35 @@ ], "title": "ResetPasswordRequest" }, + "ResourceCost": { + "properties": { + "resource": { + "type": "string", + "title": "Resource" + }, + "cost_usd": { + "type": "number", + "title": "Cost Usd" + }, + "tokens": { + "type": "integer", + "title": "Tokens" + }, + "calls": { + "type": "integer", + "title": "Calls" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "resource", + "cost_usd", + "tokens", + "calls" + ], + "title": "ResourceCost" + }, "ResourceFile": { "properties": { "id": { From 90ad0e2cc74ad42c1c6c32d358f560cfee1ad7ce Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 4 Oct 2026 09:39:35 +0000 Subject: [PATCH 2/6] feat(activity): costs for lists, estimate flags, run totals on jobs and routine runs Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AMq5CNuWmtn3Suj1EamYBT --- DEVLOG.md | 3 + fastapi_backend/app/api/v1/routes/activity.py | 32 ++- fastapi_backend/app/core/runtime.py | 17 +- fastapi_backend/app/domain/activity.py | 51 ++++- fastapi_backend/app/domain/extensions.py | 5 +- fastapi_backend/app/domain/jobs.py | 18 +- fastapi_backend/app/domain/routines.py | 36 ++- fastapi_backend/app/schemas/activity.py | 18 ++ fastapi_backend/app/schemas/jobs.py | 3 + fastapi_backend/app/schemas/routines.py | 3 + fastapi_backend/tests/api/test_activity.py | 21 ++ nextjs-frontend/app/openapi-client/index.ts | 7 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 18 ++ .../app/openapi-client/types.gen.ts | 132 +++++++++++ nextjs-frontend/openapi.json | 215 ++++++++++++++++++ 15 files changed, 556 insertions(+), 23 deletions(-) diff --git a/DEVLOG.md b/DEVLOG.md index d0acf27db..4636338a0 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -29,6 +29,9 @@ Todo: run totals, per-resource history and cost summary API, retention sweep - [ ] Activity in the web app: a history panel on routines, pipelines, workflows and recordings; Settings → Activity and costs with totals by resource +- [ ] Cost in every view (Adeel): every resource, entity and view shows what it cost and its budget; a figure that + isn't exact (no price for a model, tokens the server didn't report, a run's forecast) is marked as an estimate + (`≈`, with why on hover). The API says `exact` / `estimate` per figure - [ ] Budgets: set per routine, pipeline, workflow or namespace; spent and estimate; checked before routine runs and jobs; over budget waits for your pick (run once, skip), or the assistant decides - [ ] Budgets in the web app: set a budget, see where it stands, approve or skip a waiting run diff --git a/fastapi_backend/app/api/v1/routes/activity.py b/fastapi_backend/app/api/v1/routes/activity.py index 3773157ee..33e8092a3 100644 --- a/fastapi_backend/app/api/v1/routes/activity.py +++ b/fastapi_backend/app/api/v1/routes/activity.py @@ -11,7 +11,7 @@ from app.api.deps import Acl, AdminReader, CurrentUser, Db from app.domain import activity -from app.schemas.activity import ActivityEntry, ActivityTop, ActivityTotals +from app.schemas.activity import ActivityCosts, ActivityEntry, ActivityTop, ActivityTotals router = APIRouter(tags=["activity"]) Period = Literal["day", "week", "month", "all"] @@ -57,7 +57,10 @@ def resource_history( @router.get("/activity/totals") def resource_totals( - user: CurrentUser, acl: Acl, db: Db, resource: str | None = Query(None, description="table:id; none for everything (admins)"), + user: CurrentUser, + acl: Acl, + db: Db, + resource: str | None = Query(None, description="table:id; none for everything (admins)"), period: Period = "month", ) -> ActivityTotals: """What a resource's calls cost this day, week (from Monday), month (UTC) or all time: calls, tokens, USD, time.""" @@ -70,11 +73,34 @@ def resource_totals( return ActivityTotals(resource=resource, period=period, since=since, **activity.totals(db, resource, since)) +@router.get("/activity/costs") +def resource_costs( + user: CurrentUser, + acl: Acl, + db: Db, + resource: list[str] = Query(description="table:id, repeated (up to 500): every row of a list at once"), + period: Period = "month", +) -> ActivityCosts: + """What each of many resources cost this period, for lists; ones you can't see are left out. `estimate` says a + figure is a floor (some calls had no price or token counts).""" + seen = [] + for r in resource[:500]: + try: + _may_see(db, user, acl, r) + except HTTPException: + continue + seen.append(r) + since = activity._since(period) + return ActivityCosts(period=period, since=since, costs=activity.costs(db, seen, since)) + + @router.get("/activity/top") def top_resources( user: AdminReader, db: Db, - table: str | None = Query(None, pattern=r"^[a-z_]{1,40}$", description="only this kind of resource: routine, pipeline, workflow, space"), + table: str | None = Query( + None, pattern=r"^[a-z_]{1,40}$", description="only this kind of resource: routine, pipeline, workflow, space" + ), period: Period = "month", limit: int = Query(20, ge=1, le=200), ) -> ActivityTop: diff --git a/fastapi_backend/app/core/runtime.py b/fastapi_backend/app/core/runtime.py index f3a4dd572..c50406217 100644 --- a/fastapi_backend/app/core/runtime.py +++ b/fastapi_backend/app/core/runtime.py @@ -10,7 +10,22 @@ from typing import Any from app.config import settings as env -from app.domain import activity, auth, bridge, components, content_types, jobs, notify, routines, sensors, settings, setup, store, templates, tunnel +from app.domain import ( + activity, + auth, + bridge, + components, + content_types, + jobs, + notify, + routines, + sensors, + settings, + setup, + store, + templates, + tunnel, +) log = logging.getLogger("lens") diff --git a/fastapi_backend/app/domain/activity.py b/fastapi_backend/app/domain/activity.py index 215cc5ec2..bc3f70885 100644 --- a/fastapi_backend/app/domain/activity.py +++ b/fastapi_backend/app/domain/activity.py @@ -387,12 +387,11 @@ def totals(db, resource=None, since=None, until=None): p["until"] = until rows = db.rows( "SELECT kind, count() AS calls, math::sum(tokens_in ?? 0) AS tokens_in, math::sum(tokens_out ?? 0) AS tokens_out, " - "math::sum(cost_usd ?? 0) AS cost_usd, math::sum(ms ?? 0) AS ms, count(ok = false) AS failed FROM activity" - + (f" WHERE {' AND '.join(where)}" if where else "") - + " GROUP BY kind", + f"math::sum(cost_usd ?? 0) AS cost_usd, math::sum(ms ?? 0) AS ms, count(ok = false) AS failed, count({_unpriced_sql()}) AS unpriced " + "FROM activity" + (f" WHERE {' AND '.join(where)}" if where else "") + " GROUP BY kind", **p, ) - out = {"calls": 0, "tokens_in": 0, "tokens_out": 0, "cost_usd": 0.0, "ms": 0, "failed": 0, "by_kind": {}} + out = {"calls": 0, "tokens_in": 0, "tokens_out": 0, "cost_usd": 0.0, "ms": 0, "failed": 0, "unpriced": 0, "by_kind": {}} for r in rows: k = r.get("kind") or "?" # a run row repeats what its calls cost (its cost is their sum): counted on its own, not twice @@ -400,9 +399,10 @@ def totals(db, resource=None, since=None, until=None): out["calls"] += r.get("calls") or 0 out["failed"] += r.get("failed") or 0 if k != "run": - for x in ("tokens_in", "tokens_out", "cost_usd", "ms"): + for x in ("tokens_in", "tokens_out", "cost_usd", "ms", "unpriced"): out[x] += r.get(x) or 0 out["cost_usd"] = round(out["cost_usd"], 6) + out["estimate"] = out["unpriced"] > 0 return out @@ -435,10 +435,45 @@ def top(db, prefix=None, since=None, limit=20): return rows +PRICED = ("model.", "embeddings", "decision") # calls that cost something; one with no cost makes a total an estimate + + +def _unpriced_sql(): + return "(cost_usd = NONE AND (" + " OR ".join(f"string::starts_with(action, '{a}')" for a in PRICED) + "))" + + +def costs(db, resources, since=None): + """What each of many resources' calls cost, for lists: {resource: {cost_usd, tokens, calls, unpriced, estimate}}. + `unpriced` counts calls that cost something but have no figure (a model with no price, a reply without token + counts), so `estimate` is true: the cost shown is a floor, not to the cent.""" + want = [r for r in dict.fromkeys(resources) if REF.match(r)][:500] + out = {r: {"cost_usd": 0.0, "tokens": 0, "calls": 0, "unpriced": 0, "estimate": False} for r in want} + if not want: + return out + p = {"rs": want} + inner = "kind != 'run' AND resources CONTAINSANY $rs" + if since: + inner += " AND at >= $since" + p["since"] = since + rows = db.rows( + "SELECT res, math::sum(cost_usd ?? 0) AS cost_usd, math::sum((tokens_in ?? 0) + (tokens_out ?? 0)) AS tokens, " + "count() AS calls, count(unpriced) AS unpriced FROM " + f"(SELECT resources AS res, cost_usd, tokens_in, tokens_out, {_unpriced_sql()} AS unpriced FROM activity WHERE {inner} SPLIT res) " + "WHERE res IN $rs GROUP BY res", + **p, + ) + for r in rows: + o = out[r["res"]] + o.update(cost_usd=round(float(r.get("cost_usd") or 0), 6), tokens=int(r.get("tokens") or 0), calls=r.get("calls") or 0) + o["unpriced"] = r.get("unpriced") or 0 + o["estimate"] = o["unpriced"] > 0 + return out + + def run_total(db, resource): - """What one run's calls cost: (USD or None, tokens or None), for its run row.""" - usd, tokens = spent(db, resource) - return (usd or None), (tokens or None) + """What one run's calls cost, for its run row: (USD or None, tokens or None, whether the USD is an estimate).""" + c = costs(db, [resource])[resource] + return (c["cost_usd"] or None), (c["tokens"] or None), c["estimate"] def tidy(db, cfg, now=None): diff --git a/fastapi_backend/app/domain/extensions.py b/fastapi_backend/app/domain/extensions.py index 38c9a2c22..322cb2d64 100644 --- a/fastapi_backend/app/domain/extensions.py +++ b/fastapi_backend/app/domain/extensions.py @@ -749,7 +749,10 @@ def http_call(cfg, run, args): headers.setdefault("Content-Type", "application/json") req = urllib.request.Request(url, data=data, method=run.get("method") or "GET", headers=headers) host = urllib.parse.urlsplit(url).hostname - with activity.call("tool.http", cfg, detail={"host": host, "method": req.get_method()}), netguard.Guard(forward=True, networks=webcapture.networks(cfg), max_bytes=MAX_REPLY + 65536) as guard: + with ( + activity.call("tool.http", cfg, detail={"host": host, "method": req.get_method()}), + netguard.Guard(forward=True, networks=webcapture.networks(cfg), max_bytes=MAX_REPLY + 65536) as guard, + ): try: with feeds._opener(guard).open(req, timeout=30) as r: status, ctype, raw = r.status, r.headers.get("Content-Type", ""), r.read(MAX_REPLY) diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index 3aaa1b415..9fc186a7d 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -22,7 +22,7 @@ ACTIVE = ["queued", "running"] FIELDS = ( "record::id(id) AS id, recording, space, batch, pipeline, steps, step_index, next_step, status, worker, error, attempts, " - "created_by, created_at, started_at, finished_at, updated_at, cancel_requested, log_total" + "created_by, created_at, started_at, finished_at, updated_at, cancel_requested, log_total, cost_usd, tokens, cost_estimate" ) MEDIA_STEPS = {"transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe"} # they take longer the longer the recording FILED = ("audio", "document", "image") # sources with a file of their own for the steps to work on @@ -445,12 +445,22 @@ def run_job(db, cfg_fn, job, worker, can, log=None): with activity.scope(db, *refs): outcome = _run_job(db, cfg_fn, job, worker, can, log) if outcome in FINAL: - usd, tokens = activity.run_total(db, refs[0]) + usd, tokens, rough = activity.run_total(db, refs[0]) try: - db.q("UPDATE $j SET cost_usd = $u, tokens = $n", j=R("job", job["id"]), u=usd, n=tokens) + db.q("UPDATE $j SET cost_usd = $u, tokens = $n, cost_estimate = $e", j=R("job", job["id"]), u=usd, n=tokens, e=rough or None) except Exception: # noqa: BLE001 - the ledger never breaks the work pass - activity.record("run", f"job.{outcome}", refs, cfg_fn(), db, cost_usd=usd, tokens_in=tokens, ok=outcome != "failed") + activity.record( + "run", + f"job.{outcome}", + refs, + cfg_fn(), + db, + cost_usd=usd, + tokens_in=tokens, + ok=outcome != "failed", + detail={"estimate": rough} if rough else None, + ) return outcome diff --git a/fastapi_backend/app/domain/routines.py b/fastapi_backend/app/domain/routines.py index dbef575f8..82b7e2497 100644 --- a/fastapi_backend/app/domain/routines.py +++ b/fastapi_backend/app/domain/routines.py @@ -22,7 +22,21 @@ import datetime as dt import threading -from . import activity, fedora, jobs, organize, schedule, semantic, sensor_digests, sensor_patterns, sensors, sources, store, telemetry, workflows +from . import ( + activity, + fedora, + jobs, + organize, + schedule, + semantic, + sensor_digests, + sensor_patterns, + sensors, + sources, + store, + telemetry, + workflows, +) R = store.R ACTIONS = ("sync", "pipeline", "workflow", "sensors") @@ -226,7 +240,8 @@ def _with_changes(db, rows): def runs(db, rid, limit=20): rows = db.rows( - "SELECT record::id(id) AS id, routine, trigger, by, status, started_at, finished_at, results, error FROM routine_run " + "SELECT record::id(id) AS id, routine, trigger, by, status, started_at, finished_at, results, error, cost_usd, tokens, " + "cost_estimate FROM routine_run " "WHERE routine = $r ORDER BY id DESC LIMIT $n", r=int(rid), n=int(limit), @@ -369,10 +384,19 @@ def _run(db, cfg, rid, trigger, by, propose_only, log): refs = [f"routine:{int(rid)}", f"routine_run:{run_id}"] with activity.scope(db, *refs, cfg=cfg): status = _run_actions(db, cfg, rid, run_id, trigger, by, propose_only, log) - usd, tokens = activity.run_total(db, refs[1]) - db.q("UPDATE $r SET cost_usd = $u, tokens = $n", r=R("routine_run", run_id), u=usd, n=tokens) - activity.record("run", f"routine.{status}", refs, cfg, db, cost_usd=usd, tokens_in=tokens, ok=status != "error", - detail={"trigger": trigger}) + usd, tokens, rough = activity.run_total(db, refs[1]) + db.q("UPDATE $r SET cost_usd = $u, tokens = $n, cost_estimate = $e", r=R("routine_run", run_id), u=usd, n=tokens, e=rough or None) + activity.record( + "run", + f"routine.{status}", + refs, + cfg, + db, + cost_usd=usd, + tokens_in=tokens, + ok=status != "error", + detail=store.clean({"trigger": trigger, "estimate": rough or None}), + ) return run_id diff --git a/fastapi_backend/app/schemas/activity.py b/fastapi_backend/app/schemas/activity.py index 8f80eb9d5..687325661 100644 --- a/fastapi_backend/app/schemas/activity.py +++ b/fastapi_backend/app/schemas/activity.py @@ -49,6 +49,10 @@ class ActivityTotals(ResponseModel): tokens_out: int cost_usd: float = Field(description="what its calls cost (run rows repeat their calls' cost and aren't added again)") ms: int + unpriced: int = Field( + 0, description="calls that cost something but have no figure (a model with no price, a reply without token counts)" + ) + estimate: bool = Field(False, description="true when unpriced calls make cost_usd a floor rather than exact") by_kind: dict[str, KindTotals] @@ -59,6 +63,20 @@ class ResourceCost(ResponseModel): calls: int +class ResourceSpend(ResponseModel): + cost_usd: float + tokens: int + calls: int + unpriced: int + estimate: bool = Field(description="true: cost_usd is a floor (unpriced calls); show it as an estimate") + + +class ActivityCosts(ResponseModel): + period: str + since: str | None = None + costs: dict[str, ResourceSpend] = Field(description="by resource; ones you can't see are left out") + + class ActivityTop(ResponseModel): period: str since: str | None = None diff --git a/fastapi_backend/app/schemas/jobs.py b/fastapi_backend/app/schemas/jobs.py index e0a1b864a..3369a0d32 100644 --- a/fastapi_backend/app/schemas/jobs.py +++ b/fastapi_backend/app/schemas/jobs.py @@ -53,6 +53,9 @@ class Job(ResponseModel): finished_at: str | None = None updated_at: str | None = None cancel_requested: bool | None = None + cost_usd: float | None = Field(None, description="what its calls cost, from the activity ledger (docs/activity.md); set when it ends") + tokens: int | None = None + cost_estimate: bool | None = Field(None, description="true: some calls had no price or token counts, so cost_usd is a floor") log: list[str] | None = Field(None, description="the last 200 lines; GET /jobs/{jid}/log has them all") log_total: int | None = Field(None, description="how many lines the run has logged") step_runs: list[StepRun | None] | None = Field( diff --git a/fastapi_backend/app/schemas/routines.py b/fastapi_backend/app/schemas/routines.py index 1a12d132d..0e5fef189 100644 --- a/fastapi_backend/app/schemas/routines.py +++ b/fastapi_backend/app/schemas/routines.py @@ -76,6 +76,9 @@ class RoutineRun(ResponseModel): error: str | None = None changes: dict[str, int] | None = None log: list[str] | None = None + cost_usd: float | None = Field(None, description="what its calls cost, from the activity ledger (docs/activity.md); set when it ends") + tokens: int | None = None + cost_estimate: bool | None = Field(None, description="true: some calls had no price or token counts, so cost_usd is a floor") class SchedulePreview(ResponseModel): diff --git a/fastapi_backend/tests/api/test_activity.py b/fastapi_backend/tests/api/test_activity.py index afc249932..0931c1382 100644 --- a/fastapi_backend/tests/api/test_activity.py +++ b/fastapi_backend/tests/api/test_activity.py @@ -155,3 +155,24 @@ def values(self, *a, **k): raise RuntimeError("down") assert activity.record("out", "x", db=Broken()) is None + + +def test_costs_for_lists_say_when_they_are_estimates(client, db, cfg, h): + with activity.scope(db, "routine:1", cfg=cfg): + activity.record("out", "model.chat", model="fake", tokens_in=10, cost_usd=0.5) + with activity.scope(db, "routine:2", cfg=cfg): + activity.record("out", "model.chat", model="local", tokens_in=10) # no price + activity.record("out", "model.chat", model="fake", cost_usd=0.25) + activity.record("out", "notify.webhook") # costs nothing: not unpriced + got = client.get( + "/api/v1/activity/costs", headers=h, params=[("resource", "routine:1"), ("resource", "routine:2"), ("resource", "routine:3")] + ) + c = got.json()["costs"] + assert (c["routine:1"]["cost_usd"], c["routine:1"]["estimate"]) == (0.5, False) + assert (c["routine:2"]["cost_usd"], c["routine:2"]["unpriced"], c["routine:2"]["estimate"]) == (0.25, 1, True) + assert c["routine:3"] == {"cost_usd": 0, "tokens": 0, "calls": 0, "unpriced": 0, "estimate": False} + t = client.get("/api/v1/activity/totals", headers=h, params={"resource": "routine:2"}).json() + assert (t["unpriced"], t["estimate"]) == (1, True) + make_user(db, "v@x.io", "viewer password 1") + v = login(client, "v@x.io", "viewer password 1") + assert client.get("/api/v1/activity/costs", headers=v, params={"resource": "routine:1"}).json()["costs"] == {} diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index ce8042583..62183a899 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -54,6 +54,7 @@ export type { AcceptGraphChangeResponses, AccessRequest, AccountToken, + ActivityCosts, ActivityEntry, ActivityTop, ActivityTotals, @@ -1731,6 +1732,11 @@ export type { ResetPasswordResponse, ResetPasswordResponses, ResourceCost, + ResourceCostsData, + ResourceCostsError, + ResourceCostsErrors, + ResourceCostsResponse, + ResourceCostsResponses, ResourceFile, ResourceFiles, ResourceHistoryData, @@ -1738,6 +1744,7 @@ export type { ResourceHistoryErrors, ResourceHistoryResponse, ResourceHistoryResponses, + ResourceSpend, ResourceTotalsData, ResourceTotalsError, ResourceTotalsErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index fb765bfe5..09a6590d7 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -847,6 +847,9 @@ import type { ResetPasswordData, ResetPasswordErrors, ResetPasswordResponses, + ResourceCostsData, + ResourceCostsErrors, + ResourceCostsResponses, ResourceHistoryData, ResourceHistoryErrors, ResourceHistoryResponses, @@ -2362,6 +2365,21 @@ export class Activity { }); } + /** + * Resource Costs + * + * What each of many resources cost this period, for lists; ones you can't see are left out. `estimate` says a + * figure is a floor (some calls had no price or token counts). + */ + public static resourceCosts( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/activity/costs", + ...options, + }); + } + /** * Top Resources * diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 74b85f134..152c388dc 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -102,6 +102,29 @@ export type AccountToken = { [key: string]: unknown; }; +/** + * ActivityCosts + */ +export type ActivityCosts = { + /** + * Period + */ + period: string; + /** + * Since + */ + since?: string | null; + /** + * Costs + * + * by resource; ones you can't see are left out + */ + costs: { + [key: string]: ResourceSpend; + }; + [key: string]: unknown; +}; + /** * ActivityEntry */ @@ -244,6 +267,18 @@ export type ActivityTotals = { * Ms */ ms: number; + /** + * Unpriced + * + * calls that cost something but have no figure (a model with no price, a reply without token counts) + */ + unpriced?: number; + /** + * Estimate + * + * true when unpriced calls make cost_usd a floor rather than exact + */ + estimate?: boolean; /** * By Kind */ @@ -5500,6 +5535,22 @@ export type Job = { * Cancel Requested */ cancel_requested?: boolean | null; + /** + * Cost Usd + * + * what its calls cost, from the activity ledger (docs/activity.md); set when it ends + */ + cost_usd?: number | null; + /** + * Tokens + */ + tokens?: number | null; + /** + * Cost Estimate + * + * true: some calls had no price or token counts, so cost_usd is a floor + */ + cost_estimate?: boolean | null; /** * Log * @@ -9801,6 +9852,35 @@ export type ResourceFiles = { [key: string]: unknown; }; +/** + * ResourceSpend + */ +export type ResourceSpend = { + /** + * Cost Usd + */ + cost_usd: number; + /** + * Tokens + */ + tokens: number; + /** + * Calls + */ + calls: number; + /** + * Unpriced + */ + unpriced: number; + /** + * Estimate + * + * true: cost_usd is a floor (unpriced calls); show it as an estimate + */ + estimate: boolean; + [key: string]: unknown; +}; + /** * Reviewed */ @@ -10006,6 +10086,22 @@ export type RoutineRun = { * Log */ log?: Array | null; + /** + * Cost Usd + * + * what its calls cost, from the activity ledger (docs/activity.md); set when it ends + */ + cost_usd?: number | null; + /** + * Tokens + */ + tokens?: number | null; + /** + * Cost Estimate + * + * true: some calls had no price or token counts, so cost_usd is a floor + */ + cost_estimate?: boolean | null; [key: string]: unknown; }; @@ -15959,6 +16055,42 @@ export type ResourceTotalsResponses = { export type ResourceTotalsResponse = ResourceTotalsResponses[keyof ResourceTotalsResponses]; +export type ResourceCostsData = { + body?: never; + path?: never; + query: { + /** + * Resource + * + * table:id, repeated (up to 500): every row of a list at once + */ + resource: Array; + /** + * Period + */ + period?: "day" | "week" | "month" | "all"; + }; + url: "/api/v1/activity/costs"; +}; + +export type ResourceCostsErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ResourceCostsError = ResourceCostsErrors[keyof ResourceCostsErrors]; + +export type ResourceCostsResponses = { + /** + * Successful Response + */ + 200: ActivityCosts; +}; + +export type ResourceCostsResponse = ResourceCostsResponses[keyof ResourceCostsResponses]; + export type TopResourcesData = { body?: never; path?: never; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 982f797df..156c05cd2 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -2795,6 +2795,70 @@ } } }, + "/api/v1/activity/costs": { + "get": { + "tags": [ + "activity" + ], + "summary": "Resource Costs", + "description": "What each of many resources cost this period, for lists; ones you can't see are left out. `estimate` says a\nfigure is a floor (some calls had no price or token counts).", + "operationId": "resource_costs", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "description": "table:id, repeated (up to 500): every row of a list at once", + "title": "Resource" + }, + "description": "table:id, repeated (up to 500): every row of a list at once" + }, + { + "name": "period", + "in": "query", + "required": false, + "schema": { + "enum": [ + "day", + "week", + "month", + "all" + ], + "type": "string", + "default": "month", + "title": "Period" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ActivityCosts" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/activity/top": { "get": { "tags": [ @@ -19994,6 +20058,40 @@ "title": "AccountToken", "description": "Anyone's key, for admins." }, + "ActivityCosts": { + "properties": { + "period": { + "type": "string", + "title": "Period" + }, + "since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Since" + }, + "costs": { + "additionalProperties": { + "$ref": "#/components/schemas/ResourceSpend" + }, + "type": "object", + "title": "Costs", + "description": "by resource; ones you can't see are left out" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "period", + "costs" + ], + "title": "ActivityCosts" + }, "ActivityEntry": { "properties": { "id": { @@ -20231,6 +20329,18 @@ "type": "integer", "title": "Ms" }, + "unpriced": { + "type": "integer", + "title": "Unpriced", + "description": "calls that cost something but have no figure (a model with no price, a reply without token counts)", + "default": 0 + }, + "estimate": { + "type": "boolean", + "title": "Estimate", + "description": "true when unpriced calls make cost_usd a floor rather than exact", + "default": false + }, "by_kind": { "additionalProperties": { "$ref": "#/components/schemas/KindTotals" @@ -29355,6 +29465,41 @@ ], "title": "Cancel Requested" }, + "cost_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost Usd", + "description": "what its calls cost, from the activity ledger (docs/activity.md); set when it ends" + }, + "tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens" + }, + "cost_estimate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Cost Estimate", + "description": "true: some calls had no price or token counts, so cost_usd is a floor" + }, "log": { "anyOf": [ { @@ -36656,6 +36801,41 @@ ], "title": "ResourceFiles" }, + "ResourceSpend": { + "properties": { + "cost_usd": { + "type": "number", + "title": "Cost Usd" + }, + "tokens": { + "type": "integer", + "title": "Tokens" + }, + "calls": { + "type": "integer", + "title": "Calls" + }, + "unpriced": { + "type": "integer", + "title": "Unpriced" + }, + "estimate": { + "type": "boolean", + "title": "Estimate", + "description": "true: cost_usd is a floor (unpriced calls); show it as an estimate" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "cost_usd", + "tokens", + "calls", + "unpriced", + "estimate" + ], + "title": "ResourceSpend" + }, "Reviewed": { "properties": { "applied": { @@ -37046,6 +37226,41 @@ } ], "title": "Log" + }, + "cost_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost Usd", + "description": "what its calls cost, from the activity ledger (docs/activity.md); set when it ends" + }, + "tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens" + }, + "cost_estimate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Cost Estimate", + "description": "true: some calls had no price or token counts, so cost_usd is a floor" } }, "additionalProperties": true, From f0666a752ef83f53b89ccb69795db81e584e6cf5 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 4 Oct 2026 09:42:23 +0000 Subject: [PATCH 3/6] feat(activity): cost local models by time, by tokens or not at all (off by default) Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AMq5CNuWmtn3Suj1EamYBT --- docs/activity.md | 20 +++++++-- fastapi_backend/app/domain/activity.py | 49 +++++++++++++++------- fastapi_backend/app/domain/semantic.py | 2 +- fastapi_backend/app/domain/settings.py | 35 +++++++++++++--- fastapi_backend/app/domain/telemetry.py | 4 +- fastapi_backend/tests/api/test_activity.py | 24 ++++++++++- 6 files changed, 106 insertions(+), 28 deletions(-) diff --git a/docs/activity.md b/docs/activity.md index 13f9787c2..44046935f 100644 --- a/docs/activity.md +++ b/docs/activity.md @@ -25,9 +25,23 @@ Besides the resources, a row has its kind (`in`, `out` or `run`), what it was (` tokens in and out, the estimated cost in USD, how long it took, and how it ended (an error's type or HTTP status, never its message). It never holds prompt, reply, transcript or file content. -Cost comes from the prices you set: Settings › Telemetry › prices (USD per million tokens, per model), or the price -of the assistant's model under Settings › AI assistant. A model with no price counts tokens but no cost; local models -usually have none. `decisions.price_per_call` gives the decision model's price per decision. +## Costs + +Each model is costed the way you set it under Settings › Telemetry › prices (`telemetry.prices`), or, for the +assistant's model, Settings › AI assistant: + +| Unit | Setting | For | +| --- | --- | --- | +| Tokens | `{"input": 3.0, "output": 15.0}` (USD per million tokens) | cloud models that bill by tokens | +| Time | `{"unit": "time", "per_hour": 0.6}` (USD per hour a call takes) | a local model on your own machine | +| Off | `{"unit": "off"}`, or leave the model out | the default: no money counted, only tokens and time | + +So local models aren't costed unless you give them a price. `decisions.price_per_call` gives the decision model's +price per decision. + +A figure that should have a cost but doesn't (a model priced by tokens whose server didn't report how many it used) +marks the row `unpriced`, and every total that includes it says `estimate: true`: the cost shown is a floor, and +the web app shows it with ≈. A run row (a job or a routine run ending) repeats what its calls cost in all, and the job or run keeps it too (`cost_usd`, `tokens`). Totals don't count run rows twice. diff --git a/fastapi_backend/app/domain/activity.py b/fastapi_backend/app/domain/activity.py index bc3f70885..512085db4 100644 --- a/fastapi_backend/app/domain/activity.py +++ b/fastapi_backend/app/domain/activity.py @@ -158,7 +158,11 @@ def record(kind, action, refs=(), cfg=None, db=None, **fields): "action": action[:200], "resources": resources, "actor": fields.pop("actor", None) or _actor(s.get("asgi")), - **{k: v for k, v in fields.items() if k in ("model", "tokens_in", "tokens_out", "cost_usd", "ms", "ok", "error", "detail")}, + **{ + k: v + for k, v in fields.items() + if k in ("model", "tokens_in", "tokens_out", "cost_usd", "unpriced", "ms", "ok", "error", "detail") + }, } ) row.setdefault("ok", True) @@ -177,6 +181,7 @@ class Call: def __init__(self, action, cfg=None, model=None, refs=(), detail=None, price=None): self.action, self.cfg, self.model, self.refs, self.detail, self.price = action, cfg, model, refs, detail, price self.tin = self.tout = self.cost = None + self.unpriced = False # it costs something, but the figure is missing (see model_cost) self.t0 = time.monotonic() self.done = False @@ -193,9 +198,10 @@ def end(self, error=None): if self.done: return self.done = True - cost = self.cost + cost, unpriced = self.cost, self.unpriced + ms = int((time.monotonic() - self.t0) * 1000) if cost is None and self.price is not None: - cost = self.price(self.model, self.tin, self.tout) + cost, unpriced = self.price(self.model, self.tin, self.tout, ms) record( "out", self.action, @@ -205,7 +211,8 @@ def end(self, error=None): tokens_in=self.tin, tokens_out=self.tout, cost_usd=cost, - ms=int((time.monotonic() - self.t0) * 1000), + unpriced=unpriced or None, + ms=ms, ok=error is None, error=type(error).__name__ if error is not None else None, detail=self.detail, @@ -223,16 +230,29 @@ def call(action, cfg=None, model=None, refs=(), detail=None, price=None): return Call(action, cfg, model, refs, detail, price) -def token_cost(cfg): - """A price function for Call: USD from the per-million-token prices in Settings (telemetry.prices).""" +def model_cost(cfg): + """A price function for Call, from each model's price in Settings (telemetry.prices; docs/activity.md#costs): + + * by tokens: {input, output} USD per million tokens (the default unit), as cloud providers charge; + * by time: {unit: "time", per_hour} USD per hour the call took, for a local model on your own machine; + * off: a model with no price (or {unit: "off"}) isn't costed, so it counts tokens and time but no money. + + Returns (USD or None, unpriced): unpriced when the model has a price by tokens but the server didn't say how many + it used, so the figure is missing and totals that include it are estimates.""" from . import telemetry prices = telemetry.prices(cfg) - def price(model, tin, tout): + def price(model, tin, tout, ms): + p = prices.get(model) + unit = (p or {}).get("unit") or "tokens" + if not isinstance(p, dict) or unit == "off": + return None, False + if unit == "time": + return (round(float(p.get("per_hour") or 0) * (ms or 0) / 3_600_000, 8) if ms is not None else None), ms is None if tin is None and tout is None: - return None - return telemetry.cost(model, tin or 0, tout or 0, prices) + return None, True + return telemetry.cost(model, tin or 0, tout or 0, prices), False return price @@ -435,17 +455,16 @@ def top(db, prefix=None, since=None, limit=20): return rows -PRICED = ("model.", "embeddings", "decision") # calls that cost something; one with no cost makes a total an estimate - - def _unpriced_sql(): - return "(cost_usd = NONE AND (" + " OR ".join(f"string::starts_with(action, '{a}')" for a in PRICED) + "))" + """A row that costs something whose figure is missing: it makes the totals it is in estimates.""" + return "(unpriced = true)" def costs(db, resources, since=None): """What each of many resources' calls cost, for lists: {resource: {cost_usd, tokens, calls, unpriced, estimate}}. - `unpriced` counts calls that cost something but have no figure (a model with no price, a reply without token - counts), so `estimate` is true: the cost shown is a floor, not to the cent.""" + `unpriced` counts calls that cost something but have no figure (a priced model whose reply had no token counts), + so `estimate` is true: the cost shown is a floor, not to the cent. A model without a price (local models, by + default) isn't costed at all, and doesn't make anything an estimate.""" want = [r for r in dict.fromkeys(resources) if REF.match(r)][:500] out = {r: {"cost_usd": 0.0, "tokens": 0, "calls": 0, "unpriced": 0, "estimate": False} for r in want} if not want: diff --git a/fastapi_backend/app/domain/semantic.py b/fastapi_backend/app/domain/semantic.py index c8a087d0f..3412c8a00 100644 --- a/fastapi_backend/app/domain/semantic.py +++ b/fastapi_backend/app/domain/semantic.py @@ -125,7 +125,7 @@ def embed(cfg, texts, timeout=None): headers["Authorization"] = f"Bearer {key}" body = json.dumps({"model": model, "input": [t[:EMBED_CHARS] for t in texts]}).encode() req = urllib.request.Request(base + "/embeddings", data=body, headers=headers, method="POST") - with activity.call("embeddings", cfg, model, detail={"texts": len(texts)}, price=activity.token_cost(cfg)) as ledger: + with activity.call("embeddings", cfg, model, detail={"texts": len(texts)}, price=activity.model_cost(cfg)) as ledger: try: with urllib.request.urlopen(req, timeout=timeout or _section(cfg).get("timeout") or 60) as r: j = json.load(r) diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index 3f0c44a7c..4468af9d4 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -701,21 +701,44 @@ def _origin(url): def _prices(value): - """{model: {input, output}}: USD per million tokens, for the cost estimates.""" - msg = "telemetry.prices gives each model its input and output price in USD per million tokens" + """{model: price}: how each model's calls are costed (docs/activity.md#costs). By tokens, {input, output} USD per + million tokens (unit "tokens", the default); by time, {unit: "time", per_hour} USD per hour a call takes (a local + model on your own machine); or {unit: "off"}. A model left out isn't costed.""" + msg = ( + "telemetry.prices gives each model a price: {input, output} USD per million tokens, {unit: time, per_hour} " + "USD per hour, or {unit: off}" + ) if value is None: return {} if not isinstance(value, dict) or len(value) > 200: raise ValueError(msg) + + def num(n): + return isinstance(n, (int, float)) and not isinstance(n, bool) and 0 <= n <= 1_000_000 + out = {} for model, p in value.items(): name = str(model).strip() - if not name or len(name) > 200 or not isinstance(p, dict) or set(p) - {"input", "output"}: + if not name or len(name) > 200 or not isinstance(p, dict): raise ValueError(msg) - nums = {k: p.get(k, 0) for k in ("input", "output")} - if not all(isinstance(n, (int, float)) and not isinstance(n, bool) and 0 <= n <= 1_000_000 for n in nums.values()): + unit = p.get("unit") or "tokens" + if unit == "tokens": + if set(p) - {"unit", "input", "output"}: + raise ValueError(msg) + nums = {k: p.get(k, 0) for k in ("input", "output")} + if not all(num(n) for n in nums.values()): + raise ValueError(msg) + out[name] = {k: float(n) for k, n in nums.items()} # the unit stays implicit, as prices saved before units + elif unit == "time": + if set(p) - {"unit", "per_hour"} or not num(p.get("per_hour", 0)): + raise ValueError(msg) + out[name] = {"unit": "time", "per_hour": float(p.get("per_hour", 0))} + elif unit == "off": + if set(p) - {"unit"}: + raise ValueError(msg) + out[name] = {"unit": "off"} + else: raise ValueError(msg) - out[name] = {k: float(n) for k, n in nums.items()} return out diff --git a/fastapi_backend/app/domain/telemetry.py b/fastapi_backend/app/domain/telemetry.py index 343370cbf..830c8e2b1 100644 --- a/fastapi_backend/app/domain/telemetry.py +++ b/fastapi_backend/app/domain/telemetry.py @@ -346,7 +346,7 @@ def record(name, value, attrs=None): def cost(model, input_tokens, output_tokens, prices=None): """Estimated cost in USD from the per-million-token prices for the model, or None when it has no price.""" p = (prices if prices is not None else _state["cfg"].get("prices") or {}).get(model) - if not isinstance(p, dict): + if not isinstance(p, dict) or (p.get("unit") or "tokens") != "tokens": return None return round((input_tokens or 0) / 1e6 * float(p.get("input") or 0) + (output_tokens or 0) / 1e6 * float(p.get("output") or 0), 8) @@ -361,7 +361,7 @@ def __init__(self, cfg, payload, operation="chat", current=True): self.model = payload.get("model") self.operation, self.t0, self.done, self.response_model, self.usage = operation, time.monotonic(), False, None, None - self.ledger = activity.call(f"model.{operation}", cfg, self.model, price=activity.token_cost(cfg)) + self.ledger = activity.call(f"model.{operation}", cfg, self.model, price=activity.model_cost(cfg)) self.on = active() if not self.on: return diff --git a/fastapi_backend/tests/api/test_activity.py b/fastapi_backend/tests/api/test_activity.py index 0931c1382..40df731e9 100644 --- a/fastapi_backend/tests/api/test_activity.py +++ b/fastapi_backend/tests/api/test_activity.py @@ -161,8 +161,9 @@ def test_costs_for_lists_say_when_they_are_estimates(client, db, cfg, h): with activity.scope(db, "routine:1", cfg=cfg): activity.record("out", "model.chat", model="fake", tokens_in=10, cost_usd=0.5) with activity.scope(db, "routine:2", cfg=cfg): - activity.record("out", "model.chat", model="local", tokens_in=10) # no price + activity.record("out", "model.chat", model="local", tokens_in=10) # not costed: not an estimate activity.record("out", "model.chat", model="fake", cost_usd=0.25) + activity.record("out", "model.chat", model="fake", unpriced=True) # priced, but no token counts came back activity.record("out", "notify.webhook") # costs nothing: not unpriced got = client.get( "/api/v1/activity/costs", headers=h, params=[("resource", "routine:1"), ("resource", "routine:2"), ("resource", "routine:3")] @@ -176,3 +177,24 @@ def test_costs_for_lists_say_when_they_are_estimates(client, db, cfg, h): make_user(db, "v@x.io", "viewer password 1") v = login(client, "v@x.io", "viewer password 1") assert client.get("/api/v1/activity/costs", headers=v, params={"resource": "routine:1"}).json()["costs"] == {} + + +def test_models_are_costed_by_tokens_by_time_or_not_at_all(client, cfg, h): + cfg["telemetry"]["prices"] = { + "cloud": {"input": 3.0, "output": 15.0}, + "gpu-box": {"unit": "time", "per_hour": 0.6}, + "off": {"unit": "off"}, + } + price = activity.model_cost(cfg) + assert price("cloud", 1_000_000, 100_000, 50) == (pytest.approx(4.5), False) + assert price("cloud", None, None, 50) == (None, True) # priced, but the server didn't count + assert price("gpu-box", 10, 10, 60_000) == (pytest.approx(0.01), False) # a minute at 0.60/hour + assert price("off", 10, 10, 60_000) == (None, False) + assert price("unlisted-local", 10, 10, 60_000) == (None, False) # local models aren't costed by default + + put = lambda v: client.put("/api/v1/settings/telemetry", headers=h, json={"prices": v}) # noqa: E731 + assert put({"m": {"unit": "time", "per_hour": 1.5}, "x": {"unit": "off"}, "t": {"input": 1}}).status_code == 200 + got = client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["prices"] + assert {k: got[k] for k in "mxt"} == {"m": {"unit": "time", "per_hour": 1.5}, "x": {"unit": "off"}, "t": {"input": 1.0, "output": 0.0}} + for bad in ({"m": {"unit": "time", "input": 1}}, {"m": {"unit": "watts"}}, {"m": {"unit": "off", "per_hour": 1}}): + assert put(bad).status_code == 400, bad From 835bae47022205b215c7577185498c0117f57176 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 4 Oct 2026 09:55:54 +0000 Subject: [PATCH 4/6] feat(budgets): cap routines, pipelines, workflows and spaces; show cost and budget in views MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Budgets are off unless set. Before a routine run or a job, Lens estimates the next run from the last 30 days and holds, skips or lets the assistant weigh a run that would go over. Held runs wait for a person to pick. Lists and detail pages show spend (≈ marks estimates) and budget state. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AMq5CNuWmtn3Suj1EamYBT --- CHANGELOG.md | 7 + DEVLOG.md | 8 +- docs/budgets.md | 53 ++ fastapi_backend/app/api/v1/router.py | 2 + fastapi_backend/app/api/v1/routes/budgets.py | 46 ++ fastapi_backend/app/api/v1/routes/jobs.py | 12 + fastapi_backend/app/api/v1/routes/routines.py | 28 +- fastapi_backend/app/domain/activity.py | 14 +- fastapi_backend/app/domain/budgets.py | 269 +++++++ fastapi_backend/app/domain/jobs.py | 54 +- fastapi_backend/app/domain/routines.py | 102 ++- fastapi_backend/app/domain/store.py | 2 + fastapi_backend/app/schemas/budgets.py | 62 ++ fastapi_backend/app/schemas/jobs.py | 5 + fastapi_backend/app/schemas/routines.py | 12 + fastapi_backend/tests/api/test_budgets.py | 171 +++++ mkdocs.yml | 1 + nextjs-frontend/app/openapi-client/index.ts | 35 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 115 ++- .../app/openapi-client/types.gen.ts | 390 +++++++++++ .../components/activity/job-detail.tsx | 27 + .../components/activity/job-model.ts | 4 +- .../components/activity/job-row.tsx | 8 + nextjs-frontend/components/costs/costs.tsx | 584 ++++++++++++++++ .../components/pipelines/pipelines-page.tsx | 8 + .../components/recording/history-tab.tsx | 2 + .../components/routines/routine-detail.tsx | 54 +- .../components/routines/routine-model.ts | 3 + .../components/routines/routines-page.tsx | 104 ++- nextjs-frontend/components/ui/badge.tsx | 1 + .../components/workflows/workflows-page.tsx | 7 + nextjs-frontend/openapi.json | 660 +++++++++++++++++- 32 files changed, 2791 insertions(+), 59 deletions(-) create mode 100644 docs/budgets.md create mode 100644 fastapi_backend/app/api/v1/routes/budgets.py create mode 100644 fastapi_backend/app/domain/budgets.py create mode 100644 fastapi_backend/app/schemas/budgets.py create mode 100644 fastapi_backend/tests/api/test_budgets.py create mode 100644 nextjs-frontend/components/costs/costs.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 0460427ff..56942c6ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Budgets for routines, pipelines, workflows and namespaces.** An admin caps what a resource may cost, in USD, + tokens or both, per run, day, week or month. Before a routine runs or a job starts, Lens estimates the run from past + ones; one that would go over is held until someone picks run or skip (or is skipped, or the decision model weighs + it when asked to). Budgets near or over their cap are flagged once per period. Off unless set. See docs/budgets.md. +- **Local models cost nothing unless you price them.** A model's price is by tokens (USD per million), by time (USD + per hour, for a model on your own machine) or off, the default for a model without one. + - **Every resource has an activity history, with what it cost.** Lens now keeps a ledger of every request that changes something, every call it makes out (models, embeddings, the decision model, text to speech, webhooks, web tools) and every job and routine run, each counted for the resources it touched: a model call in a job a routine diff --git a/DEVLOG.md b/DEVLOG.md index 4636338a0..e8a982365 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -25,17 +25,19 @@ Model: Todo: -- [ ] Activity ledger: scopes, model/embedding/decision/webhook/web-tool calls, changing API requests, job and routine +- [x] Activity ledger: scopes, model/embedding/decision/webhook/web-tool calls, changing API requests, job and routine run totals, per-resource history and cost summary API, retention sweep - [ ] Activity in the web app: a history panel on routines, pipelines, workflows and recordings; Settings → Activity and costs with totals by resource - [ ] Cost in every view (Adeel): every resource, entity and view shows what it cost and its budget; a figure that isn't exact (no price for a model, tokens the server didn't report, a run's forecast) is marked as an estimate (`≈`, with why on hover). The API says `exact` / `estimate` per figure -- [ ] Budgets: set per routine, pipeline, workflow or namespace; spent and estimate; checked before routine runs and +- [x] Budgets: set per routine, pipeline, workflow or namespace; spent and estimate; checked before routine runs and jobs; over budget waits for your pick (run once, skip), or the assistant decides - [ ] Budgets in the web app: set a budget, see where it stands, approve or skip a waiting run -- [ ] Periodic check: warn at 80% and 100% of a budget (notifications and the assistant) +- [x] Periodic check: warn at 80% and 100% of a budget, once per period (on the budget and in its history) +- [x] Local model costs (Adeel): price per model by tokens, by time (per hour) or off; off by default +- [ ] Budget warnings through notifications, and the assistant answering where budgets stand Refine later: compute time as a cost (CPU seconds × a rate for local models), per-person budgets, forecasting from schedules (a routine's next runs this period), budget alerts in the weekly digest. diff --git a/docs/budgets.md b/docs/budgets.md new file mode 100644 index 000000000..aa589b8ee --- /dev/null +++ b/docs/budgets.md @@ -0,0 +1,53 @@ +# Budgets + +A budget caps what a routine, pipeline, workflow or namespace may cost. Budgets are off unless an admin sets one. +What has been spent comes from the [activity ledger](activity.md), so a budget counts everything done for that +resource: a namespace's budget counts every job in it, and a routine's counts its runs and the jobs it queued. + +## Setting one + +```http +PUT /api/v1/budgets?resource=routine:3 +{"usd": 5, "period": "month", "on_over": "ask"} +``` + +| Field | | | +| --- | --- | --- | +| `usd`, `tokens` | at least one | the cap in USD, tokens (in and out), or both | +| `period` | `month` | `run` (each run on its own), `day`, `week` (from Monday) or `month` (UTC) | +| `on_over` | `ask` | what happens to a run that would go over: see below | +| `warn_at` | `0.8` | the share spent at which the budget is `near` and gets a warning | + +`GET /api/v1/budgets` lists every budget with where it stands, `GET /api/v1/budgets/status?resource=…` gives one +(also for a resource without a budget, with its estimated next run), and `DELETE /api/v1/budgets?resource=…` +removes one. + +## Before a run + +Before a routine runs and before a job's first step, Lens looks at the budgets of everything the run counts for: the +routine, the pipelines and workflows its actions name and its namespaces; a job's pipeline, namespace and the routine +that queued it. It estimates the next run from the average of that resource's runs in the last 30 days (always an +estimate, shown with ≈). A run goes ahead unless a budget is used up, or would be once the estimated run is done +(for a `run` budget, unless the estimate alone passes the cap). Then `on_over` decides: + +- `ask` (default): the run is held. A routine gets a run with status `held`, a job the status `held`, each with + `hold.why` saying where the budget stands. Someone picks: run it now, once, whatever the budget says + (`POST /api/v1/routines/runs/{run}/decide {"run": true}`, `POST /api/v1/jobs/{jid}/release {"run": true}`), or skip + it (`{"run": false}`). Doing nothing changes nothing: a held run stays held, and a routine that comes due again adds + to the one held run (`hold.missed`) rather than piling up. +- `skip`: the run is skipped, and says why. +- `assistant`: the decision model (Settings › AI assistant, decisions) weighs it, with where the budget stands and the + estimate. It runs only when the model is sure (`decisions.act_above`); otherwise it's held as with `ask`. + +Asking a routine to run from the app or the API when it's over budget is refused with 409 and the reason; send +`{"over_budget": true}` to run it anyway. + +## Warnings + +Every five minutes the scheduler looks at every budget. One that crosses `warn_at` or its cap gets an alert, once per +period: on the budget (`alert`) and as a `budget.near` or `budget.over` row in its activity history. + +## Refine later + +Forecasting from a routine's schedule (its runs left this period), per-person budgets, budget warnings sent through +notifications and the weekly digest, and the assistant answering "where do my budgets stand?" in chat. diff --git a/fastapi_backend/app/api/v1/router.py b/fastapi_backend/app/api/v1/router.py index 06a5dee5f..70a677e31 100644 --- a/fastapi_backend/app/api/v1/router.py +++ b/fastapi_backend/app/api/v1/router.py @@ -7,6 +7,7 @@ admin, auth, batches, + budgets, chats, collections, comments, @@ -64,6 +65,7 @@ users, admin, activity, + budgets, fedora, setup, namespaces, diff --git a/fastapi_backend/app/api/v1/routes/budgets.py b/fastapi_backend/app/api/v1/routes/budgets.py new file mode 100644 index 000000000..a5e6f8f0d --- /dev/null +++ b/fastapi_backend/app/api/v1/routes/budgets.py @@ -0,0 +1,46 @@ +"""Budgets (admins set them; docs/budgets.md): caps on what a routine, pipeline, workflow or namespace may cost, and +where each stands.""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException, Query + +from app.api.deps import AdminReader, AdminWriter, Db, domain_errors +from app.domain import auth, budgets +from app.schemas.budgets import BudgetSet, BudgetStatus +from app.schemas.common import Ok + +router = APIRouter(tags=["budgets"]) +RESOURCE = Query(description="routine:, pipeline:, workflow: or space:") + + +@router.get("/budgets") +def list_budgets(user: AdminReader, db: Db) -> list[BudgetStatus]: + """Every budget, with where it stands now.""" + return [budgets.status(db, b["resource"], b) for b in budgets.all_budgets(db)] + + +@router.get("/budgets/status") +def budget_status(user: AdminReader, db: Db, resource: str = RESOURCE) -> BudgetStatus: + """Where a resource's budget stands, and what its next run will likely cost (also without a budget).""" + with domain_errors(): + budgets.check_ref(resource) + return budgets.status(db, resource) + + +@router.put("/budgets") +def set_budget(body: BudgetSet, user: AdminWriter, db: Db, resource: str = RESOURCE) -> BudgetStatus: + with domain_errors(): + budgets.put(db, resource, by=user.email, **body.model_dump()) + auth.audit(db, user.as_audit(), "budget.set", resource, body.model_dump(exclude_none=True)) + return budgets.status(db, resource) + + +@router.delete("/budgets") +def remove_budget(user: AdminWriter, db: Db, resource: str = RESOURCE) -> Ok: + with domain_errors(): + if not budgets.get(db, budgets.check_ref(resource)): + raise HTTPException(404, "this resource has no budget") + budgets.remove(db, resource) + auth.audit(db, user.as_audit(), "budget.remove", resource) + return Ok() diff --git a/fastapi_backend/app/api/v1/routes/jobs.py b/fastapi_backend/app/api/v1/routes/jobs.py index 703e585fd..0bc62aa6a 100644 --- a/fastapi_backend/app/api/v1/routes/jobs.py +++ b/fastapi_backend/app/api/v1/routes/jobs.py @@ -19,6 +19,7 @@ from app.domain.store import DB from app.schemas.common import Ok from app.schemas.jobs import Job, JobList, JobLog, JobsCreate, JobsQueued, StepQueued, WorkerInfo +from app.schemas.routines import HeldRunDecision router = APIRouter(tags=["jobs"]) log = logging.getLogger("lens") @@ -139,6 +140,17 @@ def cancel_job(jid: int, acl: Acl, user: Writer, db: Db) -> Ok: return Ok() +@router.post("/jobs/{jid}/release") +def release_job(jid: int, body: HeldRunDecision, acl: Acl, user: AdminWriter, db: Db) -> Ok: + """Pick for a job held over a budget (docs/budgets.md): run it now, once, whatever its budgets say, or skip it + (admins, who set the budgets).""" + _job(db, acl, jid, "editor") + with domain_errors(): + jobs.release(db, jid, body.run, user.email) + auth.audit(db, user.as_audit(), "job.release" if body.run else "job.skip", f"job:{jid}") + return Ok() + + @router.post("/jobs/{jid}/retry") def retry_job(jid: int, acl: Acl, user: Writer, db: Db) -> Ok: _job(db, acl, jid, "editor") diff --git a/fastapi_backend/app/api/v1/routes/routines.py b/fastapi_backend/app/api/v1/routes/routines.py index 6a8f3babf..b4f36f894 100644 --- a/fastapi_backend/app/api/v1/routes/routines.py +++ b/fastapi_backend/app/api/v1/routes/routines.py @@ -7,12 +7,13 @@ from fastapi import APIRouter, HTTPException, Query -from app.api.deps import Acl, AdminReader, AdminWriter, CurrentUser, Db, Writer, domain_errors -from app.domain import auth, organize, routines, schedule +from app.api.deps import Acl, AdminReader, AdminWriter, Cfg, CurrentUser, Db, Writer, domain_errors +from app.domain import auth, budgets, organize, routines, schedule from app.schemas.common import Created, Ok from app.schemas.routines import ( GraphChange, GraphChangeAccept, + HeldRunDecision, Routine, RoutineCatalog, RoutineCreate, @@ -83,11 +84,26 @@ def delete_routine(rid: int, user: AdminWriter, db: Db) -> Ok: @router.post("/routines/{rid}/run") -def run_routine(rid: int, body: RoutineRunRequest, user: AdminWriter, db: Db) -> Ok: - """Run it as soon as the scheduler next looks (within half a minute), even when it is off.""" +def run_routine(rid: int, body: RoutineRunRequest, user: AdminWriter, db: Db, cfg: Cfg) -> Ok: + """Run it as soon as the scheduler next looks (within half a minute), even when it is off. A run over one of its + budgets is refused (409, saying where the budget stands) unless over_budget is true (docs/budgets.md).""" with domain_errors(): - routines.request_run(db, rid, user.email, body.propose_only) - auth.audit(db, user.as_audit(), "routine.run", f"routine:{rid}") + r = routines.get(db, rid) + if not body.over_budget: + verdict = budgets.check(db, cfg, routines.refs(db, r)) + if not verdict["go"]: + raise HTTPException(409, verdict["why"] + " Run it anyway with over_budget.") + routines.request_run(db, rid, user.email, body.propose_only, body.over_budget) + auth.audit(db, user.as_audit(), "routine.run", f"routine:{rid}", {"over_budget": True} if body.over_budget else None) + return Ok() + + +@router.post("/routines/runs/{run_id}/decide") +def decide_held_run(run_id: int, body: HeldRunDecision, user: AdminWriter, db: Db) -> Ok: + """Pick for a run held over a budget: run it now, once, whatever its budgets say, or skip it.""" + with domain_errors(): + rid = routines.decide_held(db, run_id, body.run, user.email) + auth.audit(db, user.as_audit(), "routine.release" if body.run else "routine.skip", f"routine:{rid}", {"run": run_id}) return Ok() diff --git a/fastapi_backend/app/domain/activity.py b/fastapi_backend/app/domain/activity.py index 512085db4..a90a85799 100644 --- a/fastapi_backend/app/domain/activity.py +++ b/fastapi_backend/app/domain/activity.py @@ -445,9 +445,9 @@ def top(db, prefix=None, since=None, limit=20): inner += " AND at >= $since" p["since"] = since rows = db.rows( - "SELECT res AS resource, math::sum(cost_usd ?? 0) AS cost_usd, math::sum((tokens_in ?? 0) + (tokens_out ?? 0)) AS tokens, " - f"count() AS calls FROM (SELECT resources AS res, cost_usd, tokens_in, tokens_out FROM activity WHERE {inner} SPLIT res) " - "WHERE string::starts_with(res, $p) GROUP BY res ORDER BY cost_usd DESC, tokens DESC LIMIT $n", + "SELECT resources AS resource, math::sum(cost_usd ?? 0) AS cost_usd, math::sum((tokens_in ?? 0) + (tokens_out ?? 0)) AS tokens, " + f"count() AS calls FROM (SELECT resources, cost_usd, tokens_in, tokens_out FROM activity WHERE {inner} SPLIT resources) " + "WHERE string::starts_with(resources, $p) GROUP BY resource ORDER BY cost_usd DESC, tokens DESC LIMIT $n", **p, ) for r in rows: @@ -475,10 +475,10 @@ def costs(db, resources, since=None): inner += " AND at >= $since" p["since"] = since rows = db.rows( - "SELECT res, math::sum(cost_usd ?? 0) AS cost_usd, math::sum((tokens_in ?? 0) + (tokens_out ?? 0)) AS tokens, " - "count() AS calls, count(unpriced) AS unpriced FROM " - f"(SELECT resources AS res, cost_usd, tokens_in, tokens_out, {_unpriced_sql()} AS unpriced FROM activity WHERE {inner} SPLIT res) " - "WHERE res IN $rs GROUP BY res", + "SELECT resources AS res, math::sum(cost_usd ?? 0) AS cost_usd, math::sum((tokens_in ?? 0) + (tokens_out ?? 0)) AS tokens, " + "count() AS calls, count(unpriced = true) AS unpriced FROM " + f"(SELECT resources, cost_usd, tokens_in, tokens_out, unpriced FROM activity WHERE {inner} SPLIT resources) " + "WHERE resources IN $rs GROUP BY res", **p, ) for r in rows: diff --git a/fastapi_backend/app/domain/budgets.py b/fastapi_backend/app/domain/budgets.py new file mode 100644 index 000000000..2cbc362aa --- /dev/null +++ b/fastapi_backend/app/domain/budgets.py @@ -0,0 +1,269 @@ +"""Budgets (docs/budgets.md): a cap on what a routine, pipeline, workflow or namespace may cost, checked before it runs. + +A budget is off unless someone sets one. It caps cost in USD, tokens, or both, per run, day, week (from Monday) or +month (UTC), with what was spent read from the activity ledger (activity.py). Before a routine run or a job, Lens +estimates what the run will cost from that resource's past runs and looks at what is left: + +* under budget: it runs; +* over (what was spent plus the estimate passes the cap): what `on_over` says. `ask` (the default) holds the run + until someone picks: run it once, or skip it. `skip` skips it and says so. `assistant` lets the decision model weigh + it (decide.py: where the budget stands, how sure the estimate is, how far over it would go); it runs only when the + model is sure, and asks otherwise. + +Doing nothing changes nothing: a held run stays held. A run someone let through runs. `warn_at` (0.8) marks a budget +`near` once that share is spent, for the periodic check (sweep) to warn about. +""" + +from __future__ import annotations + +import datetime as dt +import logging +import re +import time + +from . import activity, store + +log = logging.getLogger("lens") +R = store.R +PERIODS = ("run", "day", "week", "month") +ON_OVER = ("ask", "skip", "assistant") +TABLES = ("routine", "pipeline", "workflow", "space") +REF = re.compile(r"^(routine|pipeline|workflow|space):(\d+)$") +FIELDS = "resource, usd, tokens, period, on_over, warn_at, note, by, updated_at, alert" +LOOKBACK_DAYS = 30 # the past runs an estimate is drawn from +SWEEP_SECONDS = 300 # how often the scheduler looks at every budget +_swept = {"at": 0.0} + + +def _key(resource): + return R("budget", resource.replace(":", "_")) + + +def check_ref(resource): + if not isinstance(resource, str) or not REF.match(resource): + raise ValueError("a budget is for routine:, pipeline:, workflow: or space:") + return resource + + +def _num(v, what, whole=False): + if v is None: + return None + if isinstance(v, bool) or not isinstance(v, (int, float)) or v < 0 or v > 1e12 or (whole and int(v) != v): + raise ValueError(f"{what} is a {'whole ' if whole else ''}number of at least 0") + return int(v) if whole else float(v) + + +def put(db, resource, usd=None, tokens=None, period="month", on_over="ask", warn_at=0.8, note=None, by=None): + """Set (or replace) a resource's budget. At least one of usd and tokens.""" + check_ref(resource) + usd, tokens = _num(usd, "usd"), _num(tokens, "tokens", whole=True) + if usd is None and tokens is None: + raise ValueError("give the budget a cap: usd, tokens or both") + if period not in PERIODS: + raise ValueError(f"period is one of {', '.join(PERIODS)}") + if on_over not in ON_OVER: + raise ValueError(f"on_over is one of {', '.join(ON_OVER)}") + warn_at = _num(warn_at, "warn_at") + if warn_at is None or not 0.1 <= warn_at <= 1: + raise ValueError("warn_at is a share of the budget from 0.1 to 1") + row = store.clean( + { + "resource": resource, + "usd": usd, + "tokens": tokens, + "period": period, + "on_over": on_over, + "warn_at": warn_at, + "note": (note or "").strip()[:300] or None, + "by": by, + "updated_at": store.now(), + } + ) + db.q("UPSERT $r CONTENT $d", r=_key(resource), d=row) + return get(db, resource) + + +def get(db, resource): + return db.one(f"SELECT {FIELDS} FROM $r", r=_key(resource)) + + +def remove(db, resource): + check_ref(resource) + db.q("DELETE $r", r=_key(resource)) + + +def all_budgets(db): + return db.rows(f"SELECT {FIELDS} FROM budget ORDER BY resource") + + +# ---------- where it stands ---------- +def _runs(db, resource, since): + """How many runs this resource had since a time: a routine's runs, or the jobs of a pipeline, workflow or namespace.""" + m = REF.match(resource) + if m and m.group(1) == "routine": + return len(db.values("SELECT VALUE id FROM routine_run WHERE routine = $r AND started_at >= $s", r=int(m.group(2)), s=since)) + return len( + db.values( + "SELECT VALUE id FROM activity WHERE kind = 'run' AND resources CONTAINS $res AND at >= $s AND string::starts_with(action, 'job.')", + res=resource, + s=since, + ) + ) + + +def estimate(db, resource, now=None): + """What one more run will likely cost: the average of the last LOOKBACK_DAYS. Always an estimate. + {usd, tokens, runs (it's drawn from), basis}.""" + now = now or dt.datetime.now(dt.timezone.utc) + since = (now - dt.timedelta(days=LOOKBACK_DAYS)).isoformat(timespec="seconds") + runs = _runs(db, resource, since) + if not runs: + return {"usd": None, "tokens": None, "runs": 0, "basis": "no runs in the last 30 days to go on"} + usd, tokens = activity.spent(db, resource, since) + return { + "usd": round(usd / runs, 6), + "tokens": int(tokens / runs), + "runs": runs, + "basis": f"the average of {runs} run{'s' if runs != 1 else ''} in the last {LOOKBACK_DAYS} days", + } + + +def status(db, resource, budget=None, now=None): + """Where a resource's budget stands: {budget, since, spent: {usd, tokens, estimate}, next: estimate, left, share, + state: none | ok | near | over}. `share` is the largest of the spent shares of its caps.""" + budget = budget if budget is not None else get(db, resource) + nxt = estimate(db, resource, now) + if not budget: + return {"resource": resource, "budget": None, "state": "none", "next": nxt} + period = budget.get("period") or "month" + since = None if period == "run" else activity._since(period, now) + if period == "run": + spent = {"usd": 0.0, "tokens": 0, "estimate": False} + else: + c = activity.costs(db, [resource], since)[resource] + spent = {"usd": c["cost_usd"], "tokens": c["tokens"], "estimate": c["estimate"]} + left, shares, after = {}, [], [] + for cap, k in (("usd", "usd"), ("tokens", "tokens")): + if budget.get(cap) is None: + continue + lim = budget[cap] + left[k] = max(0, lim - spent[k]) if k == "tokens" else round(max(0.0, lim - spent[k]), 6) + shares.append(spent[k] / lim if lim else (1.0 if spent[k] else 0.0)) + if nxt.get(k) is not None: + after.append((spent[k] + nxt[k]) / lim if lim else (1.0 if spent[k] + nxt[k] else 0.0)) + share = max(shares) if shares else 0.0 + share_after = max(after) if after else None + state = "over" if share >= 1 else "near" if share >= (budget.get("warn_at") or 0.8) else "ok" + return { + "resource": resource, + "budget": budget, + "since": since, + "spent": spent, + "next": nxt, + "left": left, + "share": round(share, 4), + "share_after_next": round(share_after, 4) if share_after is not None else None, + "state": state, + } + + +# ---------- before a run ---------- +def check(db, cfg, resources, now=None): + """Whether a run that counts for these resources may go ahead. Returns {go, action, why, resource, status}: + go True (run), or action "ask" (hold it for a person), "skip". The first budget that stops it decides.""" + for res in dict.fromkeys(r for r in resources if r and REF.match(r)): + b = get(db, res) + if not b: + continue + st = status(db, res, b, now) + over_now = st["state"] == "over" + over_after = (st["share_after_next"] or 0) > 1 + if not (over_now or over_after): + continue + why = _why(res, st, over_now) + how = b.get("on_over") or "ask" + if how == "assistant": + how = _weigh(cfg, res, st, why) + if how == "run": + return {"go": True, "action": "run", "why": why + " The assistant let it run.", "resource": res, "status": st} + return {"go": False, "action": how, "why": why, "resource": res, "status": st} + return {"go": True, "action": "run", "why": None, "resource": None, "status": None} + + +def _money(v): + return f"${v:,.4f}".rstrip("0").rstrip(".") if v < 1 else f"${v:,.2f}" + + +def _why(res, st, over_now): + b, sp, nxt = st["budget"], st["spent"], st["next"] + per = "per run" if b["period"] == "run" else f"this {b['period']}" + parts = [] + if b.get("usd") is not None: + parts.append(f"{'≈' if sp['estimate'] else ''}{_money(sp['usd'])} of {_money(b['usd'])} {per}") + if b.get("tokens") is not None: + parts.append(f"{sp['tokens']:,} of {b['tokens']:,} tokens {per}") + spent = " and ".join(parts) + if over_now: + return f"{res} has spent {spent}: its budget is used up." + est = [] + if nxt.get("usd") is not None and b.get("usd") is not None: + est.append(f"≈{_money(nxt['usd'])}") + if nxt.get("tokens") is not None and b.get("tokens") is not None: + est.append(f"≈{nxt['tokens']:,} tokens") + return f"{res} has spent {spent}; the next run ({' and '.join(est) or 'cost unknown'}, {nxt['basis']}) would go over." + + +def _weigh(cfg, res, st, why): + """The decision model's call on a run that would go over: "run" only when it's sure; else "ask".""" + from . import decide + + try: + d = decide.choose( + cfg, + "A scheduled run would go over its budget. Should it run anyway, or wait for the person to decide? Run " + "only when going over is small and the work is routine; anything large, or a budget already used up, waits.", + {"run": "run it now, over budget", "ask": "hold it and ask the person"}, + { + "why": why, + "budget": st["budget"], + "spent": st["spent"], + "next_run_estimate": st["next"], + "share_after_next": st["share_after_next"], + }, + ) + except decide.Undecided as e: + log.info("budgets: no decision engine for %s (%s); asking", res, e) + return "ask" + return d["choice"] if decide.sure(cfg, d) else "ask" + + +# ---------- the periodic check ---------- +def sweep(db, cfg, now=None, every=None): + """Look at every budget (at most once `every` seconds, when given): one that has crossed warn_at or its cap since it was last looked at gets an alert (on the + budget, and a ledger row `budget.near` / `budget.over` for it), once per period. Returns the alerts made.""" + if every is not None: + if time.monotonic() - _swept["at"] < every and _swept["at"]: + return [] + _swept["at"] = time.monotonic() + made = [] + for b in all_budgets(db): + res = b["resource"] + if b.get("period") == "run": + continue + try: + st = status(db, res, b, now) + except Exception as e: # noqa: BLE001 - one bad budget doesn't stop the rest + log.info("budgets: couldn't check %s: %s", res, e) + continue + if st["state"] not in ("near", "over"): + continue + last = b.get("alert") or {} + if last.get("since") == st["since"] and (last.get("state") == st["state"] or last.get("state") == "over"): + continue + alert = {"state": st["state"], "since": st["since"], "at": store.now(), "share": st["share"]} + db.q("UPDATE $r SET alert = $a", r=_key(res), a=alert) + activity.record( + "run", f"budget.{st['state']}", [res], cfg, db, detail={"share": st["share"], "estimate": st["spent"]["estimate"] or None} + ) + made.append({"resource": res, **alert}) + return made diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index 9fc186a7d..aa798e559 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -19,10 +19,10 @@ R = store.R PIPELINE = ["transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe", "analyze", "embed", "summarize", "report"] AFTER_IMPORT = ["analyze", "embed", "summarize", "report"] -ACTIVE = ["queued", "running"] +ACTIVE = ["queued", "running", "held"] # held: over budget, waiting for someone to pick (budgets.py) FIELDS = ( "record::id(id) AS id, recording, space, batch, pipeline, steps, step_index, next_step, status, worker, error, attempts, " - "created_by, created_at, started_at, finished_at, updated_at, cancel_requested, log_total, cost_usd, tokens, cost_estimate" + "created_by, created_at, started_at, finished_at, updated_at, cancel_requested, log_total, cost_usd, tokens, cost_estimate, hold" ) MEDIA_STEPS = {"transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe"} # they take longer the longer the recording FILED = ("audio", "document", "image") # sources with a file of their own for the steps to work on @@ -442,6 +442,8 @@ def run_job(db, cfg_fn, job, worker, can, log=None): """Run a job's steps from where it is, with its calls counted for it (activity.py); when it ends, a run row with what it cost in all, kept on the job too (cost_usd, tokens).""" refs = job_refs(job) + if not job.get("budget_ok") and (held := _budget(db, cfg_fn, job, refs)): + return held with activity.scope(db, *refs): outcome = _run_job(db, cfg_fn, job, worker, can, log) if outcome in FINAL: @@ -464,6 +466,52 @@ def run_job(db, cfg_fn, job, worker, can, log=None): return outcome +def _budget(db, cfg_fn, job, refs): + """Before a job's first step: its budgets (budgets.py). Over one, the job is held for someone to pick ("held"), or + skipped ("cancelled", saying why); else None, and it isn't checked again for this job.""" + from . import budgets + + jr, t = R("job", job["id"]), store.now() + try: + verdict = budgets.check(db, cfg_fn(), refs) + except Exception as e: # noqa: BLE001 - a budget that can't be read doesn't stop the work + verdict = {"go": True} + db.q("UPDATE $j SET budget_note = $n", j=jr, n=f"budget check failed: {type(e).__name__}: {e}"[:300]) + if verdict["go"]: + db.q("UPDATE $j SET budget_ok = true", j=jr) + return None + hold = {"why": verdict["why"], "resource": verdict["resource"], "action": verdict["action"], "at": t} + if verdict["action"] == "skip": + db.q( + "UPDATE $j SET status = 'cancelled', worker = NONE, hold = $h, error = $e, finished_at = $t, updated_at = $t", + j=jr, + h=hold, + e="skipped, over budget: " + verdict["why"], + t=t, + ) + activity.record("run", "job.skipped", refs, cfg_fn(), db, ok=False, error="OverBudget", detail={"budget": verdict["resource"]}) + return "cancelled" + db.q("UPDATE $j SET status = 'held', worker = NONE, hold = $h, updated_at = $t", j=jr, h=hold, t=t) + activity.record("run", "job.held", refs, cfg_fn(), db, detail={"budget": verdict["resource"]}) + return "held" + + +def release(db, jid, run=True, by=None): + """Someone's pick for a job held over budget: run it (once, whatever its budgets say) or skip it.""" + j = get(db, jid) + if not j: + raise KeyError(jid) + if j["status"] != "held": + raise ValueError(f"the job isn't waiting on a budget (it's {j['status']})") + t = store.now() + picked = {"run": run, "by": by, "at": t} + if run: + sets = "status = 'queued', budget_ok = true" + else: + sets = "status = 'cancelled', error = 'skipped, over budget', finished_at = $t" + db.q(f"UPDATE $j SET {sets}, hold.decided = $d, updated_at = $t WHERE status = 'held'", j=R("job", jid), d=picked, t=t) + + def _run_job(db, cfg_fn, job, worker, can, log=None): jid, rid, jr, wr = job["id"], job["recording"], R("job", job["id"]), R("worker", worker) steps, i = job["steps"], job.get("step_index") or 0 @@ -616,7 +664,7 @@ def cancel(db, jid): if not j: raise KeyError(jid) t = store.now() - if j["status"] == "queued": + if j["status"] in ("queued", "held"): db.q("UPDATE $j SET status = 'cancelled', finished_at = $t, updated_at = $t", j=R("job", jid), t=t) elif j["status"] == "running": db.q("UPDATE $j SET cancel_requested = true, updated_at = $t", j=R("job", jid), t=t) diff --git a/fastapi_backend/app/domain/routines.py b/fastapi_backend/app/domain/routines.py index 82b7e2497..83948e8ba 100644 --- a/fastapi_backend/app/domain/routines.py +++ b/fastapi_backend/app/domain/routines.py @@ -24,6 +24,7 @@ from . import ( activity, + budgets, fedora, jobs, organize, @@ -241,7 +242,7 @@ def _with_changes(db, rows): def runs(db, rid, limit=20): rows = db.rows( "SELECT record::id(id) AS id, routine, trigger, by, status, started_at, finished_at, results, error, cost_usd, tokens, " - "cost_estimate FROM routine_run " + "cost_estimate, hold FROM routine_run " "WHERE routine = $r ORDER BY id DESC LIMIT $n", r=int(rid), n=int(limit), @@ -256,15 +257,16 @@ def get_run(db, run_id): return _with_changes(db, [row])[0] -def request_run(db, rid, by=None, propose_only=False): - """Ask for a run as soon as the scheduler next looks (even when the routine is off).""" +def request_run(db, rid, by=None, propose_only=False, over_budget=False): + """Ask for a run as soon as the scheduler next looks (even when the routine is off). over_budget: someone chose to + run it whatever its budgets say (budgets.py).""" r = get(db, rid) if r["running"]: raise ValueError("this routine is running now") db.q( "UPDATE $r SET run_now = $d", r=R("routine", int(rid)), - d=store.clean({"by": by, "at": store.now(), "propose_only": propose_only or None}), + d=store.clean({"by": by, "at": store.now(), "propose_only": propose_only or None, "over_budget": over_budget or None}), ) @@ -507,12 +509,102 @@ def _idle(db, cfg, r): return not semantic.configured(cfg) or bool(semantic.failing(db, cfg)) or not semantic.unindexed(db, cfg, _spaces(db, r), 1) +def refs(db, r): + """What a routine's runs count for, for its budgets: it, the pipelines and workflows its actions name, and its + namespaces.""" + out = [f"routine:{r['id']}"] + for a in r.get("actions") or []: + if a.get("type") == "pipeline" and isinstance(a.get("pipeline"), int): + out.append(f"pipeline:{a['pipeline']}") + if a.get("type") == "workflow" and a.get("workflow") is not None: + out.append(f"workflow:{int(a['workflow'])}") + out += [f"space:{s}" for s in _spaces(db, r)] + return out + + +def _held(db, cfg, r, trigger, by, now): + """Whether a routine that's due is over a budget (budgets.py): if so its run is held for someone to pick, or + skipped, as a run row that says why, and the routine is free again (its next time stays as claimed).""" + try: + verdict = budgets.check(db, cfg, refs(db, r), now) + except Exception: # noqa: BLE001 - a budget that can't be read doesn't stop the work + return False + if verdict["go"]: + return False + held = verdict["action"] != "skip" + status = "held" if held else "skipped" + waiting = db.values("SELECT VALUE record::id(id) FROM routine_run WHERE routine = $r AND status = 'held'", r=int(r["id"])) + if held and waiting: # one held run waits for a pick; later times it was due add to it rather than pile up + db.q("UPDATE $r SET hold.missed += 1, hold.why = $w", r=R("routine_run", waiting[0]), w=verdict["why"]) + db.q("UPDATE $r SET running_since = NONE, heartbeat_at = NONE", r=R("routine", r["id"])) + return True + run_id = db.next_id("routine_run") + db.q( + "CREATE $r CONTENT $d", + r=R("routine_run", run_id), + d=store.clean( + { + "routine": int(r["id"]), + "trigger": trigger, + "by": by, + "status": status, + "started_at": _iso(now), + "finished_at": None if held else _iso(now), + "results": [], + "hold": {"why": verdict["why"], "resource": verdict["resource"], "action": verdict["action"], "missed": 0}, + } + ), + ) + db.q( + "UPDATE $r SET running_since = NONE, heartbeat_at = NONE, last_status = $s, last_run = $i", + r=R("routine", r["id"]), + s=status, + i=run_id, + ) + activity.record( + "run", + f"routine.{status}", + [f"routine:{r['id']}", f"routine_run:{run_id}"], + cfg, + db, + ok=held, + error=None if held else "OverBudget", + detail={"budget": verdict["resource"], "trigger": trigger}, + ) + return True + + +def decide_held(db, run_id, run=True, by=None): + """Someone's pick for a run held over budget: run it now (once, whatever its budgets say) or skip it.""" + row = db.one("SELECT routine, status FROM $r", r=R("routine_run", int(run_id))) + if not row: + raise KeyError(run_id) + if row["status"] != "held": + raise ValueError(f"this run isn't waiting on a budget (it's {row['status']})") + t = store.now() + db.q( + "UPDATE $r SET status = $s, finished_at = $t, hold.decided = $d WHERE status = 'held'", + r=R("routine_run", int(run_id)), + s="released" if run else "skipped", + t=t, + d={"run": run, "by": by, "at": t}, + ) + if run: + request_run(db, row["routine"], by=by, over_budget=True) + return row["routine"] + + def run_due(db, cfg, log=print, now=None): """Run every routine that is due (or asked to run now); how many ran.""" now = now or dt.datetime.now(dt.timezone.utc) stamp = _iso(now) sweep(db, now) activity.tidy(db, cfg) # the activity ledger past activity.keep_days, once an hour + try: + budgets.sweep(db, cfg, every=budgets.SWEEP_SECONDS) # budgets that crossed warn_at or their cap: warned once + except Exception as e: # noqa: BLE001 - the schedule carries on + if log: + log(f"budgets: {type(e).__name__}: {e}") due = db.rows( "SELECT record::id(id) AS id, schedule, timezone, next_run_at, run_now, namespaces, actions FROM routine " "WHERE run_now != NONE OR (enabled = true AND next_run_at != NONE AND next_run_at <= $n)", @@ -527,6 +619,8 @@ def run_due(db, cfg, log=print, now=None): if not _claim(db, r, now): continue req = r.get("run_now") or {} + if not req.get("over_budget") and _held(db, cfg, r, "manual" if req else "schedule", req.get("by"), now): + continue try: run(db, cfg, r["id"], "manual" if req else "schedule", req.get("by"), bool(req.get("propose_only")), log) done += 1 diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 78de6bec6..c4dc10f62 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -756,6 +756,8 @@ def close(self): "DEFINE TABLE IF NOT EXISTS activity SCHEMALESS", "DEFINE INDEX IF NOT EXISTS activity_at ON activity FIELDS at", "DEFINE INDEX IF NOT EXISTS activity_resources ON activity FIELDS resources", + # caps on what a routine, pipeline, workflow or namespace may cost (domain/budgets.py): budget:_ + "DEFINE TABLE IF NOT EXISTS budget SCHEMALESS", # background work "DEFINE TABLE IF NOT EXISTS job SCHEMALESS", "DEFINE INDEX IF NOT EXISTS job_status ON job FIELDS status", diff --git a/fastapi_backend/app/schemas/budgets.py b/fastapi_backend/app/schemas/budgets.py new file mode 100644 index 000000000..1009e2356 --- /dev/null +++ b/fastapi_backend/app/schemas/budgets.py @@ -0,0 +1,62 @@ +"""Budgets: caps on what a routine, pipeline, workflow or namespace may cost (docs/budgets.md).""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import Field + +from app.schemas.common import RequestModel, ResponseModel + + +class BudgetSet(RequestModel): + usd: float | None = Field(None, ge=0, description="the cap in USD (from the costs in the activity ledger)") + tokens: int | None = Field(None, ge=0, description="the cap in tokens (in and out)") + period: Literal["run", "day", "week", "month"] = Field( + "month", description="what the cap is for: one run, or a day, week or month (UTC)" + ) + on_over: Literal["ask", "skip", "assistant"] = Field( + "ask", + description="a run that would go over: ask (hold it until someone picks run or skip), skip, or assistant (the " + "decision model weighs it; it runs only when sure, else asks)", + ) + warn_at: float = Field(0.8, ge=0.1, le=1, description="the share spent at which it's marked near (and warned about)") + note: str | None = Field(None, max_length=300) + + +class Budget(ResponseModel): + resource: str + usd: float | None = None + tokens: int | None = None + period: str + on_over: str + warn_at: float + note: str | None = None + by: str | None = None + updated_at: str | None = None + alert: dict[str, Any] | None = Field(None, description="the last warning: {state near|over, since, at, share}") + + +class Spent(ResponseModel): + usd: float + tokens: int + estimate: bool = Field(description="true: some calls had no figure, so usd is a floor") + + +class NextRun(ResponseModel): + usd: float | None = None + tokens: int | None = None + runs: int = Field(description="the past runs the estimate is drawn from") + basis: str + + +class BudgetStatus(ResponseModel): + resource: str + budget: Budget | None = None + state: Literal["none", "ok", "near", "over"] + since: str | None = Field(None, description="the start of the period being counted") + spent: Spent | None = None + next: NextRun = Field(description="what one more run will likely cost: always an estimate") + left: dict[str, float] | None = None + share: float | None = Field(None, description="the largest share of a cap spent (1 is all of it)") + share_after_next: float | None = Field(None, description="the share once the next run (estimated) is done") diff --git a/fastapi_backend/app/schemas/jobs.py b/fastapi_backend/app/schemas/jobs.py index 3369a0d32..5d21b95dc 100644 --- a/fastapi_backend/app/schemas/jobs.py +++ b/fastapi_backend/app/schemas/jobs.py @@ -56,6 +56,11 @@ class Job(ResponseModel): cost_usd: float | None = Field(None, description="what its calls cost, from the activity ledger (docs/activity.md); set when it ends") tokens: int | None = None cost_estimate: bool | None = Field(None, description="true: some calls had no price or token counts, so cost_usd is a floor") + hold: dict[str, Any] | None = Field( + None, + description="held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), " + "decided: {run, by, at} once someone picked}", + ) log: list[str] | None = Field(None, description="the last 200 lines; GET /jobs/{jid}/log has them all") log_total: int | None = Field(None, description="how many lines the run has logged") step_runs: list[StepRun | None] | None = Field( diff --git a/fastapi_backend/app/schemas/routines.py b/fastapi_backend/app/schemas/routines.py index 0e5fef189..c03120219 100644 --- a/fastapi_backend/app/schemas/routines.py +++ b/fastapi_backend/app/schemas/routines.py @@ -62,6 +62,13 @@ class RoutineUpdate(RequestModel): class RoutineRunRequest(RequestModel): propose_only: bool = Field(False, description="graph workflows propose every change instead of making the sure ones") + over_budget: bool = Field( + False, description="run it even though it's over a budget (docs/budgets.md); without it, a run over budget is refused (409)" + ) + + +class HeldRunDecision(RequestModel): + run: bool = Field(description="true: run it now, once, whatever its budgets say; false: skip it") class RoutineRun(ResponseModel): @@ -79,6 +86,11 @@ class RoutineRun(ResponseModel): cost_usd: float | None = Field(None, description="what its calls cost, from the activity ledger (docs/activity.md); set when it ends") tokens: int | None = None cost_estimate: bool | None = Field(None, description="true: some calls had no price or token counts, so cost_usd is a floor") + hold: dict[str, Any] | None = Field( + None, + description="held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), " + "decided: {run, by, at} once someone picked}", + ) class SchedulePreview(ResponseModel): diff --git a/fastapi_backend/tests/api/test_budgets.py b/fastapi_backend/tests/api/test_budgets.py new file mode 100644 index 000000000..0f217f8ab --- /dev/null +++ b/fastapi_backend/tests/api/test_budgets.py @@ -0,0 +1,171 @@ +"""Budgets: off unless set; a run over one waits for someone to pick (or is skipped, or the assistant weighs it).""" + +from __future__ import annotations + +import pytest + +from app.domain import activity, budgets, decide, jobs, routines, store +from tests.helpers import drain, login, make_user, seed + +R = store.R + + +@pytest.fixture +def h(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +def spend(db, resource, usd=0.0, tokens=0): + activity.record("out", "model.chat", [resource], db=db, model="m", cost_usd=usd, tokens_in=tokens) + + +def routine(client, h, name="Sync"): + return client.post("/api/v1/routines", headers=h, json={"name": name, "actions": [{"type": "sync"}]}).json()["id"] + + +def test_setting_reading_and_removing_a_budget(client, db, h): + rid = routine(client, h) + res = f"routine:{rid}" + st = client.get("/api/v1/budgets/status", headers=h, params={"resource": res}).json() + assert st["state"] == "none" and st["next"]["runs"] == 0 # off unless set + + bad = [{"period": "month"}, {"usd": -1}, {"usd": 1, "period": "year"}, {"usd": 1, "on_over": "maybe"}, {"usd": 1, "warn_at": 2}] + for body in bad: + assert client.put("/api/v1/budgets", headers=h, params={"resource": res}, json=body).status_code in (400, 422), body + assert client.put("/api/v1/budgets", headers=h, params={"resource": "chat:1"}, json={"usd": 1}).status_code == 400 + + spend(db, res, usd=0.85, tokens=900) + st = client.put("/api/v1/budgets", headers=h, params={"resource": res}, json={"usd": 1, "tokens": 10_000}).json() + assert (st["state"], st["share"], st["spent"]["usd"], st["left"]["usd"], st["left"]["tokens"]) == ("near", 0.85, 0.85, 0.15, 9100) + assert st["budget"]["on_over"] == "ask" and st["budget"]["period"] == "month" + assert [b["resource"] for b in client.get("/api/v1/budgets", headers=h).json()] == [res] + + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + e = login(client, "ed@x.io", "editor password 1") + assert client.put("/api/v1/budgets", headers=e, params={"resource": res}, json={"usd": 9}).status_code == 403 + assert client.delete("/api/v1/budgets", headers=h, params={"resource": res}).status_code == 200 + assert client.delete("/api/v1/budgets", headers=h, params={"resource": res}).status_code == 404 + assert any(r["action"] == "budget.set" for r in activity.history(db, res)) + + +def test_a_routine_over_budget_waits_for_a_pick(client, db, cfg, h): + rid = routine(client, h) + res = f"routine:{rid}" + budgets.put(db, res, usd=0.5) + spend(db, res, usd=0.6) + + # a person asking is told where it stands, and can choose to run it anyway + r = client.post(f"/api/v1/routines/{rid}/run", headers=h, json={}) + assert r.status_code == 409 and "budget is used up" in r.json()["detail"] + + # the schedule (or anything else) asking: held, once, however many times it comes due + routines.request_run(db, rid) + assert routines.run_due(db, cfg, log=None) == 0 + routines.request_run(db, rid) + routines.run_due(db, cfg, log=None) + held = routines.runs(db, rid) + assert [x["status"] for x in held] == ["held"] and held[0]["hold"]["resource"] == res + assert held[0]["hold"]["missed"] == 1 and "$0.6 of $0.5 this month" in held[0]["hold"]["why"] + assert routines.get(db, rid)["last_status"] == "held" + + # doing nothing changes nothing; a pick to run it runs it once + assert client.post(f"/api/v1/routines/runs/{held[0]['id']}/decide", headers=h, json={"run": True}).status_code == 200 + assert routines.run_due(db, cfg, log=None) == 1 + assert [x["status"] for x in routines.runs(db, rid)] == ["done", "released"] + assert client.post(f"/api/v1/routines/runs/{held[0]['id']}/decide", headers=h, json={"run": True}).status_code == 400 + + # chosen up front by a person + assert client.post(f"/api/v1/routines/{rid}/run", headers=h, json={"over_budget": True}).status_code == 200 + assert routines.run_due(db, cfg, log=None) == 1 + + # skip instead of ask + budgets.put(db, res, usd=0.5, on_over="skip") + routines.request_run(db, rid) + routines.run_due(db, cfg, log=None) + assert routines.runs(db, rid)[0]["status"] == "skipped" + + +def test_the_next_run_is_estimated_from_past_ones(client, db, cfg, h): + rid = routine(client, h) + res = f"routine:{rid}" + for _ in range(2): + with activity.scope(db, res): + spend(db, res, usd=0.3, tokens=100) + routines.run(db, cfg, rid) + st = budgets.status(db, res) + assert (st["next"]["usd"], st["next"]["tokens"], st["next"]["runs"]) == (0.3, 100, 2) + + # 0.6 spent of 0.8 isn't over, but another ≈0.3 run would be + budgets.put(db, res, usd=0.8, period="day") + v = budgets.check(db, cfg, [res]) + assert (v["go"], v["action"], v["resource"]) == (False, "ask", res) + assert "the next run (≈$0.3" in v["why"] and "would go over" in v["why"] + budgets.put(db, res, usd=0.95, period="day") + assert budgets.check(db, cfg, [res])["go"] is True + + # per run: only the estimate counts + budgets.put(db, res, usd=0.2, period="run") + assert budgets.check(db, cfg, [res])["go"] is False + + +def test_the_assistant_weighs_it_when_asked_to(client, db, cfg, h, monkeypatch): + rid = routine(client, h) + res = f"routine:{rid}" + budgets.put(db, res, tokens=10, on_over="assistant") + spend(db, res, tokens=20) + picks = [] + + def choose(cfg_, question, options, state): + picks.append(state) + return {"choice": "run", "confidence": confidence, "ranked": [], "by": "jev"} + + monkeypatch.setattr(decide, "choose", choose) + confidence = 0.95 + v = budgets.check(db, cfg, [res]) + assert v["go"] is True and "assistant let it run" in v["why"] + assert picks[0]["spent"]["tokens"] == 20 and picks[0]["budget"]["tokens"] == 10 + confidence = 0.5 # not sure: ask + assert budgets.check(db, cfg, [res])["action"] == "ask" + + def undecided(*a): + raise decide.Undecided("nothing set up") + + monkeypatch.setattr(decide, "choose", undecided) + assert budgets.check(db, cfg, [res])["action"] == "ask" + + +def test_jobs_over_a_budget_are_held_until_released(client, db, cfg, folder, h): + rids = seed(db, cfg, folder) + pods = store.ns_id(db, "pods") + budgets.put(db, f"space:{pods}", usd=0.1, period="week") + spend(db, f"space:{pods}", usd=0.2) + for rid in rids: + jobs.enqueue(db, rid, ["analyze"]) + drain(db, cfg) + got = {j["recording"]: j for j in jobs.list_jobs(db)} + held = [j for j in got.values() if j["status"] == "held"] + assert len(held) == 2 and all(j["hold"]["resource"] == f"space:{pods}" for j in held) + assert [j["status"] for j in got.values() if j["space"] != pods] == ["succeeded"] + assert jobs.enqueue(db, held[0]["recording"], ["analyze"]) == held[0]["id"] # still the one job + + r = client.get("/api/v1/jobs", headers=h).json() + assert {j["status"] for j in r["jobs"]} >= {"held"} + assert client.post(f"/api/v1/jobs/{held[0]['id']}/release", headers=h, json={"run": True}).status_code == 200 + assert client.post(f"/api/v1/jobs/{held[1]['id']}/release", headers=h, json={"run": False}).status_code == 200 + drain(db, cfg) + assert jobs.get(db, held[0]["id"])["status"] == "succeeded" # let through once, not held again + assert jobs.get(db, held[1]["id"])["status"] == "cancelled" + assert client.post(f"/api/v1/jobs/{held[0]['id']}/release", headers=h, json={"run": True}).status_code == 400 + + +def test_the_periodic_check_warns_once_per_period(db, cfg): + budgets.put(db, "workflow:1", usd=1) + assert budgets.sweep(db, cfg) == [] + spend(db, "workflow:1", usd=0.9) + assert [a["state"] for a in budgets.sweep(db, cfg)] == ["near"] + assert budgets.sweep(db, cfg) == [] + spend(db, "workflow:1", usd=0.2) + assert [a["state"] for a in budgets.sweep(db, cfg)] == ["over"] + assert budgets.sweep(db, cfg) == [] + assert [r["action"] for r in activity.history(db, "workflow:1", kind="run")] == ["budget.over", "budget.near"] diff --git a/mkdocs.yml b/mkdocs.yml index 02571a2f5..8463f435e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -76,6 +76,7 @@ nav: - Sensors: sensors.md - Telemetry: telemetry.md - Activity and costs: activity.md + - Budgets: budgets.md - IIIF: iiif.md - Graph and Cypher: graph.md - Linked data (RDF): rdf.md diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 62183a899..0aa3def1e 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -6,6 +6,7 @@ export { Admin, Auth, Batches, + Budgets, Chats, Collections, Comments, @@ -145,6 +146,14 @@ export type { BrowseSourceErrors, BrowseSourceResponse, BrowseSourceResponses, + Budget, + BudgetSet, + BudgetStatus, + BudgetStatusData, + BudgetStatusError, + BudgetStatusErrors, + BudgetStatusResponse, + BudgetStatusResponses, BulkUpdateMetadataData, BulkUpdateMetadataError, BulkUpdateMetadataErrors, @@ -436,6 +445,11 @@ export type { DecideApprovalErrors, DecideApprovalResponse, DecideApprovalResponses, + DecideHeldRunData, + DecideHeldRunError, + DecideHeldRunErrors, + DecideHeldRunResponse, + DecideHeldRunResponses, DeclineAccessRequestData, DeclineAccessRequestError, DeclineAccessRequestErrors, @@ -1011,6 +1025,7 @@ export type { HealthDisk, HealthSource, Heard, + HeldRunDecision, HideEntityData, HideEntityError, HideEntityErrors, @@ -1117,6 +1132,9 @@ export type { ListBatchesData, ListBatchesResponse, ListBatchesResponses, + ListBudgetsData, + ListBudgetsResponse, + ListBudgetsResponses, ListChatsData, ListChatsResponse, ListChatsResponses, @@ -1475,6 +1493,7 @@ export type { NewTokenErrors, NewTokenResponse, NewTokenResponses, + NextRun, NodeParam, NodeTrace, NodeType, @@ -1673,6 +1692,16 @@ export type { ReindexSearchData, ReindexSearchResponse, ReindexSearchResponses, + ReleaseJobData, + ReleaseJobError, + ReleaseJobErrors, + ReleaseJobResponse, + ReleaseJobResponses, + RemoveBudgetData, + RemoveBudgetError, + RemoveBudgetErrors, + RemoveBudgetResponse, + RemoveBudgetResponses, RemoveLoginProviderData, RemoveLoginProviderError, RemoveLoginProviderErrors, @@ -1924,6 +1953,11 @@ export type { SensorType, SensorUpdate, SeriesPoint, + SetBudgetData, + SetBudgetError, + SetBudgetErrors, + SetBudgetResponse, + SetBudgetResponses, SetCollectionMemberData, SetCollectionMemberError, SetCollectionMemberErrors, @@ -2026,6 +2060,7 @@ export type { SpeakTextErrors, SpeakTextResponse, SpeakTextResponses, + Spent, SplitSegmentData, SplitSegmentError, SplitSegmentErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 09a6590d7..410d26816 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -48,6 +48,9 @@ import type { BrowseSourceData, BrowseSourceErrors, BrowseSourceResponses, + BudgetStatusData, + BudgetStatusErrors, + BudgetStatusResponses, BulkUpdateMetadataData, BulkUpdateMetadataErrors, BulkUpdateMetadataResponses, @@ -196,6 +199,9 @@ import type { DecideApprovalData, DecideApprovalErrors, DecideApprovalResponses, + DecideHeldRunData, + DecideHeldRunErrors, + DecideHeldRunResponses, DeclineAccessRequestData, DeclineAccessRequestErrors, DeclineAccessRequestResponses, @@ -546,6 +552,8 @@ import type { ListBackendsResponses, ListBatchesData, ListBatchesResponses, + ListBudgetsData, + ListBudgetsResponses, ListChatsData, ListChatsResponses, ListCollectionMembersData, @@ -814,6 +822,12 @@ import type { RegisterResponses, ReindexSearchData, ReindexSearchResponses, + ReleaseJobData, + ReleaseJobErrors, + ReleaseJobResponses, + RemoveBudgetData, + RemoveBudgetErrors, + RemoveBudgetResponses, RemoveLoginProviderData, RemoveLoginProviderErrors, RemoveLoginProviderResponses, @@ -944,6 +958,9 @@ import type { SendMessageData, SendMessageErrors, SendMessageResponses, + SetBudgetData, + SetBudgetErrors, + SetBudgetResponses, SetCollectionMemberData, SetCollectionMemberErrors, SetCollectionMemberResponses, @@ -2395,6 +2412,64 @@ export class Activity { } } +export class Budgets { + /** + * Remove Budget + */ + public static removeBudget( + options: Options, + ): RequestResult { + return (options.client ?? client).delete({ + url: "/api/v1/budgets", + ...options, + }); + } + + /** + * List Budgets + * + * Every budget, with where it stands now. + */ + public static listBudgets( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/budgets", + ...options, + }); + } + + /** + * Set Budget + */ + public static setBudget( + options: Options, + ): RequestResult { + return (options.client ?? client).put({ + url: "/api/v1/budgets", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Budget Status + * + * Where a resource's budget stands, and what its next run will likely cost (also without a budget). + */ + public static budgetStatus( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/budgets/status", + ...options, + }); + } +} + export class Fedora { /** * Get Fedora Status @@ -5463,6 +5538,25 @@ export class Jobs { }); } + /** + * Release Job + * + * Pick for a job held over a budget (docs/budgets.md): run it now, once, whatever its budgets say, or skip it + * (admins, who set the budgets). + */ + public static releaseJob( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/jobs/{jid}/release", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * Retry Job */ @@ -7076,7 +7170,8 @@ export class Routines { /** * Run Routine * - * Run it as soon as the scheduler next looks (within half a minute), even when it is off. + * Run it as soon as the scheduler next looks (within half a minute), even when it is off. A run over one of its + * budgets is refused (409, saying where the budget stands) unless over_budget is true (docs/budgets.md). */ public static runRoutine( options: Options, @@ -7091,6 +7186,24 @@ export class Routines { }); } + /** + * Decide Held Run + * + * Pick for a run held over a budget: run it now, once, whatever its budgets say, or skip it. + */ + public static decideHeldRun( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/routines/runs/{run_id}/decide", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * List Runs */ diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 152c388dc..5660ccad0 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -1107,6 +1107,142 @@ export type BrowseEntry = { [key: string]: unknown; }; +/** + * Budget + */ +export type Budget = { + /** + * Resource + */ + resource: string; + /** + * Usd + */ + usd?: number | null; + /** + * Tokens + */ + tokens?: number | null; + /** + * Period + */ + period: string; + /** + * On Over + */ + on_over: string; + /** + * Warn At + */ + warn_at: number; + /** + * Note + */ + note?: string | null; + /** + * By + */ + by?: string | null; + /** + * Updated At + */ + updated_at?: string | null; + /** + * Alert + * + * the last warning: {state near|over, since, at, share} + */ + alert?: { + [key: string]: unknown; + } | null; + [key: string]: unknown; +}; + +/** + * BudgetSet + */ +export type BudgetSet = { + /** + * Usd + * + * the cap in USD (from the costs in the activity ledger) + */ + usd?: number | null; + /** + * Tokens + * + * the cap in tokens (in and out) + */ + tokens?: number | null; + /** + * Period + * + * what the cap is for: one run, or a day, week or month (UTC) + */ + period?: "run" | "day" | "week" | "month"; + /** + * On Over + * + * a run that would go over: ask (hold it until someone picks run or skip), skip, or assistant (the decision model weighs it; it runs only when sure, else asks) + */ + on_over?: "ask" | "skip" | "assistant"; + /** + * Warn At + * + * the share spent at which it's marked near (and warned about) + */ + warn_at?: number; + /** + * Note + */ + note?: string | null; +}; + +/** + * BudgetStatus + */ +export type BudgetStatus = { + /** + * Resource + */ + resource: string; + budget?: Budget | null; + /** + * State + */ + state: "none" | "ok" | "near" | "over"; + /** + * Since + * + * the start of the period being counted + */ + since?: string | null; + spent?: Spent | null; + /** + * what one more run will likely cost: always an estimate + */ + next: NextRun; + /** + * Left + */ + left?: { + [key: string]: number; + } | null; + /** + * Share + * + * the largest share of a cap spent (1 is all of it) + */ + share?: number | null; + /** + * Share After Next + * + * the share once the next run (estimated) is done + */ + share_after_next?: number | null; + [key: string]: unknown; +}; + /** * ChangeAsk */ @@ -4822,6 +4958,18 @@ export type Heard = { [key: string]: unknown; }; +/** + * HeldRunDecision + */ +export type HeldRunDecision = { + /** + * Run + * + * true: run it now, once, whatever its budgets say; false: skip it + */ + run: boolean; +}; + /** * Highlight */ @@ -5551,6 +5699,14 @@ export type Job = { * true: some calls had no price or token counts, so cost_usd is a floor */ cost_estimate?: boolean | null; + /** + * Hold + * + * held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), decided: {run, by, at} once someone picked} + */ + hold?: { + [key: string]: unknown; + } | null; /** * Log * @@ -6610,6 +6766,31 @@ export type NamespaceUpdate = { } | null; }; +/** + * NextRun + */ +export type NextRun = { + /** + * Usd + */ + usd?: number | null; + /** + * Tokens + */ + tokens?: number | null; + /** + * Runs + * + * the past runs the estimate is drawn from + */ + runs: number; + /** + * Basis + */ + basis: string; + [key: string]: unknown; +}; + /** * NodeParam */ @@ -10102,6 +10283,14 @@ export type RoutineRun = { * true: some calls had no price or token counts, so cost_usd is a floor */ cost_estimate?: boolean | null; + /** + * Hold + * + * held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), decided: {run, by, at} once someone picked} + */ + hold?: { + [key: string]: unknown; + } | null; [key: string]: unknown; }; @@ -10115,6 +10304,12 @@ export type RoutineRunRequest = { * graph workflows propose every change instead of making the sure ones */ propose_only?: boolean; + /** + * Over Budget + * + * run it even though it's over a budget (docs/budgets.md); without it, a run over budget is refused (409) + */ + over_budget?: boolean; }; /** @@ -12221,6 +12416,27 @@ export type SpeakerSuggestion = { [key: string]: unknown; }; +/** + * Spent + */ +export type Spent = { + /** + * Usd + */ + usd: number; + /** + * Tokens + */ + tokens: number; + /** + * Estimate + * + * true: some calls had no figure, so usd is a floor + */ + estimate: boolean; + [key: string]: unknown; +}; + /** * Started * @@ -16131,6 +16347,120 @@ export type TopResourcesResponses = { export type TopResourcesResponse = TopResourcesResponses[keyof TopResourcesResponses]; +export type RemoveBudgetData = { + body?: never; + path?: never; + query: { + /** + * Resource + * + * routine:, pipeline:, workflow: or space: + */ + resource: string; + }; + url: "/api/v1/budgets"; +}; + +export type RemoveBudgetErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type RemoveBudgetError = RemoveBudgetErrors[keyof RemoveBudgetErrors]; + +export type RemoveBudgetResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type RemoveBudgetResponse = RemoveBudgetResponses[keyof RemoveBudgetResponses]; + +export type ListBudgetsData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/budgets"; +}; + +export type ListBudgetsResponses = { + /** + * Response Budgets-List Budgets + * + * Successful Response + */ + 200: Array; +}; + +export type ListBudgetsResponse = ListBudgetsResponses[keyof ListBudgetsResponses]; + +export type SetBudgetData = { + body: BudgetSet; + path?: never; + query: { + /** + * Resource + * + * routine:, pipeline:, workflow: or space: + */ + resource: string; + }; + url: "/api/v1/budgets"; +}; + +export type SetBudgetErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type SetBudgetError = SetBudgetErrors[keyof SetBudgetErrors]; + +export type SetBudgetResponses = { + /** + * Successful Response + */ + 200: BudgetStatus; +}; + +export type SetBudgetResponse = SetBudgetResponses[keyof SetBudgetResponses]; + +export type BudgetStatusData = { + body?: never; + path?: never; + query: { + /** + * Resource + * + * routine:, pipeline:, workflow: or space: + */ + resource: string; + }; + url: "/api/v1/budgets/status"; +}; + +export type BudgetStatusErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type BudgetStatusError = BudgetStatusErrors[keyof BudgetStatusErrors]; + +export type BudgetStatusResponses = { + /** + * Successful Response + */ + 200: BudgetStatus; +}; + +export type BudgetStatusResponse = BudgetStatusResponses[keyof BudgetStatusResponses]; + export type GetFedoraStatusData = { body?: never; path?: never; @@ -22379,6 +22709,36 @@ export type CancelJobResponses = { export type CancelJobResponse = CancelJobResponses[keyof CancelJobResponses]; +export type ReleaseJobData = { + body: HeldRunDecision; + path: { + /** + * Jid + */ + jid: number; + }; + query?: never; + url: "/api/v1/jobs/{jid}/release"; +}; + +export type ReleaseJobErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ReleaseJobError = ReleaseJobErrors[keyof ReleaseJobErrors]; + +export type ReleaseJobResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type ReleaseJobResponse = ReleaseJobResponses[keyof ReleaseJobResponses]; + export type RetryJobData = { body?: never; path: { @@ -25342,6 +25702,36 @@ export type RunRoutineResponses = { export type RunRoutineResponse = RunRoutineResponses[keyof RunRoutineResponses]; +export type DecideHeldRunData = { + body: HeldRunDecision; + path: { + /** + * Run Id + */ + run_id: number; + }; + query?: never; + url: "/api/v1/routines/runs/{run_id}/decide"; +}; + +export type DecideHeldRunErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type DecideHeldRunError = DecideHeldRunErrors[keyof DecideHeldRunErrors]; + +export type DecideHeldRunResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type DecideHeldRunResponse = DecideHeldRunResponses[keyof DecideHeldRunResponses]; + export type ListRunsData = { body?: never; path: { diff --git a/nextjs-frontend/components/activity/job-detail.tsx b/nextjs-frontend/components/activity/job-detail.tsx index 74569df8f..ed4b95b5f 100644 --- a/nextjs-frontend/components/activity/job-detail.tsx +++ b/nextjs-frontend/components/activity/job-detail.tsx @@ -8,6 +8,7 @@ import { useMemo, useState, type ReactNode } from "react"; import { Jobs, Pipelines, Resources, Templates } from "@/app/openapi-client"; import { CancelJobDialog } from "@/components/activity/cancel-dialog"; +import { Cost, HeldNotice } from "@/components/costs/costs"; import { approx, canRetry, @@ -224,6 +225,18 @@ export function JobDetail({ jobId }: { jobId: number }) { onError: (e: Error) => toast({ tone: "red", title: "Couldn’t start the run", body: e.message }), }); + const release = useMutation({ + mutationFn: (go: boolean) => data(Jobs.releaseJob({ client, path: { jid: jobId }, body: { run: go } })), + onSuccess: (_, go) => { + void qc.invalidateQueries({ queryKey: ["job", jobId] }); + void qc.invalidateQueries({ queryKey: ["jobs"] }); + toast( + go ? { tone: "green", title: "Running it once", body: "A worker picks it up next." } : { title: "Skipped" }, + ); + }, + onError: (e: Error) => toast({ tone: "red", title: "Couldn’t do that", body: e.message }), + }); + if (q.isLoading) return (
@@ -345,6 +358,15 @@ export function JobDetail({ jobId }: { jobId: number }) { ); } else if (job.status === "paused") box = ; + else if (job.status === "held") + box = ( + release.mutate(true)} + onSkip={() => release.mutate(false)} + /> + ); else box = ( @@ -449,6 +471,11 @@ export function JobDetail({ jobId }: { jobId: number }) { : ""} {left && {left}} + {(job.cost_usd != null || job.tokens != null) && ( + + Cost + + )} {ns && {ns}} {job.recording != null && ( diff --git a/nextjs-frontend/components/activity/job-model.ts b/nextjs-frontend/components/activity/job-model.ts index cda7a7995..2629415e0 100644 --- a/nextjs-frontend/components/activity/job-model.ts +++ b/nextjs-frontend/components/activity/job-model.ts @@ -11,7 +11,7 @@ import { STEP_LABEL } from "@/components/ui/loop"; /** A run as the API sends it. */ export type JobRecord = Job; -export const ACTIVE = ["queued", "running", "paused"]; +export const ACTIVE = ["queued", "running", "paused", "held"]; export const isActive = (status: string | null | undefined) => ACTIVE.includes(status ?? ""); export const canRetry = (status: string | null | undefined) => status === "failed" || status === "cancelled"; @@ -146,6 +146,8 @@ export function jobPhase(job: JobRecord, opts: { waitingFor?: string | null; now return i > 0 ? `${label} · waiting for a worker` : `Queued · ${label} first`; case "paused": return `Paused with its batch · ${label} next`; + case "held": + return `Waiting for you: over budget · ${label} first`; case "failed": { const e = plainError(job.error); return `${label} failed${e ? `: ${e}` : ""}`; diff --git a/nextjs-frontend/components/activity/job-row.tsx b/nextjs-frontend/components/activity/job-row.tsx index bb704fc07..c1ba454f3 100644 --- a/nextjs-frontend/components/activity/job-row.tsx +++ b/nextjs-frontend/components/activity/job-row.tsx @@ -28,6 +28,13 @@ const ICON: Record = { running: "bg-blue", queued: "bg-border", paused: "bg-border", + held: "bg-gold", succeeded: "bg-green", failed: "bg-red", cancelled: "bg-fg-muted", diff --git a/nextjs-frontend/components/costs/costs.tsx b/nextjs-frontend/components/costs/costs.tsx new file mode 100644 index 000000000..dc8946682 --- /dev/null +++ b/nextjs-frontend/components/costs/costs.tsx @@ -0,0 +1,584 @@ +"use client"; + +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { ArrowDownLeft, ArrowUpRight, History, Pencil, PiggyBank, Play, RotateCcw } from "lucide-react"; +import { useState } from "react"; + +import { Activity, Budgets } from "@/app/openapi-client"; +import type { ActivityEntry, BudgetSet, BudgetStatus, ResourceSpend } from "@/app/openapi-client/types.gen"; +import { Badge, type Tone } from "@/components/ui/badge"; +import { Banner } from "@/components/ui/banner"; +import { Button } from "@/components/ui/button"; +import { Field, Input, Select } from "@/components/ui/field"; +import { Panel } from "@/components/ui/panel"; +import { DateTime, Skeleton, SkeletonRows } from "@/components/ui/states"; +import { useToast } from "@/components/ui/toast"; +import { Tooltip } from "@/components/ui/tooltip"; +import { data, useApiClient } from "@/lib/api/browser"; +import { useArchive } from "@/lib/hooks/session"; +import { cn } from "@/lib/utils"; + +export type Period = "day" | "week" | "month" | "all"; +const PERIOD_TEXT: Record = { day: "today", week: "this week", month: "this month", all: "in all" }; + +/** USD the way a person reads it: cents for small sums, more digits below a cent. */ +export function usd(v: number | null | undefined): string { + if (v == null) return "—"; + if (v === 0) return "$0"; + if (v < 0.01) return `$${v.toPrecision(2)}`; + if (v < 100) return `$${v.toFixed(2)}`; + return `$${Math.round(v).toLocaleString()}`; +} + +/** 950, 12.4k, 3.1M tokens. */ +export function tokens(n: number | null | undefined): string { + if (n == null) return "—"; + if (n < 1000) return String(n); + if (n < 1_000_000) return `${(n / 1000).toFixed(n < 10_000 ? 1 : 0)}k`; + return `${(n / 1_000_000).toFixed(1)}M`; +} + +const ESTIMATE_WHY = + "An estimate: some calls had a price but no token counts from the server, so the real cost is at least this."; + +/** + * A cost, honest about what it is: "$0.42", "≈$0.42" when it's an estimate (with why on hover), or the tokens alone + * when nothing was priced (local models aren't costed unless they're given a price). + */ +export function Cost({ + usd: v, + tokens: t, + estimate, + why, + className, +}: { + usd?: number | null; + tokens?: number | null; + estimate?: boolean | null; + why?: string; + className?: string; +}) { + const priced = (v ?? 0) > 0; + const text = priced ? usd(v) : t ? `${tokens(t)} tokens` : usd(v ?? 0); + const title = [ + estimate ? (why ?? ESTIMATE_WHY) : null, + priced && t ? `${tokens(t)} tokens` : null, + !priced && t ? "No price set for these models, so they aren't costed (Settings › Telemetry › prices)." : null, + ] + .filter(Boolean) + .join(" "); + const body = ( + + {estimate ? ( + + ≈ + + ) : null} + {text} + + ); + return title ? ( + + {body} + + ) : ( + body + ); +} + +/** What each of many resources cost this period, in one request (for lists). */ +export function useCosts(resources: string[], period: Period = "month") { + const client = useApiClient(); + const list = [...new Set(resources)].sort(); + return useQuery({ + queryKey: ["activity-costs", period, list], + queryFn: () => data(Activity.resourceCosts({ client, query: { resource: list, period } })), + enabled: list.length > 0, + staleTime: 30_000, + }); +} + +/** One resource's cost this period, from a list's costs (or its own request). */ +export function ResourceCost({ + resource, + costs, + period = "month", + className, +}: { + resource: string; + costs?: Record; + period?: Period; + className?: string; +}) { + const own = useCosts(costs ? [] : [resource], period); + const c = costs ? costs[resource] : own.data?.costs[resource]; + if (!c) return —; + return ; +} + +const KIND: Record = { + in: { label: "Request", icon: , tone: "neutral" }, + out: { label: "Call out", icon: , tone: "intent" }, + run: { label: "Run", icon: , tone: "green" }, + change: { label: "Change", icon: , tone: "gate" }, +}; + +function actionText(e: ActivityEntry): string { + const a = e.action; + if (a.startsWith("model.")) + return `${a.slice(6) === "chat" ? "Model call" : `Model ${a.slice(6)}`}${e.model ? ` · ${e.model}` : ""}`; + if (a === "embeddings") return `Embeddings${e.model ? ` · ${e.model}` : ""}`; + if (a === "decision") return `Decision model${e.model ? ` · ${e.model}` : ""}`; + if (a.startsWith("notify.")) return `Notification · ${a.slice(7)}`; + if (a === "tool.http") return "Web tool"; + if (a.startsWith("budget.")) return a === "budget.over" ? "Budget used up" : "Budget nearly used"; + return a; +} + +/** A resource's history: requests that changed it, calls made for it, runs and audit entries, with their cost. */ +export function ActivityPanel({ resource, title = "Activity" }: { resource: string; title?: string }) { + const client = useApiClient(); + const [limit, setLimit] = useState(50); + const [period, setPeriod] = useState("month"); + const q = useQuery({ + queryKey: ["activity", resource, limit], + queryFn: () => data(Activity.resourceHistory({ client, query: { resource, limit } })), + staleTime: 15_000, + }); + const totals = useQuery({ + queryKey: ["activity-totals", resource, period], + queryFn: () => data(Activity.resourceTotals({ client, query: { resource, period } })), + staleTime: 30_000, + }); + const t = totals.data; + return ( + + {PERIOD_TEXT[period]} + {" · "} + {t.by_kind.out?.calls ?? 0} calls out · {t.by_kind.in?.calls ?? 0} changes + {t.failed ? ` · ${t.failed} failed` : ""} + + ) : ( + + ) + } + actions={ + setForm({ ...form, usd: e.target.value })} + placeholder="5.00" + /> + )} + + + {({ id }) => ( + setForm({ ...form, tokens: e.target.value })} + placeholder="1000000" + /> + )} + + + {({ id }) => ( + setForm({ ...form, warn: e.target.value })} + /> + )} + + + {({ id }) => ( +
+ @@ -139,6 +143,7 @@ export function PipelinesPage() { built-in + {list.map((p, i) => { @@ -178,6 +183,9 @@ export function PipelinesPage() { + ); diff --git a/nextjs-frontend/components/recording/history-tab.tsx b/nextjs-frontend/components/recording/history-tab.tsx index 0030dc4eb..83e219006 100644 --- a/nextjs-frontend/components/recording/history-tab.tsx +++ b/nextjs-frontend/components/recording/history-tab.tsx @@ -4,6 +4,7 @@ import { ExternalLink } from "lucide-react"; import Link from "next/link"; import { pipelineLabel } from "@/components/activity/job-model"; +import { ActivityPanel } from "@/components/costs/costs"; import { useRec } from "@/components/recording/context"; import { ChangeHistoryList } from "@/components/recording/edit"; import { useJob } from "@/components/recording/hooks"; @@ -46,6 +47,7 @@ export function HistoryTab() { + ); } diff --git a/nextjs-frontend/components/routines/routine-detail.tsx b/nextjs-frontend/components/routines/routine-detail.tsx index ad97e55b8..74d82d198 100644 --- a/nextjs-frontend/components/routines/routine-detail.tsx +++ b/nextjs-frontend/components/routines/routine-detail.tsx @@ -7,6 +7,7 @@ import { useRouter } from "next/navigation"; import { useState } from "react"; import { Routines } from "@/app/openapi-client"; +import { ActivityPanel, BudgetPanel, Cost, HeldNotice } from "@/components/costs/costs"; import type { RoutineRun } from "@/app/openapi-client/types.gen"; import { useNames } from "@/components/routines/data"; import { RoutineEditor } from "@/components/routines/routine-editor"; @@ -51,6 +52,22 @@ function RunLog({ id }: { id: number }) { function RunRow({ run, onUndo }: { run: RoutineRun; onUndo: (r: RoutineRun) => void }) { const [open, setOpen] = useState(false); + const client = useApiClient(); + const qc = useQueryClient(); + const toast = useToast(); + const decide = useMutation({ + mutationFn: (go: boolean) => data(Routines.decideHeldRun({ client, path: { run_id: run.id }, body: { run: go } })), + onSuccess: (_, go) => { + void qc.invalidateQueries({ queryKey: ["routine-runs", run.routine] }); + void qc.invalidateQueries({ queryKey: ["routine", run.routine] }); + toast( + go + ? { tone: "green", title: "Running it once", body: "It starts within half a minute." } + : { title: "Skipped" }, + ); + }, + onError: (e: Error) => toast({ tone: "red", title: "Couldn’t do that", body: e.message }), + }); const applied = run.changes?.applied ?? 0; const proposed = run.changes?.proposed ?? 0; const results = (run.results ?? []) as Record[]; @@ -77,6 +94,14 @@ function RunRow({ run, onUndo }: { run: RoutineRun; onUndo: (r: RoutineRun) => v {took(run.started_at, run.finished_at) ? ` · took ${took(run.started_at, run.finished_at)}` : ""} + {(run.cost_usd != null || run.tokens != null) && ( + + )} {proposed > 0 && ( v )} -
0 || run.error) && "pb-3")}> +
0 || run.error || run.hold) && "pb-3", + )} + > + {run.status === "held" ? ( + decide.mutate(true)} + onSkip={() => decide.mutate(false)} + /> + ) : run.hold?.why ? ( +

{String(run.hold.why)}

+ ) : null} {results.length > 0 && (
    {results.map((r, i) => ( @@ -112,7 +152,7 @@ function RunRow({ run, onUndo }: { run: RoutineRun; onUndo: (r: RoutineRun) => v } /** A routine: its recent runs (with their logs, and undo for graph changes) and its settings. */ -export function RoutineDetail({ id, tab: initialTab }: { id: number; tab?: "runs" | "settings" }) { +export function RoutineDetail({ id, tab: initialTab }: { id: number; tab?: "runs" | "costs" | "settings" }) { const client = useApiClient(); const qc = useQueryClient(); const router = useRouter(); @@ -120,7 +160,7 @@ export function RoutineDetail({ id, tab: initialTab }: { id: number; tab?: "runs const { admin, me } = useArchive(); const names = useNames(); const { toggle } = useRoutineActions(); - const [tab, setTab] = useState(initialTab ?? "runs"); + const [tab, setTab] = useState<"runs" | "costs" | "settings">(initialTab ?? "runs"); const [undoing, setUndoing] = useState(null); const [deleting, setDeleting] = useState(false); @@ -235,14 +275,20 @@ export function RoutineDetail({ id, tab: initialTab }: { id: number; tab?: "runs setTab(v as "runs" | "settings")} + onChange={(v) => setTab(v as "runs" | "costs" | "settings")} items={[ { value: "runs", label: "Runs", count: runs.data?.length }, + { value: "costs", label: "Costs and activity" }, { value: "settings", label: "Settings" }, ]} /> {tab === "settings" ? ( setTab("runs")} /> + ) : tab === "costs" ? ( +
    + + +
    ) : runs.isLoading ? ( ) : runs.error ? ( diff --git a/nextjs-frontend/components/routines/routine-model.ts b/nextjs-frontend/components/routines/routine-model.ts index 49d7111f0..3b500c216 100644 --- a/nextjs-frontend/components/routines/routine-model.ts +++ b/nextjs-frontend/components/routines/routine-model.ts @@ -179,6 +179,9 @@ export const STATUS: Record = { partly: { label: "Partly done", tone: "gate" }, error: { label: "Failed", tone: "red" }, running: { label: "Running", tone: "intent" }, + held: { label: "Waiting for you", tone: "gate" }, + released: { label: "Let through", tone: "neutral" }, + skipped: { label: "Skipped", tone: "neutral" }, }; export function statusOf(s: string | null | undefined) { diff --git a/nextjs-frontend/components/routines/routines-page.tsx b/nextjs-frontend/components/routines/routines-page.tsx index 14ad8d672..af829898b 100644 --- a/nextjs-frontend/components/routines/routines-page.tsx +++ b/nextjs-frontend/components/routines/routines-page.tsx @@ -9,6 +9,7 @@ import { useState } from "react"; import { Routines } from "@/app/openapi-client"; import type { Routine } from "@/app/openapi-client/types.gen"; import { useGraphChanges, useNames, useRoutines } from "@/components/routines/data"; +import { CostCell, useBudgets, useCosts } from "@/components/costs/costs"; import { routineText, statusOf, touchesGraph, type RoutineAction } from "@/components/routines/routine-model"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -19,7 +20,7 @@ import { DateTime, EmptyState, SkeletonRows } from "@/components/ui/states"; import { Table, Td, Th, THead, Tr } from "@/components/ui/table"; import { Tabs } from "@/components/ui/tabs"; import { useToast } from "@/components/ui/toast"; -import { data, useApiClient } from "@/lib/api/browser"; +import { ApiError, data, useApiClient } from "@/lib/api/browser"; import { useArchive } from "@/lib/hooks/session"; /** "Routines" with the Routines / Proposed changes tabs; people who aren't admins see only the changes. */ @@ -64,12 +65,18 @@ export function useRoutineActions() { void qc.invalidateQueries({ queryKey: ["routines"] }); void qc.invalidateQueries({ queryKey: ["routine", id] }); }; + const [overBudget, setOverBudget] = useState<{ routine: Routine; proposeOnly?: boolean; why: string } | null>(null); const run = useMutation({ - mutationFn: (v: { routine: Routine; proposeOnly?: boolean }) => + mutationFn: (v: { routine: Routine; proposeOnly?: boolean; overBudget?: boolean }) => data( - Routines.runRoutine({ client, path: { rid: v.routine.id }, body: { propose_only: Boolean(v.proposeOnly) } }), + Routines.runRoutine({ + client, + path: { rid: v.routine.id }, + body: { propose_only: Boolean(v.proposeOnly), over_budget: Boolean(v.overBudget) }, + }), ), onSuccess: (_, v) => { + setOverBudget(null); refresh(v.routine.id); void qc.invalidateQueries({ queryKey: ["routine-runs", v.routine.id] }); toast({ @@ -78,7 +85,12 @@ export function useRoutineActions() { body: `${v.routine.name} starts within half a minute.`, }); }, - onError: (e: Error) => toast({ tone: "red", title: "Couldn’t start the run", body: e.message }), + onError: (e: Error, v) => { + // over budget: say where it stands and let the person choose (docs/budgets.md) + if (e instanceof ApiError && e.status === 409) + setOverBudget({ routine: v.routine, proposeOnly: v.proposeOnly, why: e.message }); + else toast({ tone: "red", title: "Couldn’t start the run", body: e.message }); + }, }); const toggle = useMutation({ mutationFn: (v: { routine: Routine; enabled: boolean }) => @@ -89,7 +101,29 @@ export function useRoutineActions() { }, onError: (e: Error) => toast({ tone: "red", title: "Couldn’t change the routine", body: e.message }), }); - return { run, toggle }; + const overBudgetDialog = ( + !o && setOverBudget(null)} + title="Run it over budget?" + description={overBudget?.why.replace(/ Run it anyway with over_budget\.$/, "")} + actions={ + <> + + + + } + /> + ); + return { run, toggle, overBudgetDialog }; } export function DeleteRoutineDialog({ @@ -136,36 +170,42 @@ export function DeleteRoutineDialog({ /** Run now, with "propose only" next to it when the routine organises the graph. */ export function RunButton({ routine, graph, size = "sm" }: { routine: Routine; graph: boolean; size?: "xs" | "sm" }) { - const { run } = useRoutineActions(); + const { run, overBudgetDialog } = useRoutineActions(); const busy = Boolean(routine.running) || (run.isPending && run.variables?.routine.id === routine.id); const why = routine.running ? "It’s running now" : "Asking…"; if (!graph || busy) return ( - + <> + + {overBudgetDialog} + ); return ( - - - - - - run.mutate({ routine })}>Run now - run.mutate({ routine, proposeOnly: true })}> - Run now, only propose graph changes - - - + <> + + + + + + run.mutate({ routine })}>Run now + run.mutate({ routine, proposeOnly: true })}> + Run now, only propose graph changes + + + + {overBudgetDialog} + ); } @@ -194,6 +234,8 @@ export function RoutinesPage() { const { toggle } = useRoutineActions(); const [deleting, setDeleting] = useState(null); const list = routines.data?.routines ?? []; + const costs = useCosts(list.map((r) => `routine:${r.id}`)); + const budgets = useBudgets(); if (me && !admin) return ( @@ -252,6 +294,7 @@ export function RoutinesPage() {
+ + @@ -62,6 +66,9 @@ export function WorkflowsPage() { + ))} diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 156c05cd2..67bdb0a6a 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -2940,6 +2940,171 @@ } } }, + "/api/v1/budgets": { + "get": { + "tags": [ + "budgets" + ], + "summary": "List Budgets", + "description": "Every budget, with where it stands now.", + "operationId": "list_budgets", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BudgetStatus" + }, + "title": "Response Budgets-List Budgets" + } + } + } + } + } + }, + "put": { + "tags": [ + "budgets" + ], + "summary": "Set Budget", + "operationId": "set_budget", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "routine:, pipeline:, workflow: or space:", + "title": "Resource" + }, + "description": "routine:, pipeline:, workflow: or space:" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BudgetSet" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BudgetStatus" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "budgets" + ], + "summary": "Remove Budget", + "operationId": "remove_budget", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "routine:, pipeline:, workflow: or space:", + "title": "Resource" + }, + "description": "routine:, pipeline:, workflow: or space:" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ok" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/budgets/status": { + "get": { + "tags": [ + "budgets" + ], + "summary": "Budget Status", + "description": "Where a resource's budget stands, and what its next run will likely cost (also without a budget).", + "operationId": "budget_status", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "routine:, pipeline:, workflow: or space:", + "title": "Resource" + }, + "description": "routine:, pipeline:, workflow: or space:" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BudgetStatus" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/admin/fedora": { "get": { "tags": [ @@ -13445,6 +13610,59 @@ } } }, + "/api/v1/jobs/{jid}/release": { + "post": { + "tags": [ + "jobs" + ], + "summary": "Release Job", + "description": "Pick for a job held over a budget (docs/budgets.md): run it now, once, whatever its budgets say, or skip it\n(admins, who set the budgets).", + "operationId": "release_job", + "parameters": [ + { + "name": "jid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Jid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HeldRunDecision" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ok" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/jobs/{jid}/retry": { "post": { "tags": [ @@ -18108,7 +18326,7 @@ "routines" ], "summary": "Run Routine", - "description": "Run it as soon as the scheduler next looks (within half a minute), even when it is off.", + "description": "Run it as soon as the scheduler next looks (within half a minute), even when it is off. A run over one of its\nbudgets is refused (409, saying where the budget stands) unless over_budget is true (docs/budgets.md).", "operationId": "run_routine", "parameters": [ { @@ -18155,6 +18373,59 @@ } } }, + "/api/v1/routines/runs/{run_id}/decide": { + "post": { + "tags": [ + "routines" + ], + "summary": "Decide Held Run", + "description": "Pick for a run held over a budget: run it now, once, whatever its budgets say, or skip it.", + "operationId": "decide_held_run", + "parameters": [ + { + "name": "run_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Run Id" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HeldRunDecision" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ok" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/routines/{rid}/runs": { "get": { "tags": [ @@ -21777,6 +22048,279 @@ ], "title": "BrowseEntry" }, + "Budget": { + "properties": { + "resource": { + "type": "string", + "title": "Resource" + }, + "usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Usd" + }, + "tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens" + }, + "period": { + "type": "string", + "title": "Period" + }, + "on_over": { + "type": "string", + "title": "On Over" + }, + "warn_at": { + "type": "number", + "title": "Warn At" + }, + "note": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Note" + }, + "by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "By" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "alert": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Alert", + "description": "the last warning: {state near|over, since, at, share}" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "resource", + "period", + "on_over", + "warn_at" + ], + "title": "Budget" + }, + "BudgetSet": { + "properties": { + "usd": { + "anyOf": [ + { + "type": "number", + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Usd", + "description": "the cap in USD (from the costs in the activity ledger)" + }, + "tokens": { + "anyOf": [ + { + "type": "integer", + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Tokens", + "description": "the cap in tokens (in and out)" + }, + "period": { + "type": "string", + "enum": [ + "run", + "day", + "week", + "month" + ], + "title": "Period", + "description": "what the cap is for: one run, or a day, week or month (UTC)", + "default": "month" + }, + "on_over": { + "type": "string", + "enum": [ + "ask", + "skip", + "assistant" + ], + "title": "On Over", + "description": "a run that would go over: ask (hold it until someone picks run or skip), skip, or assistant (the decision model weighs it; it runs only when sure, else asks)", + "default": "ask" + }, + "warn_at": { + "type": "number", + "maximum": 1.0, + "minimum": 0.1, + "title": "Warn At", + "description": "the share spent at which it's marked near (and warned about)", + "default": 0.8 + }, + "note": { + "anyOf": [ + { + "type": "string", + "maxLength": 300 + }, + { + "type": "null" + } + ], + "title": "Note" + } + }, + "additionalProperties": false, + "type": "object", + "title": "BudgetSet" + }, + "BudgetStatus": { + "properties": { + "resource": { + "type": "string", + "title": "Resource" + }, + "budget": { + "anyOf": [ + { + "$ref": "#/components/schemas/Budget" + }, + { + "type": "null" + } + ] + }, + "state": { + "type": "string", + "enum": [ + "none", + "ok", + "near", + "over" + ], + "title": "State" + }, + "since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Since", + "description": "the start of the period being counted" + }, + "spent": { + "anyOf": [ + { + "$ref": "#/components/schemas/Spent" + }, + { + "type": "null" + } + ] + }, + "next": { + "$ref": "#/components/schemas/NextRun", + "description": "what one more run will likely cost: always an estimate" + }, + "left": { + "anyOf": [ + { + "additionalProperties": { + "type": "number" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Left" + }, + "share": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Share", + "description": "the largest share of a cap spent (1 is all of it)" + }, + "share_after_next": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Share After Next", + "description": "the share once the next run (estimated) is done" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "resource", + "state", + "next" + ], + "title": "BudgetStatus" + }, "ChangeAsk": { "properties": { "kind": { @@ -28197,6 +28741,21 @@ ], "title": "Heard" }, + "HeldRunDecision": { + "properties": { + "run": { + "type": "boolean", + "title": "Run", + "description": "true: run it now, once, whatever its budgets say; false: skip it" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "run" + ], + "title": "HeldRunDecision" + }, "Highlight": { "properties": { "id": { @@ -29500,6 +30059,19 @@ "title": "Cost Estimate", "description": "true: some calls had no price or token counts, so cost_usd is a floor" }, + "hold": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Hold", + "description": "held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), decided: {run, by, at} once someone picked}" + }, "log": { "anyOf": [ { @@ -31246,6 +31818,48 @@ "title": "NamespaceUpdate", "description": "Only the fields you send change; send ``pipeline: null`` to go back to the default pipeline." }, + "NextRun": { + "properties": { + "usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Usd" + }, + "tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens" + }, + "runs": { + "type": "integer", + "title": "Runs", + "description": "the past runs the estimate is drawn from" + }, + "basis": { + "type": "string", + "title": "Basis" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "runs", + "basis" + ], + "title": "NextRun" + }, "NodeParam": { "properties": { "name": { @@ -37261,6 +37875,19 @@ ], "title": "Cost Estimate", "description": "true: some calls had no price or token counts, so cost_usd is a floor" + }, + "hold": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Hold", + "description": "held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), decided: {run, by, at} once someone picked}" } }, "additionalProperties": true, @@ -37280,6 +37907,12 @@ "title": "Propose Only", "description": "graph workflows propose every change instead of making the sure ones", "default": false + }, + "over_budget": { + "type": "boolean", + "title": "Over Budget", + "description": "run it even though it's over a budget (docs/budgets.md); without it, a run over budget is refused (409)", + "default": false } }, "additionalProperties": false, @@ -40955,6 +41588,31 @@ ], "title": "SpeakerSuggestion" }, + "Spent": { + "properties": { + "usd": { + "type": "number", + "title": "Usd" + }, + "tokens": { + "type": "integer", + "title": "Tokens" + }, + "estimate": { + "type": "boolean", + "title": "Estimate", + "description": "true: some calls had no figure, so usd is a floor" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "usd", + "tokens", + "estimate" + ], + "title": "Spent" + }, "Started": { "properties": { "ok": { From e16b4a8ad32c43a7b95dc123ac1d40c0d5fc3f8b Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 4 Oct 2026 09:57:00 +0000 Subject: [PATCH 5/6] feat(budgets): show namespace cost, budget and activity Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AMq5CNuWmtn3Suj1EamYBT --- nextjs-frontend/components/admin/namespaces.tsx | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nextjs-frontend/components/admin/namespaces.tsx b/nextjs-frontend/components/admin/namespaces.tsx index 4466f2b55..bd93d3f75 100644 --- a/nextjs-frontend/components/admin/namespaces.tsx +++ b/nextjs-frontend/components/admin/namespaces.tsx @@ -9,6 +9,7 @@ import { Namespaces, Pipelines, Users } from "@/app/openapi-client"; import { IpGroupsSection } from "@/components/access/ip-groups"; import { AdminFrame, usePeople } from "@/components/admin/admin-frame"; import { roleLabel, type Role } from "@/components/admin/people-model"; +import { ActivityPanel, BudgetPanel, CostCell, useBudgets, useCosts } from "@/components/costs/costs"; import { isUnreachable } from "@/components/errors/error-states"; import { NotificationsSection } from "@/components/notifications/notifications-section"; import { runtime } from "@/components/iiif/iiif-model"; @@ -38,6 +39,8 @@ export function NamespacesPage() { const [creating, setCreating] = useState(false); const shown = namespaces.filter((n) => admin || roleIn(n.name) === "owner"); const members = (ns: string) => (people.data ?? []).filter((p) => !p.admin && p.roles?.[ns]).length; + const costs = useCosts(shown.map((n) => `space:${n.id}`)); + const budgets = useBudgets(); return ( Recordings {admin && } + @@ -90,6 +94,9 @@ export function NamespacesPage() { {admin && } + @@ -468,6 +475,8 @@ export function NamespaceDetail({ ns }: { ns: string }) { {known && } + {known && } + {known && }
From 17b756b6c866ba2324aab0f3ba102827f6b09855 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 4 Oct 2026 12:22:27 +0000 Subject: [PATCH 6/6] chore(openapi): regenerate the schema and client after merging main --- nextjs-frontend/app/openapi-client/index.ts | 63 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 187 +- .../app/openapi-client/types.gen.ts | 853 +++++++- nextjs-frontend/openapi.json | 1920 +++++++++++++++-- 4 files changed, 2785 insertions(+), 238 deletions(-) diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 312f6447b..b0b31ca8f 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -2,9 +2,11 @@ export { AccessRequests, + Activity, Admin, Auth, Batches, + Budgets, Chats, Collections, Comments, @@ -54,6 +56,10 @@ export type { AcceptGraphChangeResponses, AccessRequest, AccountToken, + ActivityCosts, + ActivityEntry, + ActivityTop, + ActivityTotals, AddFileData, AddFileError, AddFileErrors, @@ -146,6 +152,14 @@ export type { BrowseSourceErrors, BrowseSourceResponse, BrowseSourceResponses, + Budget, + BudgetSet, + BudgetStatus, + BudgetStatusData, + BudgetStatusError, + BudgetStatusErrors, + BudgetStatusResponse, + BudgetStatusResponses, BulkUpdateMetadataData, BulkUpdateMetadataError, BulkUpdateMetadataErrors, @@ -447,6 +461,11 @@ export type { DecideApprovalErrors, DecideApprovalResponse, DecideApprovalResponses, + DecideHeldRunData, + DecideHeldRunError, + DecideHeldRunErrors, + DecideHeldRunResponse, + DecideHeldRunResponses, DecisionStatus, DecisionStatusData, DecisionStatusResponse, @@ -1053,6 +1072,7 @@ export type { HealthDisk, HealthSource, Heard, + HeldRunDecision, HideEntityData, HideEntityError, HideEntityErrors, @@ -1117,6 +1137,7 @@ export type { JobQueued, JobsCreate, JobsQueued, + KindTotals, LanguageCount, LastSend, LayaModel, @@ -1165,6 +1186,9 @@ export type { ListBatchesData, ListBatchesResponse, ListBatchesResponses, + ListBudgetsData, + ListBudgetsResponse, + ListBudgetsResponses, ListChatsData, ListChatsResponse, ListChatsResponses, @@ -1543,6 +1567,7 @@ export type { NewTokenErrors, NewTokenResponse, NewTokenResponses, + NextRun, NodeParam, NodeTrace, NodeType, @@ -1762,6 +1787,16 @@ export type { ReindexSearchData, ReindexSearchResponse, ReindexSearchResponses, + ReleaseJobData, + ReleaseJobError, + ReleaseJobErrors, + ReleaseJobResponse, + ReleaseJobResponses, + RemoveBudgetData, + RemoveBudgetError, + RemoveBudgetErrors, + RemoveBudgetResponse, + RemoveBudgetResponses, RemoveLoginProviderData, RemoveLoginProviderError, RemoveLoginProviderErrors, @@ -1820,8 +1855,25 @@ export type { ResetPasswordRequest, ResetPasswordResponse, ResetPasswordResponses, + ResourceCost, + ResourceCostsData, + ResourceCostsError, + ResourceCostsErrors, + ResourceCostsResponse, + ResourceCostsResponses, ResourceFile, ResourceFiles, + ResourceHistoryData, + ResourceHistoryError, + ResourceHistoryErrors, + ResourceHistoryResponse, + ResourceHistoryResponses, + ResourceSpend, + ResourceTotalsData, + ResourceTotalsError, + ResourceTotalsErrors, + ResourceTotalsResponse, + ResourceTotalsResponses, RetagRecordingsData, RetagRecordingsError, RetagRecordingsErrors, @@ -1996,6 +2048,11 @@ export type { SensorType, SensorUpdate, SeriesPoint, + SetBudgetData, + SetBudgetError, + SetBudgetErrors, + SetBudgetResponse, + SetBudgetResponses, SetCollectionMemberData, SetCollectionMemberError, SetCollectionMemberErrors, @@ -2099,6 +2156,7 @@ export type { SpeakTextResponse, SpeakTextResponses, SpeechTestResult, + Spent, SplitSegmentData, SplitSegmentError, SplitSegmentErrors, @@ -2216,6 +2274,11 @@ export type { TopicRecordings, TopicRef, TopicUpdate, + TopResourcesData, + TopResourcesError, + TopResourcesErrors, + TopResourcesResponse, + TopResourcesResponses, TranscribeClipData, TranscribeClipResponse, TranscribeClipResponses, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 2a7af1c3c..5ff23ca2f 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -51,6 +51,9 @@ import type { BrowseSourceData, BrowseSourceErrors, BrowseSourceResponses, + BudgetStatusData, + BudgetStatusErrors, + BudgetStatusResponses, BulkUpdateMetadataData, BulkUpdateMetadataErrors, BulkUpdateMetadataResponses, @@ -205,6 +208,9 @@ import type { DecideApprovalData, DecideApprovalErrors, DecideApprovalResponses, + DecideHeldRunData, + DecideHeldRunErrors, + DecideHeldRunResponses, DecisionStatusData, DecisionStatusResponses, DeclineAccessRequestData, @@ -575,6 +581,8 @@ import type { ListBackendsResponses, ListBatchesData, ListBatchesResponses, + ListBudgetsData, + ListBudgetsResponses, ListChatsData, ListChatsResponses, ListCollectionMembersData, @@ -861,6 +869,12 @@ import type { RegisterResponses, ReindexSearchData, ReindexSearchResponses, + ReleaseJobData, + ReleaseJobErrors, + ReleaseJobResponses, + RemoveBudgetData, + RemoveBudgetErrors, + RemoveBudgetResponses, RemoveLoginProviderData, RemoveLoginProviderErrors, RemoveLoginProviderResponses, @@ -894,6 +908,15 @@ import type { ResetPasswordData, ResetPasswordErrors, ResetPasswordResponses, + ResourceCostsData, + ResourceCostsErrors, + ResourceCostsResponses, + ResourceHistoryData, + ResourceHistoryErrors, + ResourceHistoryResponses, + ResourceTotalsData, + ResourceTotalsErrors, + ResourceTotalsResponses, RetagRecordingsData, RetagRecordingsErrors, RetagRecordingsResponses, @@ -982,6 +1005,9 @@ import type { SendMessageData, SendMessageErrors, SendMessageResponses, + SetBudgetData, + SetBudgetErrors, + SetBudgetResponses, SetCollectionMemberData, SetCollectionMemberErrors, SetCollectionMemberResponses, @@ -1068,6 +1094,9 @@ import type { TokenLimitsData, TokenLimitsResponses, TokenResponses, + TopResourcesData, + TopResourcesErrors, + TopResourcesResponses, TranscribeClipData, TranscribeClipResponses, TryWorkflowData, @@ -2427,6 +2456,124 @@ export class Admin { } } +export class Activity { + /** + * Resource History + * + * A resource's history, newest first: API requests that changed it, calls made for it (models, embeddings, the + * decision model, webhooks, web tools) with tokens and cost, runs that ended, and its audit log entries. + */ + public static resourceHistory( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/activity", + ...options, + }); + } + + /** + * Resource Totals + * + * What a resource's calls cost this day, week (from Monday), month (UTC) or all time: calls, tokens, USD, time. + */ + public static resourceTotals( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/activity/totals", + ...options, + }); + } + + /** + * Resource Costs + * + * What each of many resources cost this period, for lists; ones you can't see are left out. `estimate` says a + * figure is a floor (some calls had no price or token counts). + */ + public static resourceCosts( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/activity/costs", + ...options, + }); + } + + /** + * Top Resources + * + * The resources that cost most this period (admins). + */ + public static topResources( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/activity/top", + ...options, + }); + } +} + +export class Budgets { + /** + * Remove Budget + */ + public static removeBudget( + options: Options, + ): RequestResult { + return (options.client ?? client).delete({ + url: "/api/v1/budgets", + ...options, + }); + } + + /** + * List Budgets + * + * Every budget, with where it stands now. + */ + public static listBudgets( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/budgets", + ...options, + }); + } + + /** + * Set Budget + */ + public static setBudget( + options: Options, + ): RequestResult { + return (options.client ?? client).put({ + url: "/api/v1/budgets", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Budget Status + * + * Where a resource's budget stands, and what its next run will likely cost (also without a budget). + */ + public static budgetStatus( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/budgets/status", + ...options, + }); + } +} + export class Fedora { /** * Get Fedora Status @@ -5786,6 +5933,25 @@ export class Jobs { }); } + /** + * Release Job + * + * Pick for a job held over a budget (docs/budgets.md): run it now, once, whatever its budgets say, or skip it + * (admins, who set the budgets). + */ + public static releaseJob( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/jobs/{jid}/release", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * Retry Job */ @@ -7399,7 +7565,8 @@ export class Routines { /** * Run Routine * - * Run it as soon as the scheduler next looks (within half a minute), even when it is off. + * Run it as soon as the scheduler next looks (within half a minute), even when it is off. A run over one of its + * budgets is refused (409, saying where the budget stands) unless over_budget is true (docs/budgets.md). */ public static runRoutine( options: Options, @@ -7414,6 +7581,24 @@ export class Routines { }); } + /** + * Decide Held Run + * + * Pick for a run held over a budget: run it now, once, whatever its budgets say, or skip it. + */ + public static decideHeldRun( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/routines/runs/{run_id}/decide", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * List Runs */ diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index f4e8dee75..3fbca9b35 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -102,6 +102,192 @@ export type AccountToken = { [key: string]: unknown; }; +/** + * ActivityCosts + */ +export type ActivityCosts = { + /** + * Period + */ + period: string; + /** + * Since + */ + since?: string | null; + /** + * Costs + * + * by resource; ones you can't see are left out + */ + costs: { + [key: string]: ResourceSpend; + }; + [key: string]: unknown; +}; + +/** + * ActivityEntry + */ +export type ActivityEntry = { + /** + * Id + * + * the ledger row's id; none for an audit log entry + */ + id?: string | null; + /** + * At + */ + at: string; + /** + * Kind + * + * in: an API request; out: a call Lens made (a model, embeddings, the decision model, a webhook, a web tool); run: a job or routine run ended; change: an audit log entry + */ + kind: "in" | "out" | "run" | "change"; + /** + * Action + * + * what it was: `POST /api/v1/routines/{rid}/run`, `model.chat`, `job.succeeded`, `routine.update` + */ + action: string; + /** + * Resources + * + * every resource it counts for, as table:id + */ + resources?: Array; + /** + * Actor + * + * the account that made it, when a person did + */ + actor?: number | null; + /** + * Email + */ + email?: string | null; + /** + * Model + */ + model?: string | null; + /** + * Tokens In + */ + tokens_in?: number | null; + /** + * Tokens Out + */ + tokens_out?: number | null; + /** + * Cost Usd + * + * estimated from the prices in Settings; none when the model has no price + */ + cost_usd?: number | null; + /** + * Ms + */ + ms?: number | null; + /** + * Ok + */ + ok?: boolean; + /** + * Error + * + * the error's type or HTTP status; never its message + */ + error?: string | null; + /** + * Detail + */ + detail?: unknown; + [key: string]: unknown; +}; + +/** + * ActivityTop + */ +export type ActivityTop = { + /** + * Period + */ + period: string; + /** + * Since + */ + since?: string | null; + /** + * Resources + */ + resources: Array; + [key: string]: unknown; +}; + +/** + * ActivityTotals + */ +export type ActivityTotals = { + /** + * Resource + */ + resource?: string | null; + /** + * Period + */ + period: string; + /** + * Since + */ + since?: string | null; + /** + * Calls + */ + calls: number; + /** + * Failed + */ + failed: number; + /** + * Tokens In + */ + tokens_in: number; + /** + * Tokens Out + */ + tokens_out: number; + /** + * Cost Usd + * + * what its calls cost (run rows repeat their calls' cost and aren't added again) + */ + cost_usd: number; + /** + * Ms + */ + ms: number; + /** + * Unpriced + * + * calls that cost something but have no figure (a model with no price, a reply without token counts) + */ + unpriced?: number; + /** + * Estimate + * + * true when unpriced calls make cost_usd a floor rather than exact + */ + estimate?: boolean; + /** + * By Kind + */ + by_kind: { + [key: string]: KindTotals; + }; + [key: string]: unknown; +}; + /** * AnswerCheck */ @@ -921,6 +1107,142 @@ export type BrowseEntry = { [key: string]: unknown; }; +/** + * Budget + */ +export type Budget = { + /** + * Resource + */ + resource: string; + /** + * Usd + */ + usd?: number | null; + /** + * Tokens + */ + tokens?: number | null; + /** + * Period + */ + period: string; + /** + * On Over + */ + on_over: string; + /** + * Warn At + */ + warn_at: number; + /** + * Note + */ + note?: string | null; + /** + * By + */ + by?: string | null; + /** + * Updated At + */ + updated_at?: string | null; + /** + * Alert + * + * the last warning: {state near|over, since, at, share} + */ + alert?: { + [key: string]: unknown; + } | null; + [key: string]: unknown; +}; + +/** + * BudgetSet + */ +export type BudgetSet = { + /** + * Usd + * + * the cap in USD (from the costs in the activity ledger) + */ + usd?: number | null; + /** + * Tokens + * + * the cap in tokens (in and out) + */ + tokens?: number | null; + /** + * Period + * + * what the cap is for: one run, or a day, week or month (UTC) + */ + period?: "run" | "day" | "week" | "month"; + /** + * On Over + * + * a run that would go over: ask (hold it until someone picks run or skip), skip, or assistant (the decision model weighs it; it runs only when sure, else asks) + */ + on_over?: "ask" | "skip" | "assistant"; + /** + * Warn At + * + * the share spent at which it's marked near (and warned about) + */ + warn_at?: number; + /** + * Note + */ + note?: string | null; +}; + +/** + * BudgetStatus + */ +export type BudgetStatus = { + /** + * Resource + */ + resource: string; + budget?: Budget | null; + /** + * State + */ + state: "none" | "ok" | "near" | "over"; + /** + * Since + * + * the start of the period being counted + */ + since?: string | null; + spent?: Spent | null; + /** + * what one more run will likely cost: always an estimate + */ + next: NextRun; + /** + * Left + */ + left?: { + [key: string]: number; + } | null; + /** + * Share + * + * the largest share of a cap spent (1 is all of it) + */ + share?: number | null; + /** + * Share After Next + * + * the share once the next run (estimated) is done + */ + share_after_next?: number | null; + [key: string]: unknown; +}; + /** * ChangeAsk */ @@ -4721,6 +5043,18 @@ export type Heard = { [key: string]: unknown; }; +/** + * HeldRunDecision + */ +export type HeldRunDecision = { + /** + * Run + * + * true: run it now, once, whatever its budgets say; false: skip it + */ + run: boolean; +}; + /** * Highlight */ @@ -5434,6 +5768,30 @@ export type Job = { * Cancel Requested */ cancel_requested?: boolean | null; + /** + * Cost Usd + * + * what its calls cost, from the activity ledger (docs/activity.md); set when it ends + */ + cost_usd?: number | null; + /** + * Tokens + */ + tokens?: number | null; + /** + * Cost Estimate + * + * true: some calls had no price or token counts, so cost_usd is a floor + */ + cost_estimate?: boolean | null; + /** + * Hold + * + * held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), decided: {run, by, at} once someone picked} + */ + hold?: { + [key: string]: unknown; + } | null; /** * Log * @@ -5625,6 +5983,37 @@ export type JobsQueued = { [key: string]: unknown; }; +/** + * KindTotals + */ +export type KindTotals = { + /** + * Calls + */ + calls?: number; + /** + * Tokens In + */ + tokens_in?: number; + /** + * Tokens Out + */ + tokens_out?: number; + /** + * Cost Usd + */ + cost_usd?: number; + /** + * Ms + */ + ms?: number; + /** + * Failed + */ + failed?: number; + [key: string]: unknown; +}; + /** * LanguageCount */ @@ -6506,6 +6895,31 @@ export type NamespaceUpdate = { } | null; }; +/** + * NextRun + */ +export type NextRun = { + /** + * Usd + */ + usd?: number | null; + /** + * Tokens + */ + tokens?: number | null; + /** + * Runs + * + * the past runs the estimate is drawn from + */ + runs: number; + /** + * Basis + */ + basis: string; + [key: string]: unknown; +}; + /** * NodeParam */ @@ -10033,6 +10447,29 @@ export type ResetPasswordRequest = { password: string; }; +/** + * ResourceCost + */ +export type ResourceCost = { + /** + * Resource + */ + resource: string; + /** + * Cost Usd + */ + cost_usd: number; + /** + * Tokens + */ + tokens: number; + /** + * Calls + */ + calls: number; + [key: string]: unknown; +}; + /** * ResourceFile */ @@ -10155,6 +10592,35 @@ export type ResourceFiles = { [key: string]: unknown; }; +/** + * ResourceSpend + */ +export type ResourceSpend = { + /** + * Cost Usd + */ + cost_usd: number; + /** + * Tokens + */ + tokens: number; + /** + * Calls + */ + calls: number; + /** + * Unpriced + */ + unpriced: number; + /** + * Estimate + * + * true: cost_usd is a floor (unpriced calls); show it as an estimate + */ + estimate: boolean; + [key: string]: unknown; +}; + /** * Reviewed */ @@ -10359,7 +10825,31 @@ export type RoutineRun = { /** * Log */ - log?: Array | null; + log?: Array | null; + /** + * Cost Usd + * + * what its calls cost, from the activity ledger (docs/activity.md); set when it ends + */ + cost_usd?: number | null; + /** + * Tokens + */ + tokens?: number | null; + /** + * Cost Estimate + * + * true: some calls had no price or token counts, so cost_usd is a floor + */ + cost_estimate?: boolean | null; + /** + * Hold + * + * held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), decided: {run, by, at} once someone picked} + */ + hold?: { + [key: string]: unknown; + } | null; [key: string]: unknown; }; @@ -10373,6 +10863,12 @@ export type RoutineRunRequest = { * graph workflows propose every change instead of making the sure ones */ propose_only?: boolean; + /** + * Over Budget + * + * run it even though it's over a budget (docs/budgets.md); without it, a run over budget is refused (409) + */ + over_budget?: boolean; }; /** @@ -12504,6 +13000,27 @@ export type SpeechTestResult = { [key: string]: unknown; }; +/** + * Spent + */ +export type Spent = { + /** + * Usd + */ + usd: number; + /** + * Tokens + */ + tokens: number; + /** + * Estimate + * + * true: some calls had no figure, so usd is a floor + */ + estimate: boolean; + [key: string]: unknown; +}; + /** * Started * @@ -16573,6 +17090,280 @@ export type ReindexSearchResponses = { export type ReindexSearchResponse = ReindexSearchResponses[keyof ReindexSearchResponses]; +export type ResourceHistoryData = { + body?: never; + path?: never; + query: { + /** + * Resource + * + * table:id, like routine:3, pipeline:2, workflow:4, recording:12, account:1 + */ + resource: string; + /** + * Kind + */ + kind?: "in" | "out" | "run" | "change" | null; + /** + * Before + * + * only entries before this time (the last `at` of the page before) + */ + before?: string | null; + /** + * Limit + */ + limit?: number; + }; + url: "/api/v1/activity"; +}; + +export type ResourceHistoryErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ResourceHistoryError = ResourceHistoryErrors[keyof ResourceHistoryErrors]; + +export type ResourceHistoryResponses = { + /** + * Response Activity-Resource History + * + * Successful Response + */ + 200: Array; +}; + +export type ResourceHistoryResponse = ResourceHistoryResponses[keyof ResourceHistoryResponses]; + +export type ResourceTotalsData = { + body?: never; + path?: never; + query?: { + /** + * Resource + * + * table:id; none for everything (admins) + */ + resource?: string | null; + /** + * Period + */ + period?: "day" | "week" | "month" | "all"; + }; + url: "/api/v1/activity/totals"; +}; + +export type ResourceTotalsErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ResourceTotalsError = ResourceTotalsErrors[keyof ResourceTotalsErrors]; + +export type ResourceTotalsResponses = { + /** + * Successful Response + */ + 200: ActivityTotals; +}; + +export type ResourceTotalsResponse = ResourceTotalsResponses[keyof ResourceTotalsResponses]; + +export type ResourceCostsData = { + body?: never; + path?: never; + query: { + /** + * Resource + * + * table:id, repeated (up to 500): every row of a list at once + */ + resource: Array; + /** + * Period + */ + period?: "day" | "week" | "month" | "all"; + }; + url: "/api/v1/activity/costs"; +}; + +export type ResourceCostsErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ResourceCostsError = ResourceCostsErrors[keyof ResourceCostsErrors]; + +export type ResourceCostsResponses = { + /** + * Successful Response + */ + 200: ActivityCosts; +}; + +export type ResourceCostsResponse = ResourceCostsResponses[keyof ResourceCostsResponses]; + +export type TopResourcesData = { + body?: never; + path?: never; + query?: { + /** + * Table + * + * only this kind of resource: routine, pipeline, workflow, space + */ + table?: string | null; + /** + * Period + */ + period?: "day" | "week" | "month" | "all"; + /** + * Limit + */ + limit?: number; + }; + url: "/api/v1/activity/top"; +}; + +export type TopResourcesErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type TopResourcesError = TopResourcesErrors[keyof TopResourcesErrors]; + +export type TopResourcesResponses = { + /** + * Successful Response + */ + 200: ActivityTop; +}; + +export type TopResourcesResponse = TopResourcesResponses[keyof TopResourcesResponses]; + +export type RemoveBudgetData = { + body?: never; + path?: never; + query: { + /** + * Resource + * + * routine:, pipeline:, workflow: or space: + */ + resource: string; + }; + url: "/api/v1/budgets"; +}; + +export type RemoveBudgetErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type RemoveBudgetError = RemoveBudgetErrors[keyof RemoveBudgetErrors]; + +export type RemoveBudgetResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type RemoveBudgetResponse = RemoveBudgetResponses[keyof RemoveBudgetResponses]; + +export type ListBudgetsData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/budgets"; +}; + +export type ListBudgetsResponses = { + /** + * Response Budgets-List Budgets + * + * Successful Response + */ + 200: Array; +}; + +export type ListBudgetsResponse = ListBudgetsResponses[keyof ListBudgetsResponses]; + +export type SetBudgetData = { + body: BudgetSet; + path?: never; + query: { + /** + * Resource + * + * routine:, pipeline:, workflow: or space: + */ + resource: string; + }; + url: "/api/v1/budgets"; +}; + +export type SetBudgetErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type SetBudgetError = SetBudgetErrors[keyof SetBudgetErrors]; + +export type SetBudgetResponses = { + /** + * Successful Response + */ + 200: BudgetStatus; +}; + +export type SetBudgetResponse = SetBudgetResponses[keyof SetBudgetResponses]; + +export type BudgetStatusData = { + body?: never; + path?: never; + query: { + /** + * Resource + * + * routine:, pipeline:, workflow: or space: + */ + resource: string; + }; + url: "/api/v1/budgets/status"; +}; + +export type BudgetStatusErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type BudgetStatusError = BudgetStatusErrors[keyof BudgetStatusErrors]; + +export type BudgetStatusResponses = { + /** + * Successful Response + */ + 200: BudgetStatus; +}; + +export type BudgetStatusResponse = BudgetStatusResponses[keyof BudgetStatusResponses]; + export type GetFedoraStatusData = { body?: never; path?: never; @@ -23396,6 +24187,36 @@ export type CancelJobResponses = { export type CancelJobResponse = CancelJobResponses[keyof CancelJobResponses]; +export type ReleaseJobData = { + body: HeldRunDecision; + path: { + /** + * Jid + */ + jid: number; + }; + query?: never; + url: "/api/v1/jobs/{jid}/release"; +}; + +export type ReleaseJobErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ReleaseJobError = ReleaseJobErrors[keyof ReleaseJobErrors]; + +export type ReleaseJobResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type ReleaseJobResponse = ReleaseJobResponses[keyof ReleaseJobResponses]; + export type RetryJobData = { body?: never; path: { @@ -26359,6 +27180,36 @@ export type RunRoutineResponses = { export type RunRoutineResponse = RunRoutineResponses[keyof RunRoutineResponses]; +export type DecideHeldRunData = { + body: HeldRunDecision; + path: { + /** + * Run Id + */ + run_id: number; + }; + query?: never; + url: "/api/v1/routines/runs/{run_id}/decide"; +}; + +export type DecideHeldRunErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type DecideHeldRunError = DecideHeldRunErrors[keyof DecideHeldRunErrors]; + +export type DecideHeldRunResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type DecideHeldRunResponse = DecideHeldRunResponses[keyof DecideHeldRunResponses]; + export type ListRunsData = { body?: never; path: { diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 707d3443d..90458414d 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -2714,6 +2714,485 @@ } } }, + "/api/v1/activity": { + "get": { + "tags": [ + "activity" + ], + "summary": "Resource History", + "description": "A resource's history, newest first: API requests that changed it, calls made for it (models, embeddings, the\ndecision model, webhooks, web tools) with tokens and cost, runs that ended, and its audit log entries.", + "operationId": "resource_history", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "table:id, like routine:3, pipeline:2, workflow:4, recording:12, account:1", + "title": "Resource" + }, + "description": "table:id, like routine:3, pipeline:2, workflow:4, recording:12, account:1" + }, + { + "name": "kind", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "enum": [ + "in", + "out", + "run", + "change" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Kind" + } + }, + { + "name": "before", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "only entries before this time (the last `at` of the page before)", + "title": "Before" + }, + "description": "only entries before this time (the last `at` of the page before)" + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 1000, + "minimum": 1, + "default": 100, + "title": "Limit" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ActivityEntry" + }, + "title": "Response Activity-Resource History" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/activity/totals": { + "get": { + "tags": [ + "activity" + ], + "summary": "Resource Totals", + "description": "What a resource's calls cost this day, week (from Monday), month (UTC) or all time: calls, tokens, USD, time.", + "operationId": "resource_totals", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "table:id; none for everything (admins)", + "title": "Resource" + }, + "description": "table:id; none for everything (admins)" + }, + { + "name": "period", + "in": "query", + "required": false, + "schema": { + "enum": [ + "day", + "week", + "month", + "all" + ], + "type": "string", + "default": "month", + "title": "Period" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ActivityTotals" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/activity/costs": { + "get": { + "tags": [ + "activity" + ], + "summary": "Resource Costs", + "description": "What each of many resources cost this period, for lists; ones you can't see are left out. `estimate` says a\nfigure is a floor (some calls had no price or token counts).", + "operationId": "resource_costs", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "description": "table:id, repeated (up to 500): every row of a list at once", + "title": "Resource" + }, + "description": "table:id, repeated (up to 500): every row of a list at once" + }, + { + "name": "period", + "in": "query", + "required": false, + "schema": { + "enum": [ + "day", + "week", + "month", + "all" + ], + "type": "string", + "default": "month", + "title": "Period" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ActivityCosts" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/activity/top": { + "get": { + "tags": [ + "activity" + ], + "summary": "Top Resources", + "description": "The resources that cost most this period (admins).", + "operationId": "top_resources", + "parameters": [ + { + "name": "table", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-z_]{1,40}$" + }, + { + "type": "null" + } + ], + "description": "only this kind of resource: routine, pipeline, workflow, space", + "title": "Table" + }, + "description": "only this kind of resource: routine, pipeline, workflow, space" + }, + { + "name": "period", + "in": "query", + "required": false, + "schema": { + "enum": [ + "day", + "week", + "month", + "all" + ], + "type": "string", + "default": "month", + "title": "Period" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 200, + "minimum": 1, + "default": 20, + "title": "Limit" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ActivityTop" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/budgets": { + "get": { + "tags": [ + "budgets" + ], + "summary": "List Budgets", + "description": "Every budget, with where it stands now.", + "operationId": "list_budgets", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BudgetStatus" + }, + "title": "Response Budgets-List Budgets" + } + } + } + } + } + }, + "put": { + "tags": [ + "budgets" + ], + "summary": "Set Budget", + "operationId": "set_budget", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "routine:, pipeline:, workflow: or space:", + "title": "Resource" + }, + "description": "routine:, pipeline:, workflow: or space:" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BudgetSet" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BudgetStatus" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "budgets" + ], + "summary": "Remove Budget", + "operationId": "remove_budget", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "routine:, pipeline:, workflow: or space:", + "title": "Resource" + }, + "description": "routine:, pipeline:, workflow: or space:" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ok" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/budgets/status": { + "get": { + "tags": [ + "budgets" + ], + "summary": "Budget Status", + "description": "Where a resource's budget stands, and what its next run will likely cost (also without a budget).", + "operationId": "budget_status", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "routine:, pipeline:, workflow: or space:", + "title": "Resource" + }, + "description": "routine:, pipeline:, workflow: or space:" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BudgetStatus" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/admin/fedora": { "get": { "tags": [ @@ -14156,6 +14635,59 @@ } } }, + "/api/v1/jobs/{jid}/release": { + "post": { + "tags": [ + "jobs" + ], + "summary": "Release Job", + "description": "Pick for a job held over a budget (docs/budgets.md): run it now, once, whatever its budgets say, or skip it\n(admins, who set the budgets).", + "operationId": "release_job", + "parameters": [ + { + "name": "jid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Jid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HeldRunDecision" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ok" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/jobs/{jid}/retry": { "post": { "tags": [ @@ -18819,7 +19351,7 @@ "routines" ], "summary": "Run Routine", - "description": "Run it as soon as the scheduler next looks (within half a minute), even when it is off.", + "description": "Run it as soon as the scheduler next looks (within half a minute), even when it is off. A run over one of its\nbudgets is refused (409, saying where the budget stands) unless over_budget is true (docs/budgets.md).", "operationId": "run_routine", "parameters": [ { @@ -18866,114 +19398,14 @@ } } }, - "/api/v1/routines/{rid}/runs": { - "get": { - "tags": [ - "routines" - ], - "summary": "List Runs", - "operationId": "list_runs", - "parameters": [ - { - "name": "rid", - "in": "path", - "required": true, - "schema": { - "type": "integer", - "title": "Rid" - } - }, - { - "name": "limit", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "maximum": 200, - "minimum": 1, - "default": 20, - "title": "Limit" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/RoutineRun" - }, - "title": "Response Routines-List Runs" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/api/v1/routine-runs/{run_id}": { - "get": { - "tags": [ - "routines" - ], - "summary": "Get Run", - "operationId": "get_run", - "parameters": [ - { - "name": "run_id", - "in": "path", - "required": true, - "schema": { - "type": "integer", - "title": "Run Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RoutineRun" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/api/v1/routine-runs/{run_id}/undo": { + "/api/v1/routines/runs/{run_id}/decide": { "post": { "tags": [ "routines" ], - "summary": "Undo Run", - "description": "Take back every graph change the run made.", - "operationId": "undo_run", + "summary": "Decide Held Run", + "description": "Pick for a run held over a budget: run it now, once, whatever its budgets say, or skip it.", + "operationId": "decide_held_run", "parameters": [ { "name": "run_id", @@ -18985,142 +19417,295 @@ } } ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Undone" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/api/v1/graph-changes": { - "get": { - "tags": [ - "routines" - ], - "summary": "List Graph Changes", - "description": "Changes to the entities of namespaces you can read, newest first.", - "operationId": "list_graph_changes", - "parameters": [ - { - "name": "status", - "in": "query", - "required": false, - "schema": { - "anyOf": [ - { - "enum": [ - "proposed", - "applied", - "dismissed", - "undone" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Status" - } - }, - { - "name": "run", - "in": "query", - "required": false, - "schema": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Run" - } - }, - { - "name": "limit", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "maximum": 1000, - "minimum": 1, - "default": 200, - "title": "Limit" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/GraphChange" - }, - "title": "Response Routines-List Graph Changes" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/api/v1/graph-changes/{cid}/accept": { - "post": { - "tags": [ - "routines" - ], - "summary": "Accept Graph Change", - "operationId": "accept_graph_change", - "parameters": [ - { - "name": "cid", - "in": "path", - "required": true, - "schema": { - "type": "integer", - "title": "Cid" - } - } - ], "requestBody": { "required": true, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/GraphChangeAccept" + "$ref": "#/components/schemas/HeldRunDecision" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ok" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/routines/{rid}/runs": { + "get": { + "tags": [ + "routines" + ], + "summary": "List Runs", + "operationId": "list_runs", + "parameters": [ + { + "name": "rid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Rid" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 200, + "minimum": 1, + "default": 20, + "title": "Limit" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/RoutineRun" + }, + "title": "Response Routines-List Runs" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/routine-runs/{run_id}": { + "get": { + "tags": [ + "routines" + ], + "summary": "Get Run", + "operationId": "get_run", + "parameters": [ + { + "name": "run_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Run Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RoutineRun" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/routine-runs/{run_id}/undo": { + "post": { + "tags": [ + "routines" + ], + "summary": "Undo Run", + "description": "Take back every graph change the run made.", + "operationId": "undo_run", + "parameters": [ + { + "name": "run_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Run Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Undone" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/graph-changes": { + "get": { + "tags": [ + "routines" + ], + "summary": "List Graph Changes", + "description": "Changes to the entities of namespaces you can read, newest first.", + "operationId": "list_graph_changes", + "parameters": [ + { + "name": "status", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "enum": [ + "proposed", + "applied", + "dismissed", + "undone" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Status" + } + }, + { + "name": "run", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Run" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 1000, + "minimum": 1, + "default": 200, + "title": "Limit" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GraphChange" + }, + "title": "Response Routines-List Graph Changes" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/graph-changes/{cid}/accept": { + "post": { + "tags": [ + "routines" + ], + "summary": "Accept Graph Change", + "operationId": "accept_graph_change", + "parameters": [ + { + "name": "cid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Cid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GraphChangeAccept" } } } @@ -20769,6 +21354,311 @@ "title": "AccountToken", "description": "Anyone's key, for admins." }, + "ActivityCosts": { + "properties": { + "period": { + "type": "string", + "title": "Period" + }, + "since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Since" + }, + "costs": { + "additionalProperties": { + "$ref": "#/components/schemas/ResourceSpend" + }, + "type": "object", + "title": "Costs", + "description": "by resource; ones you can't see are left out" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "period", + "costs" + ], + "title": "ActivityCosts" + }, + "ActivityEntry": { + "properties": { + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id", + "description": "the ledger row's id; none for an audit log entry" + }, + "at": { + "type": "string", + "title": "At" + }, + "kind": { + "type": "string", + "enum": [ + "in", + "out", + "run", + "change" + ], + "title": "Kind", + "description": "in: an API request; out: a call Lens made (a model, embeddings, the decision model, a webhook, a web tool); run: a job or routine run ended; change: an audit log entry" + }, + "action": { + "type": "string", + "title": "Action", + "description": "what it was: `POST /api/v1/routines/{rid}/run`, `model.chat`, `job.succeeded`, `routine.update`" + }, + "resources": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Resources", + "description": "every resource it counts for, as table:id" + }, + "actor": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Actor", + "description": "the account that made it, when a person did" + }, + "email": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Email" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "tokens_in": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens In" + }, + "tokens_out": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens Out" + }, + "cost_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost Usd", + "description": "estimated from the prices in Settings; none when the model has no price" + }, + "ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Ms" + }, + "ok": { + "type": "boolean", + "title": "Ok", + "default": true + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error", + "description": "the error's type or HTTP status; never its message" + }, + "detail": { + "title": "Detail" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "at", + "kind", + "action" + ], + "title": "ActivityEntry" + }, + "ActivityTop": { + "properties": { + "period": { + "type": "string", + "title": "Period" + }, + "since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Since" + }, + "resources": { + "items": { + "$ref": "#/components/schemas/ResourceCost" + }, + "type": "array", + "title": "Resources" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "period", + "resources" + ], + "title": "ActivityTop" + }, + "ActivityTotals": { + "properties": { + "resource": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Resource" + }, + "period": { + "type": "string", + "title": "Period" + }, + "since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Since" + }, + "calls": { + "type": "integer", + "title": "Calls" + }, + "failed": { + "type": "integer", + "title": "Failed" + }, + "tokens_in": { + "type": "integer", + "title": "Tokens In" + }, + "tokens_out": { + "type": "integer", + "title": "Tokens Out" + }, + "cost_usd": { + "type": "number", + "title": "Cost Usd", + "description": "what its calls cost (run rows repeat their calls' cost and aren't added again)" + }, + "ms": { + "type": "integer", + "title": "Ms" + }, + "unpriced": { + "type": "integer", + "title": "Unpriced", + "description": "calls that cost something but have no figure (a model with no price, a reply without token counts)", + "default": 0 + }, + "estimate": { + "type": "boolean", + "title": "Estimate", + "description": "true when unpriced calls make cost_usd a floor rather than exact", + "default": false + }, + "by_kind": { + "additionalProperties": { + "$ref": "#/components/schemas/KindTotals" + }, + "type": "object", + "title": "By Kind" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "period", + "calls", + "failed", + "tokens_in", + "tokens_out", + "cost_usd", + "ms", + "by_kind" + ], + "title": "ActivityTotals" + }, "AnswerCheck": { "properties": { "claims": { @@ -22183,6 +23073,279 @@ ], "title": "BrowseEntry" }, + "Budget": { + "properties": { + "resource": { + "type": "string", + "title": "Resource" + }, + "usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Usd" + }, + "tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens" + }, + "period": { + "type": "string", + "title": "Period" + }, + "on_over": { + "type": "string", + "title": "On Over" + }, + "warn_at": { + "type": "number", + "title": "Warn At" + }, + "note": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Note" + }, + "by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "By" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "alert": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Alert", + "description": "the last warning: {state near|over, since, at, share}" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "resource", + "period", + "on_over", + "warn_at" + ], + "title": "Budget" + }, + "BudgetSet": { + "properties": { + "usd": { + "anyOf": [ + { + "type": "number", + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Usd", + "description": "the cap in USD (from the costs in the activity ledger)" + }, + "tokens": { + "anyOf": [ + { + "type": "integer", + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Tokens", + "description": "the cap in tokens (in and out)" + }, + "period": { + "type": "string", + "enum": [ + "run", + "day", + "week", + "month" + ], + "title": "Period", + "description": "what the cap is for: one run, or a day, week or month (UTC)", + "default": "month" + }, + "on_over": { + "type": "string", + "enum": [ + "ask", + "skip", + "assistant" + ], + "title": "On Over", + "description": "a run that would go over: ask (hold it until someone picks run or skip), skip, or assistant (the decision model weighs it; it runs only when sure, else asks)", + "default": "ask" + }, + "warn_at": { + "type": "number", + "maximum": 1.0, + "minimum": 0.1, + "title": "Warn At", + "description": "the share spent at which it's marked near (and warned about)", + "default": 0.8 + }, + "note": { + "anyOf": [ + { + "type": "string", + "maxLength": 300 + }, + { + "type": "null" + } + ], + "title": "Note" + } + }, + "additionalProperties": false, + "type": "object", + "title": "BudgetSet" + }, + "BudgetStatus": { + "properties": { + "resource": { + "type": "string", + "title": "Resource" + }, + "budget": { + "anyOf": [ + { + "$ref": "#/components/schemas/Budget" + }, + { + "type": "null" + } + ] + }, + "state": { + "type": "string", + "enum": [ + "none", + "ok", + "near", + "over" + ], + "title": "State" + }, + "since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Since", + "description": "the start of the period being counted" + }, + "spent": { + "anyOf": [ + { + "$ref": "#/components/schemas/Spent" + }, + { + "type": "null" + } + ] + }, + "next": { + "$ref": "#/components/schemas/NextRun", + "description": "what one more run will likely cost: always an estimate" + }, + "left": { + "anyOf": [ + { + "additionalProperties": { + "type": "number" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Left" + }, + "share": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Share", + "description": "the largest share of a cap spent (1 is all of it)" + }, + "share_after_next": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Share After Next", + "description": "the share once the next run (estimated) is done" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "resource", + "state", + "next" + ], + "title": "BudgetStatus" + }, "ChangeAsk": { "properties": { "kind": { @@ -28747,6 +29910,21 @@ ], "title": "Heard" }, + "HeldRunDecision": { + "properties": { + "run": { + "type": "boolean", + "title": "Run", + "description": "true: run it now, once, whatever its budgets say; false: skip it" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "run" + ], + "title": "HeldRunDecision" + }, "Highlight": { "properties": { "id": { @@ -30015,6 +31193,54 @@ ], "title": "Cancel Requested" }, + "cost_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost Usd", + "description": "what its calls cost, from the activity ledger (docs/activity.md); set when it ends" + }, + "tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens" + }, + "cost_estimate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Cost Estimate", + "description": "true: some calls had no price or token counts, so cost_usd is a floor" + }, + "hold": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Hold", + "description": "held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), decided: {run, by, at} once someone picked}" + }, "log": { "anyOf": [ { @@ -30340,6 +31566,43 @@ ], "title": "JobsQueued" }, + "KindTotals": { + "properties": { + "calls": { + "type": "integer", + "title": "Calls", + "default": 0 + }, + "tokens_in": { + "type": "integer", + "title": "Tokens In", + "default": 0 + }, + "tokens_out": { + "type": "integer", + "title": "Tokens Out", + "default": 0 + }, + "cost_usd": { + "type": "number", + "title": "Cost Usd", + "default": 0 + }, + "ms": { + "type": "integer", + "title": "Ms", + "default": 0 + }, + "failed": { + "type": "integer", + "title": "Failed", + "default": 0 + } + }, + "additionalProperties": true, + "type": "object", + "title": "KindTotals" + }, "LanguageCount": { "properties": { "language": { @@ -31787,6 +33050,48 @@ "title": "NamespaceUpdate", "description": "Only the fields you send change; send ``pipeline: null`` to go back to the default pipeline." }, + "NextRun": { + "properties": { + "usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Usd" + }, + "tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens" + }, + "runs": { + "type": "integer", + "title": "Runs", + "description": "the past runs the estimate is drawn from" + }, + "basis": { + "type": "string", + "title": "Basis" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "runs", + "basis" + ], + "title": "NextRun" + }, "NodeParam": { "properties": { "name": { @@ -37917,6 +39222,35 @@ ], "title": "ResetPasswordRequest" }, + "ResourceCost": { + "properties": { + "resource": { + "type": "string", + "title": "Resource" + }, + "cost_usd": { + "type": "number", + "title": "Cost Usd" + }, + "tokens": { + "type": "integer", + "title": "Tokens" + }, + "calls": { + "type": "integer", + "title": "Calls" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "resource", + "cost_usd", + "tokens", + "calls" + ], + "title": "ResourceCost" + }, "ResourceFile": { "properties": { "id": { @@ -38129,6 +39463,41 @@ ], "title": "ResourceFiles" }, + "ResourceSpend": { + "properties": { + "cost_usd": { + "type": "number", + "title": "Cost Usd" + }, + "tokens": { + "type": "integer", + "title": "Tokens" + }, + "calls": { + "type": "integer", + "title": "Calls" + }, + "unpriced": { + "type": "integer", + "title": "Unpriced" + }, + "estimate": { + "type": "boolean", + "title": "Estimate", + "description": "true: cost_usd is a floor (unpriced calls); show it as an estimate" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "cost_usd", + "tokens", + "calls", + "unpriced", + "estimate" + ], + "title": "ResourceSpend" + }, "Reviewed": { "properties": { "applied": { @@ -38519,6 +39888,54 @@ } ], "title": "Log" + }, + "cost_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost Usd", + "description": "what its calls cost, from the activity ledger (docs/activity.md); set when it ends" + }, + "tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens" + }, + "cost_estimate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Cost Estimate", + "description": "true: some calls had no price or token counts, so cost_usd is a floor" + }, + "hold": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Hold", + "description": "held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), decided: {run, by, at} once someone picked}" } }, "additionalProperties": true, @@ -38538,6 +39955,12 @@ "title": "Propose Only", "description": "graph workflows propose every change instead of making the sure ones", "default": false + }, + "over_budget": { + "type": "boolean", + "title": "Over Budget", + "description": "run it even though it's over a budget (docs/budgets.md); without it, a run over budget is refused (409)", + "default": false } }, "additionalProperties": false, @@ -42261,6 +43684,31 @@ ], "title": "SpeechTestResult" }, + "Spent": { + "properties": { + "usd": { + "type": "number", + "title": "Usd" + }, + "tokens": { + "type": "integer", + "title": "Tokens" + }, + "estimate": { + "type": "boolean", + "title": "Estimate", + "description": "true: some calls had no figure, so usd is a floor" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "usd", + "tokens", + "estimate" + ], + "title": "Spent" + }, "Started": { "properties": { "ok": {
Steps Version Default inThis month Updated
{onStandard.length ? onStandard.join(", ") : "—"}— —
{used.length ? used.join(", ") : "—"} + + {relative(p.updated_at)}
When Next run Last runThis month On Actions @@ -291,6 +334,9 @@ export function RoutinesPage() { {r.last_run_at && } + + + = { failed: "red", cancelled: "neutral", review: "gate", + held: "gate", }; export function statusTone(status: string | null | undefined): Tone { diff --git a/nextjs-frontend/components/workflows/workflows-page.tsx b/nextjs-frontend/components/workflows/workflows-page.tsx index 2cfb291d8..8659b48fc 100644 --- a/nextjs-frontend/components/workflows/workflows-page.tsx +++ b/nextjs-frontend/components/workflows/workflows-page.tsx @@ -3,6 +3,7 @@ import { Workflow } from "lucide-react"; import Link from "next/link"; +import { CostCell, useBudgets, useCosts } from "@/components/costs/costs"; import { CatalogHeader, useWorkflowCatalog } from "@/components/pipelines/catalog-header"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -14,6 +15,8 @@ import { relative } from "@/lib/format"; export function WorkflowsPage() { const catalog = useWorkflowCatalog(); const list = catalog.data?.workflows ?? []; + const costs = useCosts(list.map((w) => `workflow:${w.id}`)); + const budgets = useBudgets(); return (
@@ -36,6 +39,7 @@ export function WorkflowsPage() {
Name Version Run byThis month Updated
{w.scope === "graph" ? "Routines" : w.pipelines?.length ? w.pipelines.join(", ") : "—"} + + {relative(w.updated_at)}
GraphMembersThis month Your role
{n.graph === "isolated" ? "Isolated" : "Shared"}{people.data ? count(members(n.name)) : "…"} + +