diff --git a/CHANGELOG.md b/CHANGELOG.md index b250e6342..cfa5ffdcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Budgets for routines, pipelines, workflows and namespaces.** An admin caps what a resource may cost, in USD, + tokens or both, per run, day, week or month. Before a routine runs or a job starts, Lens estimates the run from past + ones; one that would go over is held until someone picks run or skip (or is skipped, or the decision model weighs + it when asked to). Budgets near or over their cap are flagged once per period. Off unless set. See docs/budgets.md. +- **Local models cost nothing unless you price them.** A model's price is by tokens (USD per million), by time (USD + per hour, for a model on your own machine) or off, the default for a model without one. + +- **Every resource has an activity history, with what it cost.** Lens now keeps a ledger of every request that + changes something, every call it makes out (models, embeddings, the decision model, text to speech, webhooks, web + tools) and every job and routine run, each counted for the resources it touched: a model call in a job a routine + queued counts for the job, its recording, namespace, pipeline, workflow and the routine. Tokens and estimated cost + come from the prices in Settings. `GET /api/v1/activity?resource=routine:3` gives a resource's history (with its + audit log entries), `/activity/totals` what it cost this day, week or month, `/activity/top` what cost most. On by + default; `activity.keep_days` (365) keeps it in bounds. See docs/activity.md. - **Safer upgrades of existing installs.** Data upgrades are now named steps that run once per database, under a lock so only one process runs them while the others wait, with a record of when each ran and why one failed. `lens migrations` lists them. A database with data is backed up into `/backups` before it is upgraded diff --git a/DEVLOG.md b/DEVLOG.md index 34c7bfd2d..6da1d5592 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -2,6 +2,45 @@ Plans and progress for work in flight. Newest first. +## 2026-10-04 · Activity history and budgets + +Goal (Adeel): every change and every call coming in or going out is tracked, with its cost, down to the resource, so +each resource has an activity history. Pipelines, workflows and routines can be capped at a budget, and before a run +(or something larger) the assistant looks at where the budget stands and whether the run should go ahead. + +Model: + +- One ledger, `activity`: a row per call in (API requests that change something), call out (model, embeddings, + decision model, webhooks, web tools) and run (a job, a routine run). Each row names every resource it touched + (`routine:3`, `routine_run:12`, `job:40`, `recording:7`, `pipeline:2`, `workflow:4`, `chat:9`, `space:1`, + `account:1`), with tokens, estimated cost (the prices in Settings), duration and outcome; never prompt or reply text. + Who a call is for comes from the work it runs in (a scope), so a model call inside a job counts for the job, its + recording, pipeline, namespace and the routine that queued it. +- A resource's history is its ledger rows plus its audit log entries. On by default, nothing to set up; kept + `activity.keep_days` (365). Reads (GET) aren't logged unless `activity.reads` is on. +- Budgets are off unless set: a budget on a routine, pipeline, workflow or namespace caps cost (USD) and/or tokens per + run, day, week or month. Before a routine run or a job, Lens estimates the run from past runs and checks what is + left. Over budget, it asks you (the default: the run waits for your pick), skips, or lets the assistant weigh it (the + decision model; unsure means ask). Doing nothing changes nothing: a waiting run stays waiting. + +Todo: + +- [x] Activity ledger: scopes, model/embedding/decision/webhook/web-tool calls, changing API requests, job and routine + run totals, per-resource history and cost summary API, retention sweep +- [ ] Activity in the web app: a history panel on routines, pipelines, workflows and recordings; Settings → Activity + and costs with totals by resource +- [ ] Cost in every view (Adeel): every resource, entity and view shows what it cost and its budget; a figure that + isn't exact (no price for a model, tokens the server didn't report, a run's forecast) is marked as an estimate + (`≈`, with why on hover). The API says `exact` / `estimate` per figure +- [x] Budgets: set per routine, pipeline, workflow or namespace; spent and estimate; checked before routine runs and + jobs; over budget waits for your pick (run once, skip), or the assistant decides +- [ ] Budgets in the web app: set a budget, see where it stands, approve or skip a waiting run +- [x] Periodic check: warn at 80% and 100% of a budget, once per period (on the budget and in its history) +- [x] Local model costs (Adeel): price per model by tokens, by time (per hour) or off; off by default +- [ ] Budget warnings through notifications, and the assistant answering where budgets stand + +Refine later: compute time as a cost (CPU seconds × a rate for local models), per-person budgets, forecasting from +schedules (a routine's next runs this period), budget alerts in the weekly digest. ## 2026-10-04 · Local decision models: Laya on MLX Goal (Adeel): routine decisions can run on the machine itself with the Laya typed decision models on MLX diff --git a/docs/activity.md b/docs/activity.md new file mode 100644 index 000000000..44046935f --- /dev/null +++ b/docs/activity.md @@ -0,0 +1,82 @@ +# Activity and costs + +Lens keeps a ledger of what happens to each resource and what it cost: every request that changes something, every +call it makes out (a language model, embeddings, the decision model, text to speech, a webhook, a web tool), and every +job or routine run that ends. It is on by default and needs nothing set up. + +## What a row holds + +Each row names every resource it counts for, as `table:id`: + +| Resource | Example | +| --- | --- | +| A routine and one of its runs | `routine:3`, `routine_run:12` | +| A job, its recording, namespace and pipeline | `job:40`, `recording:7`, `space:1`, `pipeline:2` | +| A workflow | `workflow:4` | +| A chat | `chat:9` | +| The person who asked | `account:1` | + +A model call inside a workflow step of a job that a routine queued counts for the workflow, the job, the recording, +its namespace, its pipeline and the routine. A request counts for the resources in its path (`PATCH +/api/v1/routines/3` is `routine:3`), for what a `POST` made (from the `id` in its reply), and for who sent it. + +Besides the resources, a row has its kind (`in`, `out` or `run`), what it was (`POST /api/v1/routines/{rid}`, +`model.chat`, `embeddings`, `decision`, `notify.webhook`, `tool.http`, `job.succeeded`, `routine.done`), the model, +tokens in and out, the estimated cost in USD, how long it took, and how it ended (an error's type or HTTP status, +never its message). It never holds prompt, reply, transcript or file content. + +## Costs + +Each model is costed the way you set it under Settings › Telemetry › prices (`telemetry.prices`), or, for the +assistant's model, Settings › AI assistant: + +| Unit | Setting | For | +| --- | --- | --- | +| Tokens | `{"input": 3.0, "output": 15.0}` (USD per million tokens) | cloud models that bill by tokens | +| Time | `{"unit": "time", "per_hour": 0.6}` (USD per hour a call takes) | a local model on your own machine | +| Off | `{"unit": "off"}`, or leave the model out | the default: no money counted, only tokens and time | + +So local models aren't costed unless you give them a price. `decisions.price_per_call` gives the decision model's +price per decision. + +A figure that should have a cost but doesn't (a model priced by tokens whose server didn't report how many it used) +marks the row `unpriced`, and every total that includes it says `estimate: true`: the cost shown is a floor, and +the web app shows it with ≈. + +A run row (a job or a routine run ending) repeats what its calls cost in all, and the job or run keeps it too +(`cost_usd`, `tokens`). Totals don't count run rows twice. + +## Reading it + +A resource's history merges its ledger rows with its audit log entries (kind `change`), newest first: + +```http +GET /api/v1/activity?resource=routine:3&limit=100 +GET /api/v1/activity?resource=routine:3&kind=out&before=2026-10-04T09:00:00.000+00:00 +``` + +What it cost this day, week (from Monday) or month (UTC), or all time: + +```http +GET /api/v1/activity/totals?resource=pipeline:2&period=month +GET /api/v1/activity/totals?period=day # everything (admins) +GET /api/v1/activity/top?table=routine&period=week # what cost most (admins) +``` + +Admins see every resource. Everyone else sees their own account's activity, their own chats', and the recordings and +namespaces they can read. + +## Settings + +| Setting | Default | | +| --- | --- | --- | +| `activity.enabled` | `true` | Off stops new rows (the audit log carries on). | +| `activity.reads` | `false` | Log API reads (GET) too. The web app reads often, so this grows the ledger fast. | +| `activity.keep_days` | `365` | Rows older than this are dropped, hourly, by the routines scheduler. | + +Sign-in requests (`/api/v1/auth/`, `/api/v1/passkeys/`) aren't logged here; the audit log records sign-ins. + +## Refine later + +Compute time as a cost for local models (CPU seconds at a rate), per-person totals in the web app, and calls the +ledger doesn't see yet (Fedora sync, Matterbridge messages). diff --git a/docs/budgets.md b/docs/budgets.md new file mode 100644 index 000000000..aa589b8ee --- /dev/null +++ b/docs/budgets.md @@ -0,0 +1,53 @@ +# Budgets + +A budget caps what a routine, pipeline, workflow or namespace may cost. Budgets are off unless an admin sets one. +What has been spent comes from the [activity ledger](activity.md), so a budget counts everything done for that +resource: a namespace's budget counts every job in it, and a routine's counts its runs and the jobs it queued. + +## Setting one + +```http +PUT /api/v1/budgets?resource=routine:3 +{"usd": 5, "period": "month", "on_over": "ask"} +``` + +| Field | | | +| --- | --- | --- | +| `usd`, `tokens` | at least one | the cap in USD, tokens (in and out), or both | +| `period` | `month` | `run` (each run on its own), `day`, `week` (from Monday) or `month` (UTC) | +| `on_over` | `ask` | what happens to a run that would go over: see below | +| `warn_at` | `0.8` | the share spent at which the budget is `near` and gets a warning | + +`GET /api/v1/budgets` lists every budget with where it stands, `GET /api/v1/budgets/status?resource=…` gives one +(also for a resource without a budget, with its estimated next run), and `DELETE /api/v1/budgets?resource=…` +removes one. + +## Before a run + +Before a routine runs and before a job's first step, Lens looks at the budgets of everything the run counts for: the +routine, the pipelines and workflows its actions name and its namespaces; a job's pipeline, namespace and the routine +that queued it. It estimates the next run from the average of that resource's runs in the last 30 days (always an +estimate, shown with ≈). A run goes ahead unless a budget is used up, or would be once the estimated run is done +(for a `run` budget, unless the estimate alone passes the cap). Then `on_over` decides: + +- `ask` (default): the run is held. A routine gets a run with status `held`, a job the status `held`, each with + `hold.why` saying where the budget stands. Someone picks: run it now, once, whatever the budget says + (`POST /api/v1/routines/runs/{run}/decide {"run": true}`, `POST /api/v1/jobs/{jid}/release {"run": true}`), or skip + it (`{"run": false}`). Doing nothing changes nothing: a held run stays held, and a routine that comes due again adds + to the one held run (`hold.missed`) rather than piling up. +- `skip`: the run is skipped, and says why. +- `assistant`: the decision model (Settings › AI assistant, decisions) weighs it, with where the budget stands and the + estimate. It runs only when the model is sure (`decisions.act_above`); otherwise it's held as with `ask`. + +Asking a routine to run from the app or the API when it's over budget is refused with 409 and the reason; send +`{"over_budget": true}` to run it anyway. + +## Warnings + +Every five minutes the scheduler looks at every budget. One that crosses `warn_at` or its cap gets an alert, once per +period: on the budget (`alert`) and as a `budget.near` or `budget.over` row in its activity history. + +## Refine later + +Forecasting from a routine's schedule (its runs left this period), per-person budgets, budget warnings sent through +notifications and the weekly digest, and the assistant answering "where do my budgets stand?" in chat. diff --git a/fastapi_backend/app/api/v1/router.py b/fastapi_backend/app/api/v1/router.py index 50588f0e9..0ab80ba58 100644 --- a/fastapi_backend/app/api/v1/router.py +++ b/fastapi_backend/app/api/v1/router.py @@ -3,9 +3,11 @@ from fastapi import APIRouter from app.api.v1.routes import ( + activity, admin, auth, batches, + budgets, chats, collections, comments, @@ -64,6 +66,8 @@ for module in ( users, admin, + activity, + budgets, fedora, setup, namespaces, diff --git a/fastapi_backend/app/api/v1/routes/activity.py b/fastapi_backend/app/api/v1/routes/activity.py new file mode 100644 index 000000000..33e8092a3 --- /dev/null +++ b/fastapi_backend/app/api/v1/routes/activity.py @@ -0,0 +1,109 @@ +"""The activity ledger (docs/activity.md): a resource's history (its calls in and out, runs and changes) and what it +cost. Admins see every resource's; others see their own account's, their chats', and the recordings and namespaces they +can read.""" + +from __future__ import annotations + +import re +from typing import Literal + +from fastapi import APIRouter, HTTPException, Query + +from app.api.deps import Acl, AdminReader, CurrentUser, Db +from app.domain import activity +from app.schemas.activity import ActivityCosts, ActivityEntry, ActivityTop, ActivityTotals + +router = APIRouter(tags=["activity"]) +Period = Literal["day", "week", "month", "all"] +RESOURCE = re.compile(r"^([a-z_]{1,40}):([A-Za-z0-9_.-]{1,80})$") + + +def _may_see(db: Db, user: CurrentUser, acl: Acl, resource: str) -> None: + m = RESOURCE.match(resource) + if not m: + raise HTTPException(422, "resource is table:id, like routine:3 or recording:12") + if user.admin: + return + table, key = m.groups() + if table == "account" and key == str(user.id): + return + if key.isdigit(): + if table == "recording": + acl.recording(int(key)) + return + if table == "space": + acl.need(int(key)) + return + if table == "chat" and db.one("SELECT id FROM $r WHERE account = $a", r=activity.R("chat", int(key)), a=user.id): + return + raise HTTPException(403, "only admins see this resource's activity") + + +@router.get("/activity") +def resource_history( + user: CurrentUser, + acl: Acl, + db: Db, + resource: str = Query(description="table:id, like routine:3, pipeline:2, workflow:4, recording:12, account:1"), + kind: Literal["in", "out", "run", "change"] | None = None, + before: str | None = Query(None, description="only entries before this time (the last `at` of the page before)"), + limit: int = Query(100, ge=1, le=1000), +) -> list[ActivityEntry]: + """A resource's history, newest first: API requests that changed it, calls made for it (models, embeddings, the + decision model, webhooks, web tools) with tokens and cost, runs that ended, and its audit log entries.""" + _may_see(db, user, acl, resource) + return activity.history(db, resource, limit, before, kind) + + +@router.get("/activity/totals") +def resource_totals( + user: CurrentUser, + acl: Acl, + db: Db, + resource: str | None = Query(None, description="table:id; none for everything (admins)"), + period: Period = "month", +) -> ActivityTotals: + """What a resource's calls cost this day, week (from Monday), month (UTC) or all time: calls, tokens, USD, time.""" + if resource is None: + if not user.admin: + raise HTTPException(403, "only admins see the totals for everything") + else: + _may_see(db, user, acl, resource) + since = activity._since(period) + return ActivityTotals(resource=resource, period=period, since=since, **activity.totals(db, resource, since)) + + +@router.get("/activity/costs") +def resource_costs( + user: CurrentUser, + acl: Acl, + db: Db, + resource: list[str] = Query(description="table:id, repeated (up to 500): every row of a list at once"), + period: Period = "month", +) -> ActivityCosts: + """What each of many resources cost this period, for lists; ones you can't see are left out. `estimate` says a + figure is a floor (some calls had no price or token counts).""" + seen = [] + for r in resource[:500]: + try: + _may_see(db, user, acl, r) + except HTTPException: + continue + seen.append(r) + since = activity._since(period) + return ActivityCosts(period=period, since=since, costs=activity.costs(db, seen, since)) + + +@router.get("/activity/top") +def top_resources( + user: AdminReader, + db: Db, + table: str | None = Query( + None, pattern=r"^[a-z_]{1,40}$", description="only this kind of resource: routine, pipeline, workflow, space" + ), + period: Period = "month", + limit: int = Query(20, ge=1, le=200), +) -> ActivityTop: + """The resources that cost most this period (admins).""" + since = activity._since(period) + return ActivityTop(period=period, since=since, resources=activity.top(db, table, since, limit)) diff --git a/fastapi_backend/app/api/v1/routes/budgets.py b/fastapi_backend/app/api/v1/routes/budgets.py new file mode 100644 index 000000000..a5e6f8f0d --- /dev/null +++ b/fastapi_backend/app/api/v1/routes/budgets.py @@ -0,0 +1,46 @@ +"""Budgets (admins set them; docs/budgets.md): caps on what a routine, pipeline, workflow or namespace may cost, and +where each stands.""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException, Query + +from app.api.deps import AdminReader, AdminWriter, Db, domain_errors +from app.domain import auth, budgets +from app.schemas.budgets import BudgetSet, BudgetStatus +from app.schemas.common import Ok + +router = APIRouter(tags=["budgets"]) +RESOURCE = Query(description="routine:, pipeline:, workflow: or space:") + + +@router.get("/budgets") +def list_budgets(user: AdminReader, db: Db) -> list[BudgetStatus]: + """Every budget, with where it stands now.""" + return [budgets.status(db, b["resource"], b) for b in budgets.all_budgets(db)] + + +@router.get("/budgets/status") +def budget_status(user: AdminReader, db: Db, resource: str = RESOURCE) -> BudgetStatus: + """Where a resource's budget stands, and what its next run will likely cost (also without a budget).""" + with domain_errors(): + budgets.check_ref(resource) + return budgets.status(db, resource) + + +@router.put("/budgets") +def set_budget(body: BudgetSet, user: AdminWriter, db: Db, resource: str = RESOURCE) -> BudgetStatus: + with domain_errors(): + budgets.put(db, resource, by=user.email, **body.model_dump()) + auth.audit(db, user.as_audit(), "budget.set", resource, body.model_dump(exclude_none=True)) + return budgets.status(db, resource) + + +@router.delete("/budgets") +def remove_budget(user: AdminWriter, db: Db, resource: str = RESOURCE) -> Ok: + with domain_errors(): + if not budgets.get(db, budgets.check_ref(resource)): + raise HTTPException(404, "this resource has no budget") + budgets.remove(db, resource) + auth.audit(db, user.as_audit(), "budget.remove", resource) + return Ok() diff --git a/fastapi_backend/app/api/v1/routes/jobs.py b/fastapi_backend/app/api/v1/routes/jobs.py index 703e585fd..0bc62aa6a 100644 --- a/fastapi_backend/app/api/v1/routes/jobs.py +++ b/fastapi_backend/app/api/v1/routes/jobs.py @@ -19,6 +19,7 @@ from app.domain.store import DB from app.schemas.common import Ok from app.schemas.jobs import Job, JobList, JobLog, JobsCreate, JobsQueued, StepQueued, WorkerInfo +from app.schemas.routines import HeldRunDecision router = APIRouter(tags=["jobs"]) log = logging.getLogger("lens") @@ -139,6 +140,17 @@ def cancel_job(jid: int, acl: Acl, user: Writer, db: Db) -> Ok: return Ok() +@router.post("/jobs/{jid}/release") +def release_job(jid: int, body: HeldRunDecision, acl: Acl, user: AdminWriter, db: Db) -> Ok: + """Pick for a job held over a budget (docs/budgets.md): run it now, once, whatever its budgets say, or skip it + (admins, who set the budgets).""" + _job(db, acl, jid, "editor") + with domain_errors(): + jobs.release(db, jid, body.run, user.email) + auth.audit(db, user.as_audit(), "job.release" if body.run else "job.skip", f"job:{jid}") + return Ok() + + @router.post("/jobs/{jid}/retry") def retry_job(jid: int, acl: Acl, user: Writer, db: Db) -> Ok: _job(db, acl, jid, "editor") diff --git a/fastapi_backend/app/api/v1/routes/routines.py b/fastapi_backend/app/api/v1/routes/routines.py index 6a8f3babf..b4f36f894 100644 --- a/fastapi_backend/app/api/v1/routes/routines.py +++ b/fastapi_backend/app/api/v1/routes/routines.py @@ -7,12 +7,13 @@ from fastapi import APIRouter, HTTPException, Query -from app.api.deps import Acl, AdminReader, AdminWriter, CurrentUser, Db, Writer, domain_errors -from app.domain import auth, organize, routines, schedule +from app.api.deps import Acl, AdminReader, AdminWriter, Cfg, CurrentUser, Db, Writer, domain_errors +from app.domain import auth, budgets, organize, routines, schedule from app.schemas.common import Created, Ok from app.schemas.routines import ( GraphChange, GraphChangeAccept, + HeldRunDecision, Routine, RoutineCatalog, RoutineCreate, @@ -83,11 +84,26 @@ def delete_routine(rid: int, user: AdminWriter, db: Db) -> Ok: @router.post("/routines/{rid}/run") -def run_routine(rid: int, body: RoutineRunRequest, user: AdminWriter, db: Db) -> Ok: - """Run it as soon as the scheduler next looks (within half a minute), even when it is off.""" +def run_routine(rid: int, body: RoutineRunRequest, user: AdminWriter, db: Db, cfg: Cfg) -> Ok: + """Run it as soon as the scheduler next looks (within half a minute), even when it is off. A run over one of its + budgets is refused (409, saying where the budget stands) unless over_budget is true (docs/budgets.md).""" with domain_errors(): - routines.request_run(db, rid, user.email, body.propose_only) - auth.audit(db, user.as_audit(), "routine.run", f"routine:{rid}") + r = routines.get(db, rid) + if not body.over_budget: + verdict = budgets.check(db, cfg, routines.refs(db, r)) + if not verdict["go"]: + raise HTTPException(409, verdict["why"] + " Run it anyway with over_budget.") + routines.request_run(db, rid, user.email, body.propose_only, body.over_budget) + auth.audit(db, user.as_audit(), "routine.run", f"routine:{rid}", {"over_budget": True} if body.over_budget else None) + return Ok() + + +@router.post("/routines/runs/{run_id}/decide") +def decide_held_run(run_id: int, body: HeldRunDecision, user: AdminWriter, db: Db) -> Ok: + """Pick for a run held over a budget: run it now, once, whatever its budgets say, or skip it.""" + with domain_errors(): + rid = routines.decide_held(db, run_id, body.run, user.email) + auth.audit(db, user.as_audit(), "routine.release" if body.run else "routine.skip", f"routine:{rid}", {"run": run_id}) return Ok() diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py index cd4a83637..51311659f 100644 --- a/fastapi_backend/app/cli.py +++ b/fastapi_backend/app/cli.py @@ -8,7 +8,7 @@ import shutil import sys -from .domain import analyze, components, decide, graph, ingest, render, store +from .domain import activity, analyze, components, decide, graph, ingest, render, store from .domain import search as searchmod from .domain import speakers as spk @@ -284,6 +284,7 @@ def platform_main(argv, config): raise SystemExit(str(e)) from None return db = store.connect(cfg) + activity.bind(db) # calls made here reach the activity ledger (docs/activity.md) C = settings.Settings(db, cfg).current def password(): diff --git a/fastapi_backend/app/core/runtime.py b/fastapi_backend/app/core/runtime.py index 4587dcc21..c50406217 100644 --- a/fastapi_backend/app/core/runtime.py +++ b/fastapi_backend/app/core/runtime.py @@ -10,7 +10,22 @@ from typing import Any from app.config import settings as env -from app.domain import auth, bridge, components, content_types, jobs, notify, routines, sensors, settings, setup, store, templates, tunnel +from app.domain import ( + activity, + auth, + bridge, + components, + content_types, + jobs, + notify, + routines, + sensors, + settings, + setup, + store, + templates, + tunnel, +) log = logging.getLogger("lens") @@ -22,6 +37,7 @@ def __init__(self, cfg: dict[str, Any] | None = None, db: store.DB | None = None self.base = cfg or store.load_config(env.ARCHIVE_CONFIG) components.activate(self.base) # packages and models fetched into the data folder self.db = db or store.connect(self.base) + activity.bind(self.db) # calls made outside any job, run or request still reach the ledger self.owns_db = db is None self.settings = settings.Settings(self.db, self.base) self.stop = threading.Event() diff --git a/fastapi_backend/app/domain/activity.py b/fastapi_backend/app/domain/activity.py new file mode 100644 index 000000000..a90a85799 --- /dev/null +++ b/fastapi_backend/app/domain/activity.py @@ -0,0 +1,507 @@ +"""Activity: what happened to each resource, and what it cost (docs/activity.md). + +One ledger, the `activity` table, a row for each + +* call in: an API request that changes something (POST, PUT, PATCH, DELETE; reads too when `activity.reads` is on), + counted for the resources in its path, what it made (a POST's reply {"id"}) and who called; +* call out: a model call, embeddings, the decision model, text to speech, a webhook, a web tool; +* run: a job or a routine run ending, with what it cost in all. + +A row names every resource it touched as `table:id` strings (`routine:3`, `routine_run:12`, `job:40`, `recording:7`, +`pipeline:2`, `workflow:4`, `chat:9`, `space:1`, `account:1`), so a resource's history is the rows that name it, next +to its audit log entries. It carries tokens, the estimated cost in USD (from the prices in Settings), how long it took +and how it ended; never prompt, reply or file content. + +Who a call is for comes from the work it runs in: `with activity.scope(db, "job:40", "recording:7"):` around a job, a +routine run or a workflow, and the API request's own resources (from its path) and caller. Scopes nest, so a model +call inside a workflow step of a job queued by a routine counts for all of them. + +On by default (activity.enabled) and cheap: one small row per call. Kept activity.keep_days (365), then dropped by the +routines scheduler. Writing a row never breaks the work it describes. +""" + +from __future__ import annotations + +import contextlib +import contextvars +import datetime as dt +import json +import logging +import re +import time + +from . import store + +log = logging.getLogger("lens") +R = store.R + +KINDS = ("in", "out", "run") +TABLES = { # the API's path segments, and the table a number after one names + "recordings": "recording", + "resources": "recording", + "routines": "routine", + "runs": "routine_run", + "pipelines": "pipeline", + "workflows": "workflow", + "custom-nodes": "custom_node", + "chats": "chat", + "jobs": "job", + "namespaces": "space", + "spaces": "space", + "collections": "collection", + "extensions": "extension", + "sources": "watch_path", + "watches": "watch_path", + "sensors": "sensor", + "templates": "template", + "notes": "note", + "comments": "comment", + "entities": "entity", + "speakers": "speaker", + "users": "account", + "batches": "batch", + "budgets": "budget", +} +REF = re.compile(r"^[a-z_]{1,40}:[A-Za-z0-9_.-]{1,80}$") +CHANGING = {"POST", "PUT", "PATCH", "DELETE"} +SKIP_PATHS = ("/api/v1/auth/", "/api/v1/passkeys/") # sign-in traffic is in the audit log; its bodies are secrets +SWEEP_SECONDS = 3600 +CREATED_BYTES = 2048 # how much of a POST's reply is read for the id of what it made +FIELDS = "record::id(id) AS id, at, kind, action, resources, actor, model, tokens_in, tokens_out, cost_usd, ms, ok, error, detail" + +_scope: contextvars.ContextVar[dict | None] = contextvars.ContextVar("lens_activity", default=None) +_default = {"db": None} +_swept = {"at": 0.0} + + +def bind(db): + """The process's database, for calls made outside any scope (the API, `lens worker`).""" + _default["db"] = db + + +def _section(cfg): + return (cfg or {}).get("activity") or {} + + +def enabled(cfg): + return _section(cfg).get("enabled", True) is not False + + +def ref(table, key): + return f"{table}:{key}" + + +@contextlib.contextmanager +def scope(db, *refs, cfg=None): + """Calls made inside count for these resources too (and for the scopes around this one).""" + outer = _scope.get() + mine = [r for r in refs if r] + token = _scope.set( + { + "db": db if db is not None else (outer or {}).get("db"), + "refs": [*((outer or {}).get("refs") or []), *mine], + "cfg": cfg if cfg is not None else (outer or {}).get("cfg"), + "asgi": (outer or {}).get("asgi"), + } + ) + try: + yield + finally: + _scope.reset(token) + + +def current(): + """The resources calls made here count for, in order, without repeats.""" + s = _scope.get() or {} + out = list(s.get("refs") or []) + asgi = s.get("asgi") + if asgi is not None: + out += request_refs(asgi) + return list(dict.fromkeys(r for r in out if REF.match(r))) + + +def request_refs(asgi): + """An API request's resources: each number (or path parameter) after a known path segment, and who called.""" + out = [] + params = {str(v) for v in (asgi.get("path_params") or {}).values()} + parts = [p for p in (asgi.get("path") or "").split("/") if p] + for a, b in zip(parts, parts[1:]): + table = TABLES.get(a) + if table and (b.isdigit() or b in params) and b not in TABLES: + out.append(ref(table, b)) + p = (asgi.get("state") or {}).get("principal") + if p is not None and getattr(p, "id", None) is not None: + out.append(ref("account", p.id)) + return out + + +def _actor(asgi): + p = ((asgi or {}).get("state") or {}).get("principal") + return getattr(p, "id", None) + + +def record(kind, action, refs=(), cfg=None, db=None, **fields): + """Write one row. `fields`: model, tokens_in, tokens_out, cost_usd, ms, ok, error (a type name), detail (a small + dict of counts and ids). Returns the row's id, or None when it wasn't written.""" + s = _scope.get() or {} + cfg = cfg if cfg is not None else s.get("cfg") + if cfg is not None and not enabled(cfg): + return None + db = db or s.get("db") or _default["db"] + if db is None: + return None + resources = list(dict.fromkeys([*current(), *(r for r in refs if r and REF.match(r))])) + row = store.clean( + { + "at": dt.datetime.now(dt.timezone.utc).isoformat(timespec="milliseconds"), # in order within a second + "kind": kind, + "action": action[:200], + "resources": resources, + "actor": fields.pop("actor", None) or _actor(s.get("asgi")), + **{ + k: v + for k, v in fields.items() + if k in ("model", "tokens_in", "tokens_out", "cost_usd", "unpriced", "ms", "ok", "error", "detail") + }, + } + ) + row.setdefault("ok", True) + try: + got = db.values("CREATE activity CONTENT $d RETURN VALUE record::id(id)", d=row) + return got[0] if got else None + except Exception as e: # noqa: BLE001 - the ledger never breaks the work it describes + log.debug("activity: couldn't write a row: %s", e) + return None + + +class Call: + """A call out, timed: `with activity.call("embeddings", model=m) as c: ...; c.usage(tin, tout)`. An exception that + leaves the block marks the row failed with its type.""" + + def __init__(self, action, cfg=None, model=None, refs=(), detail=None, price=None): + self.action, self.cfg, self.model, self.refs, self.detail, self.price = action, cfg, model, refs, detail, price + self.tin = self.tout = self.cost = None + self.unpriced = False # it costs something, but the figure is missing (see model_cost) + self.t0 = time.monotonic() + self.done = False + + def usage(self, tokens_in=None, tokens_out=None, cost_usd=None): + if isinstance(tokens_in, int): + self.tin = tokens_in + if isinstance(tokens_out, int): + self.tout = tokens_out + if cost_usd is not None: + self.cost = cost_usd + return self + + def end(self, error=None): + if self.done: + return + self.done = True + cost, unpriced = self.cost, self.unpriced + ms = int((time.monotonic() - self.t0) * 1000) + if cost is None and self.price is not None: + cost, unpriced = self.price(self.model, self.tin, self.tout, ms) + record( + "out", + self.action, + self.refs, + self.cfg, + model=self.model, + tokens_in=self.tin, + tokens_out=self.tout, + cost_usd=cost, + unpriced=unpriced or None, + ms=ms, + ok=error is None, + error=type(error).__name__ if error is not None else None, + detail=self.detail, + ) + + def __enter__(self): + return self + + def __exit__(self, et, e, tb): + self.end(e) + return False + + +def call(action, cfg=None, model=None, refs=(), detail=None, price=None): + return Call(action, cfg, model, refs, detail, price) + + +def model_cost(cfg): + """A price function for Call, from each model's price in Settings (telemetry.prices; docs/activity.md#costs): + + * by tokens: {input, output} USD per million tokens (the default unit), as cloud providers charge; + * by time: {unit: "time", per_hour} USD per hour the call took, for a local model on your own machine; + * off: a model with no price (or {unit: "off"}) isn't costed, so it counts tokens and time but no money. + + Returns (USD or None, unpriced): unpriced when the model has a price by tokens but the server didn't say how many + it used, so the figure is missing and totals that include it are estimates.""" + from . import telemetry + + prices = telemetry.prices(cfg) + + def price(model, tin, tout, ms): + p = prices.get(model) + unit = (p or {}).get("unit") or "tokens" + if not isinstance(p, dict) or unit == "off": + return None, False + if unit == "time": + return (round(float(p.get("per_hour") or 0) * (ms or 0) / 3_600_000, 8) if ms is not None else None), ms is None + if tin is None and tout is None: + return None, True + return telemetry.cost(model, tin or 0, tout or 0, prices), False + + return price + + +# ---------- the API ---------- +class Middleware: + """A row for each API request that changes something (or every request, with activity.reads on), naming the + resources in its path and who called; calls the request makes count for those resources too.""" + + def __init__(self, app): + self.app = app + + async def __call__(self, scope_, receive, send): + if scope_["type"] != "http" or not scope_.get("path", "").startswith("/api/"): + await self.app(scope_, receive, send) + return + app = scope_.get("app") + db = getattr(getattr(app, "state", None), "db", None) + token = _scope.set({"db": db, "refs": [], "cfg": None, "asgi": scope_}) + status, head = [None], [b""] + + async def sending(message): + if message["type"] == "http.response.start": + status[0] = message["status"] + elif message["type"] == "http.response.body" and len(head[0]) < CREATED_BYTES and scope_.get("method") == "POST": + head[0] += message.get("body", b"")[:CREATED_BYTES] + await send(message) + + t0, error = time.monotonic(), None + try: + await self.app(scope_, receive, sending) + except Exception as e: + error = e + raise + finally: + try: + self._log(scope_, app, db, status[0], error, t0, head[0]) + finally: + _scope.reset(token) + + def _log(self, scope_, app, db, status, error, t0, head=b""): + method = scope_.get("method") or "GET" + path = scope_.get("path") or "" + if db is None or path.startswith(SKIP_PATHS): + return + try: + cfg = app.state.settings.current() + except Exception: # noqa: BLE001 + return + if not enabled(cfg) or (method not in CHANGING and not _section(cfg).get("reads")): + return + from . import telemetry + + code = status or 500 + record( + "in", + f"{method} {telemetry._route_template(scope_)}", + [created(path, head)] if method == "POST" and 200 <= code < 300 else (), + cfg=cfg, + db=db, + ms=int((time.monotonic() - t0) * 1000), + ok=error is None and code < 400, + error=type(error).__name__ if error is not None else (str(code) if code >= 400 else None), + detail={"status": code}, + ) + + +def created(path, head): + """What a POST to a collection made, from its reply ({"id": n}): `routine:7` for POST /api/v1/routines.""" + table = TABLES.get(path.rstrip("/").rsplit("/", 1)[-1]) + if not table or not head: + return None + try: + got = json.loads(head) + except ValueError: + return None + key = got.get("id") if isinstance(got, dict) else None + return ref(table, key) if isinstance(key, (int, str)) and not isinstance(key, bool) else None + + +# ---------- reading it ---------- +def _since(period, now=None): + """The start of the period now is in (UTC): day, week (from Monday), month; None for all time.""" + now = now or dt.datetime.now(dt.timezone.utc) + if period == "day": + start = now.replace(hour=0, minute=0, second=0, microsecond=0) + elif period == "week": + start = (now - dt.timedelta(days=now.weekday())).replace(hour=0, minute=0, second=0, microsecond=0) + elif period == "month": + start = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + else: + return None + return start.isoformat(timespec="seconds") + + +def _when(at): + """A row's time, for ordering ledger rows (milliseconds) and audit entries (seconds) together.""" + try: + t = dt.datetime.fromisoformat(str(at)) + return t if t.tzinfo else t.replace(tzinfo=dt.timezone.utc) + except ValueError: + return dt.datetime.min.replace(tzinfo=dt.timezone.utc) + + +def history(db, resource, limit=100, before=None, kind=None): + """A resource's activity, newest first: its ledger rows and its audit log entries, merged.""" + where, p = ["resources CONTAINS $res"], {"res": resource, "n": limit} + if before: + where.append("at < $before") + p["before"] = before + if kind: + where.append("kind = $kind") + p["kind"] = kind + rows = db.rows(f"SELECT {FIELDS} FROM activity WHERE {' AND '.join(where)} ORDER BY at DESC LIMIT $n", **p) + if kind in (None, "change"): + audit = db.rows( + "SELECT at, account, email, action, target, detail FROM audit_log WHERE target = $res" + + (" AND at < $before" if before else "") + + " ORDER BY at DESC LIMIT $n", + **p, + ) + rows += [ + { + "id": None, + "at": a["at"], + "kind": "change", + "action": a["action"], + "resources": [resource], + "actor": a.get("account"), + "email": a.get("email"), + "ok": True, + "detail": a.get("detail") if isinstance(a.get("detail"), dict) else ({"items": a["detail"]} if a.get("detail") else None), + } + for a in audit + ] + rows.sort(key=lambda r: _when(r.get("at")), reverse=True) + rows = rows[:limit] + return rows + + +def totals(db, resource=None, since=None, until=None): + """What a resource (or everything) cost: calls, tokens, USD and time, by kind of call.""" + where, p = [], {} + if resource: + where.append("resources CONTAINS $res") + p["res"] = resource + if since: + where.append("at >= $since") + p["since"] = since + if until: + where.append("at < $until") + p["until"] = until + rows = db.rows( + "SELECT kind, count() AS calls, math::sum(tokens_in ?? 0) AS tokens_in, math::sum(tokens_out ?? 0) AS tokens_out, " + f"math::sum(cost_usd ?? 0) AS cost_usd, math::sum(ms ?? 0) AS ms, count(ok = false) AS failed, count({_unpriced_sql()}) AS unpriced " + "FROM activity" + (f" WHERE {' AND '.join(where)}" if where else "") + " GROUP BY kind", + **p, + ) + out = {"calls": 0, "tokens_in": 0, "tokens_out": 0, "cost_usd": 0.0, "ms": 0, "failed": 0, "unpriced": 0, "by_kind": {}} + for r in rows: + k = r.get("kind") or "?" + # a run row repeats what its calls cost (its cost is their sum): counted on its own, not twice + out["by_kind"][k] = {x: r.get(x) or 0 for x in ("calls", "tokens_in", "tokens_out", "cost_usd", "ms", "failed")} + out["calls"] += r.get("calls") or 0 + out["failed"] += r.get("failed") or 0 + if k != "run": + for x in ("tokens_in", "tokens_out", "cost_usd", "ms", "unpriced"): + out[x] += r.get(x) or 0 + out["cost_usd"] = round(out["cost_usd"], 6) + out["estimate"] = out["unpriced"] > 0 + return out + + +def spent(db, resource, since=None): + """(USD, tokens) a resource's calls cost since a time (not counting run rows, which repeat them).""" + p = {"res": resource} + q = "SELECT math::sum(cost_usd ?? 0) AS usd, math::sum((tokens_in ?? 0) + (tokens_out ?? 0)) AS tokens FROM activity WHERE resources CONTAINS $res AND kind != 'run'" + if since: + q += " AND at >= $since" + p["since"] = since + row = db.one(q + " GROUP ALL", **p) or {} + return round(float(row.get("usd") or 0), 6), int(row.get("tokens") or 0) + + +def top(db, prefix=None, since=None, limit=20): + """The resources that cost most since a time, of one table (`routine`) or any: [{resource, cost_usd, tokens, calls}].""" + p = {"n": limit, "p": f"{prefix}:" if prefix else ""} + inner = "kind != 'run' AND array::len(resources ?? []) > 0" + if since: + inner += " AND at >= $since" + p["since"] = since + rows = db.rows( + "SELECT resources AS resource, math::sum(cost_usd ?? 0) AS cost_usd, math::sum((tokens_in ?? 0) + (tokens_out ?? 0)) AS tokens, " + f"count() AS calls FROM (SELECT resources, cost_usd, tokens_in, tokens_out FROM activity WHERE {inner} SPLIT resources) " + "WHERE string::starts_with(resources, $p) GROUP BY resource ORDER BY cost_usd DESC, tokens DESC LIMIT $n", + **p, + ) + for r in rows: + r["cost_usd"] = round(float(r.get("cost_usd") or 0), 6) + return rows + + +def _unpriced_sql(): + """A row that costs something whose figure is missing: it makes the totals it is in estimates.""" + return "(unpriced = true)" + + +def costs(db, resources, since=None): + """What each of many resources' calls cost, for lists: {resource: {cost_usd, tokens, calls, unpriced, estimate}}. + `unpriced` counts calls that cost something but have no figure (a priced model whose reply had no token counts), + so `estimate` is true: the cost shown is a floor, not to the cent. A model without a price (local models, by + default) isn't costed at all, and doesn't make anything an estimate.""" + want = [r for r in dict.fromkeys(resources) if REF.match(r)][:500] + out = {r: {"cost_usd": 0.0, "tokens": 0, "calls": 0, "unpriced": 0, "estimate": False} for r in want} + if not want: + return out + p = {"rs": want} + inner = "kind != 'run' AND resources CONTAINSANY $rs" + if since: + inner += " AND at >= $since" + p["since"] = since + rows = db.rows( + "SELECT resources AS res, math::sum(cost_usd ?? 0) AS cost_usd, math::sum((tokens_in ?? 0) + (tokens_out ?? 0)) AS tokens, " + "count() AS calls, count(unpriced = true) AS unpriced FROM " + f"(SELECT resources, cost_usd, tokens_in, tokens_out, unpriced FROM activity WHERE {inner} SPLIT resources) " + "WHERE resources IN $rs GROUP BY res", + **p, + ) + for r in rows: + o = out[r["res"]] + o.update(cost_usd=round(float(r.get("cost_usd") or 0), 6), tokens=int(r.get("tokens") or 0), calls=r.get("calls") or 0) + o["unpriced"] = r.get("unpriced") or 0 + o["estimate"] = o["unpriced"] > 0 + return out + + +def run_total(db, resource): + """What one run's calls cost, for its run row: (USD or None, tokens or None, whether the USD is an estimate).""" + c = costs(db, [resource])[resource] + return (c["cost_usd"] or None), (c["tokens"] or None), c["estimate"] + + +def tidy(db, cfg, now=None): + """Drop rows older than activity.keep_days, at most once an hour per process; the cut-off, or 0 when it didn't look.""" + t = time.monotonic() + if now is None and t - _swept["at"] < SWEEP_SECONDS and _swept["at"]: + return 0 + _swept["at"] = t + days = int(_section(cfg).get("keep_days") or 365) + cut = ((now or dt.datetime.now(dt.timezone.utc)) - dt.timedelta(days=days)).isoformat(timespec="seconds") + db.q("DELETE activity WHERE at < $cut", cut=cut) + return cut diff --git a/fastapi_backend/app/domain/budgets.py b/fastapi_backend/app/domain/budgets.py new file mode 100644 index 000000000..2cbc362aa --- /dev/null +++ b/fastapi_backend/app/domain/budgets.py @@ -0,0 +1,269 @@ +"""Budgets (docs/budgets.md): a cap on what a routine, pipeline, workflow or namespace may cost, checked before it runs. + +A budget is off unless someone sets one. It caps cost in USD, tokens, or both, per run, day, week (from Monday) or +month (UTC), with what was spent read from the activity ledger (activity.py). Before a routine run or a job, Lens +estimates what the run will cost from that resource's past runs and looks at what is left: + +* under budget: it runs; +* over (what was spent plus the estimate passes the cap): what `on_over` says. `ask` (the default) holds the run + until someone picks: run it once, or skip it. `skip` skips it and says so. `assistant` lets the decision model weigh + it (decide.py: where the budget stands, how sure the estimate is, how far over it would go); it runs only when the + model is sure, and asks otherwise. + +Doing nothing changes nothing: a held run stays held. A run someone let through runs. `warn_at` (0.8) marks a budget +`near` once that share is spent, for the periodic check (sweep) to warn about. +""" + +from __future__ import annotations + +import datetime as dt +import logging +import re +import time + +from . import activity, store + +log = logging.getLogger("lens") +R = store.R +PERIODS = ("run", "day", "week", "month") +ON_OVER = ("ask", "skip", "assistant") +TABLES = ("routine", "pipeline", "workflow", "space") +REF = re.compile(r"^(routine|pipeline|workflow|space):(\d+)$") +FIELDS = "resource, usd, tokens, period, on_over, warn_at, note, by, updated_at, alert" +LOOKBACK_DAYS = 30 # the past runs an estimate is drawn from +SWEEP_SECONDS = 300 # how often the scheduler looks at every budget +_swept = {"at": 0.0} + + +def _key(resource): + return R("budget", resource.replace(":", "_")) + + +def check_ref(resource): + if not isinstance(resource, str) or not REF.match(resource): + raise ValueError("a budget is for routine:, pipeline:, workflow: or space:") + return resource + + +def _num(v, what, whole=False): + if v is None: + return None + if isinstance(v, bool) or not isinstance(v, (int, float)) or v < 0 or v > 1e12 or (whole and int(v) != v): + raise ValueError(f"{what} is a {'whole ' if whole else ''}number of at least 0") + return int(v) if whole else float(v) + + +def put(db, resource, usd=None, tokens=None, period="month", on_over="ask", warn_at=0.8, note=None, by=None): + """Set (or replace) a resource's budget. At least one of usd and tokens.""" + check_ref(resource) + usd, tokens = _num(usd, "usd"), _num(tokens, "tokens", whole=True) + if usd is None and tokens is None: + raise ValueError("give the budget a cap: usd, tokens or both") + if period not in PERIODS: + raise ValueError(f"period is one of {', '.join(PERIODS)}") + if on_over not in ON_OVER: + raise ValueError(f"on_over is one of {', '.join(ON_OVER)}") + warn_at = _num(warn_at, "warn_at") + if warn_at is None or not 0.1 <= warn_at <= 1: + raise ValueError("warn_at is a share of the budget from 0.1 to 1") + row = store.clean( + { + "resource": resource, + "usd": usd, + "tokens": tokens, + "period": period, + "on_over": on_over, + "warn_at": warn_at, + "note": (note or "").strip()[:300] or None, + "by": by, + "updated_at": store.now(), + } + ) + db.q("UPSERT $r CONTENT $d", r=_key(resource), d=row) + return get(db, resource) + + +def get(db, resource): + return db.one(f"SELECT {FIELDS} FROM $r", r=_key(resource)) + + +def remove(db, resource): + check_ref(resource) + db.q("DELETE $r", r=_key(resource)) + + +def all_budgets(db): + return db.rows(f"SELECT {FIELDS} FROM budget ORDER BY resource") + + +# ---------- where it stands ---------- +def _runs(db, resource, since): + """How many runs this resource had since a time: a routine's runs, or the jobs of a pipeline, workflow or namespace.""" + m = REF.match(resource) + if m and m.group(1) == "routine": + return len(db.values("SELECT VALUE id FROM routine_run WHERE routine = $r AND started_at >= $s", r=int(m.group(2)), s=since)) + return len( + db.values( + "SELECT VALUE id FROM activity WHERE kind = 'run' AND resources CONTAINS $res AND at >= $s AND string::starts_with(action, 'job.')", + res=resource, + s=since, + ) + ) + + +def estimate(db, resource, now=None): + """What one more run will likely cost: the average of the last LOOKBACK_DAYS. Always an estimate. + {usd, tokens, runs (it's drawn from), basis}.""" + now = now or dt.datetime.now(dt.timezone.utc) + since = (now - dt.timedelta(days=LOOKBACK_DAYS)).isoformat(timespec="seconds") + runs = _runs(db, resource, since) + if not runs: + return {"usd": None, "tokens": None, "runs": 0, "basis": "no runs in the last 30 days to go on"} + usd, tokens = activity.spent(db, resource, since) + return { + "usd": round(usd / runs, 6), + "tokens": int(tokens / runs), + "runs": runs, + "basis": f"the average of {runs} run{'s' if runs != 1 else ''} in the last {LOOKBACK_DAYS} days", + } + + +def status(db, resource, budget=None, now=None): + """Where a resource's budget stands: {budget, since, spent: {usd, tokens, estimate}, next: estimate, left, share, + state: none | ok | near | over}. `share` is the largest of the spent shares of its caps.""" + budget = budget if budget is not None else get(db, resource) + nxt = estimate(db, resource, now) + if not budget: + return {"resource": resource, "budget": None, "state": "none", "next": nxt} + period = budget.get("period") or "month" + since = None if period == "run" else activity._since(period, now) + if period == "run": + spent = {"usd": 0.0, "tokens": 0, "estimate": False} + else: + c = activity.costs(db, [resource], since)[resource] + spent = {"usd": c["cost_usd"], "tokens": c["tokens"], "estimate": c["estimate"]} + left, shares, after = {}, [], [] + for cap, k in (("usd", "usd"), ("tokens", "tokens")): + if budget.get(cap) is None: + continue + lim = budget[cap] + left[k] = max(0, lim - spent[k]) if k == "tokens" else round(max(0.0, lim - spent[k]), 6) + shares.append(spent[k] / lim if lim else (1.0 if spent[k] else 0.0)) + if nxt.get(k) is not None: + after.append((spent[k] + nxt[k]) / lim if lim else (1.0 if spent[k] + nxt[k] else 0.0)) + share = max(shares) if shares else 0.0 + share_after = max(after) if after else None + state = "over" if share >= 1 else "near" if share >= (budget.get("warn_at") or 0.8) else "ok" + return { + "resource": resource, + "budget": budget, + "since": since, + "spent": spent, + "next": nxt, + "left": left, + "share": round(share, 4), + "share_after_next": round(share_after, 4) if share_after is not None else None, + "state": state, + } + + +# ---------- before a run ---------- +def check(db, cfg, resources, now=None): + """Whether a run that counts for these resources may go ahead. Returns {go, action, why, resource, status}: + go True (run), or action "ask" (hold it for a person), "skip". The first budget that stops it decides.""" + for res in dict.fromkeys(r for r in resources if r and REF.match(r)): + b = get(db, res) + if not b: + continue + st = status(db, res, b, now) + over_now = st["state"] == "over" + over_after = (st["share_after_next"] or 0) > 1 + if not (over_now or over_after): + continue + why = _why(res, st, over_now) + how = b.get("on_over") or "ask" + if how == "assistant": + how = _weigh(cfg, res, st, why) + if how == "run": + return {"go": True, "action": "run", "why": why + " The assistant let it run.", "resource": res, "status": st} + return {"go": False, "action": how, "why": why, "resource": res, "status": st} + return {"go": True, "action": "run", "why": None, "resource": None, "status": None} + + +def _money(v): + return f"${v:,.4f}".rstrip("0").rstrip(".") if v < 1 else f"${v:,.2f}" + + +def _why(res, st, over_now): + b, sp, nxt = st["budget"], st["spent"], st["next"] + per = "per run" if b["period"] == "run" else f"this {b['period']}" + parts = [] + if b.get("usd") is not None: + parts.append(f"{'≈' if sp['estimate'] else ''}{_money(sp['usd'])} of {_money(b['usd'])} {per}") + if b.get("tokens") is not None: + parts.append(f"{sp['tokens']:,} of {b['tokens']:,} tokens {per}") + spent = " and ".join(parts) + if over_now: + return f"{res} has spent {spent}: its budget is used up." + est = [] + if nxt.get("usd") is not None and b.get("usd") is not None: + est.append(f"≈{_money(nxt['usd'])}") + if nxt.get("tokens") is not None and b.get("tokens") is not None: + est.append(f"≈{nxt['tokens']:,} tokens") + return f"{res} has spent {spent}; the next run ({' and '.join(est) or 'cost unknown'}, {nxt['basis']}) would go over." + + +def _weigh(cfg, res, st, why): + """The decision model's call on a run that would go over: "run" only when it's sure; else "ask".""" + from . import decide + + try: + d = decide.choose( + cfg, + "A scheduled run would go over its budget. Should it run anyway, or wait for the person to decide? Run " + "only when going over is small and the work is routine; anything large, or a budget already used up, waits.", + {"run": "run it now, over budget", "ask": "hold it and ask the person"}, + { + "why": why, + "budget": st["budget"], + "spent": st["spent"], + "next_run_estimate": st["next"], + "share_after_next": st["share_after_next"], + }, + ) + except decide.Undecided as e: + log.info("budgets: no decision engine for %s (%s); asking", res, e) + return "ask" + return d["choice"] if decide.sure(cfg, d) else "ask" + + +# ---------- the periodic check ---------- +def sweep(db, cfg, now=None, every=None): + """Look at every budget (at most once `every` seconds, when given): one that has crossed warn_at or its cap since it was last looked at gets an alert (on the + budget, and a ledger row `budget.near` / `budget.over` for it), once per period. Returns the alerts made.""" + if every is not None: + if time.monotonic() - _swept["at"] < every and _swept["at"]: + return [] + _swept["at"] = time.monotonic() + made = [] + for b in all_budgets(db): + res = b["resource"] + if b.get("period") == "run": + continue + try: + st = status(db, res, b, now) + except Exception as e: # noqa: BLE001 - one bad budget doesn't stop the rest + log.info("budgets: couldn't check %s: %s", res, e) + continue + if st["state"] not in ("near", "over"): + continue + last = b.get("alert") or {} + if last.get("since") == st["since"] and (last.get("state") == st["state"] or last.get("state") == "over"): + continue + alert = {"state": st["state"], "since": st["since"], "at": store.now(), "share": st["share"]} + db.q("UPDATE $r SET alert = $a", r=_key(res), a=alert) + activity.record( + "run", f"budget.{st['state']}", [res], cfg, db, detail={"share": st["share"], "estimate": st["spent"]["estimate"] or None} + ) + made.append({"resource": res, **alert}) + return made diff --git a/fastapi_backend/app/domain/decide.py b/fastapi_backend/app/domain/decide.py index 205e2396d..e1c6a9621 100644 --- a/fastapi_backend/app/domain/decide.py +++ b/fastapi_backend/app/domain/decide.py @@ -20,7 +20,7 @@ import urllib.error import urllib.request -from . import llm, machine +from . import activity, llm, machine log = logging.getLogger(__name__) # Jev reads up to about 32k tokens; a state is cut well below that @@ -74,7 +74,11 @@ def _system_one(base_url, body, api_key=None, timeout=10): def _jev(cfg, questions, state): d = cfg.get("decisions") or {} body = {"state": _state(state), "model": d.get("model") or "jev-latest", "questions": questions} - return _system_one(d.get("base_url") or "https://api.typesafe.ai/v1", body, d.get("api_key"), d.get("timeout") or 10) + with activity.call("decision", cfg, body["model"], detail={"questions": len(questions)}) as ledger: + answers = _system_one(d.get("base_url") or "https://api.typesafe.ai/v1", body, d.get("api_key"), d.get("timeout") or 10) + if d.get("price_per_call") is not None: + ledger.usage(cost_usd=float(d["price_per_call"])) + return answers # ---------- Laya on MLX ---------- diff --git a/fastapi_backend/app/domain/extensions.py b/fastapi_backend/app/domain/extensions.py index 25466b2ac..322cb2d64 100644 --- a/fastapi_backend/app/domain/extensions.py +++ b/fastapi_backend/app/domain/extensions.py @@ -32,7 +32,7 @@ import yaml -from . import auth, store +from . import activity, auth, store R = store.R KINDS = ("tool", "skill", "hook", "plugin") @@ -748,7 +748,11 @@ def http_call(cfg, run, args): data = json.dumps(fill_json(body, args)).encode("utf-8") headers.setdefault("Content-Type", "application/json") req = urllib.request.Request(url, data=data, method=run.get("method") or "GET", headers=headers) - with netguard.Guard(forward=True, networks=webcapture.networks(cfg), max_bytes=MAX_REPLY + 65536) as guard: + host = urllib.parse.urlsplit(url).hostname + with ( + activity.call("tool.http", cfg, detail={"host": host, "method": req.get_method()}), + netguard.Guard(forward=True, networks=webcapture.networks(cfg), max_bytes=MAX_REPLY + 65536) as guard, + ): try: with feeds._opener(guard).open(req, timeout=30) as r: status, ctype, raw = r.status, r.headers.get("Content-Type", ""), r.read(MAX_REPLY) diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index af33d0fce..aa798e559 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -14,15 +14,15 @@ import threading import time -from . import analyze, ingest, keyring, pipelines, render, speakers as spk, store, telemetry +from . import activity, analyze, ingest, keyring, pipelines, render, speakers as spk, store, telemetry R = store.R PIPELINE = ["transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe", "analyze", "embed", "summarize", "report"] AFTER_IMPORT = ["analyze", "embed", "summarize", "report"] -ACTIVE = ["queued", "running"] +ACTIVE = ["queued", "running", "held"] # held: over budget, waiting for someone to pick (budgets.py) FIELDS = ( "record::id(id) AS id, recording, space, batch, pipeline, steps, step_index, next_step, status, worker, error, attempts, " - "created_by, created_at, started_at, finished_at, updated_at, cancel_requested, log_total" + "created_by, created_at, started_at, finished_at, updated_at, cancel_requested, log_total, cost_usd, tokens, cost_estimate, hold" ) MEDIA_STEPS = {"transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe"} # they take longer the longer the recording FILED = ("audio", "document", "image") # sources with a file of their own for the steps to work on @@ -116,7 +116,8 @@ def _export(db, cfg, rid, say, spec=None): def _workflow(db, cfg, rid, say, spec=None): from . import workflows - done = workflows.run(db, cfg, rid, int(spec["workflow"]), spec.get("version"), say) + with activity.scope(db, f"workflow:{int(spec['workflow'])}"): + done = workflows.run(db, cfg, rid, int(spec["workflow"]), spec.get("version"), say) ran = sum(1 for v in done.values() if v == "done") - 1 # not counting the input say(f"workflow ran {ran} of {len(done) - 1} nodes") @@ -421,7 +422,97 @@ def eta(job, est, now=None): return round(left + sum(est[i + 1 :]), 1) +FINAL = ("succeeded", "failed", "cancelled") + + +def job_refs(job): + """The resources a job's calls count for: it, its recording, namespace and pipeline, and the routine that queued it.""" + pipe = job.get("pipeline") if isinstance(job.get("pipeline"), dict) else {} + by = job.get("created_by") + return [ + f"job:{job['id']}", + f"recording:{job['recording']}", + f"space:{job['space']}" if job.get("space") is not None else None, + f"pipeline:{pipe['id']}" if pipe.get("id") is not None else None, + by if isinstance(by, str) and by.startswith("routine:") else None, + ] + + def run_job(db, cfg_fn, job, worker, can, log=None): + """Run a job's steps from where it is, with its calls counted for it (activity.py); when it ends, a run row with + what it cost in all, kept on the job too (cost_usd, tokens).""" + refs = job_refs(job) + if not job.get("budget_ok") and (held := _budget(db, cfg_fn, job, refs)): + return held + with activity.scope(db, *refs): + outcome = _run_job(db, cfg_fn, job, worker, can, log) + if outcome in FINAL: + usd, tokens, rough = activity.run_total(db, refs[0]) + try: + db.q("UPDATE $j SET cost_usd = $u, tokens = $n, cost_estimate = $e", j=R("job", job["id"]), u=usd, n=tokens, e=rough or None) + except Exception: # noqa: BLE001 - the ledger never breaks the work + pass + activity.record( + "run", + f"job.{outcome}", + refs, + cfg_fn(), + db, + cost_usd=usd, + tokens_in=tokens, + ok=outcome != "failed", + detail={"estimate": rough} if rough else None, + ) + return outcome + + +def _budget(db, cfg_fn, job, refs): + """Before a job's first step: its budgets (budgets.py). Over one, the job is held for someone to pick ("held"), or + skipped ("cancelled", saying why); else None, and it isn't checked again for this job.""" + from . import budgets + + jr, t = R("job", job["id"]), store.now() + try: + verdict = budgets.check(db, cfg_fn(), refs) + except Exception as e: # noqa: BLE001 - a budget that can't be read doesn't stop the work + verdict = {"go": True} + db.q("UPDATE $j SET budget_note = $n", j=jr, n=f"budget check failed: {type(e).__name__}: {e}"[:300]) + if verdict["go"]: + db.q("UPDATE $j SET budget_ok = true", j=jr) + return None + hold = {"why": verdict["why"], "resource": verdict["resource"], "action": verdict["action"], "at": t} + if verdict["action"] == "skip": + db.q( + "UPDATE $j SET status = 'cancelled', worker = NONE, hold = $h, error = $e, finished_at = $t, updated_at = $t", + j=jr, + h=hold, + e="skipped, over budget: " + verdict["why"], + t=t, + ) + activity.record("run", "job.skipped", refs, cfg_fn(), db, ok=False, error="OverBudget", detail={"budget": verdict["resource"]}) + return "cancelled" + db.q("UPDATE $j SET status = 'held', worker = NONE, hold = $h, updated_at = $t", j=jr, h=hold, t=t) + activity.record("run", "job.held", refs, cfg_fn(), db, detail={"budget": verdict["resource"]}) + return "held" + + +def release(db, jid, run=True, by=None): + """Someone's pick for a job held over budget: run it (once, whatever its budgets say) or skip it.""" + j = get(db, jid) + if not j: + raise KeyError(jid) + if j["status"] != "held": + raise ValueError(f"the job isn't waiting on a budget (it's {j['status']})") + t = store.now() + picked = {"run": run, "by": by, "at": t} + if run: + sets = "status = 'queued', budget_ok = true" + else: + sets = "status = 'cancelled', error = 'skipped, over budget', finished_at = $t" + db.q(f"UPDATE $j SET {sets}, hold.decided = $d, updated_at = $t WHERE status = 'held'", j=R("job", jid), d=picked, t=t) + + +def _run_job(db, cfg_fn, job, worker, can, log=None): jid, rid, jr, wr = job["id"], job["recording"], R("job", job["id"]), R("worker", worker) steps, i = job["steps"], job.get("step_index") or 0 out = RunLog(db, jid, job.get("space"), job.get("log"), job.get("log_total")) @@ -573,7 +664,7 @@ def cancel(db, jid): if not j: raise KeyError(jid) t = store.now() - if j["status"] == "queued": + if j["status"] in ("queued", "held"): db.q("UPDATE $j SET status = 'cancelled', finished_at = $t, updated_at = $t", j=R("job", jid), t=t) elif j["status"] == "running": db.q("UPDATE $j SET cancel_requested = true, updated_at = $t", j=R("job", jid), t=t) diff --git a/fastapi_backend/app/domain/notify.py b/fastapi_backend/app/domain/notify.py index b6a78ad2b..c032d4bec 100644 --- a/fastapi_backend/app/domain/notify.py +++ b/fastapi_backend/app/domain/notify.py @@ -39,7 +39,7 @@ import time import urllib.parse -from . import netguard, settings, store +from . import activity, netguard, settings, store R = store.R EVENTS = { @@ -382,6 +382,16 @@ def send(db, cfg, t, event, msg_id): error = f"couldn't reach it: {type(e).__name__}: {e}"[:300] last = store.clean({"at": store.now(), "ok": error is None, "code": code, "error": error, "event": event["type"]}) db.q("UPDATE $r SET last = $l", r=R("notify_target", int(t["id"])), l=last) + activity.record( + "out", + f"notify.{t.get('kind') or 'webhook'}", + [f"notify_target:{t['id']}", f"space:{t['space']}" if t.get("space") is not None else None], + cfg, + db, + ok=error is None, + error=None if error is None else (f"HTTP {code}" if code else "unreachable"), + detail=store.clean({"event": event["type"], "status": code}), + ) return {"ok": error is None, "code": code, "error": error} diff --git a/fastapi_backend/app/domain/routines.py b/fastapi_backend/app/domain/routines.py index abe149d91..93bf0e1e3 100644 --- a/fastapi_backend/app/domain/routines.py +++ b/fastapi_backend/app/domain/routines.py @@ -23,6 +23,8 @@ import threading from . import ( + activity, + budgets, fedora, jobs, notebook, @@ -240,7 +242,8 @@ def _with_changes(db, rows): def runs(db, rid, limit=20): rows = db.rows( - "SELECT record::id(id) AS id, routine, trigger, by, status, started_at, finished_at, results, error FROM routine_run " + "SELECT record::id(id) AS id, routine, trigger, by, status, started_at, finished_at, results, error, cost_usd, tokens, " + "cost_estimate, hold FROM routine_run " "WHERE routine = $r ORDER BY id DESC LIMIT $n", r=int(rid), n=int(limit), @@ -255,15 +258,16 @@ def get_run(db, run_id): return _with_changes(db, [row])[0] -def request_run(db, rid, by=None, propose_only=False): - """Ask for a run as soon as the scheduler next looks (even when the routine is off).""" +def request_run(db, rid, by=None, propose_only=False, over_budget=False): + """Ask for a run as soon as the scheduler next looks (even when the routine is off). over_budget: someone chose to + run it whatever its budgets say (budgets.py).""" r = get(db, rid) if r["running"]: raise ValueError("this routine is running now") db.q( "UPDATE $r SET run_now = $d", r=R("routine", int(rid)), - d=store.clean({"by": by, "at": store.now(), "propose_only": propose_only or None}), + d=store.clean({"by": by, "at": store.now(), "propose_only": propose_only or None, "over_budget": over_budget or None}), ) @@ -349,7 +353,8 @@ def _action(db, cfg, routine, run_id, a, seen, propose_only, say): w = workflows.get(db, int(a["workflow"]), a.get("version")) if w["scope"] == "graph": origin = {"routine": routine["id"], "run": run_id} - done, stats = organize.run(db, cfg, w["id"], spaces, w["version"], say, origin, propose_only or a.get("propose_only", False)) + with activity.scope(db, f"workflow:{w['id']}"): + done, stats = organize.run(db, cfg, w["id"], spaces, w["version"], say, origin, propose_only or a.get("propose_only", False)) return {"workflow": w["name"], "version": w["version"], "nodes": sum(v == "done" for v in done.values()), **stats} rids = _recordings(db, spaces, a.get("recordings", "new"), seen, limit, cfg) step = {"type": "workflow", "workflow": w["id"], "version": w["version"]} @@ -377,10 +382,32 @@ def run(db, cfg, rid, trigger="manual", by=None, propose_only=False, log=None): def _run(db, cfg, rid, trigger, by, propose_only, log): + """A run, with its calls counted for the routine and the run (activity.py); its run row says what it cost.""" + run_id = db.next_id("routine_run") + refs = [f"routine:{int(rid)}", f"routine_run:{run_id}"] + with activity.scope(db, *refs, cfg=cfg): + status = _run_actions(db, cfg, rid, run_id, trigger, by, propose_only, log) + usd, tokens, rough = activity.run_total(db, refs[1]) + db.q("UPDATE $r SET cost_usd = $u, tokens = $n, cost_estimate = $e", r=R("routine_run", run_id), u=usd, n=tokens, e=rough or None) + activity.record( + "run", + f"routine.{status}", + refs, + cfg, + db, + cost_usd=usd, + tokens_in=tokens, + ok=status != "error", + detail=store.clean({"trigger": trigger, "estimate": rough or None}), + ) + return run_id + + +def _run_actions(db, cfg, rid, run_id, trigger, by, propose_only, log): routine = get(db, rid) since = routine.get("seen_recording") or 0 seen = {"since": since, "marks": [], "failed": False} - run_id, lines = db.next_id("routine_run"), [] + lines = [] started = store.now() def say(msg): @@ -430,7 +457,7 @@ def say(msg): s=status, i=run_id, ) - return run_id + return status def _claim(db, r, now): @@ -483,11 +510,102 @@ def _idle(db, cfg, r): return not semantic.configured(cfg) or bool(semantic.failing(db, cfg)) or not semantic.unindexed(db, cfg, _spaces(db, r), 1) +def refs(db, r): + """What a routine's runs count for, for its budgets: it, the pipelines and workflows its actions name, and its + namespaces.""" + out = [f"routine:{r['id']}"] + for a in r.get("actions") or []: + if a.get("type") == "pipeline" and isinstance(a.get("pipeline"), int): + out.append(f"pipeline:{a['pipeline']}") + if a.get("type") == "workflow" and a.get("workflow") is not None: + out.append(f"workflow:{int(a['workflow'])}") + out += [f"space:{s}" for s in _spaces(db, r)] + return out + + +def _held(db, cfg, r, trigger, by, now): + """Whether a routine that's due is over a budget (budgets.py): if so its run is held for someone to pick, or + skipped, as a run row that says why, and the routine is free again (its next time stays as claimed).""" + try: + verdict = budgets.check(db, cfg, refs(db, r), now) + except Exception: # noqa: BLE001 - a budget that can't be read doesn't stop the work + return False + if verdict["go"]: + return False + held = verdict["action"] != "skip" + status = "held" if held else "skipped" + waiting = db.values("SELECT VALUE record::id(id) FROM routine_run WHERE routine = $r AND status = 'held'", r=int(r["id"])) + if held and waiting: # one held run waits for a pick; later times it was due add to it rather than pile up + db.q("UPDATE $r SET hold.missed += 1, hold.why = $w", r=R("routine_run", waiting[0]), w=verdict["why"]) + db.q("UPDATE $r SET running_since = NONE, heartbeat_at = NONE", r=R("routine", r["id"])) + return True + run_id = db.next_id("routine_run") + db.q( + "CREATE $r CONTENT $d", + r=R("routine_run", run_id), + d=store.clean( + { + "routine": int(r["id"]), + "trigger": trigger, + "by": by, + "status": status, + "started_at": _iso(now), + "finished_at": None if held else _iso(now), + "results": [], + "hold": {"why": verdict["why"], "resource": verdict["resource"], "action": verdict["action"], "missed": 0}, + } + ), + ) + db.q( + "UPDATE $r SET running_since = NONE, heartbeat_at = NONE, last_status = $s, last_run = $i", + r=R("routine", r["id"]), + s=status, + i=run_id, + ) + activity.record( + "run", + f"routine.{status}", + [f"routine:{r['id']}", f"routine_run:{run_id}"], + cfg, + db, + ok=held, + error=None if held else "OverBudget", + detail={"budget": verdict["resource"], "trigger": trigger}, + ) + return True + + +def decide_held(db, run_id, run=True, by=None): + """Someone's pick for a run held over budget: run it now (once, whatever its budgets say) or skip it.""" + row = db.one("SELECT routine, status FROM $r", r=R("routine_run", int(run_id))) + if not row: + raise KeyError(run_id) + if row["status"] != "held": + raise ValueError(f"this run isn't waiting on a budget (it's {row['status']})") + if run: # first, so a routine that can't start now (it's running) leaves the run held to pick again + request_run(db, row["routine"], by=by, over_budget=True) + t = store.now() + db.q( + "UPDATE $r SET status = $s, finished_at = $t, hold.decided = $d WHERE status = 'held'", + r=R("routine_run", int(run_id)), + s="released" if run else "skipped", + t=t, + d={"run": run, "by": by, "at": t}, + ) + return row["routine"] + + def run_due(db, cfg, log=print, now=None): """Run every routine that is due (or asked to run now); how many ran.""" now = now or dt.datetime.now(dt.timezone.utc) stamp = _iso(now) sweep(db, now) + activity.tidy(db, cfg) # the activity ledger past activity.keep_days, once an hour + try: + budgets.sweep(db, cfg, every=budgets.SWEEP_SECONDS) # budgets that crossed warn_at or their cap: warned once + except Exception as e: # noqa: BLE001 - the schedule carries on + if log: + log(f"budgets: {type(e).__name__}: {e}") due = db.rows( "SELECT record::id(id) AS id, schedule, timezone, next_run_at, run_now, namespaces, actions FROM routine " "WHERE run_now != NONE OR (enabled = true AND next_run_at != NONE AND next_run_at <= $n)", @@ -502,6 +620,8 @@ def run_due(db, cfg, log=print, now=None): if not _claim(db, r, now): continue req = r.get("run_now") or {} + if not req.get("over_budget") and _held(db, cfg, r, "manual" if req else "schedule", req.get("by"), now): + continue try: run(db, cfg, r["id"], "manual" if req else "schedule", req.get("by"), bool(req.get("propose_only")), log) done += 1 diff --git a/fastapi_backend/app/domain/semantic.py b/fastapi_backend/app/domain/semantic.py index dff804435..3412c8a00 100644 --- a/fastapi_backend/app/domain/semantic.py +++ b/fastapi_backend/app/domain/semantic.py @@ -24,7 +24,7 @@ import urllib.request from collections import OrderedDict -from . import store +from . import activity, store R = store.R log = logging.getLogger("lens") @@ -125,15 +125,18 @@ def embed(cfg, texts, timeout=None): headers["Authorization"] = f"Bearer {key}" body = json.dumps({"model": model, "input": [t[:EMBED_CHARS] for t in texts]}).encode() req = urllib.request.Request(base + "/embeddings", data=body, headers=headers, method="POST") - try: - with urllib.request.urlopen(req, timeout=timeout or _section(cfg).get("timeout") or 60) as r: - j = json.load(r) - except urllib.error.HTTPError as e: - raise EmbedError(f"{e.code} from the embeddings server for {model}: {e.read().decode('utf-8', 'replace')[:300]}") from None - except (urllib.error.URLError, TimeoutError, OSError) as e: - raise EmbedError(f"can't reach the embeddings server at {base}: {e}") from None - except ValueError: - raise EmbedError("the embeddings server's reply wasn't JSON") from None + with activity.call("embeddings", cfg, model, detail={"texts": len(texts)}, price=activity.model_cost(cfg)) as ledger: + try: + with urllib.request.urlopen(req, timeout=timeout or _section(cfg).get("timeout") or 60) as r: + j = json.load(r) + except urllib.error.HTTPError as e: + raise EmbedError(f"{e.code} from the embeddings server for {model}: {e.read().decode('utf-8', 'replace')[:300]}") from None + except (urllib.error.URLError, TimeoutError, OSError) as e: + raise EmbedError(f"can't reach the embeddings server at {base}: {e}") from None + except ValueError: + raise EmbedError("the embeddings server's reply wasn't JSON") from None + u = j.get("usage") if isinstance(j, dict) and isinstance(j.get("usage"), dict) else {} + ledger.usage(u.get("prompt_tokens", u.get("total_tokens"))) try: data = sorted(j["data"], key=lambda d: d.get("index", 0)) vecs = [[float(x) for x in d["embedding"]] for d in data] diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index c3fd6eb3d..1728861c6 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -43,6 +43,7 @@ "bridge": None, "notifications": None, "telemetry": None, + "activity": None, "encryption": None, "fedora": None, "tunnel": None, @@ -358,6 +359,8 @@ def _check(section, key, value, default): return _notify_setting(key, value) if section == "telemetry": return _telemetry_setting(key, value) + if section == "activity": + return _activity_setting(key, value) if section == "fedora": return _fedora_setting(key, value) if section == "sensors": @@ -551,9 +554,27 @@ def _decision_setting(key, value): if not (isinstance(value, (int, float)) and not isinstance(value, bool) and 1 <= value <= 120): raise ValueError("decisions.timeout is a number of seconds from 1 to 120") return value + if key == "price_per_call": + if value is None or value == "": + return None + if not (isinstance(value, (int, float)) and not isinstance(value, bool) and 0 <= value <= 100): + raise ValueError("decisions.price_per_call is what one decision costs, in USD, from 0 to 100") + return float(value) raise ValueError(f"unknown setting decisions.{key}") +def _activity_setting(key, value): + if key in ("enabled", "reads"): + if not isinstance(value, bool): + raise ValueError(f"activity.{key} is true or false") + return value + if key == "keep_days": + if not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 3650): + raise ValueError("activity.keep_days is a whole number of days from 1 to 3650") + return value + raise ValueError(f"unknown setting activity.{key}") + + EMBED_RANGES = {"passage_chars": (200, 4000), "batch_size": (1, 256), "neighbours": (5, 500), "timeout": (5, 600)} @@ -718,21 +739,44 @@ def _origin(url): def _prices(value): - """{model: {input, output}}: USD per million tokens, for the cost estimates.""" - msg = "telemetry.prices gives each model its input and output price in USD per million tokens" + """{model: price}: how each model's calls are costed (docs/activity.md#costs). By tokens, {input, output} USD per + million tokens (unit "tokens", the default); by time, {unit: "time", per_hour} USD per hour a call takes (a local + model on your own machine); or {unit: "off"}. A model left out isn't costed.""" + msg = ( + "telemetry.prices gives each model a price: {input, output} USD per million tokens, {unit: time, per_hour} " + "USD per hour, or {unit: off}" + ) if value is None: return {} if not isinstance(value, dict) or len(value) > 200: raise ValueError(msg) + + def num(n): + return isinstance(n, (int, float)) and not isinstance(n, bool) and 0 <= n <= 1_000_000 + out = {} for model, p in value.items(): name = str(model).strip() - if not name or len(name) > 200 or not isinstance(p, dict) or set(p) - {"input", "output"}: + if not name or len(name) > 200 or not isinstance(p, dict): raise ValueError(msg) - nums = {k: p.get(k, 0) for k in ("input", "output")} - if not all(isinstance(n, (int, float)) and not isinstance(n, bool) and 0 <= n <= 1_000_000 for n in nums.values()): + unit = p.get("unit") or "tokens" + if unit == "tokens": + if set(p) - {"unit", "input", "output"}: + raise ValueError(msg) + nums = {k: p.get(k, 0) for k in ("input", "output")} + if not all(num(n) for n in nums.values()): + raise ValueError(msg) + out[name] = {k: float(n) for k, n in nums.items()} # the unit stays implicit, as prices saved before units + elif unit == "time": + if set(p) - {"unit", "per_hour"} or not num(p.get("per_hour", 0)): + raise ValueError(msg) + out[name] = {"unit": "time", "per_hour": float(p.get("per_hour", 0))} + elif unit == "off": + if set(p) - {"unit"}: + raise ValueError(msg) + out[name] = {"unit": "off"} + else: raise ValueError(msg) - out[name] = {k: float(n) for k, n in nums.items()} return out diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index f159a1a15..14be9d633 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -305,6 +305,7 @@ def labels(): "api_key": None, "act_above": 0.8, "timeout": 10, + "price_per_call": None, # USD per decision, for the activity ledger's cost (none: not counted) "laya_model": "aac6fef/laya-mlx", "laya_url": None, }, @@ -378,6 +379,9 @@ def labels(): # encryption at rest (docs/encryption.md): files Lens keeps under data_dir, encrypted with their namespace's key; # work_minutes: how long a plain working copy for ffmpeg and the other tools is kept after its last use "encryption": {"files": False, "work_minutes": 30, "vault_minutes": 60}, + # the activity ledger (docs/activity.md): every change and call in or out, with its cost, per resource. On by + # default; reads: log API reads (GET) too; keep_days: how long rows are kept + "activity": {"enabled": True, "reads": False, "keep_days": 365}, "telemetry": { "enabled": False, "endpoint": None, @@ -808,6 +812,13 @@ def close(self): "DEFINE INDEX IF NOT EXISTS share_embed_share ON share_embed FIELDS share", "DEFINE TABLE IF NOT EXISTS audit_log SCHEMALESS", "DEFINE INDEX IF NOT EXISTS audit_at ON audit_log FIELDS at", + "DEFINE INDEX IF NOT EXISTS audit_target ON audit_log FIELDS target", + # what happened to each resource and what it cost (domain/activity.py): a row per call in, call out and run + "DEFINE TABLE IF NOT EXISTS activity SCHEMALESS", + "DEFINE INDEX IF NOT EXISTS activity_at ON activity FIELDS at", + "DEFINE INDEX IF NOT EXISTS activity_resources ON activity FIELDS resources", + # caps on what a routine, pipeline, workflow or namespace may cost (domain/budgets.py): budget:_ + "DEFINE TABLE IF NOT EXISTS budget SCHEMALESS", # background work "DEFINE TABLE IF NOT EXISTS job SCHEMALESS", "DEFINE INDEX IF NOT EXISTS job_status ON job FIELDS status", diff --git a/fastapi_backend/app/domain/telemetry.py b/fastapi_backend/app/domain/telemetry.py index 3bcad2c3a..830c8e2b1 100644 --- a/fastapi_backend/app/domain/telemetry.py +++ b/fastapi_backend/app/domain/telemetry.py @@ -346,22 +346,26 @@ def record(name, value, attrs=None): def cost(model, input_tokens, output_tokens, prices=None): """Estimated cost in USD from the per-million-token prices for the model, or None when it has no price.""" p = (prices if prices is not None else _state["cfg"].get("prices") or {}).get(model) - if not isinstance(p, dict): + if not isinstance(p, dict) or (p.get("unit") or "tokens") != "tokens": return None return round((input_tokens or 0) / 1e6 * float(p.get("input") or 0) + (output_tokens or 0) / 1e6 * float(p.get("output") or 0), 8) class ModelCall: - """One call to the model server: a client span (gen_ai.* attributes), its duration, tokens and estimated cost. - Neither the prompt nor the reply is recorded.""" + """One call to the model server: a client span (gen_ai.* attributes), its duration, tokens and estimated cost, and + a row in the activity ledger (activity.py) whether or not telemetry is on. Neither the prompt nor the reply is + recorded.""" def __init__(self, cfg, payload, operation="chat", current=True): + from . import activity # the ledger counts every call, whether or not telemetry is on + + self.model = payload.get("model") + self.operation, self.t0, self.done, self.response_model, self.usage = operation, time.monotonic(), False, None, None + self.ledger = activity.call(f"model.{operation}", cfg, self.model, price=activity.model_cost(cfg)) self.on = active() if not self.on: return l = (cfg or {}).get("llm") or {} - self.model = payload.get("model") - self.operation, self.t0, self.done, self.response_model, self.usage = operation, time.monotonic(), False, None, None server = urllib.parse.urlsplit(l.get("base_url") or "") attrs = { "gen_ai.operation.name": operation, @@ -386,7 +390,7 @@ def __init__(self, cfg, payload, operation="chat", current=True): def reply(self, j): """Note the usage and model of a reply (a parsed JSON body or a streamed chunk); returns it.""" - if self.on and isinstance(j, dict): + if isinstance(j, dict): if isinstance(j.get("usage"), dict): self.usage = j["usage"] if isinstance(j.get("model"), str): @@ -394,9 +398,15 @@ def reply(self, j): return j def end(self, error=None): - if not self.on or self.done: + if self.done: return self.done = True + u = self.usage or {} + tin, tout = u.get("prompt_tokens", u.get("input_tokens")), u.get("completion_tokens", u.get("output_tokens")) + self.ledger.usage(tin, tout) + self.ledger.end(error) + if not self.on: + return seconds = time.monotonic() - self.t0 base = {"gen_ai.operation.name": self.operation, "gen_ai.provider.name": "openai", "gen_ai.request.model": self.model} if self.response_model: @@ -407,8 +417,6 @@ def end(self, error=None): record("gen_ai.client.operation.duration", seconds, {**base, "error.type": type(error).__name__}) else: record("gen_ai.client.operation.duration", seconds, base) - u = self.usage or {} - tin, tout = u.get("prompt_tokens", u.get("input_tokens")), u.get("completion_tokens", u.get("output_tokens")) if isinstance(tin, int): self.span.set_attribute("gen_ai.usage.input_tokens", tin) record("gen_ai.client.token.usage", tin, {**base, "gen_ai.token.type": "input"}) @@ -434,7 +442,8 @@ def __exit__(self, et, e, tb): def model_call(cfg, payload, operation="chat", current=True): - """`with model_call(cfg, payload) as call: ... call.reply(json)`: a measured model call (a no-op while off).""" + """`with model_call(cfg, payload) as call: ... call.reply(json)`: a measured model call (only the ledger's row while + telemetry is off).""" return ModelCall(cfg, payload, operation, current) diff --git a/fastapi_backend/app/domain/voice.py b/fastapi_backend/app/domain/voice.py index 9f5ec6e19..65e67764e 100644 --- a/fastapi_backend/app/domain/voice.py +++ b/fastapi_backend/app/domain/voice.py @@ -16,7 +16,7 @@ import threading import time -from . import ingest, speech +from . import activity, ingest, speech log = logging.getLogger(__name__) IDLE_SECONDS = 600 @@ -128,8 +128,13 @@ def speak(cfg, text): as (bytes, media type); None when there's none or it failed, so the browser reads it instead.""" if not can_speak(cfg) or not text.strip(): return None + v = cfg.get("voice") or {} + ledger = activity.call("model.speech", cfg, v.get("tts_model") or v.get("tts_provider"), detail={"chars": len(text[:4000])}) try: - return speech.speak(cfg, text) + out = speech.speak(cfg, text) + ledger.end() + return out except speech.ProviderError as e: + ledger.end(e) log.info("voice: text-to-speech failed, the browser reads it: %s", e) return None diff --git a/fastapi_backend/app/main.py b/fastapi_backend/app/main.py index bcad215a1..96e31f1df 100644 --- a/fastapi_backend/app/main.py +++ b/fastapi_backend/app/main.py @@ -26,7 +26,7 @@ from app.config import settings from app.core import middleware from app.core.runtime import Archive -from app.domain import __version__, store, telemetry +from app.domain import __version__, activity, store, telemetry from app.domain.render import WEB_DIR from app.utils import simple_generate_unique_route_id @@ -73,6 +73,8 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # innermost, so the router has picked the route it names spans by; a no-op while telemetry is off app.add_middleware(telemetry.Middleware) + # a row in the activity ledger for each request that changes something, naming its resources (docs/activity.md) + app.add_middleware(activity.Middleware) app.add_middleware( CORSMiddleware, allow_origins=sorted(settings.CORS_ORIGINS), diff --git a/fastapi_backend/app/schemas/activity.py b/fastapi_backend/app/schemas/activity.py new file mode 100644 index 000000000..687325661 --- /dev/null +++ b/fastapi_backend/app/schemas/activity.py @@ -0,0 +1,83 @@ +"""The activity ledger: what happened to a resource and what it cost (docs/activity.md).""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import Field + +from app.schemas.common import ResponseModel + + +class ActivityEntry(ResponseModel): + id: str | None = Field(None, description="the ledger row's id; none for an audit log entry") + at: str + kind: Literal["in", "out", "run", "change"] = Field( + description="in: an API request; out: a call Lens made (a model, embeddings, the decision model, a webhook, a " + "web tool); run: a job or routine run ended; change: an audit log entry" + ) + action: str = Field(description="what it was: `POST /api/v1/routines/{rid}/run`, `model.chat`, `job.succeeded`, `routine.update`") + resources: list[str] = Field(default_factory=list, description="every resource it counts for, as table:id") + actor: int | None = Field(None, description="the account that made it, when a person did") + email: str | None = None + model: str | None = None + tokens_in: int | None = None + tokens_out: int | None = None + cost_usd: float | None = Field(None, description="estimated from the prices in Settings; none when the model has no price") + ms: int | None = None + ok: bool = True + error: str | None = Field(None, description="the error's type or HTTP status; never its message") + detail: Any = None + + +class KindTotals(ResponseModel): + calls: int = 0 + tokens_in: int = 0 + tokens_out: int = 0 + cost_usd: float = 0 + ms: int = 0 + failed: int = 0 + + +class ActivityTotals(ResponseModel): + resource: str | None = None + period: str + since: str | None = None + calls: int + failed: int + tokens_in: int + tokens_out: int + cost_usd: float = Field(description="what its calls cost (run rows repeat their calls' cost and aren't added again)") + ms: int + unpriced: int = Field( + 0, description="calls that cost something but have no figure (a model with no price, a reply without token counts)" + ) + estimate: bool = Field(False, description="true when unpriced calls make cost_usd a floor rather than exact") + by_kind: dict[str, KindTotals] + + +class ResourceCost(ResponseModel): + resource: str + cost_usd: float + tokens: int + calls: int + + +class ResourceSpend(ResponseModel): + cost_usd: float + tokens: int + calls: int + unpriced: int + estimate: bool = Field(description="true: cost_usd is a floor (unpriced calls); show it as an estimate") + + +class ActivityCosts(ResponseModel): + period: str + since: str | None = None + costs: dict[str, ResourceSpend] = Field(description="by resource; ones you can't see are left out") + + +class ActivityTop(ResponseModel): + period: str + since: str | None = None + resources: list[ResourceCost] diff --git a/fastapi_backend/app/schemas/budgets.py b/fastapi_backend/app/schemas/budgets.py new file mode 100644 index 000000000..1009e2356 --- /dev/null +++ b/fastapi_backend/app/schemas/budgets.py @@ -0,0 +1,62 @@ +"""Budgets: caps on what a routine, pipeline, workflow or namespace may cost (docs/budgets.md).""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import Field + +from app.schemas.common import RequestModel, ResponseModel + + +class BudgetSet(RequestModel): + usd: float | None = Field(None, ge=0, description="the cap in USD (from the costs in the activity ledger)") + tokens: int | None = Field(None, ge=0, description="the cap in tokens (in and out)") + period: Literal["run", "day", "week", "month"] = Field( + "month", description="what the cap is for: one run, or a day, week or month (UTC)" + ) + on_over: Literal["ask", "skip", "assistant"] = Field( + "ask", + description="a run that would go over: ask (hold it until someone picks run or skip), skip, or assistant (the " + "decision model weighs it; it runs only when sure, else asks)", + ) + warn_at: float = Field(0.8, ge=0.1, le=1, description="the share spent at which it's marked near (and warned about)") + note: str | None = Field(None, max_length=300) + + +class Budget(ResponseModel): + resource: str + usd: float | None = None + tokens: int | None = None + period: str + on_over: str + warn_at: float + note: str | None = None + by: str | None = None + updated_at: str | None = None + alert: dict[str, Any] | None = Field(None, description="the last warning: {state near|over, since, at, share}") + + +class Spent(ResponseModel): + usd: float + tokens: int + estimate: bool = Field(description="true: some calls had no figure, so usd is a floor") + + +class NextRun(ResponseModel): + usd: float | None = None + tokens: int | None = None + runs: int = Field(description="the past runs the estimate is drawn from") + basis: str + + +class BudgetStatus(ResponseModel): + resource: str + budget: Budget | None = None + state: Literal["none", "ok", "near", "over"] + since: str | None = Field(None, description="the start of the period being counted") + spent: Spent | None = None + next: NextRun = Field(description="what one more run will likely cost: always an estimate") + left: dict[str, float] | None = None + share: float | None = Field(None, description="the largest share of a cap spent (1 is all of it)") + share_after_next: float | None = Field(None, description="the share once the next run (estimated) is done") diff --git a/fastapi_backend/app/schemas/jobs.py b/fastapi_backend/app/schemas/jobs.py index e0a1b864a..5d21b95dc 100644 --- a/fastapi_backend/app/schemas/jobs.py +++ b/fastapi_backend/app/schemas/jobs.py @@ -53,6 +53,14 @@ class Job(ResponseModel): finished_at: str | None = None updated_at: str | None = None cancel_requested: bool | None = None + cost_usd: float | None = Field(None, description="what its calls cost, from the activity ledger (docs/activity.md); set when it ends") + tokens: int | None = None + cost_estimate: bool | None = Field(None, description="true: some calls had no price or token counts, so cost_usd is a floor") + hold: dict[str, Any] | None = Field( + None, + description="held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), " + "decided: {run, by, at} once someone picked}", + ) log: list[str] | None = Field(None, description="the last 200 lines; GET /jobs/{jid}/log has them all") log_total: int | None = Field(None, description="how many lines the run has logged") step_runs: list[StepRun | None] | None = Field( diff --git a/fastapi_backend/app/schemas/routines.py b/fastapi_backend/app/schemas/routines.py index 1a12d132d..c03120219 100644 --- a/fastapi_backend/app/schemas/routines.py +++ b/fastapi_backend/app/schemas/routines.py @@ -62,6 +62,13 @@ class RoutineUpdate(RequestModel): class RoutineRunRequest(RequestModel): propose_only: bool = Field(False, description="graph workflows propose every change instead of making the sure ones") + over_budget: bool = Field( + False, description="run it even though it's over a budget (docs/budgets.md); without it, a run over budget is refused (409)" + ) + + +class HeldRunDecision(RequestModel): + run: bool = Field(description="true: run it now, once, whatever its budgets say; false: skip it") class RoutineRun(ResponseModel): @@ -76,6 +83,14 @@ class RoutineRun(ResponseModel): error: str | None = None changes: dict[str, int] | None = None log: list[str] | None = None + cost_usd: float | None = Field(None, description="what its calls cost, from the activity ledger (docs/activity.md); set when it ends") + tokens: int | None = None + cost_estimate: bool | None = Field(None, description="true: some calls had no price or token counts, so cost_usd is a floor") + hold: dict[str, Any] | None = Field( + None, + description="held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), " + "decided: {run, by, at} once someone picked}", + ) class SchedulePreview(ResponseModel): diff --git a/fastapi_backend/tests/api/test_activity.py b/fastapi_backend/tests/api/test_activity.py new file mode 100644 index 000000000..40df731e9 --- /dev/null +++ b/fastapi_backend/tests/api/test_activity.py @@ -0,0 +1,200 @@ +"""The activity ledger: every changing request and every call out, with its cost, counted for each resource it touched.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from app.domain import activity, routines, store +from tests import fake_llm +from tests.helpers import drain, login, make_user, seed + + +@pytest.fixture +def model(cfg): + srv, url = fake_llm.start() + cfg["llm"].update(base_url=url, model="fake") + cfg["telemetry"]["prices"] = {"fake": {"input": 1.0, "output": 2.0}} # USD per million tokens + cfg["ai"]["tools"] = False # chats get the plain, streamed answer + fake_llm.Handler.usage = {"prompt_tokens": 1000, "completion_tokens": 500, "total_tokens": 1500} + yield fake_llm.Handler + fake_llm.Handler.usage = None + srv.shutdown() + + +@pytest.fixture +def app(cfg, db, model): + from app.main import create_app + + return create_app(cfg, db, background=False) + + +@pytest.fixture +def h(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +def uid(db, email): + return db.one("SELECT record::id(id) AS id FROM account WHERE email = $e", e=email)["id"] + + +def rows(db, resource, kind=None): + return [r for r in activity.history(db, resource, 500) if kind is None or r["kind"] == kind] + + +def test_a_routines_calls_count_for_it_its_run_and_the_jobs_it_queued(client, db, cfg, folder, h): + seed(db, cfg, folder) + rid = client.post( + "/api/v1/routines", + headers=h, + json={"name": "Summaries", "actions": [{"type": "pipeline", "steps": ["summarize"], "recordings": "all"}]}, + ).json()["id"] + run_id = routines.run(db, cfg, rid, by="test") + assert drain(db, cfg) == 3 + + calls = rows(db, f"routine:{rid}", "out") + assert len(calls) == 3 and {c["action"] for c in calls} == {"model.chat"} + for c in calls: + assert (c["model"], c["tokens_in"], c["tokens_out"], c["ok"]) == ("fake", 1000, 500, True) + assert c["cost_usd"] == pytest.approx(0.002) + job = next(r for r in c["resources"] if r.startswith("job:")) + assert {f"routine:{rid}", "pipeline:standard"} - set(c["resources"]) <= {"pipeline:standard"} + assert any(r.startswith("recording:") for r in c["resources"]) and any(r.startswith("space:") for r in c["resources"]) + # each job's own total, on the job and as its run row + jid = int(job.split(":")[1]) + assert db.one("SELECT cost_usd, tokens FROM $j", j=store.R("job", jid)) == {"cost_usd": pytest.approx(0.002), "tokens": 1500} + assert [r["action"] for r in rows(db, job, "run")] == ["job.succeeded"] + + # the routine's history: the request that made it, its run, and the audit entry + hist = rows(db, f"routine:{rid}") + assert {"in", "out", "run", "change"} <= {r["kind"] for r in hist} + assert any(r["kind"] == "change" and r["action"] == "routine.create" for r in hist) + assert db.one("SELECT cost_usd FROM $r", r=store.R("routine_run", run_id))["cost_usd"] is None # queued, not run, in the run + assert [r["action"] for r in rows(db, f"routine_run:{run_id}", "run")] == ["routine.done"] + + t = client.get("/api/v1/activity/totals", headers=h, params={"resource": f"routine:{rid}", "period": "day"}).json() + assert (t["tokens_in"], t["tokens_out"]) == (3000, 1500) and t["cost_usd"] == pytest.approx(0.006) + assert t["by_kind"]["out"]["calls"] == 3 and t["by_kind"]["run"]["calls"] >= 4 # three jobs and the routine run + top = client.get("/api/v1/activity/top", headers=h, params={"table": "routine"}).json()["resources"] + assert top[0]["resource"] == f"routine:{rid}" and top[0]["cost_usd"] == pytest.approx(0.006) and top[0]["tokens"] == 4500 + everything = client.get("/api/v1/activity/totals", headers=h).json() + assert everything["cost_usd"] == pytest.approx(0.006) + + +def test_changing_requests_are_logged_with_their_resources_and_caller(client, db, h): + rid = client.post("/api/v1/routines", headers=h, json={"name": "Nightly", "actions": [{"type": "sync"}]}).json()["id"] + assert client.patch(f"/api/v1/routines/{rid}", headers=h, json={"name": "Nightly sync"}).status_code == 200 + assert client.get(f"/api/v1/routines/{rid}", headers=h).status_code == 200 # reads aren't logged + assert client.patch("/api/v1/routines/999", headers=h, json={"name": "x"}).status_code == 404 + me = uid(db, "root@x.io") + + got = rows(db, f"routine:{rid}", "in") + assert [r["action"] for r in got] == ["PATCH /api/v1/routines/{rid}", "POST /api/v1/routines"] # made, then changed + assert got[0]["actor"] == me and f"account:{me}" in got[0]["resources"] and got[0]["detail"] == {"status": 200} + failed = rows(db, "routine:999", "in") + assert (failed[0]["ok"], failed[0]["error"]) == (False, "404") + mine = [r["action"] for r in rows(db, f"account:{me}", "in")] + assert "POST /api/v1/routines" in mine + assert not any("/auth/" in a for a in mine) # signing in is in the audit log, never with its body + + # reads too, once asked + assert client.put("/api/v1/settings/activity", headers=h, json={"reads": True}).status_code == 200 + client.get(f"/api/v1/routines/{rid}", headers=h) + assert "GET /api/v1/routines/{rid}" in [r["action"] for r in rows(db, f"routine:{rid}", "in")] + + # off: nothing more is written + assert client.put("/api/v1/settings/activity", headers=h, json={"enabled": False, "reads": False}).status_code == 200 + ins, changes = len(rows(db, f"routine:{rid}", "in")), len(rows(db, f"routine:{rid}", "change")) + client.patch(f"/api/v1/routines/{rid}", headers=h, json={"name": "again"}) + assert len(rows(db, f"routine:{rid}", "in")) == ins and len(rows(db, f"routine:{rid}", "change")) == changes + 1 # audit only + + +def test_chats_and_people_see_only_their_own(client, db, cfg, folder, h, model): + seed(db, cfg, folder) + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + e = login(client, "ed@x.io", "editor password 1") + cid = client.post("/api/v1/chats", headers=e, json={"scope": {"namespaces": ["pods"]}}).json()["id"] + assert client.post(f"/api/v1/chats/{cid}/messages", headers=e, json={"content": "What was it about?"}).status_code == 200 + me = uid(db, "ed@x.io") + + got = client.get("/api/v1/activity", headers=e, params={"resource": f"chat:{cid}", "kind": "out"}).json() + assert [r["action"] for r in got] == ["model.chat"] and f"account:{me}" in got[0]["resources"] + assert got[0]["cost_usd"] == pytest.approx(0.002), [(r["tokens_in"], r["model"]) for r in got] + assert client.get("/api/v1/activity", headers=e, params={"resource": f"account:{me}"}).status_code == 200 + pods = store.ns_id(db, "pods") + assert client.get("/api/v1/activity", headers=e, params={"resource": f"space:{pods}"}).status_code == 200 + for other in ("routine:1", "account:1", f"space:{store.ns_id(db, 'calls')}"): + assert client.get("/api/v1/activity", headers=e, params={"resource": other}).status_code in (403, 404), other + assert client.get("/api/v1/activity", headers=e, params={"resource": "not a resource"}).status_code == 422 + assert client.get("/api/v1/activity/totals", headers=e).status_code == 403 + assert client.get("/api/v1/activity/top", headers=e).status_code == 403 + # an admin sees anyone's + assert client.get("/api/v1/activity", headers=h, params={"resource": f"chat:{cid}"}).status_code == 200 + + +def test_old_rows_go_after_keep_days(db, cfg): + old = (dt.datetime.now(dt.UTC) - dt.timedelta(days=400)).isoformat(timespec="seconds") + db.q("CREATE activity CONTENT $d", d={"at": old, "kind": "out", "action": "model.chat", "resources": ["routine:1"]}) + activity.record("out", "model.chat", ["routine:1"], cfg, db) + activity.tidy(db, cfg, now=dt.datetime.now(dt.UTC)) + assert [r["at"] > old for r in rows(db, "routine:1")] == [True] + + +def test_scopes_nest_and_writing_never_breaks_the_work(db, cfg): + with activity.scope(db, "routine:1", cfg=cfg): + with activity.scope(None, "job:2", None, "not a ref"): + assert activity.current() == ["routine:1", "job:2"] + activity.record("out", "x") + assert activity.current() == ["routine:1"] + assert rows(db, "job:2")[0]["resources"] == ["routine:1", "job:2"] + + class Broken: + def values(self, *a, **k): + raise RuntimeError("down") + + assert activity.record("out", "x", db=Broken()) is None + + +def test_costs_for_lists_say_when_they_are_estimates(client, db, cfg, h): + with activity.scope(db, "routine:1", cfg=cfg): + activity.record("out", "model.chat", model="fake", tokens_in=10, cost_usd=0.5) + with activity.scope(db, "routine:2", cfg=cfg): + activity.record("out", "model.chat", model="local", tokens_in=10) # not costed: not an estimate + activity.record("out", "model.chat", model="fake", cost_usd=0.25) + activity.record("out", "model.chat", model="fake", unpriced=True) # priced, but no token counts came back + activity.record("out", "notify.webhook") # costs nothing: not unpriced + got = client.get( + "/api/v1/activity/costs", headers=h, params=[("resource", "routine:1"), ("resource", "routine:2"), ("resource", "routine:3")] + ) + c = got.json()["costs"] + assert (c["routine:1"]["cost_usd"], c["routine:1"]["estimate"]) == (0.5, False) + assert (c["routine:2"]["cost_usd"], c["routine:2"]["unpriced"], c["routine:2"]["estimate"]) == (0.25, 1, True) + assert c["routine:3"] == {"cost_usd": 0, "tokens": 0, "calls": 0, "unpriced": 0, "estimate": False} + t = client.get("/api/v1/activity/totals", headers=h, params={"resource": "routine:2"}).json() + assert (t["unpriced"], t["estimate"]) == (1, True) + make_user(db, "v@x.io", "viewer password 1") + v = login(client, "v@x.io", "viewer password 1") + assert client.get("/api/v1/activity/costs", headers=v, params={"resource": "routine:1"}).json()["costs"] == {} + + +def test_models_are_costed_by_tokens_by_time_or_not_at_all(client, cfg, h): + cfg["telemetry"]["prices"] = { + "cloud": {"input": 3.0, "output": 15.0}, + "gpu-box": {"unit": "time", "per_hour": 0.6}, + "off": {"unit": "off"}, + } + price = activity.model_cost(cfg) + assert price("cloud", 1_000_000, 100_000, 50) == (pytest.approx(4.5), False) + assert price("cloud", None, None, 50) == (None, True) # priced, but the server didn't count + assert price("gpu-box", 10, 10, 60_000) == (pytest.approx(0.01), False) # a minute at 0.60/hour + assert price("off", 10, 10, 60_000) == (None, False) + assert price("unlisted-local", 10, 10, 60_000) == (None, False) # local models aren't costed by default + + put = lambda v: client.put("/api/v1/settings/telemetry", headers=h, json={"prices": v}) # noqa: E731 + assert put({"m": {"unit": "time", "per_hour": 1.5}, "x": {"unit": "off"}, "t": {"input": 1}}).status_code == 200 + got = client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["prices"] + assert {k: got[k] for k in "mxt"} == {"m": {"unit": "time", "per_hour": 1.5}, "x": {"unit": "off"}, "t": {"input": 1.0, "output": 0.0}} + for bad in ({"m": {"unit": "time", "input": 1}}, {"m": {"unit": "watts"}}, {"m": {"unit": "off", "per_hour": 1}}): + assert put(bad).status_code == 400, bad diff --git a/fastapi_backend/tests/api/test_budgets.py b/fastapi_backend/tests/api/test_budgets.py new file mode 100644 index 000000000..0f217f8ab --- /dev/null +++ b/fastapi_backend/tests/api/test_budgets.py @@ -0,0 +1,171 @@ +"""Budgets: off unless set; a run over one waits for someone to pick (or is skipped, or the assistant weighs it).""" + +from __future__ import annotations + +import pytest + +from app.domain import activity, budgets, decide, jobs, routines, store +from tests.helpers import drain, login, make_user, seed + +R = store.R + + +@pytest.fixture +def h(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +def spend(db, resource, usd=0.0, tokens=0): + activity.record("out", "model.chat", [resource], db=db, model="m", cost_usd=usd, tokens_in=tokens) + + +def routine(client, h, name="Sync"): + return client.post("/api/v1/routines", headers=h, json={"name": name, "actions": [{"type": "sync"}]}).json()["id"] + + +def test_setting_reading_and_removing_a_budget(client, db, h): + rid = routine(client, h) + res = f"routine:{rid}" + st = client.get("/api/v1/budgets/status", headers=h, params={"resource": res}).json() + assert st["state"] == "none" and st["next"]["runs"] == 0 # off unless set + + bad = [{"period": "month"}, {"usd": -1}, {"usd": 1, "period": "year"}, {"usd": 1, "on_over": "maybe"}, {"usd": 1, "warn_at": 2}] + for body in bad: + assert client.put("/api/v1/budgets", headers=h, params={"resource": res}, json=body).status_code in (400, 422), body + assert client.put("/api/v1/budgets", headers=h, params={"resource": "chat:1"}, json={"usd": 1}).status_code == 400 + + spend(db, res, usd=0.85, tokens=900) + st = client.put("/api/v1/budgets", headers=h, params={"resource": res}, json={"usd": 1, "tokens": 10_000}).json() + assert (st["state"], st["share"], st["spent"]["usd"], st["left"]["usd"], st["left"]["tokens"]) == ("near", 0.85, 0.85, 0.15, 9100) + assert st["budget"]["on_over"] == "ask" and st["budget"]["period"] == "month" + assert [b["resource"] for b in client.get("/api/v1/budgets", headers=h).json()] == [res] + + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + e = login(client, "ed@x.io", "editor password 1") + assert client.put("/api/v1/budgets", headers=e, params={"resource": res}, json={"usd": 9}).status_code == 403 + assert client.delete("/api/v1/budgets", headers=h, params={"resource": res}).status_code == 200 + assert client.delete("/api/v1/budgets", headers=h, params={"resource": res}).status_code == 404 + assert any(r["action"] == "budget.set" for r in activity.history(db, res)) + + +def test_a_routine_over_budget_waits_for_a_pick(client, db, cfg, h): + rid = routine(client, h) + res = f"routine:{rid}" + budgets.put(db, res, usd=0.5) + spend(db, res, usd=0.6) + + # a person asking is told where it stands, and can choose to run it anyway + r = client.post(f"/api/v1/routines/{rid}/run", headers=h, json={}) + assert r.status_code == 409 and "budget is used up" in r.json()["detail"] + + # the schedule (or anything else) asking: held, once, however many times it comes due + routines.request_run(db, rid) + assert routines.run_due(db, cfg, log=None) == 0 + routines.request_run(db, rid) + routines.run_due(db, cfg, log=None) + held = routines.runs(db, rid) + assert [x["status"] for x in held] == ["held"] and held[0]["hold"]["resource"] == res + assert held[0]["hold"]["missed"] == 1 and "$0.6 of $0.5 this month" in held[0]["hold"]["why"] + assert routines.get(db, rid)["last_status"] == "held" + + # doing nothing changes nothing; a pick to run it runs it once + assert client.post(f"/api/v1/routines/runs/{held[0]['id']}/decide", headers=h, json={"run": True}).status_code == 200 + assert routines.run_due(db, cfg, log=None) == 1 + assert [x["status"] for x in routines.runs(db, rid)] == ["done", "released"] + assert client.post(f"/api/v1/routines/runs/{held[0]['id']}/decide", headers=h, json={"run": True}).status_code == 400 + + # chosen up front by a person + assert client.post(f"/api/v1/routines/{rid}/run", headers=h, json={"over_budget": True}).status_code == 200 + assert routines.run_due(db, cfg, log=None) == 1 + + # skip instead of ask + budgets.put(db, res, usd=0.5, on_over="skip") + routines.request_run(db, rid) + routines.run_due(db, cfg, log=None) + assert routines.runs(db, rid)[0]["status"] == "skipped" + + +def test_the_next_run_is_estimated_from_past_ones(client, db, cfg, h): + rid = routine(client, h) + res = f"routine:{rid}" + for _ in range(2): + with activity.scope(db, res): + spend(db, res, usd=0.3, tokens=100) + routines.run(db, cfg, rid) + st = budgets.status(db, res) + assert (st["next"]["usd"], st["next"]["tokens"], st["next"]["runs"]) == (0.3, 100, 2) + + # 0.6 spent of 0.8 isn't over, but another ≈0.3 run would be + budgets.put(db, res, usd=0.8, period="day") + v = budgets.check(db, cfg, [res]) + assert (v["go"], v["action"], v["resource"]) == (False, "ask", res) + assert "the next run (≈$0.3" in v["why"] and "would go over" in v["why"] + budgets.put(db, res, usd=0.95, period="day") + assert budgets.check(db, cfg, [res])["go"] is True + + # per run: only the estimate counts + budgets.put(db, res, usd=0.2, period="run") + assert budgets.check(db, cfg, [res])["go"] is False + + +def test_the_assistant_weighs_it_when_asked_to(client, db, cfg, h, monkeypatch): + rid = routine(client, h) + res = f"routine:{rid}" + budgets.put(db, res, tokens=10, on_over="assistant") + spend(db, res, tokens=20) + picks = [] + + def choose(cfg_, question, options, state): + picks.append(state) + return {"choice": "run", "confidence": confidence, "ranked": [], "by": "jev"} + + monkeypatch.setattr(decide, "choose", choose) + confidence = 0.95 + v = budgets.check(db, cfg, [res]) + assert v["go"] is True and "assistant let it run" in v["why"] + assert picks[0]["spent"]["tokens"] == 20 and picks[0]["budget"]["tokens"] == 10 + confidence = 0.5 # not sure: ask + assert budgets.check(db, cfg, [res])["action"] == "ask" + + def undecided(*a): + raise decide.Undecided("nothing set up") + + monkeypatch.setattr(decide, "choose", undecided) + assert budgets.check(db, cfg, [res])["action"] == "ask" + + +def test_jobs_over_a_budget_are_held_until_released(client, db, cfg, folder, h): + rids = seed(db, cfg, folder) + pods = store.ns_id(db, "pods") + budgets.put(db, f"space:{pods}", usd=0.1, period="week") + spend(db, f"space:{pods}", usd=0.2) + for rid in rids: + jobs.enqueue(db, rid, ["analyze"]) + drain(db, cfg) + got = {j["recording"]: j for j in jobs.list_jobs(db)} + held = [j for j in got.values() if j["status"] == "held"] + assert len(held) == 2 and all(j["hold"]["resource"] == f"space:{pods}" for j in held) + assert [j["status"] for j in got.values() if j["space"] != pods] == ["succeeded"] + assert jobs.enqueue(db, held[0]["recording"], ["analyze"]) == held[0]["id"] # still the one job + + r = client.get("/api/v1/jobs", headers=h).json() + assert {j["status"] for j in r["jobs"]} >= {"held"} + assert client.post(f"/api/v1/jobs/{held[0]['id']}/release", headers=h, json={"run": True}).status_code == 200 + assert client.post(f"/api/v1/jobs/{held[1]['id']}/release", headers=h, json={"run": False}).status_code == 200 + drain(db, cfg) + assert jobs.get(db, held[0]["id"])["status"] == "succeeded" # let through once, not held again + assert jobs.get(db, held[1]["id"])["status"] == "cancelled" + assert client.post(f"/api/v1/jobs/{held[0]['id']}/release", headers=h, json={"run": True}).status_code == 400 + + +def test_the_periodic_check_warns_once_per_period(db, cfg): + budgets.put(db, "workflow:1", usd=1) + assert budgets.sweep(db, cfg) == [] + spend(db, "workflow:1", usd=0.9) + assert [a["state"] for a in budgets.sweep(db, cfg)] == ["near"] + assert budgets.sweep(db, cfg) == [] + spend(db, "workflow:1", usd=0.2) + assert [a["state"] for a in budgets.sweep(db, cfg)] == ["over"] + assert budgets.sweep(db, cfg) == [] + assert [r["action"] for r in activity.history(db, "workflow:1", kind="run")] == ["budget.over", "budget.near"] diff --git a/mkdocs.yml b/mkdocs.yml index b06b86a25..39cfe3c91 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -76,6 +76,8 @@ nav: - Chat rooms: chat-rooms.md - Sensors: sensors.md - Telemetry: telemetry.md + - Activity and costs: activity.md + - Budgets: budgets.md - IIIF: iiif.md - Graph and Cypher: graph.md - Topics: topics.md diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 312f6447b..b0b31ca8f 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -2,9 +2,11 @@ export { AccessRequests, + Activity, Admin, Auth, Batches, + Budgets, Chats, Collections, Comments, @@ -54,6 +56,10 @@ export type { AcceptGraphChangeResponses, AccessRequest, AccountToken, + ActivityCosts, + ActivityEntry, + ActivityTop, + ActivityTotals, AddFileData, AddFileError, AddFileErrors, @@ -146,6 +152,14 @@ export type { BrowseSourceErrors, BrowseSourceResponse, BrowseSourceResponses, + Budget, + BudgetSet, + BudgetStatus, + BudgetStatusData, + BudgetStatusError, + BudgetStatusErrors, + BudgetStatusResponse, + BudgetStatusResponses, BulkUpdateMetadataData, BulkUpdateMetadataError, BulkUpdateMetadataErrors, @@ -447,6 +461,11 @@ export type { DecideApprovalErrors, DecideApprovalResponse, DecideApprovalResponses, + DecideHeldRunData, + DecideHeldRunError, + DecideHeldRunErrors, + DecideHeldRunResponse, + DecideHeldRunResponses, DecisionStatus, DecisionStatusData, DecisionStatusResponse, @@ -1053,6 +1072,7 @@ export type { HealthDisk, HealthSource, Heard, + HeldRunDecision, HideEntityData, HideEntityError, HideEntityErrors, @@ -1117,6 +1137,7 @@ export type { JobQueued, JobsCreate, JobsQueued, + KindTotals, LanguageCount, LastSend, LayaModel, @@ -1165,6 +1186,9 @@ export type { ListBatchesData, ListBatchesResponse, ListBatchesResponses, + ListBudgetsData, + ListBudgetsResponse, + ListBudgetsResponses, ListChatsData, ListChatsResponse, ListChatsResponses, @@ -1543,6 +1567,7 @@ export type { NewTokenErrors, NewTokenResponse, NewTokenResponses, + NextRun, NodeParam, NodeTrace, NodeType, @@ -1762,6 +1787,16 @@ export type { ReindexSearchData, ReindexSearchResponse, ReindexSearchResponses, + ReleaseJobData, + ReleaseJobError, + ReleaseJobErrors, + ReleaseJobResponse, + ReleaseJobResponses, + RemoveBudgetData, + RemoveBudgetError, + RemoveBudgetErrors, + RemoveBudgetResponse, + RemoveBudgetResponses, RemoveLoginProviderData, RemoveLoginProviderError, RemoveLoginProviderErrors, @@ -1820,8 +1855,25 @@ export type { ResetPasswordRequest, ResetPasswordResponse, ResetPasswordResponses, + ResourceCost, + ResourceCostsData, + ResourceCostsError, + ResourceCostsErrors, + ResourceCostsResponse, + ResourceCostsResponses, ResourceFile, ResourceFiles, + ResourceHistoryData, + ResourceHistoryError, + ResourceHistoryErrors, + ResourceHistoryResponse, + ResourceHistoryResponses, + ResourceSpend, + ResourceTotalsData, + ResourceTotalsError, + ResourceTotalsErrors, + ResourceTotalsResponse, + ResourceTotalsResponses, RetagRecordingsData, RetagRecordingsError, RetagRecordingsErrors, @@ -1996,6 +2048,11 @@ export type { SensorType, SensorUpdate, SeriesPoint, + SetBudgetData, + SetBudgetError, + SetBudgetErrors, + SetBudgetResponse, + SetBudgetResponses, SetCollectionMemberData, SetCollectionMemberError, SetCollectionMemberErrors, @@ -2099,6 +2156,7 @@ export type { SpeakTextResponse, SpeakTextResponses, SpeechTestResult, + Spent, SplitSegmentData, SplitSegmentError, SplitSegmentErrors, @@ -2216,6 +2274,11 @@ export type { TopicRecordings, TopicRef, TopicUpdate, + TopResourcesData, + TopResourcesError, + TopResourcesErrors, + TopResourcesResponse, + TopResourcesResponses, TranscribeClipData, TranscribeClipResponse, TranscribeClipResponses, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 2a7af1c3c..5ff23ca2f 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -51,6 +51,9 @@ import type { BrowseSourceData, BrowseSourceErrors, BrowseSourceResponses, + BudgetStatusData, + BudgetStatusErrors, + BudgetStatusResponses, BulkUpdateMetadataData, BulkUpdateMetadataErrors, BulkUpdateMetadataResponses, @@ -205,6 +208,9 @@ import type { DecideApprovalData, DecideApprovalErrors, DecideApprovalResponses, + DecideHeldRunData, + DecideHeldRunErrors, + DecideHeldRunResponses, DecisionStatusData, DecisionStatusResponses, DeclineAccessRequestData, @@ -575,6 +581,8 @@ import type { ListBackendsResponses, ListBatchesData, ListBatchesResponses, + ListBudgetsData, + ListBudgetsResponses, ListChatsData, ListChatsResponses, ListCollectionMembersData, @@ -861,6 +869,12 @@ import type { RegisterResponses, ReindexSearchData, ReindexSearchResponses, + ReleaseJobData, + ReleaseJobErrors, + ReleaseJobResponses, + RemoveBudgetData, + RemoveBudgetErrors, + RemoveBudgetResponses, RemoveLoginProviderData, RemoveLoginProviderErrors, RemoveLoginProviderResponses, @@ -894,6 +908,15 @@ import type { ResetPasswordData, ResetPasswordErrors, ResetPasswordResponses, + ResourceCostsData, + ResourceCostsErrors, + ResourceCostsResponses, + ResourceHistoryData, + ResourceHistoryErrors, + ResourceHistoryResponses, + ResourceTotalsData, + ResourceTotalsErrors, + ResourceTotalsResponses, RetagRecordingsData, RetagRecordingsErrors, RetagRecordingsResponses, @@ -982,6 +1005,9 @@ import type { SendMessageData, SendMessageErrors, SendMessageResponses, + SetBudgetData, + SetBudgetErrors, + SetBudgetResponses, SetCollectionMemberData, SetCollectionMemberErrors, SetCollectionMemberResponses, @@ -1068,6 +1094,9 @@ import type { TokenLimitsData, TokenLimitsResponses, TokenResponses, + TopResourcesData, + TopResourcesErrors, + TopResourcesResponses, TranscribeClipData, TranscribeClipResponses, TryWorkflowData, @@ -2427,6 +2456,124 @@ export class Admin { } } +export class Activity { + /** + * Resource History + * + * A resource's history, newest first: API requests that changed it, calls made for it (models, embeddings, the + * decision model, webhooks, web tools) with tokens and cost, runs that ended, and its audit log entries. + */ + public static resourceHistory( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/activity", + ...options, + }); + } + + /** + * Resource Totals + * + * What a resource's calls cost this day, week (from Monday), month (UTC) or all time: calls, tokens, USD, time. + */ + public static resourceTotals( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/activity/totals", + ...options, + }); + } + + /** + * Resource Costs + * + * What each of many resources cost this period, for lists; ones you can't see are left out. `estimate` says a + * figure is a floor (some calls had no price or token counts). + */ + public static resourceCosts( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/activity/costs", + ...options, + }); + } + + /** + * Top Resources + * + * The resources that cost most this period (admins). + */ + public static topResources( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/activity/top", + ...options, + }); + } +} + +export class Budgets { + /** + * Remove Budget + */ + public static removeBudget( + options: Options, + ): RequestResult { + return (options.client ?? client).delete({ + url: "/api/v1/budgets", + ...options, + }); + } + + /** + * List Budgets + * + * Every budget, with where it stands now. + */ + public static listBudgets( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/budgets", + ...options, + }); + } + + /** + * Set Budget + */ + public static setBudget( + options: Options, + ): RequestResult { + return (options.client ?? client).put({ + url: "/api/v1/budgets", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Budget Status + * + * Where a resource's budget stands, and what its next run will likely cost (also without a budget). + */ + public static budgetStatus( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/budgets/status", + ...options, + }); + } +} + export class Fedora { /** * Get Fedora Status @@ -5786,6 +5933,25 @@ export class Jobs { }); } + /** + * Release Job + * + * Pick for a job held over a budget (docs/budgets.md): run it now, once, whatever its budgets say, or skip it + * (admins, who set the budgets). + */ + public static releaseJob( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/jobs/{jid}/release", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * Retry Job */ @@ -7399,7 +7565,8 @@ export class Routines { /** * Run Routine * - * Run it as soon as the scheduler next looks (within half a minute), even when it is off. + * Run it as soon as the scheduler next looks (within half a minute), even when it is off. A run over one of its + * budgets is refused (409, saying where the budget stands) unless over_budget is true (docs/budgets.md). */ public static runRoutine( options: Options, @@ -7414,6 +7581,24 @@ export class Routines { }); } + /** + * Decide Held Run + * + * Pick for a run held over a budget: run it now, once, whatever its budgets say, or skip it. + */ + public static decideHeldRun( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/routines/runs/{run_id}/decide", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * List Runs */ diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index f4e8dee75..3fbca9b35 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -102,6 +102,192 @@ export type AccountToken = { [key: string]: unknown; }; +/** + * ActivityCosts + */ +export type ActivityCosts = { + /** + * Period + */ + period: string; + /** + * Since + */ + since?: string | null; + /** + * Costs + * + * by resource; ones you can't see are left out + */ + costs: { + [key: string]: ResourceSpend; + }; + [key: string]: unknown; +}; + +/** + * ActivityEntry + */ +export type ActivityEntry = { + /** + * Id + * + * the ledger row's id; none for an audit log entry + */ + id?: string | null; + /** + * At + */ + at: string; + /** + * Kind + * + * in: an API request; out: a call Lens made (a model, embeddings, the decision model, a webhook, a web tool); run: a job or routine run ended; change: an audit log entry + */ + kind: "in" | "out" | "run" | "change"; + /** + * Action + * + * what it was: `POST /api/v1/routines/{rid}/run`, `model.chat`, `job.succeeded`, `routine.update` + */ + action: string; + /** + * Resources + * + * every resource it counts for, as table:id + */ + resources?: Array; + /** + * Actor + * + * the account that made it, when a person did + */ + actor?: number | null; + /** + * Email + */ + email?: string | null; + /** + * Model + */ + model?: string | null; + /** + * Tokens In + */ + tokens_in?: number | null; + /** + * Tokens Out + */ + tokens_out?: number | null; + /** + * Cost Usd + * + * estimated from the prices in Settings; none when the model has no price + */ + cost_usd?: number | null; + /** + * Ms + */ + ms?: number | null; + /** + * Ok + */ + ok?: boolean; + /** + * Error + * + * the error's type or HTTP status; never its message + */ + error?: string | null; + /** + * Detail + */ + detail?: unknown; + [key: string]: unknown; +}; + +/** + * ActivityTop + */ +export type ActivityTop = { + /** + * Period + */ + period: string; + /** + * Since + */ + since?: string | null; + /** + * Resources + */ + resources: Array; + [key: string]: unknown; +}; + +/** + * ActivityTotals + */ +export type ActivityTotals = { + /** + * Resource + */ + resource?: string | null; + /** + * Period + */ + period: string; + /** + * Since + */ + since?: string | null; + /** + * Calls + */ + calls: number; + /** + * Failed + */ + failed: number; + /** + * Tokens In + */ + tokens_in: number; + /** + * Tokens Out + */ + tokens_out: number; + /** + * Cost Usd + * + * what its calls cost (run rows repeat their calls' cost and aren't added again) + */ + cost_usd: number; + /** + * Ms + */ + ms: number; + /** + * Unpriced + * + * calls that cost something but have no figure (a model with no price, a reply without token counts) + */ + unpriced?: number; + /** + * Estimate + * + * true when unpriced calls make cost_usd a floor rather than exact + */ + estimate?: boolean; + /** + * By Kind + */ + by_kind: { + [key: string]: KindTotals; + }; + [key: string]: unknown; +}; + /** * AnswerCheck */ @@ -921,6 +1107,142 @@ export type BrowseEntry = { [key: string]: unknown; }; +/** + * Budget + */ +export type Budget = { + /** + * Resource + */ + resource: string; + /** + * Usd + */ + usd?: number | null; + /** + * Tokens + */ + tokens?: number | null; + /** + * Period + */ + period: string; + /** + * On Over + */ + on_over: string; + /** + * Warn At + */ + warn_at: number; + /** + * Note + */ + note?: string | null; + /** + * By + */ + by?: string | null; + /** + * Updated At + */ + updated_at?: string | null; + /** + * Alert + * + * the last warning: {state near|over, since, at, share} + */ + alert?: { + [key: string]: unknown; + } | null; + [key: string]: unknown; +}; + +/** + * BudgetSet + */ +export type BudgetSet = { + /** + * Usd + * + * the cap in USD (from the costs in the activity ledger) + */ + usd?: number | null; + /** + * Tokens + * + * the cap in tokens (in and out) + */ + tokens?: number | null; + /** + * Period + * + * what the cap is for: one run, or a day, week or month (UTC) + */ + period?: "run" | "day" | "week" | "month"; + /** + * On Over + * + * a run that would go over: ask (hold it until someone picks run or skip), skip, or assistant (the decision model weighs it; it runs only when sure, else asks) + */ + on_over?: "ask" | "skip" | "assistant"; + /** + * Warn At + * + * the share spent at which it's marked near (and warned about) + */ + warn_at?: number; + /** + * Note + */ + note?: string | null; +}; + +/** + * BudgetStatus + */ +export type BudgetStatus = { + /** + * Resource + */ + resource: string; + budget?: Budget | null; + /** + * State + */ + state: "none" | "ok" | "near" | "over"; + /** + * Since + * + * the start of the period being counted + */ + since?: string | null; + spent?: Spent | null; + /** + * what one more run will likely cost: always an estimate + */ + next: NextRun; + /** + * Left + */ + left?: { + [key: string]: number; + } | null; + /** + * Share + * + * the largest share of a cap spent (1 is all of it) + */ + share?: number | null; + /** + * Share After Next + * + * the share once the next run (estimated) is done + */ + share_after_next?: number | null; + [key: string]: unknown; +}; + /** * ChangeAsk */ @@ -4721,6 +5043,18 @@ export type Heard = { [key: string]: unknown; }; +/** + * HeldRunDecision + */ +export type HeldRunDecision = { + /** + * Run + * + * true: run it now, once, whatever its budgets say; false: skip it + */ + run: boolean; +}; + /** * Highlight */ @@ -5434,6 +5768,30 @@ export type Job = { * Cancel Requested */ cancel_requested?: boolean | null; + /** + * Cost Usd + * + * what its calls cost, from the activity ledger (docs/activity.md); set when it ends + */ + cost_usd?: number | null; + /** + * Tokens + */ + tokens?: number | null; + /** + * Cost Estimate + * + * true: some calls had no price or token counts, so cost_usd is a floor + */ + cost_estimate?: boolean | null; + /** + * Hold + * + * held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), decided: {run, by, at} once someone picked} + */ + hold?: { + [key: string]: unknown; + } | null; /** * Log * @@ -5625,6 +5983,37 @@ export type JobsQueued = { [key: string]: unknown; }; +/** + * KindTotals + */ +export type KindTotals = { + /** + * Calls + */ + calls?: number; + /** + * Tokens In + */ + tokens_in?: number; + /** + * Tokens Out + */ + tokens_out?: number; + /** + * Cost Usd + */ + cost_usd?: number; + /** + * Ms + */ + ms?: number; + /** + * Failed + */ + failed?: number; + [key: string]: unknown; +}; + /** * LanguageCount */ @@ -6506,6 +6895,31 @@ export type NamespaceUpdate = { } | null; }; +/** + * NextRun + */ +export type NextRun = { + /** + * Usd + */ + usd?: number | null; + /** + * Tokens + */ + tokens?: number | null; + /** + * Runs + * + * the past runs the estimate is drawn from + */ + runs: number; + /** + * Basis + */ + basis: string; + [key: string]: unknown; +}; + /** * NodeParam */ @@ -10033,6 +10447,29 @@ export type ResetPasswordRequest = { password: string; }; +/** + * ResourceCost + */ +export type ResourceCost = { + /** + * Resource + */ + resource: string; + /** + * Cost Usd + */ + cost_usd: number; + /** + * Tokens + */ + tokens: number; + /** + * Calls + */ + calls: number; + [key: string]: unknown; +}; + /** * ResourceFile */ @@ -10155,6 +10592,35 @@ export type ResourceFiles = { [key: string]: unknown; }; +/** + * ResourceSpend + */ +export type ResourceSpend = { + /** + * Cost Usd + */ + cost_usd: number; + /** + * Tokens + */ + tokens: number; + /** + * Calls + */ + calls: number; + /** + * Unpriced + */ + unpriced: number; + /** + * Estimate + * + * true: cost_usd is a floor (unpriced calls); show it as an estimate + */ + estimate: boolean; + [key: string]: unknown; +}; + /** * Reviewed */ @@ -10359,7 +10825,31 @@ export type RoutineRun = { /** * Log */ - log?: Array | null; + log?: Array | null; + /** + * Cost Usd + * + * what its calls cost, from the activity ledger (docs/activity.md); set when it ends + */ + cost_usd?: number | null; + /** + * Tokens + */ + tokens?: number | null; + /** + * Cost Estimate + * + * true: some calls had no price or token counts, so cost_usd is a floor + */ + cost_estimate?: boolean | null; + /** + * Hold + * + * held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), decided: {run, by, at} once someone picked} + */ + hold?: { + [key: string]: unknown; + } | null; [key: string]: unknown; }; @@ -10373,6 +10863,12 @@ export type RoutineRunRequest = { * graph workflows propose every change instead of making the sure ones */ propose_only?: boolean; + /** + * Over Budget + * + * run it even though it's over a budget (docs/budgets.md); without it, a run over budget is refused (409) + */ + over_budget?: boolean; }; /** @@ -12504,6 +13000,27 @@ export type SpeechTestResult = { [key: string]: unknown; }; +/** + * Spent + */ +export type Spent = { + /** + * Usd + */ + usd: number; + /** + * Tokens + */ + tokens: number; + /** + * Estimate + * + * true: some calls had no figure, so usd is a floor + */ + estimate: boolean; + [key: string]: unknown; +}; + /** * Started * @@ -16573,6 +17090,280 @@ export type ReindexSearchResponses = { export type ReindexSearchResponse = ReindexSearchResponses[keyof ReindexSearchResponses]; +export type ResourceHistoryData = { + body?: never; + path?: never; + query: { + /** + * Resource + * + * table:id, like routine:3, pipeline:2, workflow:4, recording:12, account:1 + */ + resource: string; + /** + * Kind + */ + kind?: "in" | "out" | "run" | "change" | null; + /** + * Before + * + * only entries before this time (the last `at` of the page before) + */ + before?: string | null; + /** + * Limit + */ + limit?: number; + }; + url: "/api/v1/activity"; +}; + +export type ResourceHistoryErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ResourceHistoryError = ResourceHistoryErrors[keyof ResourceHistoryErrors]; + +export type ResourceHistoryResponses = { + /** + * Response Activity-Resource History + * + * Successful Response + */ + 200: Array; +}; + +export type ResourceHistoryResponse = ResourceHistoryResponses[keyof ResourceHistoryResponses]; + +export type ResourceTotalsData = { + body?: never; + path?: never; + query?: { + /** + * Resource + * + * table:id; none for everything (admins) + */ + resource?: string | null; + /** + * Period + */ + period?: "day" | "week" | "month" | "all"; + }; + url: "/api/v1/activity/totals"; +}; + +export type ResourceTotalsErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ResourceTotalsError = ResourceTotalsErrors[keyof ResourceTotalsErrors]; + +export type ResourceTotalsResponses = { + /** + * Successful Response + */ + 200: ActivityTotals; +}; + +export type ResourceTotalsResponse = ResourceTotalsResponses[keyof ResourceTotalsResponses]; + +export type ResourceCostsData = { + body?: never; + path?: never; + query: { + /** + * Resource + * + * table:id, repeated (up to 500): every row of a list at once + */ + resource: Array; + /** + * Period + */ + period?: "day" | "week" | "month" | "all"; + }; + url: "/api/v1/activity/costs"; +}; + +export type ResourceCostsErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ResourceCostsError = ResourceCostsErrors[keyof ResourceCostsErrors]; + +export type ResourceCostsResponses = { + /** + * Successful Response + */ + 200: ActivityCosts; +}; + +export type ResourceCostsResponse = ResourceCostsResponses[keyof ResourceCostsResponses]; + +export type TopResourcesData = { + body?: never; + path?: never; + query?: { + /** + * Table + * + * only this kind of resource: routine, pipeline, workflow, space + */ + table?: string | null; + /** + * Period + */ + period?: "day" | "week" | "month" | "all"; + /** + * Limit + */ + limit?: number; + }; + url: "/api/v1/activity/top"; +}; + +export type TopResourcesErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type TopResourcesError = TopResourcesErrors[keyof TopResourcesErrors]; + +export type TopResourcesResponses = { + /** + * Successful Response + */ + 200: ActivityTop; +}; + +export type TopResourcesResponse = TopResourcesResponses[keyof TopResourcesResponses]; + +export type RemoveBudgetData = { + body?: never; + path?: never; + query: { + /** + * Resource + * + * routine:, pipeline:, workflow: or space: + */ + resource: string; + }; + url: "/api/v1/budgets"; +}; + +export type RemoveBudgetErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type RemoveBudgetError = RemoveBudgetErrors[keyof RemoveBudgetErrors]; + +export type RemoveBudgetResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type RemoveBudgetResponse = RemoveBudgetResponses[keyof RemoveBudgetResponses]; + +export type ListBudgetsData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/budgets"; +}; + +export type ListBudgetsResponses = { + /** + * Response Budgets-List Budgets + * + * Successful Response + */ + 200: Array; +}; + +export type ListBudgetsResponse = ListBudgetsResponses[keyof ListBudgetsResponses]; + +export type SetBudgetData = { + body: BudgetSet; + path?: never; + query: { + /** + * Resource + * + * routine:, pipeline:, workflow: or space: + */ + resource: string; + }; + url: "/api/v1/budgets"; +}; + +export type SetBudgetErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type SetBudgetError = SetBudgetErrors[keyof SetBudgetErrors]; + +export type SetBudgetResponses = { + /** + * Successful Response + */ + 200: BudgetStatus; +}; + +export type SetBudgetResponse = SetBudgetResponses[keyof SetBudgetResponses]; + +export type BudgetStatusData = { + body?: never; + path?: never; + query: { + /** + * Resource + * + * routine:, pipeline:, workflow: or space: + */ + resource: string; + }; + url: "/api/v1/budgets/status"; +}; + +export type BudgetStatusErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type BudgetStatusError = BudgetStatusErrors[keyof BudgetStatusErrors]; + +export type BudgetStatusResponses = { + /** + * Successful Response + */ + 200: BudgetStatus; +}; + +export type BudgetStatusResponse = BudgetStatusResponses[keyof BudgetStatusResponses]; + export type GetFedoraStatusData = { body?: never; path?: never; @@ -23396,6 +24187,36 @@ export type CancelJobResponses = { export type CancelJobResponse = CancelJobResponses[keyof CancelJobResponses]; +export type ReleaseJobData = { + body: HeldRunDecision; + path: { + /** + * Jid + */ + jid: number; + }; + query?: never; + url: "/api/v1/jobs/{jid}/release"; +}; + +export type ReleaseJobErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ReleaseJobError = ReleaseJobErrors[keyof ReleaseJobErrors]; + +export type ReleaseJobResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type ReleaseJobResponse = ReleaseJobResponses[keyof ReleaseJobResponses]; + export type RetryJobData = { body?: never; path: { @@ -26359,6 +27180,36 @@ export type RunRoutineResponses = { export type RunRoutineResponse = RunRoutineResponses[keyof RunRoutineResponses]; +export type DecideHeldRunData = { + body: HeldRunDecision; + path: { + /** + * Run Id + */ + run_id: number; + }; + query?: never; + url: "/api/v1/routines/runs/{run_id}/decide"; +}; + +export type DecideHeldRunErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type DecideHeldRunError = DecideHeldRunErrors[keyof DecideHeldRunErrors]; + +export type DecideHeldRunResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type DecideHeldRunResponse = DecideHeldRunResponses[keyof DecideHeldRunResponses]; + export type ListRunsData = { body?: never; path: { diff --git a/nextjs-frontend/components/activity/job-detail.tsx b/nextjs-frontend/components/activity/job-detail.tsx index 74569df8f..ed4b95b5f 100644 --- a/nextjs-frontend/components/activity/job-detail.tsx +++ b/nextjs-frontend/components/activity/job-detail.tsx @@ -8,6 +8,7 @@ import { useMemo, useState, type ReactNode } from "react"; import { Jobs, Pipelines, Resources, Templates } from "@/app/openapi-client"; import { CancelJobDialog } from "@/components/activity/cancel-dialog"; +import { Cost, HeldNotice } from "@/components/costs/costs"; import { approx, canRetry, @@ -224,6 +225,18 @@ export function JobDetail({ jobId }: { jobId: number }) { onError: (e: Error) => toast({ tone: "red", title: "Couldn’t start the run", body: e.message }), }); + const release = useMutation({ + mutationFn: (go: boolean) => data(Jobs.releaseJob({ client, path: { jid: jobId }, body: { run: go } })), + onSuccess: (_, go) => { + void qc.invalidateQueries({ queryKey: ["job", jobId] }); + void qc.invalidateQueries({ queryKey: ["jobs"] }); + toast( + go ? { tone: "green", title: "Running it once", body: "A worker picks it up next." } : { title: "Skipped" }, + ); + }, + onError: (e: Error) => toast({ tone: "red", title: "Couldn’t do that", body: e.message }), + }); + if (q.isLoading) return (
@@ -345,6 +358,15 @@ export function JobDetail({ jobId }: { jobId: number }) { ); } else if (job.status === "paused") box = ; + else if (job.status === "held") + box = ( + release.mutate(true)} + onSkip={() => release.mutate(false)} + /> + ); else box = ( @@ -449,6 +471,11 @@ export function JobDetail({ jobId }: { jobId: number }) { : ""} {left && {left}} + {(job.cost_usd != null || job.tokens != null) && ( + + Cost + + )} {ns && {ns}} {job.recording != null && ( diff --git a/nextjs-frontend/components/activity/job-model.ts b/nextjs-frontend/components/activity/job-model.ts index cda7a7995..2629415e0 100644 --- a/nextjs-frontend/components/activity/job-model.ts +++ b/nextjs-frontend/components/activity/job-model.ts @@ -11,7 +11,7 @@ import { STEP_LABEL } from "@/components/ui/loop"; /** A run as the API sends it. */ export type JobRecord = Job; -export const ACTIVE = ["queued", "running", "paused"]; +export const ACTIVE = ["queued", "running", "paused", "held"]; export const isActive = (status: string | null | undefined) => ACTIVE.includes(status ?? ""); export const canRetry = (status: string | null | undefined) => status === "failed" || status === "cancelled"; @@ -146,6 +146,8 @@ export function jobPhase(job: JobRecord, opts: { waitingFor?: string | null; now return i > 0 ? `${label} · waiting for a worker` : `Queued · ${label} first`; case "paused": return `Paused with its batch · ${label} next`; + case "held": + return `Waiting for you: over budget · ${label} first`; case "failed": { const e = plainError(job.error); return `${label} failed${e ? `: ${e}` : ""}`; diff --git a/nextjs-frontend/components/activity/job-row.tsx b/nextjs-frontend/components/activity/job-row.tsx index bb704fc07..c1ba454f3 100644 --- a/nextjs-frontend/components/activity/job-row.tsx +++ b/nextjs-frontend/components/activity/job-row.tsx @@ -28,6 +28,13 @@ const ICON: Record = { running: "bg-blue", queued: "bg-border", paused: "bg-border", + held: "bg-gold", succeeded: "bg-green", failed: "bg-red", cancelled: "bg-fg-muted", diff --git a/nextjs-frontend/components/admin/namespaces.tsx b/nextjs-frontend/components/admin/namespaces.tsx index 4466f2b55..bd93d3f75 100644 --- a/nextjs-frontend/components/admin/namespaces.tsx +++ b/nextjs-frontend/components/admin/namespaces.tsx @@ -9,6 +9,7 @@ import { Namespaces, Pipelines, Users } from "@/app/openapi-client"; import { IpGroupsSection } from "@/components/access/ip-groups"; import { AdminFrame, usePeople } from "@/components/admin/admin-frame"; import { roleLabel, type Role } from "@/components/admin/people-model"; +import { ActivityPanel, BudgetPanel, CostCell, useBudgets, useCosts } from "@/components/costs/costs"; import { isUnreachable } from "@/components/errors/error-states"; import { NotificationsSection } from "@/components/notifications/notifications-section"; import { runtime } from "@/components/iiif/iiif-model"; @@ -38,6 +39,8 @@ export function NamespacesPage() { const [creating, setCreating] = useState(false); const shown = namespaces.filter((n) => admin || roleIn(n.name) === "owner"); const members = (ns: string) => (people.data ?? []).filter((p) => !p.admin && p.roles?.[ns]).length; + const costs = useCosts(shown.map((n) => `space:${n.id}`)); + const budgets = useBudgets(); return ( Recordings
{admin && } + @@ -90,6 +94,9 @@ export function NamespacesPage() { {admin && } + @@ -468,6 +475,8 @@ export function NamespaceDetail({ ns }: { ns: string }) { {known && } + {known && } + {known && }
diff --git a/nextjs-frontend/components/costs/costs.tsx b/nextjs-frontend/components/costs/costs.tsx new file mode 100644 index 000000000..dc8946682 --- /dev/null +++ b/nextjs-frontend/components/costs/costs.tsx @@ -0,0 +1,584 @@ +"use client"; + +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { ArrowDownLeft, ArrowUpRight, History, Pencil, PiggyBank, Play, RotateCcw } from "lucide-react"; +import { useState } from "react"; + +import { Activity, Budgets } from "@/app/openapi-client"; +import type { ActivityEntry, BudgetSet, BudgetStatus, ResourceSpend } from "@/app/openapi-client/types.gen"; +import { Badge, type Tone } from "@/components/ui/badge"; +import { Banner } from "@/components/ui/banner"; +import { Button } from "@/components/ui/button"; +import { Field, Input, Select } from "@/components/ui/field"; +import { Panel } from "@/components/ui/panel"; +import { DateTime, Skeleton, SkeletonRows } from "@/components/ui/states"; +import { useToast } from "@/components/ui/toast"; +import { Tooltip } from "@/components/ui/tooltip"; +import { data, useApiClient } from "@/lib/api/browser"; +import { useArchive } from "@/lib/hooks/session"; +import { cn } from "@/lib/utils"; + +export type Period = "day" | "week" | "month" | "all"; +const PERIOD_TEXT: Record = { day: "today", week: "this week", month: "this month", all: "in all" }; + +/** USD the way a person reads it: cents for small sums, more digits below a cent. */ +export function usd(v: number | null | undefined): string { + if (v == null) return "—"; + if (v === 0) return "$0"; + if (v < 0.01) return `$${v.toPrecision(2)}`; + if (v < 100) return `$${v.toFixed(2)}`; + return `$${Math.round(v).toLocaleString()}`; +} + +/** 950, 12.4k, 3.1M tokens. */ +export function tokens(n: number | null | undefined): string { + if (n == null) return "—"; + if (n < 1000) return String(n); + if (n < 1_000_000) return `${(n / 1000).toFixed(n < 10_000 ? 1 : 0)}k`; + return `${(n / 1_000_000).toFixed(1)}M`; +} + +const ESTIMATE_WHY = + "An estimate: some calls had a price but no token counts from the server, so the real cost is at least this."; + +/** + * A cost, honest about what it is: "$0.42", "≈$0.42" when it's an estimate (with why on hover), or the tokens alone + * when nothing was priced (local models aren't costed unless they're given a price). + */ +export function Cost({ + usd: v, + tokens: t, + estimate, + why, + className, +}: { + usd?: number | null; + tokens?: number | null; + estimate?: boolean | null; + why?: string; + className?: string; +}) { + const priced = (v ?? 0) > 0; + const text = priced ? usd(v) : t ? `${tokens(t)} tokens` : usd(v ?? 0); + const title = [ + estimate ? (why ?? ESTIMATE_WHY) : null, + priced && t ? `${tokens(t)} tokens` : null, + !priced && t ? "No price set for these models, so they aren't costed (Settings › Telemetry › prices)." : null, + ] + .filter(Boolean) + .join(" "); + const body = ( + + {estimate ? ( + + ≈ + + ) : null} + {text} + + ); + return title ? ( + + {body} + + ) : ( + body + ); +} + +/** What each of many resources cost this period, in one request (for lists). */ +export function useCosts(resources: string[], period: Period = "month") { + const client = useApiClient(); + const list = [...new Set(resources)].sort(); + return useQuery({ + queryKey: ["activity-costs", period, list], + queryFn: () => data(Activity.resourceCosts({ client, query: { resource: list, period } })), + enabled: list.length > 0, + staleTime: 30_000, + }); +} + +/** One resource's cost this period, from a list's costs (or its own request). */ +export function ResourceCost({ + resource, + costs, + period = "month", + className, +}: { + resource: string; + costs?: Record; + period?: Period; + className?: string; +}) { + const own = useCosts(costs ? [] : [resource], period); + const c = costs ? costs[resource] : own.data?.costs[resource]; + if (!c) return —; + return ; +} + +const KIND: Record = { + in: { label: "Request", icon: , tone: "neutral" }, + out: { label: "Call out", icon: , tone: "intent" }, + run: { label: "Run", icon: , tone: "green" }, + change: { label: "Change", icon: , tone: "gate" }, +}; + +function actionText(e: ActivityEntry): string { + const a = e.action; + if (a.startsWith("model.")) + return `${a.slice(6) === "chat" ? "Model call" : `Model ${a.slice(6)}`}${e.model ? ` · ${e.model}` : ""}`; + if (a === "embeddings") return `Embeddings${e.model ? ` · ${e.model}` : ""}`; + if (a === "decision") return `Decision model${e.model ? ` · ${e.model}` : ""}`; + if (a.startsWith("notify.")) return `Notification · ${a.slice(7)}`; + if (a === "tool.http") return "Web tool"; + if (a.startsWith("budget.")) return a === "budget.over" ? "Budget used up" : "Budget nearly used"; + return a; +} + +/** A resource's history: requests that changed it, calls made for it, runs and audit entries, with their cost. */ +export function ActivityPanel({ resource, title = "Activity" }: { resource: string; title?: string }) { + const client = useApiClient(); + const [limit, setLimit] = useState(50); + const [period, setPeriod] = useState("month"); + const q = useQuery({ + queryKey: ["activity", resource, limit], + queryFn: () => data(Activity.resourceHistory({ client, query: { resource, limit } })), + staleTime: 15_000, + }); + const totals = useQuery({ + queryKey: ["activity-totals", resource, period], + queryFn: () => data(Activity.resourceTotals({ client, query: { resource, period } })), + staleTime: 30_000, + }); + const t = totals.data; + return ( + + {PERIOD_TEXT[period]} + {" · "} + {t.by_kind.out?.calls ?? 0} calls out · {t.by_kind.in?.calls ?? 0} changes + {t.failed ? ` · ${t.failed} failed` : ""} + + ) : ( + + ) + } + actions={ + setForm({ ...form, usd: e.target.value })} + placeholder="5.00" + /> + )} + + + {({ id }) => ( + setForm({ ...form, tokens: e.target.value })} + placeholder="1000000" + /> + )} + + + {({ id }) => ( + setForm({ ...form, warn: e.target.value })} + /> + )} + + + {({ id }) => ( +
+ @@ -139,6 +143,7 @@ export function PipelinesPage() { built-in + {list.map((p, i) => { @@ -178,6 +183,9 @@ export function PipelinesPage() { + ); diff --git a/nextjs-frontend/components/recording/history-tab.tsx b/nextjs-frontend/components/recording/history-tab.tsx index 0030dc4eb..83e219006 100644 --- a/nextjs-frontend/components/recording/history-tab.tsx +++ b/nextjs-frontend/components/recording/history-tab.tsx @@ -4,6 +4,7 @@ import { ExternalLink } from "lucide-react"; import Link from "next/link"; import { pipelineLabel } from "@/components/activity/job-model"; +import { ActivityPanel } from "@/components/costs/costs"; import { useRec } from "@/components/recording/context"; import { ChangeHistoryList } from "@/components/recording/edit"; import { useJob } from "@/components/recording/hooks"; @@ -46,6 +47,7 @@ export function HistoryTab() { + ); } diff --git a/nextjs-frontend/components/routines/routine-detail.tsx b/nextjs-frontend/components/routines/routine-detail.tsx index ad97e55b8..74d82d198 100644 --- a/nextjs-frontend/components/routines/routine-detail.tsx +++ b/nextjs-frontend/components/routines/routine-detail.tsx @@ -7,6 +7,7 @@ import { useRouter } from "next/navigation"; import { useState } from "react"; import { Routines } from "@/app/openapi-client"; +import { ActivityPanel, BudgetPanel, Cost, HeldNotice } from "@/components/costs/costs"; import type { RoutineRun } from "@/app/openapi-client/types.gen"; import { useNames } from "@/components/routines/data"; import { RoutineEditor } from "@/components/routines/routine-editor"; @@ -51,6 +52,22 @@ function RunLog({ id }: { id: number }) { function RunRow({ run, onUndo }: { run: RoutineRun; onUndo: (r: RoutineRun) => void }) { const [open, setOpen] = useState(false); + const client = useApiClient(); + const qc = useQueryClient(); + const toast = useToast(); + const decide = useMutation({ + mutationFn: (go: boolean) => data(Routines.decideHeldRun({ client, path: { run_id: run.id }, body: { run: go } })), + onSuccess: (_, go) => { + void qc.invalidateQueries({ queryKey: ["routine-runs", run.routine] }); + void qc.invalidateQueries({ queryKey: ["routine", run.routine] }); + toast( + go + ? { tone: "green", title: "Running it once", body: "It starts within half a minute." } + : { title: "Skipped" }, + ); + }, + onError: (e: Error) => toast({ tone: "red", title: "Couldn’t do that", body: e.message }), + }); const applied = run.changes?.applied ?? 0; const proposed = run.changes?.proposed ?? 0; const results = (run.results ?? []) as Record[]; @@ -77,6 +94,14 @@ function RunRow({ run, onUndo }: { run: RoutineRun; onUndo: (r: RoutineRun) => v {took(run.started_at, run.finished_at) ? ` · took ${took(run.started_at, run.finished_at)}` : ""} + {(run.cost_usd != null || run.tokens != null) && ( + + )} {proposed > 0 && ( v )} -
0 || run.error) && "pb-3")}> +
0 || run.error || run.hold) && "pb-3", + )} + > + {run.status === "held" ? ( + decide.mutate(true)} + onSkip={() => decide.mutate(false)} + /> + ) : run.hold?.why ? ( +

{String(run.hold.why)}

+ ) : null} {results.length > 0 && (
    {results.map((r, i) => ( @@ -112,7 +152,7 @@ function RunRow({ run, onUndo }: { run: RoutineRun; onUndo: (r: RoutineRun) => v } /** A routine: its recent runs (with their logs, and undo for graph changes) and its settings. */ -export function RoutineDetail({ id, tab: initialTab }: { id: number; tab?: "runs" | "settings" }) { +export function RoutineDetail({ id, tab: initialTab }: { id: number; tab?: "runs" | "costs" | "settings" }) { const client = useApiClient(); const qc = useQueryClient(); const router = useRouter(); @@ -120,7 +160,7 @@ export function RoutineDetail({ id, tab: initialTab }: { id: number; tab?: "runs const { admin, me } = useArchive(); const names = useNames(); const { toggle } = useRoutineActions(); - const [tab, setTab] = useState(initialTab ?? "runs"); + const [tab, setTab] = useState<"runs" | "costs" | "settings">(initialTab ?? "runs"); const [undoing, setUndoing] = useState(null); const [deleting, setDeleting] = useState(false); @@ -235,14 +275,20 @@ export function RoutineDetail({ id, tab: initialTab }: { id: number; tab?: "runs setTab(v as "runs" | "settings")} + onChange={(v) => setTab(v as "runs" | "costs" | "settings")} items={[ { value: "runs", label: "Runs", count: runs.data?.length }, + { value: "costs", label: "Costs and activity" }, { value: "settings", label: "Settings" }, ]} /> {tab === "settings" ? ( setTab("runs")} /> + ) : tab === "costs" ? ( +
    + + +
    ) : runs.isLoading ? ( ) : runs.error ? ( diff --git a/nextjs-frontend/components/routines/routine-model.ts b/nextjs-frontend/components/routines/routine-model.ts index 49d7111f0..3b500c216 100644 --- a/nextjs-frontend/components/routines/routine-model.ts +++ b/nextjs-frontend/components/routines/routine-model.ts @@ -179,6 +179,9 @@ export const STATUS: Record = { partly: { label: "Partly done", tone: "gate" }, error: { label: "Failed", tone: "red" }, running: { label: "Running", tone: "intent" }, + held: { label: "Waiting for you", tone: "gate" }, + released: { label: "Let through", tone: "neutral" }, + skipped: { label: "Skipped", tone: "neutral" }, }; export function statusOf(s: string | null | undefined) { diff --git a/nextjs-frontend/components/routines/routines-page.tsx b/nextjs-frontend/components/routines/routines-page.tsx index 14ad8d672..af829898b 100644 --- a/nextjs-frontend/components/routines/routines-page.tsx +++ b/nextjs-frontend/components/routines/routines-page.tsx @@ -9,6 +9,7 @@ import { useState } from "react"; import { Routines } from "@/app/openapi-client"; import type { Routine } from "@/app/openapi-client/types.gen"; import { useGraphChanges, useNames, useRoutines } from "@/components/routines/data"; +import { CostCell, useBudgets, useCosts } from "@/components/costs/costs"; import { routineText, statusOf, touchesGraph, type RoutineAction } from "@/components/routines/routine-model"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -19,7 +20,7 @@ import { DateTime, EmptyState, SkeletonRows } from "@/components/ui/states"; import { Table, Td, Th, THead, Tr } from "@/components/ui/table"; import { Tabs } from "@/components/ui/tabs"; import { useToast } from "@/components/ui/toast"; -import { data, useApiClient } from "@/lib/api/browser"; +import { ApiError, data, useApiClient } from "@/lib/api/browser"; import { useArchive } from "@/lib/hooks/session"; /** "Routines" with the Routines / Proposed changes tabs; people who aren't admins see only the changes. */ @@ -64,12 +65,18 @@ export function useRoutineActions() { void qc.invalidateQueries({ queryKey: ["routines"] }); void qc.invalidateQueries({ queryKey: ["routine", id] }); }; + const [overBudget, setOverBudget] = useState<{ routine: Routine; proposeOnly?: boolean; why: string } | null>(null); const run = useMutation({ - mutationFn: (v: { routine: Routine; proposeOnly?: boolean }) => + mutationFn: (v: { routine: Routine; proposeOnly?: boolean; overBudget?: boolean }) => data( - Routines.runRoutine({ client, path: { rid: v.routine.id }, body: { propose_only: Boolean(v.proposeOnly) } }), + Routines.runRoutine({ + client, + path: { rid: v.routine.id }, + body: { propose_only: Boolean(v.proposeOnly), over_budget: Boolean(v.overBudget) }, + }), ), onSuccess: (_, v) => { + setOverBudget(null); refresh(v.routine.id); void qc.invalidateQueries({ queryKey: ["routine-runs", v.routine.id] }); toast({ @@ -78,7 +85,12 @@ export function useRoutineActions() { body: `${v.routine.name} starts within half a minute.`, }); }, - onError: (e: Error) => toast({ tone: "red", title: "Couldn’t start the run", body: e.message }), + onError: (e: Error, v) => { + // over budget: say where it stands and let the person choose (docs/budgets.md) + if (e instanceof ApiError && e.status === 409) + setOverBudget({ routine: v.routine, proposeOnly: v.proposeOnly, why: e.message }); + else toast({ tone: "red", title: "Couldn’t start the run", body: e.message }); + }, }); const toggle = useMutation({ mutationFn: (v: { routine: Routine; enabled: boolean }) => @@ -89,7 +101,29 @@ export function useRoutineActions() { }, onError: (e: Error) => toast({ tone: "red", title: "Couldn’t change the routine", body: e.message }), }); - return { run, toggle }; + const overBudgetDialog = ( + !o && setOverBudget(null)} + title="Run it over budget?" + description={overBudget?.why.replace(/ Run it anyway with over_budget\.$/, "")} + actions={ + <> + + + + } + /> + ); + return { run, toggle, overBudgetDialog }; } export function DeleteRoutineDialog({ @@ -136,36 +170,42 @@ export function DeleteRoutineDialog({ /** Run now, with "propose only" next to it when the routine organises the graph. */ export function RunButton({ routine, graph, size = "sm" }: { routine: Routine; graph: boolean; size?: "xs" | "sm" }) { - const { run } = useRoutineActions(); + const { run, overBudgetDialog } = useRoutineActions(); const busy = Boolean(routine.running) || (run.isPending && run.variables?.routine.id === routine.id); const why = routine.running ? "It’s running now" : "Asking…"; if (!graph || busy) return ( - + <> + + {overBudgetDialog} + ); return ( - - - - - - run.mutate({ routine })}>Run now - run.mutate({ routine, proposeOnly: true })}> - Run now, only propose graph changes - - - + <> + + + + + + run.mutate({ routine })}>Run now + run.mutate({ routine, proposeOnly: true })}> + Run now, only propose graph changes + + + + {overBudgetDialog} + ); } @@ -194,6 +234,8 @@ export function RoutinesPage() { const { toggle } = useRoutineActions(); const [deleting, setDeleting] = useState(null); const list = routines.data?.routines ?? []; + const costs = useCosts(list.map((r) => `routine:${r.id}`)); + const budgets = useBudgets(); if (me && !admin) return ( @@ -252,6 +294,7 @@ export function RoutinesPage() {
+ + @@ -62,6 +66,9 @@ export function WorkflowsPage() { + ))} diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 707d3443d..90458414d 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -2714,6 +2714,485 @@ } } }, + "/api/v1/activity": { + "get": { + "tags": [ + "activity" + ], + "summary": "Resource History", + "description": "A resource's history, newest first: API requests that changed it, calls made for it (models, embeddings, the\ndecision model, webhooks, web tools) with tokens and cost, runs that ended, and its audit log entries.", + "operationId": "resource_history", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "table:id, like routine:3, pipeline:2, workflow:4, recording:12, account:1", + "title": "Resource" + }, + "description": "table:id, like routine:3, pipeline:2, workflow:4, recording:12, account:1" + }, + { + "name": "kind", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "enum": [ + "in", + "out", + "run", + "change" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Kind" + } + }, + { + "name": "before", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "only entries before this time (the last `at` of the page before)", + "title": "Before" + }, + "description": "only entries before this time (the last `at` of the page before)" + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 1000, + "minimum": 1, + "default": 100, + "title": "Limit" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ActivityEntry" + }, + "title": "Response Activity-Resource History" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/activity/totals": { + "get": { + "tags": [ + "activity" + ], + "summary": "Resource Totals", + "description": "What a resource's calls cost this day, week (from Monday), month (UTC) or all time: calls, tokens, USD, time.", + "operationId": "resource_totals", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "table:id; none for everything (admins)", + "title": "Resource" + }, + "description": "table:id; none for everything (admins)" + }, + { + "name": "period", + "in": "query", + "required": false, + "schema": { + "enum": [ + "day", + "week", + "month", + "all" + ], + "type": "string", + "default": "month", + "title": "Period" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ActivityTotals" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/activity/costs": { + "get": { + "tags": [ + "activity" + ], + "summary": "Resource Costs", + "description": "What each of many resources cost this period, for lists; ones you can't see are left out. `estimate` says a\nfigure is a floor (some calls had no price or token counts).", + "operationId": "resource_costs", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "description": "table:id, repeated (up to 500): every row of a list at once", + "title": "Resource" + }, + "description": "table:id, repeated (up to 500): every row of a list at once" + }, + { + "name": "period", + "in": "query", + "required": false, + "schema": { + "enum": [ + "day", + "week", + "month", + "all" + ], + "type": "string", + "default": "month", + "title": "Period" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ActivityCosts" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/activity/top": { + "get": { + "tags": [ + "activity" + ], + "summary": "Top Resources", + "description": "The resources that cost most this period (admins).", + "operationId": "top_resources", + "parameters": [ + { + "name": "table", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-z_]{1,40}$" + }, + { + "type": "null" + } + ], + "description": "only this kind of resource: routine, pipeline, workflow, space", + "title": "Table" + }, + "description": "only this kind of resource: routine, pipeline, workflow, space" + }, + { + "name": "period", + "in": "query", + "required": false, + "schema": { + "enum": [ + "day", + "week", + "month", + "all" + ], + "type": "string", + "default": "month", + "title": "Period" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 200, + "minimum": 1, + "default": 20, + "title": "Limit" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ActivityTop" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/budgets": { + "get": { + "tags": [ + "budgets" + ], + "summary": "List Budgets", + "description": "Every budget, with where it stands now.", + "operationId": "list_budgets", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BudgetStatus" + }, + "title": "Response Budgets-List Budgets" + } + } + } + } + } + }, + "put": { + "tags": [ + "budgets" + ], + "summary": "Set Budget", + "operationId": "set_budget", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "routine:, pipeline:, workflow: or space:", + "title": "Resource" + }, + "description": "routine:, pipeline:, workflow: or space:" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BudgetSet" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BudgetStatus" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "budgets" + ], + "summary": "Remove Budget", + "operationId": "remove_budget", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "routine:, pipeline:, workflow: or space:", + "title": "Resource" + }, + "description": "routine:, pipeline:, workflow: or space:" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ok" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/budgets/status": { + "get": { + "tags": [ + "budgets" + ], + "summary": "Budget Status", + "description": "Where a resource's budget stands, and what its next run will likely cost (also without a budget).", + "operationId": "budget_status", + "parameters": [ + { + "name": "resource", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "routine:, pipeline:, workflow: or space:", + "title": "Resource" + }, + "description": "routine:, pipeline:, workflow: or space:" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BudgetStatus" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/admin/fedora": { "get": { "tags": [ @@ -14156,6 +14635,59 @@ } } }, + "/api/v1/jobs/{jid}/release": { + "post": { + "tags": [ + "jobs" + ], + "summary": "Release Job", + "description": "Pick for a job held over a budget (docs/budgets.md): run it now, once, whatever its budgets say, or skip it\n(admins, who set the budgets).", + "operationId": "release_job", + "parameters": [ + { + "name": "jid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Jid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HeldRunDecision" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ok" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/jobs/{jid}/retry": { "post": { "tags": [ @@ -18819,7 +19351,7 @@ "routines" ], "summary": "Run Routine", - "description": "Run it as soon as the scheduler next looks (within half a minute), even when it is off.", + "description": "Run it as soon as the scheduler next looks (within half a minute), even when it is off. A run over one of its\nbudgets is refused (409, saying where the budget stands) unless over_budget is true (docs/budgets.md).", "operationId": "run_routine", "parameters": [ { @@ -18866,114 +19398,14 @@ } } }, - "/api/v1/routines/{rid}/runs": { - "get": { - "tags": [ - "routines" - ], - "summary": "List Runs", - "operationId": "list_runs", - "parameters": [ - { - "name": "rid", - "in": "path", - "required": true, - "schema": { - "type": "integer", - "title": "Rid" - } - }, - { - "name": "limit", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "maximum": 200, - "minimum": 1, - "default": 20, - "title": "Limit" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/RoutineRun" - }, - "title": "Response Routines-List Runs" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/api/v1/routine-runs/{run_id}": { - "get": { - "tags": [ - "routines" - ], - "summary": "Get Run", - "operationId": "get_run", - "parameters": [ - { - "name": "run_id", - "in": "path", - "required": true, - "schema": { - "type": "integer", - "title": "Run Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RoutineRun" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/api/v1/routine-runs/{run_id}/undo": { + "/api/v1/routines/runs/{run_id}/decide": { "post": { "tags": [ "routines" ], - "summary": "Undo Run", - "description": "Take back every graph change the run made.", - "operationId": "undo_run", + "summary": "Decide Held Run", + "description": "Pick for a run held over a budget: run it now, once, whatever its budgets say, or skip it.", + "operationId": "decide_held_run", "parameters": [ { "name": "run_id", @@ -18985,142 +19417,295 @@ } } ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Undone" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/api/v1/graph-changes": { - "get": { - "tags": [ - "routines" - ], - "summary": "List Graph Changes", - "description": "Changes to the entities of namespaces you can read, newest first.", - "operationId": "list_graph_changes", - "parameters": [ - { - "name": "status", - "in": "query", - "required": false, - "schema": { - "anyOf": [ - { - "enum": [ - "proposed", - "applied", - "dismissed", - "undone" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Status" - } - }, - { - "name": "run", - "in": "query", - "required": false, - "schema": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Run" - } - }, - { - "name": "limit", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "maximum": 1000, - "minimum": 1, - "default": 200, - "title": "Limit" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/GraphChange" - }, - "title": "Response Routines-List Graph Changes" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/api/v1/graph-changes/{cid}/accept": { - "post": { - "tags": [ - "routines" - ], - "summary": "Accept Graph Change", - "operationId": "accept_graph_change", - "parameters": [ - { - "name": "cid", - "in": "path", - "required": true, - "schema": { - "type": "integer", - "title": "Cid" - } - } - ], "requestBody": { "required": true, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/GraphChangeAccept" + "$ref": "#/components/schemas/HeldRunDecision" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ok" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/routines/{rid}/runs": { + "get": { + "tags": [ + "routines" + ], + "summary": "List Runs", + "operationId": "list_runs", + "parameters": [ + { + "name": "rid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Rid" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 200, + "minimum": 1, + "default": 20, + "title": "Limit" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/RoutineRun" + }, + "title": "Response Routines-List Runs" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/routine-runs/{run_id}": { + "get": { + "tags": [ + "routines" + ], + "summary": "Get Run", + "operationId": "get_run", + "parameters": [ + { + "name": "run_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Run Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RoutineRun" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/routine-runs/{run_id}/undo": { + "post": { + "tags": [ + "routines" + ], + "summary": "Undo Run", + "description": "Take back every graph change the run made.", + "operationId": "undo_run", + "parameters": [ + { + "name": "run_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Run Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Undone" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/graph-changes": { + "get": { + "tags": [ + "routines" + ], + "summary": "List Graph Changes", + "description": "Changes to the entities of namespaces you can read, newest first.", + "operationId": "list_graph_changes", + "parameters": [ + { + "name": "status", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "enum": [ + "proposed", + "applied", + "dismissed", + "undone" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Status" + } + }, + { + "name": "run", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Run" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 1000, + "minimum": 1, + "default": 200, + "title": "Limit" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GraphChange" + }, + "title": "Response Routines-List Graph Changes" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/graph-changes/{cid}/accept": { + "post": { + "tags": [ + "routines" + ], + "summary": "Accept Graph Change", + "operationId": "accept_graph_change", + "parameters": [ + { + "name": "cid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Cid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GraphChangeAccept" } } } @@ -20769,6 +21354,311 @@ "title": "AccountToken", "description": "Anyone's key, for admins." }, + "ActivityCosts": { + "properties": { + "period": { + "type": "string", + "title": "Period" + }, + "since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Since" + }, + "costs": { + "additionalProperties": { + "$ref": "#/components/schemas/ResourceSpend" + }, + "type": "object", + "title": "Costs", + "description": "by resource; ones you can't see are left out" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "period", + "costs" + ], + "title": "ActivityCosts" + }, + "ActivityEntry": { + "properties": { + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id", + "description": "the ledger row's id; none for an audit log entry" + }, + "at": { + "type": "string", + "title": "At" + }, + "kind": { + "type": "string", + "enum": [ + "in", + "out", + "run", + "change" + ], + "title": "Kind", + "description": "in: an API request; out: a call Lens made (a model, embeddings, the decision model, a webhook, a web tool); run: a job or routine run ended; change: an audit log entry" + }, + "action": { + "type": "string", + "title": "Action", + "description": "what it was: `POST /api/v1/routines/{rid}/run`, `model.chat`, `job.succeeded`, `routine.update`" + }, + "resources": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Resources", + "description": "every resource it counts for, as table:id" + }, + "actor": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Actor", + "description": "the account that made it, when a person did" + }, + "email": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Email" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "tokens_in": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens In" + }, + "tokens_out": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens Out" + }, + "cost_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost Usd", + "description": "estimated from the prices in Settings; none when the model has no price" + }, + "ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Ms" + }, + "ok": { + "type": "boolean", + "title": "Ok", + "default": true + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error", + "description": "the error's type or HTTP status; never its message" + }, + "detail": { + "title": "Detail" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "at", + "kind", + "action" + ], + "title": "ActivityEntry" + }, + "ActivityTop": { + "properties": { + "period": { + "type": "string", + "title": "Period" + }, + "since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Since" + }, + "resources": { + "items": { + "$ref": "#/components/schemas/ResourceCost" + }, + "type": "array", + "title": "Resources" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "period", + "resources" + ], + "title": "ActivityTop" + }, + "ActivityTotals": { + "properties": { + "resource": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Resource" + }, + "period": { + "type": "string", + "title": "Period" + }, + "since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Since" + }, + "calls": { + "type": "integer", + "title": "Calls" + }, + "failed": { + "type": "integer", + "title": "Failed" + }, + "tokens_in": { + "type": "integer", + "title": "Tokens In" + }, + "tokens_out": { + "type": "integer", + "title": "Tokens Out" + }, + "cost_usd": { + "type": "number", + "title": "Cost Usd", + "description": "what its calls cost (run rows repeat their calls' cost and aren't added again)" + }, + "ms": { + "type": "integer", + "title": "Ms" + }, + "unpriced": { + "type": "integer", + "title": "Unpriced", + "description": "calls that cost something but have no figure (a model with no price, a reply without token counts)", + "default": 0 + }, + "estimate": { + "type": "boolean", + "title": "Estimate", + "description": "true when unpriced calls make cost_usd a floor rather than exact", + "default": false + }, + "by_kind": { + "additionalProperties": { + "$ref": "#/components/schemas/KindTotals" + }, + "type": "object", + "title": "By Kind" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "period", + "calls", + "failed", + "tokens_in", + "tokens_out", + "cost_usd", + "ms", + "by_kind" + ], + "title": "ActivityTotals" + }, "AnswerCheck": { "properties": { "claims": { @@ -22183,6 +23073,279 @@ ], "title": "BrowseEntry" }, + "Budget": { + "properties": { + "resource": { + "type": "string", + "title": "Resource" + }, + "usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Usd" + }, + "tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens" + }, + "period": { + "type": "string", + "title": "Period" + }, + "on_over": { + "type": "string", + "title": "On Over" + }, + "warn_at": { + "type": "number", + "title": "Warn At" + }, + "note": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Note" + }, + "by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "By" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "alert": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Alert", + "description": "the last warning: {state near|over, since, at, share}" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "resource", + "period", + "on_over", + "warn_at" + ], + "title": "Budget" + }, + "BudgetSet": { + "properties": { + "usd": { + "anyOf": [ + { + "type": "number", + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Usd", + "description": "the cap in USD (from the costs in the activity ledger)" + }, + "tokens": { + "anyOf": [ + { + "type": "integer", + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Tokens", + "description": "the cap in tokens (in and out)" + }, + "period": { + "type": "string", + "enum": [ + "run", + "day", + "week", + "month" + ], + "title": "Period", + "description": "what the cap is for: one run, or a day, week or month (UTC)", + "default": "month" + }, + "on_over": { + "type": "string", + "enum": [ + "ask", + "skip", + "assistant" + ], + "title": "On Over", + "description": "a run that would go over: ask (hold it until someone picks run or skip), skip, or assistant (the decision model weighs it; it runs only when sure, else asks)", + "default": "ask" + }, + "warn_at": { + "type": "number", + "maximum": 1.0, + "minimum": 0.1, + "title": "Warn At", + "description": "the share spent at which it's marked near (and warned about)", + "default": 0.8 + }, + "note": { + "anyOf": [ + { + "type": "string", + "maxLength": 300 + }, + { + "type": "null" + } + ], + "title": "Note" + } + }, + "additionalProperties": false, + "type": "object", + "title": "BudgetSet" + }, + "BudgetStatus": { + "properties": { + "resource": { + "type": "string", + "title": "Resource" + }, + "budget": { + "anyOf": [ + { + "$ref": "#/components/schemas/Budget" + }, + { + "type": "null" + } + ] + }, + "state": { + "type": "string", + "enum": [ + "none", + "ok", + "near", + "over" + ], + "title": "State" + }, + "since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Since", + "description": "the start of the period being counted" + }, + "spent": { + "anyOf": [ + { + "$ref": "#/components/schemas/Spent" + }, + { + "type": "null" + } + ] + }, + "next": { + "$ref": "#/components/schemas/NextRun", + "description": "what one more run will likely cost: always an estimate" + }, + "left": { + "anyOf": [ + { + "additionalProperties": { + "type": "number" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Left" + }, + "share": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Share", + "description": "the largest share of a cap spent (1 is all of it)" + }, + "share_after_next": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Share After Next", + "description": "the share once the next run (estimated) is done" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "resource", + "state", + "next" + ], + "title": "BudgetStatus" + }, "ChangeAsk": { "properties": { "kind": { @@ -28747,6 +29910,21 @@ ], "title": "Heard" }, + "HeldRunDecision": { + "properties": { + "run": { + "type": "boolean", + "title": "Run", + "description": "true: run it now, once, whatever its budgets say; false: skip it" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "run" + ], + "title": "HeldRunDecision" + }, "Highlight": { "properties": { "id": { @@ -30015,6 +31193,54 @@ ], "title": "Cancel Requested" }, + "cost_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost Usd", + "description": "what its calls cost, from the activity ledger (docs/activity.md); set when it ends" + }, + "tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens" + }, + "cost_estimate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Cost Estimate", + "description": "true: some calls had no price or token counts, so cost_usd is a floor" + }, + "hold": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Hold", + "description": "held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), decided: {run, by, at} once someone picked}" + }, "log": { "anyOf": [ { @@ -30340,6 +31566,43 @@ ], "title": "JobsQueued" }, + "KindTotals": { + "properties": { + "calls": { + "type": "integer", + "title": "Calls", + "default": 0 + }, + "tokens_in": { + "type": "integer", + "title": "Tokens In", + "default": 0 + }, + "tokens_out": { + "type": "integer", + "title": "Tokens Out", + "default": 0 + }, + "cost_usd": { + "type": "number", + "title": "Cost Usd", + "default": 0 + }, + "ms": { + "type": "integer", + "title": "Ms", + "default": 0 + }, + "failed": { + "type": "integer", + "title": "Failed", + "default": 0 + } + }, + "additionalProperties": true, + "type": "object", + "title": "KindTotals" + }, "LanguageCount": { "properties": { "language": { @@ -31787,6 +33050,48 @@ "title": "NamespaceUpdate", "description": "Only the fields you send change; send ``pipeline: null`` to go back to the default pipeline." }, + "NextRun": { + "properties": { + "usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Usd" + }, + "tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens" + }, + "runs": { + "type": "integer", + "title": "Runs", + "description": "the past runs the estimate is drawn from" + }, + "basis": { + "type": "string", + "title": "Basis" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "runs", + "basis" + ], + "title": "NextRun" + }, "NodeParam": { "properties": { "name": { @@ -37917,6 +39222,35 @@ ], "title": "ResetPasswordRequest" }, + "ResourceCost": { + "properties": { + "resource": { + "type": "string", + "title": "Resource" + }, + "cost_usd": { + "type": "number", + "title": "Cost Usd" + }, + "tokens": { + "type": "integer", + "title": "Tokens" + }, + "calls": { + "type": "integer", + "title": "Calls" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "resource", + "cost_usd", + "tokens", + "calls" + ], + "title": "ResourceCost" + }, "ResourceFile": { "properties": { "id": { @@ -38129,6 +39463,41 @@ ], "title": "ResourceFiles" }, + "ResourceSpend": { + "properties": { + "cost_usd": { + "type": "number", + "title": "Cost Usd" + }, + "tokens": { + "type": "integer", + "title": "Tokens" + }, + "calls": { + "type": "integer", + "title": "Calls" + }, + "unpriced": { + "type": "integer", + "title": "Unpriced" + }, + "estimate": { + "type": "boolean", + "title": "Estimate", + "description": "true: cost_usd is a floor (unpriced calls); show it as an estimate" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "cost_usd", + "tokens", + "calls", + "unpriced", + "estimate" + ], + "title": "ResourceSpend" + }, "Reviewed": { "properties": { "applied": { @@ -38519,6 +39888,54 @@ } ], "title": "Log" + }, + "cost_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost Usd", + "description": "what its calls cost, from the activity ledger (docs/activity.md); set when it ends" + }, + "tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tokens" + }, + "cost_estimate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Cost Estimate", + "description": "true: some calls had no price or token counts, so cost_usd is a floor" + }, + "hold": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Hold", + "description": "held or skipped over a budget (docs/budgets.md): {why, resource (the budget), action (ask, skip), decided: {run, by, at} once someone picked}" } }, "additionalProperties": true, @@ -38538,6 +39955,12 @@ "title": "Propose Only", "description": "graph workflows propose every change instead of making the sure ones", "default": false + }, + "over_budget": { + "type": "boolean", + "title": "Over Budget", + "description": "run it even though it's over a budget (docs/budgets.md); without it, a run over budget is refused (409)", + "default": false } }, "additionalProperties": false, @@ -42261,6 +43684,31 @@ ], "title": "SpeechTestResult" }, + "Spent": { + "properties": { + "usd": { + "type": "number", + "title": "Usd" + }, + "tokens": { + "type": "integer", + "title": "Tokens" + }, + "estimate": { + "type": "boolean", + "title": "Estimate", + "description": "true: some calls had no figure, so usd is a floor" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "usd", + "tokens", + "estimate" + ], + "title": "Spent" + }, "Started": { "properties": { "ok": {
GraphMembersThis month Your role
{n.graph === "isolated" ? "Isolated" : "Shared"}{people.data ? count(members(n.name)) : "…"} + + Steps Version Default inThis month Updated
{onStandard.length ? onStandard.join(", ") : "—"}— —
{used.length ? used.join(", ") : "—"} + + {relative(p.updated_at)}
When Next run Last runThis month On Actions @@ -291,6 +334,9 @@ export function RoutinesPage() { {r.last_run_at && } + + + = { failed: "red", cancelled: "neutral", review: "gate", + held: "gate", }; export function statusTone(status: string | null | undefined): Tone { diff --git a/nextjs-frontend/components/workflows/workflows-page.tsx b/nextjs-frontend/components/workflows/workflows-page.tsx index 2cfb291d8..8659b48fc 100644 --- a/nextjs-frontend/components/workflows/workflows-page.tsx +++ b/nextjs-frontend/components/workflows/workflows-page.tsx @@ -3,6 +3,7 @@ import { Workflow } from "lucide-react"; import Link from "next/link"; +import { CostCell, useBudgets, useCosts } from "@/components/costs/costs"; import { CatalogHeader, useWorkflowCatalog } from "@/components/pipelines/catalog-header"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -14,6 +15,8 @@ import { relative } from "@/lib/format"; export function WorkflowsPage() { const catalog = useWorkflowCatalog(); const list = catalog.data?.workflows ?? []; + const costs = useCosts(list.map((w) => `workflow:${w.id}`)); + const budgets = useBudgets(); return (
@@ -36,6 +39,7 @@ export function WorkflowsPage() {
Name Version Run byThis month Updated
{w.scope === "graph" ? "Routines" : w.pipelines?.length ? w.pipelines.join(", ") : "—"} + + {relative(w.updated_at)}