diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index d2137b2a2..0187595bb 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -97,7 +97,8 @@ jobs: - name: Answer it and install run: | jq -n --arg dir "$LENS_DIR" --arg ref "$LENS_REF" '{backend: "docker", target: "lean", dir: $dir, ref: $ref, - settings: {llm: {base_url: "http://localhost:11434/v1", model: "llama3", api_key: "sk-ci"}}}' >answers.json + settings: {llm: {base_url: "http://localhost:11434/v1", model: "llama3", api_key: "sk-ci"}, + budget: {hard_usd: 25, period: "month", price_in: 3, price_out: 15}}}' >answers.json curl -fsS -b jar -H 'X-Lens-Installer: 1' -H 'Content-Type: application/json' -d @answers.json \ http://127.0.0.1:3990/api/install since=0 @@ -120,7 +121,7 @@ jobs: curl -fsS -o /dev/null "$url" curl -fsS "$base/api/v1/auth/status" | grep -q '"setup_required":true' cd "$LENS_DIR" - docker compose -f docker-compose.prod.yml logs backend | grep "Applied the installer's settings: llm" + docker compose -f docker-compose.prod.yml logs backend | grep "Applied the installer's settings: llm, budget" # saved in Lens: the key doesn't stay in .env ! grep -q '^LENS_SETUP_SETTINGS=' .env diff --git a/CHANGELOG.md b/CHANGELOG.md index 5501cca3c..8f4df322c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **A spending limit from the start.** The setup wizard has a Budget step, and the web installer asks for one with a + cloud provider: a scale for a hard limit on everything Lens spends on AI, with a soft limit at 80% that warns. At + the hard limit runs wait (or are skipped) and paid model calls stop until it goes up; models on your own computer + keep working. The model's price comes with it, so the spending is counted. It's the new `all` budget, changeable in + Settings › Budgets ([Budgets](docs/budgets.md#one-budget-for-everything)). The web installer now asks only the + essentials: the folder, port and network under **Advanced**, the optional extras under **More options** on the + review. + - **The graph as files.** Graph nodes are folders, nested along its links: collections inside collections, topics inside their broader topics, entities by type. Each folder holds its assets' files, with a `.json` sidecar of metadata and a `.txt` of the text; everyone sees what the web app shows them. Browse and read it through the API diff --git a/ci/clicks/journeys.mjs b/ci/clicks/journeys.mjs index 25045c193..c44ade7fa 100644 --- a/ci/clicks/journeys.mjs +++ b/ci/clicks/journeys.mjs @@ -46,8 +46,6 @@ export const JOURNEYS = [ if (!(await found.evaluate((el) => el.classList.contains("on")))) await meter.click(found, "pick the model server found on this computer"); await meter.click(page.locator("#next"), "Continue (AI model)"); - await page.getByRole("heading", { name: "Extras" }).waitFor(); - await meter.click(page.locator("#next"), "Continue (extras)"); await page.getByRole("heading", { name: "Ready to install" }).waitFor(); await meter.click(page.locator("#next"), "Install Lens"); } diff --git a/docs/budgets.md b/docs/budgets.md index 66071284c..1d4028d31 100644 --- a/docs/budgets.md +++ b/docs/budgets.md @@ -22,6 +22,26 @@ PUT /api/v1/budgets?resource=routine:3 (also for a resource without a budget, with its estimated next run), and `DELETE /api/v1/budgets?resource=…` removes one. +## One budget for everything + +Besides budgets for single resources, `all` caps everything Lens spends: every model call, run and job, whatever it is +for. The setup wizard and the web installer ask for it after a cloud model (one on your own machine costs nothing), as a +scale for the hard limit with the soft limit at 80% of it; Settings › Budgets changes it. + +```http +PUT /api/v1/setup/budget +{"hard_usd": 25, "soft_usd": 20, "period": "month", "on_over": "ask", "price_in": 3, "price_out": 15} +``` + +- **Soft limit**: past it, the budget is `near` and warns, as `warn_at` does (it is saved as `warn_at` = soft ÷ hard). +- **Hard limit**: at it, routines and jobs are held (or skipped, with `on_over: "skip"`) as any budget's are, and model + calls that cost money stop with the reason until the limit goes up or the period turns. Models without a price + (one on your own computer) keep working, unless the budget also caps `tokens`. +- `price_in`, `price_out`: the chat model's price in USD per million tokens (`ai.price_in`, `ai.price_out`), so + what it spends is counted. Without a price a budget in USD counts nothing for that model. +- `hard_usd: null` removes the budget. `GET /api/v1/setup` shows it under `budget`. `PUT /api/v1/budgets?resource=all` + sets it like any other budget, with `tokens` too. + ## Before a run Before a routine runs and before a job's first step, Lens looks at the budgets of everything the run counts for: the diff --git a/docs/configuration.md b/docs/configuration.md index b0a4f1640..863a10982 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -31,7 +31,7 @@ with AES-GCM and are write-only: the API reports whether one is set, never its v | `LENS_ADMIN_PASSWORD` | | give that admin a password instead (passwords stay on) | | `LENS_NAMESPACE` | | create the first namespace at startup, while there is none | | `LENS_LLM_BASE_URL` / `LENS_LLM_MODEL` / `LENS_LLM_API_KEY` / `LENS_LLM_VISION_MODEL` | | the model provider; wins over Settings, which show these locked | -| `LENS_SETUP_SETTINGS` | | settings to start with, as JSON `{"section": {"key": value}}` (or that JSON base64url-encoded): saved once at startup as if made in the app, so unlike the variables above they stay editable there. The [web installer](get-started.md#the-web-installer) writes its answers here | +| `LENS_SETUP_SETTINGS` | | settings to start with, as JSON `{"section": {"key": value}}` (or that JSON base64url-encoded): saved once at startup as if made in the app, so unlike the variables above they stay editable there. The [web installer](get-started.md#the-web-installer) writes its answers here; a `budget` section (`hard_usd`, `soft_usd`, `period`, `on_over`, `price_in`, `price_out`) sets [the overall budget](budgets.md#one-budget-for-everything) | | `LENS_SETUP_WIZARD` | | `off`: never show the setup wizard | | `LENS_TELEMETRY` / `LENS_TELEMETRY_ENDPOINT` / `LENS_TELEMETRY_HEADERS` | off | opt-in OpenTelemetry traces and metrics, sent only to this OTLP/HTTP endpoint ([Telemetry](telemetry.md)); `LENS_TELEMETRY=off` keeps it off whatever Settings say | | `FRONTEND_URL` | `http://localhost:3000` | the web app's address as people use it: links in emails, and where apps send people to sign in ([OAuth](authentication.md#oauth)) | diff --git a/docs/get-started.md b/docs/get-started.md index bd7667a02..8205f7d89 100644 --- a/docs/get-started.md +++ b/docs/get-started.md @@ -26,21 +26,26 @@ On a first install run at a terminal (on Linux or a Mac with Python 3.8 or later above opens an installer in your browser instead of asking anything in the terminal: 1. **This computer**: the system, memory, free disk, whether Docker is there and running, and whether microsandbox can - run. On a first install, **Install with the defaults** skips the next steps: it picks a model server it finds on - the computer, if there is one, and installs straight away. **Choose options** goes through them. -2. **How it runs**: Docker or [microsandbox](microsandbox.md), the full or lean edition (lean is picked on a computer - with less than 4 GB of memory, such as a Raspberry Pi), the folder, the port, and whether other devices on the - network may reach it. It lists what it will install; when that needs administrator rights, it asks for your - password here, keeps it in memory for the install and hands it to `sudo` (it is never written down). + run. On a first install, **Install with the defaults** installs right away (Docker, the edition that suits the + computer, the model server found on it, if any) and shows its progress; **Choose options** goes through the steps + below, as it does when installing needs your password. +2. **How it runs**: Docker or [microsandbox](microsandbox.md) and the full or lean edition (lean is picked on a computer + with less than 4 GB of memory, such as a Raspberry Pi). It lists what it will install; when that needs + administrator rights, it asks for your password here, keeps it in memory for the install and hands it to `sudo` (it + is never written down). The folder, the port, whether other devices on the network may reach it and the version + wait under **Advanced**. 3. **AI model**: model servers already running on the computer (Ollama, LM Studio, llama.cpp, vLLM, LocalAI) with their models, or a provider (OpenAI, Anthropic, Google Gemini, OpenRouter, Groq, Mistral or any OpenAI-compatible server) - with its API key; **Test** lists the models. It can be skipped and set in Lens later. -4. **Extras**, all optional: the first namespace, search by meaning (an embeddings server), the typesafe.ai decision - model's key, cloud transcription and its key, email (SMTP), remote access through a Cloudflare tunnel, anytopdf, - and telemetry. -5. **Review**, then **Install**: it runs `install.sh` with those answers and shows its progress (and the full output - under **Details**). If something fails, it says what, and **Try again** runs it again (what's done stays done). -6. **Open Lens** goes to the setup page with the setup code filled in, where you create the admin account with a + with its API key; **Test** lists the models. For a provider, a scale sets a monthly spending limit (none unless you + pick one) with the model's price, so Lens can count what it spends; **More** sets a soft limit (80% unless changed) + and a day or week instead ([One budget for everything](budgets.md#one-budget-for-everything)). It can all be + skipped and set in Lens later. +4. **Review**, then **Install**: **More options** under the review holds what's optional, each with its default: the + first namespace, search by meaning (an embeddings server), the typesafe.ai decision model's key, cloud + transcription and its key, email (SMTP), remote access through a Cloudflare tunnel, anytopdf, and telemetry. + **Install** runs `install.sh` with those answers and shows its progress (and the full output under **Details**). If + something fails, it says what, and **Try again** runs it again (what's done stays done). +5. **Open Lens** goes to the setup page with the setup code filled in, where you create the admin account with a passkey. The answers that are settings go to Lens as `LENS_SETUP_SETTINGS`, which it saves once at its first start as if they @@ -120,10 +125,10 @@ On first start the API log prints a setup code, and a link that fills it in (`ma ``` No accounts yet. Create the first admin in the web app with setup code: … (or open http://localhost:3000/setup?code=…, which fills it in) ``` - Open the link (or the web app, and enter the code) and create the admin account. A short wizard then asks for the first -namespace, the model provider, storage, apps and AI, and telemetry. Once a model is set, **Finish with defaults** skips -the rest; change any of it later in Settings ([Configuration](configuration.md#first-run-setup) lists the `.env` values -that answer it instead). +Open the link (or the web app, and enter the code) and create the admin account. A short wizard then asks for the first +namespace, the model provider, a [budget](budgets.md#one-budget-for-everything) (for a cloud model), storage, apps and +AI, and telemetry. Once a model is set, **Finish with defaults** skips the rest; change any of it later in Settings +([Configuration](configuration.md#first-run-setup) lists the `.env` values that answer it instead). After the first sign-in, a short welcome card points to the two places most people come back for: the assistant, which you can ask straight from the card, and **Reports**, with the charts of what is in the archive. Asking, opening diff --git a/fastapi_backend/app/api/v1/routes/budgets.py b/fastapi_backend/app/api/v1/routes/budgets.py index 05796e60a..42f3759ad 100644 --- a/fastapi_backend/app/api/v1/routes/budgets.py +++ b/fastapi_backend/app/api/v1/routes/budgets.py @@ -13,7 +13,7 @@ from app.schemas.common import Ok router = APIRouter(tags=["budgets"]) -RESOURCE = Query(description="routine:, pipeline:, workflow: or space:") +RESOURCE = Query(description="all (everything Lens spends), routine:, pipeline:, workflow: or space:") @router.get("/budgets") @@ -46,7 +46,7 @@ def budget_meter( resource = f"space:{acl.namespace(ns)}" ref = resource elif user.admin: - resource, ref = None, budgets.GLOBAL + resource, ref = None, budgets.ALL else: raise HTTPException(403, "only admins see what all of Lens spent; pick a namespace") out = budgets.meter(db, resource, ref, period, detail) diff --git a/fastapi_backend/app/api/v1/routes/setup.py b/fastapi_backend/app/api/v1/routes/setup.py index cf8f085ba..f60d5d0ba 100644 --- a/fastapi_backend/app/api/v1/routes/setup.py +++ b/fastapi_backend/app/api/v1/routes/setup.py @@ -13,6 +13,7 @@ from app.domain import auth, setup from app.schemas.setup import ( LocalModelServer, + SetupBudget, SetupFinish, SetupLlm, SetupNamespace, @@ -96,6 +97,20 @@ def save_telemetry(body: SetupTelemetry, user: AdminWriter, request: Request, db return SetupSaved(saved=saved) +@router.put("/budget") +def save_budget(body: SetupBudget, user: AdminWriter, request: Request, db: Db) -> SetupSaved: + """The overall AI budget: a soft limit that warns and a hard limit where runs wait and paid model calls stop + (docs/budgets.md#one-budget-for-everything), with the chat model's price so its spending is counted. No hard + limit: no budget.""" + try: + saved = setup.save_budget(db, request.app.state.archive.base, user=user.email, **body.model_dump()) + except ValueError as e: + raise HTTPException(400, str(e)) from None + if saved: + auth.audit(db, user.as_audit(), "budget.set", "all", body.model_dump(exclude_none=True)) + return SetupSaved(saved=saved) + + @router.put("/oauth") def save_oauth(body: SetupOAuth, user: AdminWriter, request: Request, db: Db) -> SetupSaved: """Let apps and AI assistants (MCP clients such as Claude or Cursor) sign people in with their Lens account, or diff --git a/fastapi_backend/app/domain/activity.py b/fastapi_backend/app/domain/activity.py index 3accd5e91..026bbb99e 100644 --- a/fastapi_backend/app/domain/activity.py +++ b/fastapi_backend/app/domain/activity.py @@ -183,6 +183,10 @@ def record(kind, action, refs=(), cfg=None, db=None, **fields): return None +class OverBudget(RuntimeError): + """A call refused because the overall budget's hard limit is reached (budgets.hard_stop).""" + + class Call: """A call out, timed: `with activity.call("embeddings", model=m) as c: ...; c.usage(tin, tout)`. An exception that leaves the block marks the row failed with its type.""" @@ -193,6 +197,16 @@ def __init__(self, action, cfg=None, model=None, refs=(), detail=None, price=Non self.unpriced = False # it costs something, but the figure is missing (see model_cost) self.t0 = time.monotonic() self.done = False + if model: # a call to a model: the overall budget's hard limit (budgets.py) stops one that would cost money + sc = _scope.get() or {} + db = sc.get("db") or _default["db"] + if db is not None: + from . import budgets + + why = budgets.hard_stop(db, cfg if cfg is not None else sc.get("cfg"), model) + if why: + self.done = True # nothing was called: no ledger row + raise OverBudget(why) def usage(self, tokens_in=None, tokens_out=None, cost_usd=None): if isinstance(tokens_in, int): diff --git a/fastapi_backend/app/domain/budgets.py b/fastapi_backend/app/domain/budgets.py index 5b7cac50b..805b72ad3 100644 --- a/fastapi_backend/app/domain/budgets.py +++ b/fastapi_backend/app/domain/budgets.py @@ -12,6 +12,11 @@ Doing nothing changes nothing: a held run stays held. A run someone let through runs. `warn_at` (0.8) marks a budget `near` once that share is spent, for the periodic check (sweep) to warn about. + +One budget, `all`, covers everything Lens spends (the setup wizard asks for it): its `warn_at` share is the soft limit, +which warns, and its cap the hard limit. At the hard limit every run waits as above, and calls to models that cost +money are refused (activity.Call asks `hard_stop`); models without a price (local ones) keep working, unless the cap +is in tokens. """ from __future__ import annotations @@ -28,8 +33,9 @@ PERIODS = ("run", "day", "week", "month") ON_OVER = ("ask", "skip", "assistant") TABLES = ("routine", "pipeline", "workflow", "space") -GLOBAL = "all" # the budget for all of Lens (set in setup); the meter measures "all namespaces" against it -REF = re.compile(r"^(routine|pipeline|workflow|space):(\d+)$") +REF = re.compile(r"^(?:(routine|pipeline|workflow|space):(\d+)|all)$") +ALL = "all" # everything Lens spends (set in setup); the meter measures all of Lens against it +HARD_SECONDS = 30 # how long hard_stop trusts what it last found FIELDS = "resource, usd, tokens, period, on_over, warn_at, note, by, updated_at, alert" LOOKBACK_DAYS = 30 # the past runs an estimate is drawn from SWEEP_SECONDS = 300 # how often the scheduler looks at every budget @@ -42,7 +48,7 @@ def _key(resource): def check_ref(resource): if not isinstance(resource, str) or not REF.match(resource): - raise ValueError("a budget is for routine:, pipeline:, workflow: or space:") + raise ValueError("a budget is for all, routine:, pipeline:, workflow: or space:") return resource @@ -81,6 +87,7 @@ def put(db, resource, usd=None, tokens=None, period="month", on_over="ask", warn } ) db.q("UPSERT $r CONTENT $d", r=_key(resource), d=row) + _hard.clear() return get(db, resource) @@ -91,6 +98,7 @@ def get(db, resource): def remove(db, resource): check_ref(resource) db.q("DELETE $r", r=_key(resource)) + _hard.clear() def all_budgets(db): @@ -98,9 +106,31 @@ def all_budgets(db): # ---------- where it stands ---------- +def _spent(db, resource, since): + """(USD, tokens, whether the USD is an estimate) since a time: one resource's calls, or every call for `all`.""" + if resource == ALL: + p, q = ( + {}, + "SELECT math::sum(cost_usd ?? 0) AS usd, math::sum((tokens_in ?? 0) + (tokens_out ?? 0)) AS tokens, count(unpriced = true) AS unpriced FROM activity WHERE kind != 'run'", + ) + if since: + q += " AND at >= $since" + p["since"] = since + row = db.one(q + " GROUP ALL", **p) or {} + return round(float(row.get("usd") or 0), 6), int(row.get("tokens") or 0), bool(row.get("unpriced")) + c = activity.costs(db, [resource], since)[resource] + return c["cost_usd"], c["tokens"], c["estimate"] + + def _runs(db, resource, since): """How many runs this resource had since a time: a routine's runs, or the jobs of a pipeline, workflow or namespace.""" m = REF.match(resource) + if resource == ALL: + routine_runs = len(db.values("SELECT VALUE id FROM routine_run WHERE started_at >= $s", s=since)) + job_runs = db.values( + "SELECT VALUE id FROM activity WHERE kind = 'run' AND at >= $s AND string::starts_with(action, 'job.')", s=since + ) + return routine_runs + len(job_runs) if m and m.group(1) == "routine": return len(db.values("SELECT VALUE id FROM routine_run WHERE routine = $r AND started_at >= $s", r=int(m.group(2)), s=since)) return len( @@ -120,7 +150,7 @@ def estimate(db, resource, now=None): runs = _runs(db, resource, since) if not runs: return {"usd": None, "tokens": None, "runs": 0, "basis": "no runs in the last 30 days to go on"} - usd, tokens = activity.spent(db, resource, since) + usd, tokens = _spent(db, resource, since)[:2] return { "usd": round(usd / runs, 6), "tokens": int(tokens / runs), @@ -141,8 +171,8 @@ def status(db, resource, budget=None, now=None): if period == "run": spent = {"usd": 0.0, "tokens": 0, "estimate": False} else: - c = activity.costs(db, [resource], since)[resource] - spent = {"usd": c["cost_usd"], "tokens": c["tokens"], "estimate": c["estimate"]} + usd, tokens, est = _spent(db, resource, since) + spent = {"usd": usd, "tokens": tokens, "estimate": est} left, shares, after = {}, [], [] for cap, k in (("usd", "usd"), ("tokens", "tokens")): if budget.get(cap) is None: @@ -172,7 +202,7 @@ def status(db, resource, budget=None, now=None): def check(db, cfg, resources, now=None): """Whether a run that counts for these resources may go ahead. Returns {go, action, why, resource, status}: go True (run), or action "ask" (hold it for a person), "skip". The first budget that stops it decides.""" - for res in dict.fromkeys(r for r in resources if r and REF.match(r)): + for res in dict.fromkeys([ALL, *(r for r in resources if r and REF.match(r))]): b = get(db, res) if not b: continue @@ -238,6 +268,40 @@ def _weigh(cfg, res, st, why): return d["choice"] if decide.sure(cfg, d) else "ask" +# ---------- the hard limit, for calls ---------- +_hard: dict = {} + + +def hard_stop(db, cfg, model=None): + """Why a call out must not go ahead, or None: the `all` budget is used up and the call would cost money (a model + with a price), or its cap is in tokens (any call counts). Looked up at most every HARD_SECONDS.""" + now = time.monotonic() + if _hard.get("db") != id(db) or now - _hard.get("at", 0) > HARD_SECONDS: + try: + b = get(db, ALL) + st = status(db, ALL, b) if b and b.get("period") != "run" else None + except Exception as e: # noqa: BLE001 - a budget that can't be read stops nothing + log.info("budgets: couldn't check the overall budget: %s", e) + b = st = None + _hard.update( + db=id(db), + at=now, + budget=b, + over=bool(st and st["state"] == "over"), + why=_why(ALL, st, True) if st and st["state"] == "over" else None, + ) + if not _hard.get("over"): + return None + b = _hard["budget"] + if b.get("tokens") is None: # a cap in money: only calls that cost money stop + from . import telemetry + + p = telemetry.prices(cfg or {}).get(model) if model else None + if not isinstance(p, dict) or (p.get("unit") or "tokens") == "off": + return None + return _hard["why"].replace(f"{ALL} has spent", "Lens has spent") + " Raise the limit in Settings › Budgets to go on." + + # ---------- the periodic check ---------- def sweep(db, cfg, now=None, every=None): """Look at every budget (at most once `every` seconds, when given): one that has crossed warn_at or its cap since it was last looked at gets an alert (on the diff --git a/fastapi_backend/app/domain/setup.py b/fastapi_backend/app/domain/setup.py index a4ab3255a..fdab9a739 100644 --- a/fastapi_backend/app/domain/setup.py +++ b/fastapi_backend/app/domain/setup.py @@ -30,7 +30,7 @@ import urllib.request from concurrent.futures import ThreadPoolExecutor -from . import auth, settings, sources, store +from . import auth, budgets, settings, sources, store R = store.R WIZARD = ("setup", "wizard") @@ -107,6 +107,14 @@ def apply_settings(db, cfg): return [] saved = [] for section, values in wanted.items(): + if section == "budget": # the overall budget, not a settings section + try: + save_budget(db, cfg, user="install", **{k: v for k, v in values.items() if k in BUDGET_FIELDS}) + except (ValueError, TypeError) as e: + log.error("LENS_SETUP_SETTINGS budget: %s; set it in Settings › Budgets instead", e) + else: + saved.append(section) + continue keep = {k: v for k, v in values.items() if k not in settings.locked(section)} # the environment wins anyway if not keep: continue @@ -179,6 +187,7 @@ def view(db, cfg): "endpoint": shown["telemetry"].get("endpoint"), "locked": [k for k in settings.locked("telemetry") if k in ("enabled", "endpoint")], }, + "budget": budget_view(db, shown), "oauth": { "enabled": bool(shown["tokens"].get("oauth_enabled", True)), "access_minutes": int(shown["tokens"]["oauth_access_minutes"]), @@ -206,6 +215,49 @@ def save_llm(db, cfg, values, user=None): return sorted(keep) +BUDGET_FIELDS = ("hard_usd", "soft_usd", "period", "on_over", "price_in", "price_out") + + +def budget_view(db, shown): + b = budgets.get(db, budgets.ALL) or {} + hard = b.get("usd") + return { + "hard_usd": hard, + "soft_usd": round(hard * (b.get("warn_at") or 0.8), 2) if hard is not None else None, + "period": b.get("period") or "month", + "on_over": b.get("on_over") or "ask", + "price_in": shown["ai"].get("price_in"), + "price_out": shown["ai"].get("price_out"), + "model": shown["llm"].get("model"), + } + + +def save_budget(db, cfg, hard_usd=None, soft_usd=None, period="month", on_over="ask", price_in=None, price_out=None, user=None): + """The overall AI budget (budgets.ALL): a soft limit that warns and a hard limit where runs wait (or are skipped) + and paid model calls stop, per day, week or month. No hard limit: no budget. With the chat model's price (USD per + million tokens), so the money it spends is counted. Returns what was saved.""" + saved = [] + if price_in is not None or price_out is not None: + if price_in is None or price_out is None: + raise ValueError("give the model's price for both input and output tokens") + settings.save(db, cfg, "ai", {"price_in": float(price_in), "price_out": float(price_out)}, user) + saved.append("price") + if hard_usd is None: + if budgets.get(db, budgets.ALL): + budgets.remove(db, budgets.ALL) + saved.append("budget") + return saved + hard = float(hard_usd) + if hard < 0 or (soft_usd is not None and float(soft_usd) < 0): + raise ValueError("limits are 0 or more") + if soft_usd is not None and float(soft_usd) > hard: + raise ValueError("the soft limit is at most the hard limit") + warn_at = 0.8 if soft_usd is None else (float(soft_usd) / hard if hard else 1.0) + budgets.put(db, budgets.ALL, usd=hard, period=period, on_over=on_over, warn_at=min(1.0, max(0.1, warn_at)), by=user) + saved.append("budget") + return saved + + def save_telemetry(db, cfg, enabled, endpoint, user=None): """Opt in to telemetry (or stay out), leaving out what the environment sets. On needs an endpoint.""" locked = settings.locked("telemetry") diff --git a/fastapi_backend/app/schemas/setup.py b/fastapi_backend/app/schemas/setup.py index 4f9498f74..ec30f3b5f 100644 --- a/fastapi_backend/app/schemas/setup.py +++ b/fastapi_backend/app/schemas/setup.py @@ -44,6 +44,19 @@ class SetupOAuthView(ResponseModel): refresh_days: int +class SetupBudgetView(ResponseModel): + # the overall budget (budgets.py, resource `all`): none set is no limit + hard_usd: float | None = None + soft_usd: float | None = None + period: str = "month" + on_over: str = "ask" + # the chat model's price, USD per million tokens in and out (ai.price_in, ai.price_out): a budget in money + # counts only what has a price + price_in: float | None = None + price_out: float | None = None + model: str | None = None + + class SetupView(ResponseModel): pending: bool admin: SetupAdmin @@ -52,6 +65,7 @@ class SetupView(ResponseModel): storage: SetupStorageView telemetry: SetupTelemetryView oauth: SetupOAuthView + budget: SetupBudgetView class SetupNamespace(RequestModel): @@ -94,6 +108,18 @@ class SetupOAuth(RequestModel): refresh_days: int | None = None +class SetupBudget(RequestModel): + # the hard limit in USD per period; none: no limit (the overall budget is removed) + hard_usd: float | None = Field(default=None, ge=0, le=1_000_000) + # the soft limit, which warns; none: 80% of the hard limit + soft_usd: float | None = Field(default=None, ge=0, le=1_000_000) + period: Literal["day", "week", "month"] = "month" + # at the hard limit, runs wait for a person (ask) or are skipped (skip); paid model calls stop either way + on_over: Literal["ask", "skip"] = "ask" + price_in: float | None = Field(default=None, ge=0, le=100_000) + price_out: float | None = Field(default=None, ge=0, le=100_000) + + class SetupFinish(RequestModel): skipped: bool = False diff --git a/fastapi_backend/tests/api/test_budgets.py b/fastapi_backend/tests/api/test_budgets.py index 9007caf61..4763d969b 100644 --- a/fastapi_backend/tests/api/test_budgets.py +++ b/fastapi_backend/tests/api/test_budgets.py @@ -171,6 +171,33 @@ def test_the_periodic_check_warns_once_per_period(db, cfg): assert [r["action"] for r in activity.history(db, "workflow:1", kind="run")] == ["budget.over", "budget.near"] +def test_one_budget_for_everything_with_a_soft_and_a_hard_limit(client, db, cfg, h): + """`all`: every call counts; the soft limit warns, and at the hard limit runs wait and paid model calls stop.""" + st = client.put("/api/v1/budgets", headers=h, params={"resource": "all"}, json={"usd": 1, "warn_at": 0.5}).json() + assert st["state"] == "ok" and st["budget"]["resource"] == "all" + spend(db, "routine:7", usd=0.3) + spend(db, "space:2", usd=0.3) # different resources, the same overall budget + assert budgets.status(db, "all")["state"] == "near" + assert [a["resource"] for a in budgets.sweep(db, cfg)] == ["all"] + priced = {**cfg, "ai": {**cfg["ai"], "price_in": 1.0, "price_out": 2.0}, "llm": {**cfg["llm"], "model": "m"}} + assert budgets.hard_stop(db, priced, "m") is None # under the hard limit: calls go ahead + + spend(db, "routine:7", usd=0.5) + verdict = budgets.check(db, cfg, ["pipeline:3"]) # a run of anything waits now + assert not verdict["go"] and verdict["resource"] == "all" and verdict["action"] == "ask" + budgets._hard.clear() + why = budgets.hard_stop(db, priced, "m") + assert why.startswith("Lens has spent $1.10 of $1.00 this month") + assert budgets.hard_stop(db, priced, "local-model") is None # no price: costs nothing, so it still works + activity.bind(db) + with pytest.raises(activity.OverBudget, match="Raise the limit"): + activity.call("model.chat", priced, "m") + activity.call("model.chat", priced, "local-model").end() # not refused + + client.delete("/api/v1/budgets", headers=h, params={"resource": "all"}) + assert budgets.hard_stop(db, priced, "m") is None and budgets.check(db, cfg, ["pipeline:3"])["go"] + + def test_the_meter_follows_the_namespace_and_its_budget(client, db, h): sid = store.ns_id(db, "pods") other = store.ns_id(db, "news") @@ -195,6 +222,9 @@ def test_the_meter_follows_the_namespace_and_its_budget(client, db, h): assert m["resource"] is None and m["spent"]["usd"] == 2.3 and m["budget"] is None assert [x["resource"] for x in m["by_namespace"]] == [f"space:{other}", f"space:{sid}"] assert m["by_namespace"][0]["name"] == "news" + budgets.put(db, "all", usd=5) # the overall budget from setup + m = client.get("/api/v1/budgets/meter", headers=h).json() + assert m["budget"]["budget"]["resource"] == "all" and m["budget"]["left"]["usd"] == 2.7 make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) e = login(client, "ed@x.io", "editor password 1") diff --git a/fastapi_backend/tests/api/test_setup.py b/fastapi_backend/tests/api/test_setup.py index 76a67c878..4157e524a 100644 --- a/fastapi_backend/tests/api/test_setup.py +++ b/fastapi_backend/tests/api/test_setup.py @@ -202,6 +202,7 @@ def test_installer_settings_are_saved_once_and_stay_editable(fresh, monkeypatch, seed = { "llm": {"base_url": "http://ollama:11434/v1", "model": "llama3", "api_key": "sk-seed"}, "decisions": {"api_key": "ts-seed"}, + "budget": {"hard_usd": 25, "period": "month", "price_in": 3, "price_out": 15}, # the overall budget "nope": {"x": 1}, # not a section: logged and left out, the rest still applies } monkeypatch.setenv("LENS_SETUP_SETTINGS", base64.urlsafe_b64encode(json.dumps(seed).encode()).decode().rstrip("=")) @@ -216,6 +217,15 @@ def test_installer_settings_are_saved_once_and_stay_editable(fresh, monkeypatch, h = login(client, "ops@x.io", "ops password 123") view = client.get("/api/v1/setup", headers=h).json() assert view["llm"]["locked"] == [] and view["llm"]["values"]["api_key"] == {"secret": True, "set": True} + assert view["budget"] == { + "hard_usd": 25.0, + "soft_usd": 20.0, + "period": "month", + "on_over": "ask", + "price_in": 3.0, + "price_out": 15.0, + "model": "llama3", + } assert "sk-seed" not in client.get("/api/v1/settings", headers=h).text # changed in the app, then the next start: the installer's value doesn't come back diff --git a/installer/lens_installer.py b/installer/lens_installer.py index 3ef6f9756..02d4d0277 100755 --- a/installer/lens_installer.py +++ b/installer/lens_installer.py @@ -65,6 +65,8 @@ "tunnel": {"mode", "token", "hostname", "api_token"}, "documents": {"anytopdf_url", "anytopdf_token"}, "telemetry": {"enabled", "endpoint"}, + # not a settings section: Lens's overall budget (Settings › Budgets), with the chat model's price + "budget": {"hard_usd", "soft_usd", "period", "on_over", "price_in", "price_out"}, } # Things the installer can do for an existing install besides installing again, by id: {"label", "detail", "run": # fn(installer) -> None}. Empty for now: the upgrade and backup work adds its actions here, and the page shows a diff --git a/installer/page.html b/installer/page.html index dd2ec9d80..f3c343367 100644 --- a/installer/page.html +++ b/installer/page.html @@ -101,7 +101,10 @@

Install Lens

diff --git a/installer/test_lens_installer.py b/installer/test_lens_installer.py index 877961517..068dbe384 100644 --- a/installer/test_lens_installer.py +++ b/installer/test_lens_installer.py @@ -50,6 +50,7 @@ def test_settings_are_kept_to_what_lens_takes(self): "server": {"allowed_hosts": "*"}, # not something the installer sets "mail": {"port": 587, "password": {"nested": 1}}, "embeddings": {"enabled": False}, + "budget": {"hard_usd": 25, "period": "month", "price_in": 3.0, "price_out": 15, "usd": 1}, } ) self.assertEqual( @@ -58,6 +59,7 @@ def test_settings_are_kept_to_what_lens_takes(self): "llm": {"base_url": "http://host.docker.internal:11434/v1", "model": "llama3"}, "mail": {"port": 587}, "embeddings": {"enabled": False}, + "budget": {"hard_usd": 25, "period": "month", "price_in": 3.0, "price_out": 15}, }, ) diff --git a/nextjs-frontend/__tests__/setupWizard.test.tsx b/nextjs-frontend/__tests__/setupWizard.test.tsx index 0fcb1de53..26b4a4d40 100644 --- a/nextjs-frontend/__tests__/setupWizard.test.tsx +++ b/nextjs-frontend/__tests__/setupWizard.test.tsx @@ -4,7 +4,7 @@ import "@testing-library/jest-dom"; import type { ReactNode } from "react"; import { Admin, Setup, type SetupView } from "@/app/openapi-client"; -import { SetupWizard } from "@/components/setup/setup-wizard"; +import { SetupWizard, isLocalModel } from "@/components/setup/setup-wizard"; import { TooltipProvider } from "@/components/ui/tooltip"; jest.mock("@/app/openapi-client", () => ({ @@ -16,6 +16,7 @@ jest.mock("@/app/openapi-client", () => ({ saveStorage: jest.fn(), saveTelemetry: jest.fn(), saveOauth: jest.fn(), + saveBudget: jest.fn(), finish: jest.fn(), }, Admin: { testLlm: jest.fn(), testTelemetry: jest.fn() }, @@ -43,6 +44,15 @@ const VIEW = (over: Partial = {}): SetupView => ({ }, telemetry: { enabled: false, endpoint: null, locked: [] }, oauth: { enabled: true, access_minutes: 60, refresh_days: 30 }, + budget: { + hard_usd: null, + soft_usd: null, + period: "month", + on_over: "ask", + price_in: null, + price_out: null, + model: null, + }, ...over, }); @@ -125,6 +135,10 @@ describe("SetupWizard", () => { api_key: null, }); + // a model on this machine costs nothing: no budget step + expect(await screen.findByLabelText(/Largest upload/)).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /Budget/ })).not.toBeInTheDocument(); + fireEvent.change(await screen.findByLabelText(/Largest upload/), { target: { value: "2048" } }); fireEvent.click(screen.getByRole("button", { name: "Save and continue" })); await waitFor(() => expect(Setup.saveStorage).toHaveBeenCalled()); @@ -147,6 +161,44 @@ describe("SetupWizard", () => { await waitFor(() => expect(replace).toHaveBeenCalledWith("/")); }); + it("asks for a budget after a cloud model: one scale for the hard limit, with the model's price", async () => { + m(Setup.getSetup).mockImplementation(() => + ok( + VIEW({ + namespace: { existing: ["media"], locked: false }, + llm: { + values: { base_url: "https://api.openai.com/v1", model: "gpt-x", api_key: { secret: true, set: true } }, + locked: [], + }, + }), + ), + ); + m(Setup.saveLlm).mockImplementation(() => ok({ ok: true })); + m(Setup.saveBudget).mockImplementation(() => ok(["budget"])); + wrap(); + fireEvent.click(await screen.findByRole("button", { name: "Save and continue" })); + + const limit = await screen.findByLabelText("Hard limit"); + expect(screen.getByText("No limit")).toBeInTheDocument(); + fireEvent.change(limit, { target: { value: "3" } }); + expect(screen.getByText("$25 a month")).toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("Input price"), { target: { value: "3" } }); + expect(screen.getByRole("button", { name: "Save and continue" })).toBeDisabled(); // both prices or none + fireEvent.change(screen.getByLabelText("Output price"), { target: { value: "15" } }); + fireEvent.click(screen.getByRole("button", { name: "Save and continue" })); + await waitFor(() => expect(Setup.saveBudget).toHaveBeenCalled()); + expect(m(Setup.saveBudget).mock.calls[0][0].body).toEqual({ + hard_usd: 25, + soft_usd: null, + period: "month", + on_over: "ask", + price_in: 3, + price_out: 15, + }); + + expect(await screen.findByLabelText(/Largest upload/)).toBeInTheDocument(); + }); + it("opts in to telemetry with an endpoint, after a test span", async () => { m(Setup.getSetup).mockImplementation(() => ok(VIEW({ namespace: { existing: ["media"], locked: false } }))); m(Setup.saveTelemetry).mockImplementation(() => ok({ ok: true })); @@ -272,4 +324,17 @@ describe("SetupWizard", () => { expect(Setup.finish).toHaveBeenCalledWith(expect.objectContaining({ body: { skipped: true } })), ); }); + + it("tells a model on this machine or its network from a cloud provider", () => { + for (const u of [ + "http://localhost:11434/v1", + "http://host.docker.internal:1234/v1", + "http://ollama:11434/v1", + "http://192.168.1.5:8080/v1", + "http://[::1]:11434", + ]) + expect(isLocalModel(u)).toBe(true); + for (const u of ["https://api.openai.com/v1", "https://openrouter.ai/api/v1", "", null]) + expect(isLocalModel(u)).toBe(false); + }); }); diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 25196f9b0..012bb07b6 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -2407,6 +2407,11 @@ export type { RunWorkflowErrors, RunWorkflowResponse, RunWorkflowResponses, + SaveBudgetData, + SaveBudgetError, + SaveBudgetErrors, + SaveBudgetResponse, + SaveBudgetResponses, SaveCollectionFieldsData, SaveCollectionFieldsError, SaveCollectionFieldsErrors, @@ -2542,6 +2547,8 @@ export type { SetRecordingContentTypeResponse, SetRecordingContentTypeResponses, SetupAdmin, + SetupBudget, + SetupBudgetView, SetupData, SetupError, SetupErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 4fd670a0b..fe3c70fd1 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -1182,6 +1182,9 @@ import type { RunWorkflowData, RunWorkflowErrors, RunWorkflowResponses, + SaveBudgetData, + SaveBudgetErrors, + SaveBudgetResponses, SaveCollectionFieldsData, SaveCollectionFieldsErrors, SaveCollectionFieldsResponses, @@ -3310,6 +3313,26 @@ export class Setup { }); } + /** + * Save Budget + * + * The overall AI budget: a soft limit that warns and a hard limit where runs wait and paid model calls stop + * (docs/budgets.md#one-budget-for-everything), with the chat model's price so its spending is counted. No hard + * limit: no budget. + */ + public static saveBudget( + options: Options, + ): RequestResult { + return (options.client ?? client).put({ + url: "/api/v1/setup/budget", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * Save Oauth * diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 30bada334..35dc1b95b 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -14602,6 +14602,71 @@ export type SetupAdmin = { [key: string]: unknown; }; +/** + * SetupBudget + */ +export type SetupBudget = { + /** + * Hard Usd + */ + hard_usd?: number | null; + /** + * Soft Usd + */ + soft_usd?: number | null; + /** + * Period + */ + period?: "day" | "week" | "month"; + /** + * On Over + */ + on_over?: "ask" | "skip"; + /** + * Price In + */ + price_in?: number | null; + /** + * Price Out + */ + price_out?: number | null; +}; + +/** + * SetupBudgetView + */ +export type SetupBudgetView = { + /** + * Hard Usd + */ + hard_usd?: number | null; + /** + * Soft Usd + */ + soft_usd?: number | null; + /** + * Period + */ + period?: string; + /** + * On Over + */ + on_over?: string; + /** + * Price In + */ + price_in?: number | null; + /** + * Price Out + */ + price_out?: number | null; + /** + * Model + */ + model?: string | null; + [key: string]: unknown; +}; + /** * SetupFinish */ @@ -14850,6 +14915,7 @@ export type SetupView = { storage: SetupStorageView; telemetry: SetupTelemetryView; oauth: SetupOAuthView; + budget: SetupBudgetView; [key: string]: unknown; }; @@ -21069,7 +21135,7 @@ export type RemoveBudgetData = { /** * Resource * - * routine:, pipeline:, workflow: or space: + * all (everything Lens spends), routine:, pipeline:, workflow: or space: */ resource: string; }; @@ -21119,7 +21185,7 @@ export type SetBudgetData = { /** * Resource * - * routine:, pipeline:, workflow: or space: + * all (everything Lens spends), routine:, pipeline:, workflow: or space: */ resource: string; }; @@ -21195,7 +21261,7 @@ export type BudgetStatusData = { /** * Resource * - * routine:, pipeline:, workflow: or space: + * all (everything Lens spends), routine:, pipeline:, workflow: or space: */ resource: string; }; @@ -21386,6 +21452,31 @@ export type SaveTelemetryResponses = { export type SaveTelemetryResponse = SaveTelemetryResponses[keyof SaveTelemetryResponses]; +export type SaveBudgetData = { + body: SetupBudget; + path?: never; + query?: never; + url: "/api/v1/setup/budget"; +}; + +export type SaveBudgetErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type SaveBudgetError = SaveBudgetErrors[keyof SaveBudgetErrors]; + +export type SaveBudgetResponses = { + /** + * Successful Response + */ + 200: SetupSaved; +}; + +export type SaveBudgetResponse = SaveBudgetResponses[keyof SaveBudgetResponses]; + export type SaveOauthData = { body: SetupOAuth; path?: never; diff --git a/nextjs-frontend/components/setup/setup-wizard.tsx b/nextjs-frontend/components/setup/setup-wizard.tsx index c94939a3b..b4670a45e 100644 --- a/nextjs-frontend/components/setup/setup-wizard.tsx +++ b/nextjs-frontend/components/setup/setup-wizard.tsx @@ -29,6 +29,7 @@ const STEPS = [ { id: "admin", label: "Admin account" }, { id: "namespace", label: "Namespace" }, { id: "llm", label: "Model provider" }, + { id: "budget", label: "Budget" }, { id: "storage", label: "Storage" }, { id: "apps", label: "Apps and AI" }, { id: "telemetry", label: "Telemetry" }, @@ -44,13 +45,31 @@ const LLM_ENV: Record = { const TELEMETRY_ENV: Record = { enabled: "LENS_TELEMETRY", endpoint: "LENS_TELEMETRY_ENDPOINT" }; const NS = /^[a-z0-9][a-z0-9_-]{0,40}$/; const URL_RX = /^https?:\/\/[^\s]+$/; +/** The budget scale: the hard limit in USD (0: no limit). The soft limit is 80% of it unless set under More. */ +const LIMITS = [0, 5, 10, 25, 50, 100, 250, 500]; + +/** Whether a model server is on this machine or its network (free to call), not a paid cloud provider. */ +export function isLocalModel(baseUrl: string | null | undefined): boolean { + let host: string; + try { + host = new URL(baseUrl ?? "").hostname.replace(/^\[|\]$/g, ""); + } catch { + return false; + } + return ( + !host.includes(".") || // a container next to Lens (ollama), localhost, ::1 + /^(127\.|10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.|0\.0\.0\.0$)/.test(host) || + /\.(local|internal|lan|home\.arpa)$/.test(host) + ); +} function errorText(e: unknown): string { return e instanceof ApiError ? e.message : "Something went wrong. Please try again."; } /** - * First-run setup for a fresh install (the admin already exists): the first namespace, the model provider, storage, + * First-run setup for a fresh install (the admin already exists): the first namespace, the model provider, an overall + * AI budget (a soft limit that warns, a hard limit that stops paid model calls; none unless chosen), storage, * whether apps and AI assistants may sign people in (OAuth, on unless turned off) and whether to send telemetry (off * unless chosen), one step at a time. Each step can be skipped, and the whole wizard too; everything stays in Settings. * Fields .env sets are shown locked, since the environment wins. @@ -100,12 +119,21 @@ export function SetupWizard() { }, }); - const next = (from: StepId, saved: boolean) => { + // The budget step is for a paid (cloud) model; one on this machine or its network costs nothing, so it's skipped. + const [modelUrl, setModelUrl] = useState(null); + const url = modelUrl ?? (view.data?.llm.values as { base_url?: string | null } | undefined)?.base_url ?? null; + const paid = Boolean(url) && !isLocalModel(url); + const steps = STEPS.filter((s) => s.id !== "budget" || paid); + + const next = (from: StepId, saved: boolean, baseUrl?: string) => { if (saved) setDone((d) => new Set(d).add(from)); queryClient.invalidateQueries({ queryKey: ["setup"] }); - const i = STEPS.findIndex((s) => s.id === from); - if (i === STEPS.length - 1) finish.mutate(false); - else setStep(STEPS[i + 1].id); + const shown = + baseUrl === undefined ? steps : STEPS.filter((s) => s.id !== "budget" || (baseUrl && !isLocalModel(baseUrl))); + if (baseUrl !== undefined) setModelUrl(baseUrl); + const i = shown.findIndex((s) => s.id === from); + if (i === shown.length - 1) finish.mutate(false); + else setStep(shown[i + 1].id); }; return ( @@ -131,7 +159,7 @@ export function SetupWizard() {
    - {STEPS.map((s, i) => { + {steps.map((s, i) => { const on = s.id === step; const ok = done.has(s.id); return ( @@ -199,7 +227,9 @@ export function SetupWizard() { ) : step === "namespace" ? ( next("namespace", saved)} /> ) : step === "llm" ? ( - next("llm", saved)} /> + next("llm", saved, baseUrl)} /> + ) : step === "budget" ? ( + next("budget", saved)} /> ) : step === "storage" ? ( next("storage", saved)} /> ) : step === "apps" ? ( @@ -355,7 +385,7 @@ function NamespaceStep({ view, onNext }: { view: SetupView; onNext: (saved: bool ); } -function LlmStep({ view, onNext }: { view: SetupView; onNext: (saved: boolean) => void }) { +function LlmStep({ view, onNext }: { view: SetupView; onNext: (saved: boolean, baseUrl?: string) => void }) { const client = useApiClient(); const v = view.llm.values as { base_url?: string | null; model?: string | null; api_key?: { set?: boolean } }; const locked = new Set(view.llm.locked); @@ -517,7 +547,9 @@ function LlmStep({ view, onNext }: { view: SetupView; onNext: (saved: boolean) =