From 0b824ee33a880b29449507eb6f116238971f6ea1 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Thu, 15 Oct 2026 22:40:38 +0000 Subject: [PATCH] feat(automations): run a workflow, a pipeline or a routine Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CyjR2zEfmTXBewKAx9Taw5 --- docs/automations.md | 16 ++ fastapi_backend/app/domain/actions.py | 53 +++++- fastapi_backend/app/domain/actions_run.py | 172 ++++++++++++++++++ fastapi_backend/tests/api/test_actions_run.py | 142 +++++++++++++++ 4 files changed, 381 insertions(+), 2 deletions(-) create mode 100644 fastapi_backend/app/domain/actions_run.py create mode 100644 fastapi_backend/tests/api/test_actions_run.py diff --git a/docs/automations.md b/docs/automations.md index cb7ff09c7..148ecec21 100644 --- a/docs/automations.md +++ b/docs/automations.md @@ -21,6 +21,7 @@ merged is called built. | Telling a person: the `notify` action and `channels.tell` (their own chat, a notification target, a Matterbridge room) | built | | The model and the agent: the `ask_model`, `call_tool` and `ask_agent` actions, and prompts with outside text marked | built | | Calling a web address: the `http_request` action, with credentials in headers kept sealed | built | +| Running Lens work: the `run_workflow`, `run_pipeline` and `run_routine` actions | built | | Settings → Automations: how often followers look and how long the event log is kept | built | | The Timeline page with filters, grouping, review keys and new items, and Home's Latest card ([Timeline](timeline.md)) | built | | Storing and running automations, schedules and timers, joins, the other actions, approvals, variables, the studio | planned | @@ -292,6 +293,9 @@ or removed does nothing. What it writes is by the automation itself, `automation | `call_tool` | calls one of the assistant's tools (`tool`, with `args`) as the automation | read: a change becomes an approval | `{tool, result, summary, approvals}` | | `ask_agent` | hands what happened (`about`, by default the event) to the agent with your `instructions`, limited to some `tools` if you like, for up to `max_steps` rounds (6, at most 12) | read: changes become approvals | `{answer, chat, approvals, steps}` | | `http_request` | sends a `method` (GET, POST or PUT) to a `url`, with `headers` and a `body` (text, or an object sent as JSON) | out | `{status, ok, body, json, content_type, truncated, method, url, headers}`: the answer, as outside text | +| `run_workflow` | runs a recording `workflow` (its published `version`, or the one named) on an `asset` as a job | lens | `{job, workflow, version, asset}` | +| `run_pipeline` | processes an `asset` again with a `pipeline` (or the one its namespace uses), keeping its outputs | lens | `{job, pipeline, asset}` | +| `run_routine` | asks the scheduler to run a `routine` now (`propose_only` to only propose its changes) | lens, so it asks | `{routine, requested, propose_only}` | #### Telling a person @@ -345,6 +349,16 @@ key ([Encryption](encryption.md)), bound to the automation and to the scheme, ho sealed value can't be copied into another automation or sent to another host; it is opened only to send. A locked vault stops the request until it is unlocked. +#### Running Lens work + +`run_workflow` and `run_pipeline` take an `asset`: an asset id, or the activity's object (`lens:recording/`, which +is the default, `{{ event.object.id }}`). On the asset the activity is about, in the automation's namespaces, they run +without asking; on any other, they ask first. They need the automation's person to be able to process assets in that +namespace, queue a job (after whatever the asset's job still has to do) that shows as by "automation: ", and pass +on the job's id. `run_routine` asks the scheduler to start a routine's run within half a minute, as Run in Settings +does: routines are an admin's, the routine must work only in the automation's namespaces, a run over one of its budgets +is refused, and it always asks first. The run's id shows in the routine's runs. + ## Refine later - More operators: `in`, `matches` (bounded regular expressions), `between`, times against now, `changed`. @@ -369,3 +383,5 @@ vault stops the request until it is unlocked. - Calling a web address: secrets in the address itself (a Slack or Discord webhook URL) sealed like headers; PATCH and DELETE (DELETE would count as deleting); OAuth sign-in for an API; form-encoded bodies; retries with backoff; networks allowed for automations apart from notifications. +- Running Lens work: a routine run's id passed on (today the scheduler starts it later); waiting for the job and passing + on its outputs; roles on collections counting for `process`; running a pipeline on every asset a join collected. diff --git a/fastapi_backend/app/domain/actions.py b/fastapi_backend/app/domain/actions.py index de8545977..1f01a84c6 100644 --- a/fastapi_backend/app/domain/actions.py +++ b/fastapi_backend/app/domain/actions.py @@ -14,7 +14,7 @@ (`actor`). Params marked `"format": "prompt"` are prompts: the engine renders them with `render_prompt`, which wraps what the run read from outside Lens as data. `log_line` ships here; the other actions are registered by a module per group, loaded with this one: actions_messages.py (notify), actions_ai.py (ask_model, call_tool, ask_agent), -actions_web.py (http_request). +actions_web.py (http_request), actions_run.py (run_workflow, run_pipeline, run_routine). """ from __future__ import annotations @@ -243,6 +243,55 @@ def reach(db, intent, account): return readable, {s for s in readable if auth.allows(roles, s, "editor")} +def allowed(db, account, space, perm): + """Whether the person's role in a namespace now has a permission (roles.py: edit, process, …).""" + return perm in auth.permissions(db, account).get(space, ()) + + +# ---------- the asset an action is about ---------- +EVENT_OBJECT = "{{ event.object.id }}" +ASSET_RX = re.compile(r"^(?:lens:recording/)?(\d{1,12})$") + + +def asset_id(v): + """An asset's id from a param: an id, or an activity's object id (lens:recording/). ValueError otherwise.""" + m = ASSET_RX.match(str("" if v is None else v).strip()) + if not m: + raise ValueError(f"{v} isn't an asset: give an asset id, or run it on an activity about an asset") + return int(m.group(1)) + + +def asset_subject(p, key="asset"): + """The object an action on an asset changes, as activities name it, for resolve_risk; None when it can't tell.""" + try: + return f"lens:recording/{asset_id(p.get(key))}" + except ValueError: + return None + + +def asset_param(title="Asset"): + return {"title": title, "default": EVENT_OBJECT, "description": "an asset id, or the activity's object (lens:recording/)"} + + +def on_asset(p, key="asset"): + """The asset an action names, in words.""" + a = p.get(key) + return "the asset it's about" if a in (None, EVENT_OBJECT) else f"asset {a}" + + +def asset_for(db, intent, account, rid, perm): + """The asset {id, space, title, collection}, when it is in one of the automation's namespaces and the person's + role there now has `perm`. ValueError naming the problem.""" + rec = db.one("SELECT space, title, collection FROM $r", r=store.R("recording", int(rid))) + if not rec: + raise ValueError(f"there's no asset {rid}") + if intent["spaces"] and rec["space"] not in intent["spaces"]: + raise ValueError(f"asset {rid} isn't in this automation's namespaces") + if not allowed(db, account, rec["space"], perm): + raise ValueError(f"{account['name']} can't {perm} assets in that namespace") + return {"id": int(rid), **rec} + + # ---------- prompts ---------- def prompt_params(spec): """The names of an action's params that are prompts (format "prompt").""" @@ -342,4 +391,4 @@ def _log_line(db, cfg, intent, user, dry): ) -from . import actions_ai, actions_messages, actions_web # noqa: E402, F401 - they register their actions; loaded with the catalog +from . import actions_ai, actions_messages, actions_run, actions_web # noqa: E402, F401 - they register their actions; loaded with the catalog diff --git a/fastapi_backend/app/domain/actions_run.py b/fastapi_backend/app/domain/actions_run.py new file mode 100644 index 000000000..2acb80162 --- /dev/null +++ b/fastapi_backend/app/domain/actions_run.py @@ -0,0 +1,172 @@ +"""Automation actions that run Lens's own work (docs/automations.md, Actions): `run_workflow`, `run_pipeline` and +`run_routine`. + +Each changes something inside Lens (effect `lens`): a workflow or a pipeline on the asset the run is about, in the +automation's namespaces, runs without asking; on any other asset, and a routine always, it asks first. The work is +queued by the automation (its jobs and runs are by "automation: "), with its person's permission to process +assets in that namespace; a routine is an admin's to run, as in Settings. + +`asset` takes the activity's object id (`lens:recording/`, the default `{{ event.object.id }}`) or an asset id. +""" + +from __future__ import annotations + +from . import actions, budgets, jobs, pipelines, routines, workflows + + +def _int(p, key, what): + try: + return int(p[key]) + except (KeyError, TypeError, ValueError): + raise ValueError(f"{what} is an id") from None + + +# ---------- run_workflow ---------- +def _run_workflow(db, cfg, intent, user, dry): + p = intent["params"] + acc = actions.person(db, intent, user) + wid = _int(p, "workflow", "the workflow") + rec = actions.asset_for(db, intent, acc, actions.asset_id(p.get("asset")), "process") + try: + w = workflows.get(db, wid, p.get("version")) + except KeyError: + raise ValueError(f"there's no workflow {wid}" + (f" version {p['version']}" if p.get("version") else "")) from None + if w["scope"] != "recording": + raise ValueError(f"the workflow {w['name']} organises the graph; run it with a routine") + out = {"workflow": wid, "version": w["version"], "asset": rec["id"]} + if dry: + return {**out, "job": None, "would": f"run the workflow {w['name']} on {rec.get('title') or rec['id']}"} + step = {"type": "workflow", "workflow": wid, "version": w["version"]} + return {**out, "job": jobs.add_steps(db, rec["id"], [step], by=actions.actor(db, intent)["email"])} + + +# ---------- run_pipeline ---------- +def _run_pipeline(db, cfg, intent, user, dry): + p = intent["params"] + acc = actions.person(db, intent, user) + rec = actions.asset_for(db, intent, acc, actions.asset_id(p.get("asset")), "process") + pid = _int(p, "pipeline", "the pipeline") if p.get("pipeline") is not None else None + name = "its namespace's pipeline" + if pid is not None: + try: + name = f"the pipeline {pipelines.get(db, pid)['name']}" + except KeyError: + raise ValueError(f"there's no pipeline {pid}") from None + out = {"pipeline": pid, "asset": rec["id"]} + if dry: + return {**out, "job": None, "would": f"run {name} on {rec.get('title') or rec['id']}"} + try: + jid = jobs.enqueue(db, rec["id"], None, by=actions.actor(db, intent)["email"], pipeline=pid) + except KeyError: + raise ValueError(f"there's no pipeline {pid}") from None + return {**out, "job": jid} + + +# ---------- run_routine ---------- +def _run_routine(db, cfg, intent, user, dry): + p = intent["params"] + acc = actions.person(db, intent, user) + rid = _int(p, "routine", "the routine") + try: + r = routines.get(db, rid) + except KeyError: + raise ValueError(f"there's no routine {rid}") from None + if not acc["admin"]: + raise ValueError(f"routines are run by admins, and {acc['name']} isn't one") + spaces = {int(s.split(":")[1]) for s in routines.refs(db, r) if s.startswith("space:")} + if intent["spaces"] and not spaces <= set(intent["spaces"]): + raise ValueError(f"the routine {r['name']} works in namespaces outside this automation's") + propose = bool(p.get("propose_only")) + out = {"routine": rid, "propose_only": propose} + if dry: + return {**out, "requested": False, "would": f"run the routine {r['name']}" + (", proposing its changes" if propose else "")} + verdict = budgets.check(db, cfg, routines.refs(db, r)) + if not verdict["go"]: + raise ValueError(verdict["why"]) + try: + routines.request_run(db, rid, actions.actor(db, intent)["email"], propose) + except ValueError as e: + raise ValueError(f"the routine {r['name']}: {e}") from None + return {**out, "requested": True} + + +actions.register( + actions.ActionSpec( + name="run_workflow", + label="Run a workflow", + group="Run", + description="Runs a workflow on an asset as a job: at once on the asset the activity is about, else asking first.", + params={ + "type": "object", + "properties": { + "workflow": {"type": "integer", "title": "Workflow"}, + "version": {"type": "integer", "title": "Version", "description": "empty: the published one"}, + "asset": actions.asset_param(), + }, + }, + required=("workflow",), + effect="lens", + subject=actions.asset_subject, + output={ + "type": "object", + "properties": { + "job": {"type": "integer"}, + "workflow": {"type": "integer"}, + "version": {"type": "integer"}, + "asset": {"type": "integer"}, + }, + }, + perform=_run_workflow, + describe=lambda p: f"Run workflow {p.get('workflow')} on {actions.on_asset(p)}", + ) +) + +actions.register( + actions.ActionSpec( + name="run_pipeline", + label="Run a pipeline", + group="Run", + description="Processes an asset again with a pipeline (or its namespace's own), keeping its outputs.", + params={ + "type": "object", + "properties": { + "pipeline": {"type": "integer", "title": "Pipeline", "description": "empty: the one its namespace uses"}, + "asset": actions.asset_param(), + }, + }, + effect="lens", + subject=actions.asset_subject, + output={ + "type": "object", + "properties": {"job": {"type": "integer"}, "pipeline": {"type": "integer"}, "asset": {"type": "integer"}}, + }, + perform=_run_pipeline, + describe=lambda p: f"Run {'pipeline ' + str(p['pipeline']) if p.get('pipeline') else 'its pipeline'} on {actions.on_asset(p)}", + ) +) + +actions.register( + actions.ActionSpec( + name="run_routine", + label="Run a routine", + group="Run", + description="Asks the scheduler to run a routine now (an admin's); always asks you first.", + params={ + "type": "object", + "properties": { + "routine": {"type": "integer", "title": "Routine"}, + "propose_only": {"type": "boolean", "title": "Only propose its changes", "default": False}, + }, + }, + required=("routine",), + effect="lens", + subject=lambda p: None, + output={ + "type": "object", + "description": "the scheduler starts the run within half a minute; its id shows in the routine's runs", + "properties": {"routine": {"type": "integer"}, "requested": {"type": "boolean"}, "propose_only": {"type": "boolean"}}, + }, + perform=_run_routine, + describe=lambda p: f"Run routine {p.get('routine')}" + (", proposing its changes" if p.get("propose_only") else ""), + ) +) diff --git a/fastapi_backend/tests/api/test_actions_run.py b/fastapi_backend/tests/api/test_actions_run.py new file mode 100644 index 000000000..d3d6d22e7 --- /dev/null +++ b/fastapi_backend/tests/api/test_actions_run.py @@ -0,0 +1,142 @@ +"""Automation actions that run Lens's own work (docs/automations.md, Actions): run_workflow and run_pipeline queue a +job on an asset by the automation, with its person's permission to process it, at once on the asset the activity is +about and asking first otherwise; run_routine asks the scheduler for a routine's run, an admin's to ask for.""" + +from __future__ import annotations + +import pytest + +from app.domain import actions, budgets, pipelines, routines, store, workflows +from tests import fake_llm +from tests.api._assist import start_llm +from tests.api.test_budgets import spend +from tests.helpers import make_user, seed + +GRAPH = { + "nodes": [{"id": "i", "type": "input", "config": {}}, {"id": "o", "type": "output", "config": {"key": "who"}}], + "edges": [{"source": "i", "target": "o"}], +} + + +@pytest.fixture +def app(cfg, db): + from app.main import create_app + + srv = start_llm(cfg) + yield create_app(cfg, db, background=False) + fake_llm.Handler.usage = None + srv.shutdown() + + +@pytest.fixture +def env(app, db, cfg, folder): + a, b, call = seed(db, cfg, folder) + return { + "a": a, + "call": call, + "me": make_user(db, "me@x.io", "my password 1", roles={"pods": "editor"}), + "viewer": make_user(db, "vi@x.io", "viewer password 1", roles={"pods": "viewer"}), + "root": make_user(db, "root@x.io", "root password 1", admin=True), + "pods": store.ns_id(db, "pods"), + "workflow": workflows.create(db, "People", GRAPH), + "pipeline": pipelines.create(db, "Just analyze", ["analyze"]), + } + + +def run(db, cfg, env, action, params, dry=False, **kw): + intent = { + "key": "51-n5", + "action": action, + "params": params, + "effect": "lens", + "risk": "routine", + "untrusted": False, + "spaces": [env["pods"]], + "run_as": env["me"], + "node": "n5", + "automation": 4, + "run": 51, + **kw, + } + return actions.perform(db, cfg, intent, None, dry) + + +def ctx(env, rid): + return { + "event": {"object": {"id": f"lens:recording/{rid}"}, "lens": {"spaces": [env["pods"]]}}, + "automation": {"spaces": [env["pods"]]}, + } + + +def test_on_the_asset_the_activity_is_about_they_run_at_once_else_they_ask(db, cfg, env): + wf, pl, rt = (actions.get(n) for n in ("run_workflow", "run_pipeline", "run_routine")) + p = actions.check_params(wf, {"workflow": 1}) + assert p["asset"] == "{{ event.object.id }}" and actions.effect_of(wf, p) == "lens" + mine = ctx(env, env["a"]) + assert actions.resolve_risk(wf, {"workflow": 1, "asset": f"lens:recording/{env['a']}"}, mine) == "routine" + assert actions.resolve_risk(pl, {"asset": env["a"]}, mine) == "routine" # an id names the same asset + assert actions.resolve_risk(wf, {"workflow": 1, "asset": env["call"]}, mine) == "ask" + assert actions.resolve_risk(rt, {"routine": 1}, mine, {"n5": "auto"}) == "ask" + assert actions.describe("run_workflow", p) == "Run workflow 1 on the asset it's about" + assert actions.describe("run_pipeline", {"pipeline": 3, "asset": 9}) == "Run pipeline 3 on asset 9" + + +def test_run_workflow_queues_it_on_the_asset_as_the_automation(db, cfg, env): + rid, wid = env["a"], env["workflow"] + got = run(db, cfg, env, "run_workflow", {"workflow": wid, "asset": f"lens:recording/{rid}"}, dry=True) + assert got["job"] is None and got["would"].startswith("run the workflow People on ") and got["version"] == 1 + assert not db.values("SELECT VALUE id FROM job WHERE created_by = 'automation: Automation 4'") + got = run(db, cfg, env, "run_workflow", {"workflow": wid, "asset": f"lens:recording/{rid}"}) + job = db.one("SELECT recording, steps, created_by FROM $r", r=store.R("job", got["job"])) + assert got == {"workflow": wid, "version": 1, "asset": rid, "job": got["job"]} + assert job["recording"] == rid and job["steps"][-1] == {"type": "workflow", "workflow": wid, "version": 1} + assert job["created_by"] == "automation: Automation 4" + for params, said in ( + ({"workflow": 999, "asset": rid}, "there's no workflow 999"), + ({"workflow": wid, "asset": "lens:email/3"}, "isn't an asset"), + ({"workflow": wid, "asset": 99999}, "there's no asset 99999"), + ({"workflow": wid, "asset": env["call"]}, "isn't in this automation's namespaces"), + ): + with pytest.raises(ValueError, match=said): + run(db, cfg, env, "run_workflow", params, dry=True) + with pytest.raises(ValueError, match="vi can't process assets in that namespace"): + run(db, cfg, env, "run_workflow", {"workflow": wid, "asset": rid}, run_as=env["viewer"]) + + +def test_run_pipeline_processes_the_asset_again(db, cfg, env): + rid, pid = env["a"], env["pipeline"] + assert run(db, cfg, env, "run_pipeline", {"pipeline": pid, "asset": rid}, dry=True)["would"].startswith("run the pipeline Just analyze") + assert run(db, cfg, env, "run_pipeline", {"asset": rid}, dry=True)["would"].startswith("run its namespace's pipeline") + got = run(db, cfg, env, "run_pipeline", {"pipeline": pid, "asset": str(rid)}) + assert got["job"] and got["asset"] == rid and got["pipeline"] == pid + assert db.one("SELECT created_by FROM $r", r=store.R("job", got["job"]))["created_by"] == "automation: Automation 4" + with pytest.raises(ValueError, match="there's no pipeline 999"): + run(db, cfg, env, "run_pipeline", {"pipeline": 999, "asset": rid}) + + +def test_run_routine_asks_the_scheduler_for_an_admins_routine_in_the_automations_namespaces(db, cfg, env): + rid = routines.create(db, "Small batches", [{"type": "pipeline", "steps": ["analyze"], "limit": 1}], namespaces=[env["pods"]]) + with pytest.raises(ValueError, match="routines are run by admins, and me isn't one"): + run(db, cfg, env, "run_routine", {"routine": rid}) + got = run(db, cfg, env, "run_routine", {"routine": rid, "propose_only": True}, dry=True, run_as=env["root"]) + assert got == { + "routine": rid, + "propose_only": True, + "requested": False, + "would": "run the routine Small batches, proposing its changes", + } + assert not routines.get(db, rid).get("run_now") + assert run(db, cfg, env, "run_routine", {"routine": rid}, run_as=env["root"])["requested"] + assert routines.get(db, rid)["run_now"]["by"] == "automation: Automation 4" + everywhere = routines.create(db, "All", [{"type": "pipeline", "steps": ["analyze"], "limit": 1}]) + with pytest.raises(ValueError, match="works in namespaces outside this automation's"): + run(db, cfg, env, "run_routine", {"routine": everywhere}, run_as=env["root"]) + with pytest.raises(ValueError, match="there's no routine 999"): + run(db, cfg, env, "run_routine", {"routine": 999}, run_as=env["root"]) + # over one of its budgets, it doesn't ask for the run + other = routines.create(db, "Other", [{"type": "pipeline", "steps": ["analyze"], "limit": 1}], namespaces=[env["pods"]]) + budgets.put(db, f"routine:{other}", usd=0.5) + spend(db, f"routine:{other}", usd=0.6) + with pytest.raises(ValueError, match="budget is used up"): + run(db, cfg, env, "run_routine", {"routine": other}, run_as=env["root"]) + assert not routines.get(db, other).get("run_now")