From 05835352c8da412e7e632cb701c4b0035c046f5e Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 06:43:37 +0000 Subject: [PATCH 1/2] feat(workflows): draw an automation's workflow on the canvas, with action and decide nodes Event workflows open on the workflow canvas: What happened starts them, the palette lists every action in the catalog, an action node's settings are drawn from its params by SchemaForm with a badge for whether it asks first, Decide gets a port per option and unsure, the canvas checks the leaf rule and settings as the backend does, and Try on a sample runs the graph on a kind's sample and says what each action would do. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AjJqXgjj1cNdKGRQbX5frb --- README.md | 8 +- docs/automations.md | 15 +- docs/processing.md | 4 + docs/requirements.md | 1 + .../__tests__/automation-nodes-ui.test.tsx | 89 +++++++++ .../__tests__/workflow-canvas.test.ts | 142 ++++++++++++- .../components/workflows/action-settings.tsx | 167 ++++++++++++++++ .../components/workflows/graph-editor.tsx | 35 +++- .../components/workflows/node-settings.tsx | 15 ++ .../components/workflows/workflow-editor.tsx | 108 ++++++++-- .../components/workflows/workflow-model.ts | 188 ++++++++++++++++-- 11 files changed, 735 insertions(+), 37 deletions(-) create mode 100644 nextjs-frontend/__tests__/automation-nodes-ui.test.tsx create mode 100644 nextjs-frontend/components/workflows/action-settings.tsx diff --git a/README.md b/README.md index 7868786b1..223642f57 100644 --- a/README.md +++ b/README.md @@ -167,8 +167,9 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p content types or topics, acting only on a sure answer, and an Entity filter node keeps the entities you want. Anything that reaches outside Lens or changes more than what set it off asks first, until you let it run on its own: on the Approvals page or, for a change inside Lens, with a yes in the chat it told you in; agents can decline an approval but never give one. Each automation has limits and budgets, - and agents make, try and run them over MCP and in the assistant, or draft one from a sentence; the studio to draw them - in the web app is planned. ([Processing](docs/processing.md#workflows-and-the-canvas), + and agents make, try and run them over MCP and in the assistant, or draft one from a sentence. An automation's + workflow is drawn and tried on the workflow canvas, with its actions in the palette; the studio's builder is + planned. ([Processing](docs/processing.md#workflows-and-the-canvas), [converting assets](docs/processing.md#converting-assets), [notifications](docs/notifications.md), [automations](docs/automations.md), [approvals](docs/automations.md#approvals), [timeline](docs/timeline.md); [checks](docs/requirements.md#automate-it)) @@ -252,7 +253,8 @@ These are open pull requests, not yet part of Lens: Next, and not in a pull request yet; each links to where it is tracked: -- **The automations studio:** drawing automations in the web app ([automations](docs/automations.md)). +- **The automations studio:** a When, If and Then builder for automations, with their runs and costs, in the web app + ([automations](docs/automations.md)). - **Timeline:** what each item caused, and Approve and Decline on waiting approvals ([timeline](docs/timeline.md)). - **Notes:** nesting a note under its project or area without a click, pages in the graph, and wiki addresses for topics, recordings and the other things that have pages ([notes](docs/notes.md)). diff --git a/docs/automations.md b/docs/automations.md index f436277e5..435296558 100644 --- a/docs/automations.md +++ b/docs/automations.md @@ -26,6 +26,7 @@ merged is called built. | The shapes of triggers, joins, an If, policy and intents, checked (`app/domain/automation_spec.py`) | built | | The event workflow scope: what an automation does, drawn with action nodes and tried on an activity (`app/domain/automation_nodes.py`) | built | | The decide node: the decision model picks a branch, one port per option and `unsure` ([The decide node](#the-decide-node)) | built | +| Drawing an automation's workflow on the canvas: the actions in the palette, their settings from each action's params, Decide's ports, the leaf rule and a try on a kind's sample ([On the canvas](#on-the-canvas)) | built | | The action registry with risk from each action's effect, and one action, `log_line` (`app/domain/actions.py`) | built | | Telling a person: the `notify` action and `channels.tell` (their own chat, a notification target, a Matterbridge room) | built | | The model and the agent: the `ask_model`, `call_tool` and `ask_agent` actions, and prompts with outside text marked | built | @@ -567,6 +568,19 @@ lists each one with its params, effect, risk, status (`done`, `waiting`, `schedu back. Text from outside Lens (`lens.untrusted` on the activity, or what a node passed on when its output says `untrusted`, like a web page's answer) that reaches an action's params marks its intent `untrusted`. +#### On the canvas + +Built: an event workflow opens on the workflow canvas (`/workflows/`) like any other. It starts from **What +happened**, the run context. The palette lists every action in the catalog under **Actions**; dropping one adds an +action node set to it, and its settings are a form drawn from the action's params (a template field takes `{{ }}`), +with a badge saying whether it runs on its own or asks you first, and **Wait first** for a delay. **Decide** sits +under AI: write its question and its options one a line as `name: what it means`, and each option becomes an output +port, with `unsure` last. The canvas says what the backend would refuse before you publish: an action not chosen or +a required param left empty, a wait that isn't like `30m`, a decide node without a question or with fewer than two +options, and the leaf rule (nothing connected after an action that reaches outside Lens or waits). **Try on a +sample** runs the graph on the sample of the kind you pick: nothing is kept or sent, each node shows what it passed +on, and each action says what it would do (run at once, ask you first or wait for its time). + #### The decide node Built (`match.check_decide`, `match.run_decide`). A decide node asks the decision model (Jev, then Laya, then the @@ -1245,7 +1259,6 @@ held run until you pick. channel; web push; private chats through Matterbridge (it can't tell them from rooms); with several people let in, "me" is whoever last wrote privately, until each chat user has a Lens account of their own; telling which links in a message the person's template wrote, so they stay when the run read outside text. -- The canvas draws and edits event workflows (today they are saved and tried through the API). - Variables and templates: secrets as variables (sealed, never rendered into prompts); typed variable declarations with defaults per automation; a time zone of each person's own, rather than the triggers'; wrapping outside text the same way in `tool_nodes.ask_model` (pipeline workflows) and extensions' prompt tools; variables in the diff --git a/docs/processing.md b/docs/processing.md index bdf8d9908..9829e5f70 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -751,6 +751,10 @@ A pipeline runs a workflow as a Workflow step, pinned to the workflow's publishe means "runs after", and the graph is put in order (ties left to right) and kept as the version's steps, so runs and Activity work as before. A pipeline saved as a list is drawn as a chain. +An automation's workflow (scope `event`) is drawn on the same canvas, from What happened (the run) to Action and +Decide nodes, with every action in the catalog listed in the palette and its settings drawn from the action's params +([On the canvas](automations.md#on-the-canvas)). + ## Content types Every resource is video, audio, image or text (transcripts, documents, web pages, emails and calendar events are diff --git a/docs/requirements.md b/docs/requirements.md index b42919974..a54521074 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -198,6 +198,7 @@ request, to another check that proves the same thing; loosening a requirement or | Joins wait for an automation's triggers together: all of them, a number of times, one that doesn't come in time, or a batch, on timers that outlast a restart | [test_match_joins.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_match_joins.py) `test_all_waits_for_every_trigger_within_the_window_on_the_same_key`, `test_count_fires_once_per_crossing_then_counts_again`, `test_absent_waits_with_a_timer_and_the_missing_one_cancels_it`, `test_batch_collects_then_runs_once`, `test_two_writers_at_once_lose_nothing`; [test_automation_conditions.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_automation_conditions.py) `test_an_absent_join_waits_on_a_timer_and_switching_off_drops_it` | | Each automation has limits and budgets | [test_automation_runs.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_automation_runs.py) `test_the_if_limits_and_dry_runs`, `test_a_run_over_budget_is_held_for_the_persons_pick` | | Agents make, try and run automations over MCP and in the assistant, or draft one from a sentence | [test_automation_tools.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_automation_tools.py) `test_agents_save_try_and_run_automations_over_mcp`, `test_the_assistant_saves_at_once_and_asks_to_switch_on_or_run`; [test_automations.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_automations.py) `test_an_automation_is_drafted_from_a_sentence` | +| An automation's workflow is drawn and tried on the canvas: its actions in the palette, their settings drawn from each action's params, Decide's ports and the leaf rule | [workflow-canvas.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/workflow-canvas.test.ts) `give a decide node a port per option, then unsure`, `keep the leaf rule: nothing after an action that reaches outside Lens or waits`, `say what the backend would reject in their settings`; [automation-nodes-ui.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/automation-nodes-ui.test.tsx) `sets an action up from the catalog, its params drawn from its schema`; [schema-form.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/schema-form.test.tsx) `opens the variable picker on {{ and puts the pick in with Enter` | ## See what it costs diff --git a/nextjs-frontend/__tests__/automation-nodes-ui.test.tsx b/nextjs-frontend/__tests__/automation-nodes-ui.test.tsx new file mode 100644 index 000000000..99a36a3a7 --- /dev/null +++ b/nextjs-frontend/__tests__/automation-nodes-ui.test.tsx @@ -0,0 +1,89 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import "@testing-library/jest-dom"; +import { useState } from "react"; + +import { ActionSettings, DecideSettings } from "@/components/workflows/action-settings"; +import type { ActionDef } from "@/components/workflows/workflow-model"; + +const actions: ActionDef[] = [ + { + name: "notify", + label: "Send a message", + group: "Messages", + description: "Tells you in your own chat app", + params: { + type: "object", + properties: { to: { type: "string", title: "To", default: "me" }, text: { type: "string", title: "Message" } }, + }, + required: ["text"], + effect: "depends", + }, + { + name: "http_request", + label: "Call a web address", + group: "Web", + params: { type: "object", properties: { url: { type: "string", title: "Address" } } }, + required: ["url"], + effect: "out", + }, +]; + +function Harness({ + which, + spy, + start, +}: { + which: "action" | "decide"; + spy: jest.Mock; + start: Record; +}) { + const [c, setC] = useState(start); + const set = (patch: Record) => { + const next = { ...c, ...patch }; + for (const k of Object.keys(next)) if (next[k] === undefined) delete next[k]; + spy(next); + setC(next); + }; + return which === "action" ? ( + + ) : ( + + ); +} + +describe("an automation's nodes on the canvas", () => { + it("sets an action up from the catalog, its params drawn from its schema", () => { + const spy = jest.fn(); + render(); + expect(screen.getByText("Tells you in your own chat app")).toBeInTheDocument(); + expect(screen.getByText("Asks you first when it reaches other people")).toBeInTheDocument(); + fireEvent.change(screen.getByLabelText(/^Message/), { target: { value: "New: {{ event.summary }}" } }); + expect(spy).toHaveBeenLastCalledWith({ action: "notify", params: { text: "New: {{ event.summary }}" } }); + fireEvent.change(screen.getByLabelText(/^Wait first/), { target: { value: "30m" } }); + expect(spy).toHaveBeenLastCalledWith(expect.objectContaining({ delay: "30m" })); + }); + + it("clears the params when the action changes, and says an outside action asks first", () => { + const spy = jest.fn(); + render(); + fireEvent.change(screen.getByLabelText(/^Action/), { target: { value: "http_request" } }); + expect(spy).toHaveBeenLastCalledWith({ action: "http_request", params: {} }); + expect(screen.getByText("Asks you first")).toBeInTheDocument(); + expect(screen.getByLabelText(/^Address/)).toHaveValue(""); + }); + + it("writes a decide node's options one a line, as name: what it means", () => { + const spy = jest.fn(); + render(); + fireEvent.change(screen.getByLabelText(/^Question/), { target: { value: "Does it need me today?" } }); + fireEvent.change(screen.getByLabelText(/^Options/), { + target: { value: "urgent: needs an answer today\nlater: can wait\n" }, + }); + fireEvent.change(screen.getByLabelText(/^Sure enough at/), { target: { value: "0.8" } }); + expect(spy).toHaveBeenLastCalledWith({ + question: "Does it need me today?", + options: { urgent: "needs an answer today", later: "can wait" }, + min_confidence: 0.8, + }); + }); +}); diff --git a/nextjs-frontend/__tests__/workflow-canvas.test.ts b/nextjs-frontend/__tests__/workflow-canvas.test.ts index 586e48b3f..cf1339fd5 100644 --- a/nextjs-frontend/__tests__/workflow-canvas.test.ts +++ b/nextjs-frontend/__tests__/workflow-canvas.test.ts @@ -1,6 +1,18 @@ import { rulesText } from "@/components/content-types/model"; import { graphOrder, type PlGraph } from "@/components/pipelines/pipeline-model"; -import { cleanGraph, problems, starter, type WfGraph } from "@/components/workflows/workflow-model"; +import { + cleanGraph, + effectBadge, + inScope, + infoFor, + nodeSummary, + portsOf, + problems, + starter, + type ActionDef, + type WfGraph, + wouldDo, +} from "@/components/workflows/workflow-model"; const kind = (id: number) => ({ 1: "prompt", 2: "export" })[id]; @@ -92,3 +104,131 @@ describe("content types", () => { expect(rulesText(null)).toBe(""); }); }); + +// as GET /api/v1/automations/actions lists them +const actions: ActionDef[] = [ + { + name: "notify", + label: "Send a message", + group: "Messages", + params: { type: "object", properties: { to: { type: "string" }, text: { type: "string", title: "Message" } } }, + required: ["text"], + effect: "depends", + }, + { + name: "http_request", + label: "Call a web address", + group: "Web", + params: { type: "object", properties: { url: { type: "string", title: "Address" } } }, + required: ["url"], + effect: "out", + }, + { name: "log_line", label: "Write to the run log", group: "Run", params: {}, required: [], effect: "read" }, +]; +const names = { template: () => undefined, field: () => undefined, actions }; + +describe("automation workflows (scope event)", () => { + it("start from what happened with a message to you, and that's valid", () => { + const g = starter("event"); + expect(g.nodes.map((n) => n.type)).toEqual(["input", "action"]); + expect(problems(g, kind, "event", "workflow", undefined, actions)).toEqual({}); + expect(infoFor("input", "event")?.label).toBe("What happened"); + expect(nodeSummary(g.nodes[1], names)).toBe("Send a message: {{ event.summary }}"); + }); + + it("have action and decide nodes, which other scopes don't", () => { + expect(inScope("action", "event")).toBe(true); + expect(inScope("decide", "event")).toBe(true); + expect(inScope("extract_rules", "event")).toBe(false); + expect(inScope("action", "recording")).toBe(false); + const g: WfGraph = { + nodes: [ + { id: "in", type: "input", config: {} }, + { id: "a", type: "action", config: { action: "log_line", params: { text: "hi" } } }, + ], + edges: [{ source: "in", target: "a" }], + }; + expect(problems(g, kind, "recording")).toMatchObject({ a: "This node is for automations." }); + expect(problems(starter(), kind, "event")).toMatchObject({ rules: "This node isn’t for automations." }); + }); + + it("give a decide node a port per option, then unsure", () => { + const n = { id: "d", type: "decide", config: { question: "Today?", options: { urgent: "now", later: "wait" } } }; + expect(portsOf(n)).toEqual({ inputs: ["in"], outputs: ["urgent", "later", "unsure"], many: false }); + expect(nodeSummary(n, names)).toBe("Today? · urgent / later"); + }); + + it("say what the backend would reject in their settings", () => { + const g: WfGraph = { + nodes: [ + { id: "in", type: "input", config: {} }, + { id: "none", type: "action", config: { params: {} } }, + { id: "bare", type: "action", config: { action: "notify", params: {} } }, + { + id: "both", + type: "action", + config: { action: "log_line", params: {}, delay: "5m", when: "2026-10-18T07:00:00Z" }, + }, + { id: "slow", type: "action", config: { action: "log_line", params: {}, delay: "soon" } }, + { id: "q", type: "decide", config: { question: "", options: { yes: "a", no: "b" } } }, + { id: "few", type: "decide", config: { question: "Ok?", options: { yes: "a" } } }, + { id: "bad", type: "decide", config: { question: "Ok?", options: { yes: "a", unsure: "b" } } }, + { id: "mean", type: "decide", config: { question: "Ok?", options: { yes: "a", no: "" } } }, + { id: "sure", type: "decide", config: { question: "Ok?", options: { yes: "a", no: "b" }, min_confidence: 2 } }, + ], + edges: ["none", "bare", "both", "slow", "q", "few", "bad", "mean", "sure"].map((t) => ({ + source: "in", + target: t, + })), + }; + expect(problems(g, kind, "event", "workflow", undefined, actions)).toEqual({ + none: "Choose an action.", + bare: "Fill in Message.", + both: "Wait for a delay or until a time, not both.", + slow: "A wait is a whole number and s, m, h or d, like 30m.", + q: "Ask its question.", + few: "Give 2 to 10 options.", + bad: "Name each option in lowercase letters, digits and _ (not unsure).", + mean: "Say what each option means.", + sure: "How sure is a number from 0 to 1.", + }); + }); + + it("keep the leaf rule: nothing after an action that reaches outside Lens or waits", () => { + const g: WfGraph = { + nodes: [ + { id: "in", type: "input", config: {} }, + { id: "web", type: "action", config: { action: "http_request", params: { url: "https://example.org" } } }, + { id: "later", type: "action", config: { action: "log_line", params: { text: "x" }, delay: "30m" } }, + { id: "log", type: "action", config: { action: "log_line", params: { text: "y" } } }, + { id: "end", type: "action", config: { action: "log_line", params: { text: "z" } } }, + ], + edges: [ + { source: "in", target: "web" }, + { source: "web", target: "end" }, + { source: "in", target: "later" }, + { source: "later", target: "log" }, + ], + }; + const p = problems(g, kind, "event", "workflow", undefined, actions); + expect(p.web).toBe( + "It reaches outside Lens, so it asks you first and passes nothing on: connect nothing after it.", + ); + expect(p.later).toBe("It waits before it acts, so it passes nothing on: connect nothing after it."); + expect(p.log).toBeUndefined(); + expect(problems({ nodes: [g.nodes[0]], edges: [] }, kind, "event")).toEqual({ + "": "Add an action, or the automation does nothing.", + }); + }); + + it("say whether an action runs on its own, and what a try would do", () => { + expect(effectBadge("read").label).toBe("Runs on its own"); + expect(effectBadge("out").label).toBe("Asks you first"); + expect(wouldDo({ node: "web", action: "http_request", status: "waiting", untrusted: true }, actions)).toBe( + "web · Call a web address: would ask you first (reads outside text)", + ); + expect(wouldDo({ node: "log", action: "log_line", status: "done" }, actions)).toBe( + "log · Write to the run log: would run at once", + ); + }); +}); diff --git a/nextjs-frontend/components/workflows/action-settings.tsx b/nextjs-frontend/components/workflows/action-settings.tsx new file mode 100644 index 000000000..cca540e00 --- /dev/null +++ b/nextjs-frontend/components/workflows/action-settings.tsx @@ -0,0 +1,167 @@ +"use client"; + +/** The settings of an automation's own nodes on the canvas: an action (its params from the actions catalog, drawn + * by SchemaForm) and a decide node (its question and options). */ +import { useState } from "react"; + +import { SchemaForm, type Variable } from "@/components/forms/schema-form"; +import type { JsonSchema } from "@/components/forms/schema-model"; +import { Badge } from "@/components/ui/badge"; +import { Field, Input, Select, Textarea } from "@/components/ui/field"; +import { decideOptions, effectBadge, type ActionDef } from "@/components/workflows/workflow-model"; + +export function ActionSettings({ + config: c, + set, + actions, + variables, + nodeId, +}: { + config: Record; + set: (patch: Record) => void; + actions: ActionDef[]; + variables?: Variable[]; + nodeId: string; +}) { + const a = actions.find((x) => x.name === c.action); + const badge = effectBadge(a?.effect); + return ( + <> + + {({ id, describedBy }) => ( + set({ delay: e.target.value.trim() || undefined })} + /> + )} + + + ); +} + +/** Options written one a line as name: what it means. */ +const optionLines = (c: Record) => + decideOptions(c) + .map((k) => `${k}: ${String((c.options as Record)[k] ?? "")}`) + .join("\n"); + +export function DecideSettings({ + config: c, + set, +}: { + config: Record; + set: (patch: Record) => void; +}) { + const [lines, setLines] = useState(() => optionLines(c)); + return ( + <> + + {({ id, describedBy }) => ( +