From 8765a36905bdba52b910e50f6ed8076330e97085 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 06:20:06 +0000 Subject: [PATCH 1/2] feat(workflows): an HTTP call node behind the web tools' guard Asset and graph workflows call a web address through netguard, as the assistant's web tools do: public addresses on the web's ports (or networks an admin allowed), the host written out so what flows through can't move it, credential headers sealed to their header and host, and a try on the canvas that only reads. Redirects now keep any credential header, not only Authorization, from another server. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016j3Y4Dh3DLGwyFYBH23QML --- DEVLOG.md | 2 +- README.md | 3 +- docs/activity.md | 2 +- docs/processing.md | 14 ++ docs/requirements.md | 1 + fastapi_backend/app/domain/feeds.py | 13 +- fastapi_backend/app/domain/http_nodes.py | 233 ++++++++++++++++++ fastapi_backend/app/domain/workflows.py | 24 +- fastapi_backend/tests/api/test_http_nodes.py | 164 ++++++++++++ .../__tests__/workflow-flow.test.ts | 24 ++ nextjs-frontend/components/costs/costs.tsx | 4 + .../components/workflows/node-settings.tsx | 89 +++++++ .../components/workflows/workflow-model.ts | 42 +++- threat-model.json | 8 +- threat-model.md | 6 +- threat-model.yaml | 8 +- 16 files changed, 614 insertions(+), 23 deletions(-) create mode 100644 fastapi_backend/app/domain/http_nodes.py create mode 100644 fastapi_backend/tests/api/test_http_nodes.py diff --git a/DEVLOG.md b/DEVLOG.md index 718447286..c5fdc3044 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -620,7 +620,7 @@ Todo: - [x] Attach workflows from the pipeline editor - [x] Content-type mapping on the Pipelines page (By content type) - [ ] Asset converter steps (video → audio, document → images) as pipeline steps -- [ ] Workflow node types: HTTP call, template render, entity filter +- [x] Workflow node types: HTTP call, template render, entity filter (HTTP call and entity filter added 2026-10-18; template render is the template primitive) - [x] Docs: processing.md section on workflows Content types (2026-10-02, follow-up in the same PR): diff --git a/README.md b/README.md index 61034b8ed..ab9748317 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,8 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p namespaces) and let a decision model pick the branch, then tell you in your own chat app, ask the model or the agent, use a tool, call a web address (its credentials kept sealed), run a workflow, pipeline or routine, or change assets and notes. A Classify node in a workflow sorts an asset into labels, - content types or topics, acting only on a sure answer, and an Entity filter node keeps the entities you want. Anything that reaches outside Lens or changes more than what + content types or topics, acting only on a sure answer, and an Entity filter node keeps the entities you want; an HTTP call node calls a web address through the same + guard as the assistant's web tools, with its keys sealed to their host. Anything that reaches outside Lens or changes more than what set it off asks first, until you let it run on its own: on the Approvals page or, for a change inside Lens, with a yes in the chat it told you in; agents can decline an approval but never give one. Each automation has limits and budgets, and agents make, try and run them over MCP and in the assistant, or draft one from a sentence; the studio to draw them diff --git a/docs/activity.md b/docs/activity.md index 02a681f66..7ebce0f62 100644 --- a/docs/activity.md +++ b/docs/activity.md @@ -24,7 +24,7 @@ its namespace, its pipeline and the routine. A request counts for the resources Besides the resources, a row has its kind (`in`, `out` or `run`), what it was (`POST /api/v1/routines/{rid}`, `model.chat`, `model.describe` (a picture Florence-2 or Moondream described on this server), `embeddings`, `decision`, -`notify.webhook`, `tool.http`, `web.search`, `web.read`, `google.token`, `gmail.list`, `job.succeeded`, +`notify.webhook`, `tool.http`, `workflow.http`, `web.search`, `web.read`, `google.token`, `gmail.list`, `job.succeeded`, `routine.done`), the model, tokens in and out, the estimated cost in USD, how long it took, and how it ended (an error's type or HTTP status, never its message). It never holds prompt, reply, asset text or file content. diff --git a/docs/processing.md b/docs/processing.md index 852dd8901..d1950fbc7 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -720,6 +720,20 @@ node passes what it makes along its connections. `confidence`, `sure` and `ranked`, so a Condition or Switch can branch on them; [classifying](#classifying)), Pick (a path like `action_items.0.text`), Condition (yes and no branches; one test, or a [match tree](automations.md#match-trees) of several) and Merge. +- **Web:** HTTP call sends GET, POST, PUT, PATCH or DELETE to a web address and passes on the answer (`status`, `ok`, + `url`, `content_type`, `body`, and `json` when it is JSON, up to 200,000 characters), so a Condition can branch on + `ok` and a Pick or template can read the JSON. The address, header values and body are templates (what came in is + `{{ input }}`: `https://api.example.com/search?q={{ input | urlencode }}`), but the scheme, host and port are written + out, so what flows through a workflow never decides where it goes. It goes out as the assistant's web tools do: + through Lens's network guard, to public addresses on ports 80 and 443 only (or the private networks an admin allowed + in Settings → Documents), every redirect checked again. Headers that carry credentials (Authorization, cookies, API + keys and tokens) are sealed with the server's key when the workflow is saved, bound to that header and that host, and + shown as `••••`; a redirect to another server doesn't get them. A 4xx or 5xx answer is passed on with `ok` false; an + address that can't be reached fails the node. Each call is a `workflow.http` row in the [activity ledger](activity.md) + (the host and method, never what was sent). A try on the canvas sends a GET but only says what it would send for the + other methods. Asset and graph workflows have it; automations call web addresses with their own action, which asks + first ([actions](automations.md#actions-and-intents)). Refine later: credentials passed in through a custom node's parameter + aren't sealed, and a sealed header is bound to its host rather than to one workflow. - **Keeping results:** Save output (`outputs.`), Set field (a custom field, kept in the metadata history), Save entities, and a Classify node set to act on a sure answer (it sets the content type, suggests the topic or adds the tag; a try on the canvas only says what it would do). diff --git a/docs/requirements.md b/docs/requirements.md index 57adfe777..f1b9da4fd 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -182,6 +182,7 @@ request, to another check that proves the same thing; loosening a requirement or | Pipelines and workflows on a canvas | [test_workflows.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_workflows.py) `test_workflows_run_in_pipelines_drawn_as_graphs`; [workflow-canvas.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/workflow-canvas.test.ts); [journeys.mjs](https://github.com/adeelahmad/lense/blob/main/ci/clicks/journeys.mjs) `pipeline-create`, `workflow-create` | | A Classify node in workflows chooses from labels, content types or topics, and acting on a sure answer keeps something | [test_classify.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_classify.py) `test_a_workflow_classifies`; [workflow-flow.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/workflow-flow.test.ts) `chooses from labels, content types or topics, and acting on a sure answer keeps something` | | An Entity filter node keeps a list's entities by type, stop list, length, pattern or how often they're said | [test_flow.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_flow.py) `test_entity_filter_keeps_the_entities_asked_for`; [workflow-flow.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/workflow-flow.test.ts) `an entity filter says what it keeps and refuses lowercase types` | +| An HTTP call node calls a web address through the web tools' guard, with its host written out, its credentials sealed to their host, and a try that only reads | [test_http_nodes.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_http_nodes.py) `test_it_calls_the_address_and_passes_the_answer_on`, `test_where_it_goes_is_written_out_and_guarded`, `test_a_try_only_reads`; [workflow-flow.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/workflow-flow.test.ts) `an HTTP call node's host is written out` | | Routines on a schedule | [test_routines.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_routines.py) `test_routines_are_checked_and_run_when_due`, `test_schedules`; [journeys.mjs](https://github.com/adeelahmad/lense/blob/main/ci/clicks/journeys.mjs) `routine-create`; [baseline.json](https://github.com/adeelahmad/lense/blob/main/ci/clicks/baseline.json) `routine-create` ≤ 2 clicks | | Batch runs over a collection | [test_batches.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_batches.py) `test_collections_and_batches` | | Notifications to webhooks, Slack, Discord and Matterbridge | [test_notifications.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_notifications.py) `test_owners_manage_targets_and_a_test_is_signed`, `test_runs_that_end_are_sent_once_to_matterbridge` | diff --git a/fastapi_backend/app/domain/feeds.py b/fastapi_backend/app/domain/feeds.py index 86a6e71c2..5bb713be5 100644 --- a/fastapi_backend/app/domain/feeds.py +++ b/fastapi_backend/app/domain/feeds.py @@ -428,17 +428,24 @@ def _url(cfg, src): return url +class OtherServer(urllib.error.HTTPError): + """A redirect not followed: it goes to another server, which isn't given the request's credentials.""" + + class _Redirects(urllib.request.HTTPRedirectHandler): - """Follows a calendar's redirects, but not with its password to another server (or to plain http).""" + """Follows redirects, but not with a password or any other credential header (an API key, a token, a cookie: + actions_web.secret) to another server (or to plain http).""" def redirect_request(self, req, fp, code, msg, headers, newurl): + from .actions_web import secret # loaded here: the automation actions load what loads this + new = super().redirect_request(req, fp, code, msg, headers, newurl) - if new is not None and req.has_header("Authorization"): + if new is not None and any(secret(k) for k in req.headers): a, b = urllib.parse.urlsplit(req.full_url), urllib.parse.urlsplit(new.full_url) same = (a.hostname, a.port) == (b.hostname, b.port) and (a.scheme == b.scheme or b.scheme == "https") if not same: why = "it redirects to another server, which isn't given its password: use the address it ends at" - raise urllib.error.HTTPError(newurl, code, why, headers, fp) + raise OtherServer(newurl, code, why, headers, fp) return new diff --git a/fastapi_backend/app/domain/http_nodes.py b/fastapi_backend/app/domain/http_nodes.py new file mode 100644 index 000000000..4f575c4b0 --- /dev/null +++ b/fastapi_backend/app/domain/http_nodes.py @@ -0,0 +1,233 @@ +"""The HTTP call node of asset and graph workflows (scopes `recording` and `graph`; docs/processing.md, Workflows). + +`http` sends GET, POST, PUT, PATCH or DELETE to a web address and passes on the answer: {status, ok, url (where it +ended), content_type, body (its text, up to MAX_ANSWER), json (when it is JSON), truncated, untrusted}. A 4xx or 5xx +answer is passed on with `ok` false, so a condition can branch on it; an address that can't be reached fails the node. + +It goes out the way the assistant's web tools do (extensions.http_call, web.read): through netguard's proxy, to public +addresses on the web's ports only (or networks an admin allowed for web pages, `documents.web_networks`), every +redirect checked again, and never on to another server with its credentials. Each call is a `workflow.http` row in +the activity ledger, with the host and method, never what was sent or answered. + +The address, the header values and the body are templates, rendered like a template node's (what came in is +`{{ input }}`, beside the asset or the namespaces), but the scheme, host and port are written out and can't come from +a template, so what flows through a workflow never decides where it is sent. Header values that carry credentials +(Authorization, cookies, API keys and tokens: actions_web.secret) are sealed with the server's key when the workflow is +saved, bound to the header and to the scheme, host and port they were saved for, so a sealed value goes nowhere else; +they aren't templates, and are opened only to send. + +A try on the canvas sends a GET (it only reads) and says what it would send for the other methods. Automations call +web addresses with their own action (actions_web.py, `http_request`), which asks first; this node is not in their +scope. +""" + +from __future__ import annotations + +import json +import re +import urllib.error +import urllib.parse +import urllib.request + +from . import activity, flow, mcp_servers, settings, templates + +OWN_NODES = ("http",) +CONFIG = {"http": {"method", "url", "headers", "body", "seconds"}} +METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE") +READS = ("GET",) # what a try on the canvas sends +URL_RX = re.compile(r"^https?://[^/?#{}\s]+(?:[/?#]|$)") # the host is written out, not a template +MAX_URL, MAX_BODY, MAX_HEADERS = 2000, 64 * 1024, 30 +MAX_ANSWER = 200_000 # the most of an answer that is read, as the assistant's web tools read +SECONDS = 20 +AGENT = "Lens workflow" + + +def _secret(name): + from .actions_web import secret # loaded here: the automation actions load what workflows load + + return secret(name) + + +def _origin(url): + u = urllib.parse.urlsplit(url) + return f"{u.scheme}://{(u.hostname or '').lower()}:{u.port or (443 if u.scheme == 'https' else 80)}" + + +def _context(name, url): + return f"workflow-http:{name.lower()}@{_origin(url)}" + + +def _key(db): + return {"data_dir": db.data_dir} + + +def check_config(db, nid, cfg): + """ValueError naming what is wrong with an HTTP node's settings. Credential headers given in plain text are sealed + in `cfg` as it is kept; sealed ones must have been sealed for this address.""" + from .actions_web import FIXED, NAME_RX + + method = str(cfg.get("method") or "GET").upper() + if method not in METHODS: + raise ValueError(f"http node {nid}: the method is one of {', '.join(METHODS)}") + cfg["method"] = method + url = cfg.get("url") + if not isinstance(url, str) or len(url) > MAX_URL or not URL_RX.match(url.strip()): + raise ValueError( + f"http node {nid}: give a web address that starts with http:// or https:// and a host written out " + "(templates go after it, in its path or query)" + ) + cfg["url"] = url.strip() + _template(nid, "address", cfg["url"]) + headers = cfg.get("headers") or {} + if not isinstance(headers, dict) or len(headers) > MAX_HEADERS: + raise ValueError(f"http node {nid}: headers are up to {MAX_HEADERS} names and values") + kept = {} + for k, v in headers.items(): + if not NAME_RX.match(str(k)) or str(k).lower() in FIXED: + raise ValueError(f"http node {nid}: {k} isn't a header it can set") + if not isinstance(v, str) or "\r" in v or "\n" in v or len(v) > 8000: + raise ValueError(f"http node {nid}: the {k} header is one line of text") + if _secret(k) and v: + v = _seal(db, nid, k, v, cfg["url"]) + elif not mcp_servers.is_sealed(v): + _template(nid, f"{k} header", v) + kept[str(k)] = v + if kept: + cfg["headers"] = kept + else: + cfg.pop("headers", None) + body = cfg.get("body") + if body is not None: + if not isinstance(body, str) or len(body) > MAX_BODY: + raise ValueError(f"http node {nid}: the body is a template of up to {MAX_BODY // 1024} KB") + if body and method in ("GET", "DELETE"): + raise ValueError(f"http node {nid}: a {method} sends no body") + _template(nid, "body", body) + seconds = cfg.get("seconds") + if seconds is not None and not (isinstance(seconds, int) and not isinstance(seconds, bool) and 1 <= seconds <= 60): + raise ValueError(f"http node {nid}: it waits 1 to 60 seconds") + + +def _template(nid, what, text): + try: + templates.check(text) + except templates.TemplateProblem as e: + raise ValueError(f"http node {nid}: its {what}: {e}") from None + + +def _seal(db, nid, name, value, url): + """A credential header's value as it is kept: sealed for this header and address (a sealed one is checked).""" + ctx = _context(name, url) + if not mcp_servers.is_sealed(value): + return mcp_servers.SEALED + settings.seal(_key(db), value, ctx) + try: + settings.unseal(_key(db), value[len(mcp_servers.SEALED) :], ctx) + except Exception: # noqa: BLE001 - sealed for another header or address, or with another server key + raise ValueError(f"http node {nid}: enter the {name} header again: it was sealed for another address") from None + return value + + +def _open(db, name, value, url): + try: + return settings.unseal(_key(db), value[len(mcp_servers.SEALED) :], _context(name, url)) + except Exception: # noqa: BLE001 + raise ValueError(f"enter the {name} header again: it was sealed for another address") from None + + +def shown(headers): + """Headers as they're said: credentials, sealed or not, as "••••".""" + return {k: mcp_servers.HIDDEN if _secret(k) or mcp_servers.is_sealed(v) else v for k, v in (headers or {}).items()} + + +def run_node(r, f, node, value, vals): + """What an HTTP node does: {out: the answer}.""" + from . import webcapture + + c, name = node["config"], flow._label(node) + tctx = {**flow._tctx(r), "input": None if value is r.ctx else value} + method = c.get("method") or "GET" + url = templates.render_body(c["url"], tctx, "prompt").strip() + if urllib.parse.urlsplit(url).netloc.lower() != urllib.parse.urlsplit(c["url"]).netloc.lower() or re.search(r"\s", url): + raise ValueError(f"{name}: what it was given can't change where its request goes") + try: + url = webcapture.check_url(r.cfg, url) + except ValueError as e: + raise ValueError(f"{name}: {e}") from None + headers = {"User-Agent": AGENT} + for k, v in (c.get("headers") or {}).items(): + headers[k] = _open(r.db, k, v, c["url"]) if mcp_servers.is_sealed(v) else templates.render_body(v, tctx, "prompt") + if "\r" in headers[k] or "\n" in headers[k]: + raise ValueError(f"{name}: the {k} header came out as more than one line") + data = None + if c.get("body"): + text = templates.render_body(c["body"], tctx, "prompt") + data = text.encode("utf-8") + if len(data) > MAX_BODY: + raise ValueError(f"{name}: the body is over {MAX_BODY // 1024} KB") + if not any(k.lower() == "content-type" for k in headers): + headers["Content-Type"] = "application/json" if _is_json(text) else "text/plain; charset=utf-8" + sent = {"method": method, "url": url, "headers": shown({k: v for k, v in headers.items() if k != "User-Agent"})} + if r.dry and method not in READS: + r.say(f"{name}: would {method} {url}") + return {"out": {**sent, "status": None, "would": f"{method} {url}"}} + out = call(r.cfg, method, url, headers, data, c.get("seconds") or SECONDS, _refs(r)) + r.say(f"{name}: {method} {urllib.parse.urlsplit(url).hostname} answered {out['status']}") + return {"out": out} + + +def _is_json(text): + if text.lstrip()[:1] not in ("{", "["): + return False + try: + json.loads(text) + except ValueError: + return False + return True + + +def _refs(r): + rid = r.extra.get("rid") + return [f"recording:{int(rid)}"] if rid is not None else [f"space:{s}" for s in sorted(r.extra.get("spaces") or [])] + + +def call(cfg, method, url, headers, data, seconds, refs=()): + """One request through netguard (url already checked by webcapture.check_url): the answer as the node passes it + on. ValueError when it can't be reached.""" + from . import feeds, netguard, webcapture + + host = urllib.parse.urlsplit(url).hostname + req = urllib.request.Request(url, data=data, method=method, headers=headers) + with ( + activity.call("workflow.http", cfg, refs=refs, detail={"host": host, "method": method}), + netguard.Guard(forward=True, networks=webcapture.networks(cfg), max_bytes=MAX_ANSWER + 65536) as guard, + ): + try: + with feeds._opener(guard).open(req, timeout=seconds) as resp: + status, final, ctype, raw = resp.status, resp.geturl(), resp.headers.get("Content-Type", ""), resp.read(MAX_ANSWER + 1) + except feeds.OtherServer as e: + raise ValueError(f"{host}: {e.reason}") from None + except urllib.error.HTTPError as e: + if guard.refused: # the proxy turned it away (a redirect to an inside address, another port) + raise ValueError(f"{host} couldn't be reached; refused: {guard.refused[0]}") from None + status, final, raw = e.code, url, e.read(MAX_ANSWER + 1) if e.fp is not None else b"" + ctype = e.headers.get("Content-Type", "") if e.headers else "" + except (urllib.error.URLError, OSError) as e: + refused = f"; refused: {guard.refused[0]}" if guard.refused else "" + raise ValueError(f"couldn't reach {host}: {getattr(e, 'reason', e)}{refused}") from None + more = len(raw) > MAX_ANSWER + text = raw[:MAX_ANSWER].decode("utf-8", "replace") + out = { + "status": status, + "ok": 200 <= status < 300, + "url": final, + "content_type": ctype, + "body": text, + "truncated": more, + "untrusted": True, # what another server answered: outside text + } + if not more and "json" in ctype.lower(): + try: + out["json"] = json.loads(text) + except ValueError: + pass + return out diff --git a/fastapi_backend/app/domain/workflows.py b/fastapi_backend/app/domain/workflows.py index 6519aa49e..e3a056043 100644 --- a/fastapi_backend/app/domain/workflows.py +++ b/fastapi_backend/app/domain/workflows.py @@ -16,6 +16,8 @@ - `field`: sets a custom field of the recording to the value, checked like an edit (and kept in its history). - `save_entities`: makes the entities passed in the recording's entities (in place of what analyze found), with people's corrections kept, then redoes keywords and sections as analyze does. +- `http`: calls a web address, through the same guard as the assistant's web tools, and passes on the answer + (http_nodes.py; graph workflows have it too). - `classify`: which of some labels (or the content types, or the namespace's topics) fits the asset, or what came in as text, as classify.py says: passes on {choice, confidence, sure, ranked, by}. With `apply` it acts on a sure answer (sets the content type, suggests the topic, adds the tag), so it keeps something. @@ -41,6 +43,7 @@ feed_nodes, fields as fieldmod, flow, + http_nodes, llm, metadata, organize, @@ -52,7 +55,7 @@ ) R = store.R -OWN_NODES = ("llm", "extract_rules", "extract_llm", "classify", "output", "field", "save_entities") +OWN_NODES = ("llm", "extract_rules", "extract_llm", "classify", "output", "field", "save_entities") + http_nodes.OWN_NODES TERMINAL = {"output", "field", "save_entities", "apply_changes", *feed_nodes.TERMINAL} OPS = flow.OPS OWN_CONFIG = { @@ -63,6 +66,7 @@ "field": {"field"}, "save_entities": set(), "classify": {"labels", "vocabulary", "question", "of", "apply"}, + **http_nodes.CONFIG, } ENTITY_TYPES = ("PERSON", "ORG", "PRODUCT", "PLACE", "EVENT", "WORK", "TERM", "DATE", "NUMBER") PATH_RX = flow.PATH_RX @@ -101,7 +105,9 @@ def validate_graph(db, graph, scope="recording"): def _check_config(db, nid, t, cfg): - if t in organize.CONFIG: + if t in http_nodes.CONFIG: + http_nodes.check_config(db, nid, cfg) + elif t in organize.CONFIG: organize.check_config(nid, t, cfg) elif t in feed_nodes.CONFIG: feed_nodes.check_config(db, nid, t, cfg) @@ -364,6 +370,8 @@ def _run_node(r, f, node, value, vals): """What a recording workflow's own nodes do (flow.py runs the rest): {output port: value}.""" db, cfg, rid, t, c, nid = r.db, r.cfg, r.extra["rid"], node["type"], node["config"], node["id"] name, upstream = flow._label(node), None if value is r.ctx else value + if t in http_nodes.CONFIG: + return http_nodes.run_node(r, f, node, value, vals) if t == "llm": tpl = templates.get(db, int(c["template"]), c.get("version")) prompt = templates.render_body(tpl["body"], {**r.ctx, "input": upstream}, "prompt") @@ -434,6 +442,13 @@ def _classify(r, c, upstream, name): return {k: v for k, v in res.items() if k not in ("vocabulary", "question", "applied") and v is not None} +def _graph_node(t): + """What runs a graph workflow's own node of type t.""" + if t in http_nodes.CONFIG: + return http_nodes.run_node + return feed_nodes.run_node if t in feed_nodes.OWN_NODES else organize.run_node + + RUN_BY = {"graph": "organises the graph; a routine runs it", "event": "is what an automation does; its automation runs it"} @@ -504,8 +519,9 @@ def _fit(f, v): ), # organising the graph (organize.py) and publishing from it (feed_nodes.py), run over namespaces by a routine "graph": flow.Kit( - "graph", organize.OWN_NODES + feed_nodes.OWN_NODES, {**organize.CONFIG, **feed_nodes.CONFIG}, _check_config, - lambda r, f, n, v, vals: (feed_nodes.run_node if n["type"] in feed_nodes.OWN_NODES else organize.run_node)(r, f, n, v, vals), + "graph", organize.OWN_NODES + feed_nodes.OWN_NODES + http_nodes.OWN_NODES, + {**organize.CONFIG, **feed_nodes.CONFIG, **http_nodes.CONFIG}, _check_config, + lambda r, f, n, v, vals: _graph_node(n["type"])(r, f, n, v, vals), {"apply_changes", *feed_nodes.TERMINAL}, "a graph workflow needs an apply changes, publish to feed or podcast episode node, or it changes nothing", ), diff --git a/fastapi_backend/tests/api/test_http_nodes.py b/fastapi_backend/tests/api/test_http_nodes.py new file mode 100644 index 000000000..0b1b1fe8a --- /dev/null +++ b/fastapi_backend/tests/api/test_http_nodes.py @@ -0,0 +1,164 @@ +"""The HTTP call node of asset and graph workflows: out through the web tools' guard, with the host written out, the +answer passed on as outside text, credentials sealed to their address, and a try on the canvas that only reads.""" + +from __future__ import annotations + +import http.server +import json +import threading + +import pytest + +from app.domain import activity, analyze, http_nodes, ingest, mcp_servers, netguard, workflows +from tests.helpers import login, make_user + + +class _Api(http.server.BaseHTTPRequestHandler): + got: list = [] + + def _answer(self, status, body, ctype="application/json"): + data = body.encode() + self.send_response(status) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _take(self): + n = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(n).decode() if n else None + _Api.got.append({"method": self.command, "path": self.path, "headers": dict(self.headers), "body": body}) + + def do_GET(self): + self._take() + if self.path.startswith("/away"): # to the same server under another name: another host to urllib + self.send_response(302) + self.send_header("Location", f"http://localhost:{self.server.server_address[1]}/lookup") + self.end_headers() + elif self.path.startswith("/missing"): + self._answer(404, '{"error": "no such thing"}') + else: + self._answer(200, json.dumps({"path": self.path, "key": self.headers.get("X-Api-Key")})) + + def do_POST(self): + self._take() + self._answer(201, "made", "text/plain") + + def log_message(self, *a): + pass + + +@pytest.fixture +def api(cfg, monkeypatch): + srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _Api) + threading.Thread(target=srv.serve_forever, daemon=True).start() + _Api.got = [] + monkeypatch.setattr(netguard, "PORTS", netguard.PORTS | {srv.server_address[1]}) # the test server's port, as if 443 + cfg["documents"]["web_networks"] = ["127.0.0.0/8"] # a network an admin allowed: the test server's + yield f"http://127.0.0.1:{srv.server_address[1]}" + srv.shutdown() + + +@pytest.fixture +def rec(db, cfg): + rid = ingest.import_text(db, cfg, "pods", "Alice: Ship the capsid samples Friday.\nDave: Thanks.", title="Courier call") + analyze.analyze_recording(db, cfg, rid) + return rid + + +def node(nid, t, **config): + return {"id": nid, "type": t, "config": config} + + +def edge(a, b): + return {"source": a, "target": b} + + +def call_graph(http): + return { + "nodes": [node("in", "input"), node("call", "http", **http), node("o", "output", key="answer")], + "edges": [edge("in", "call"), edge("call", "o")], + } + + +def answer(db, rid): + return db.one("SELECT VALUE value FROM output WHERE recording = $r AND key = 'answer'", r=rid) + + +def test_it_calls_the_address_and_passes_the_answer_on(api, db, cfg, rec): + graph = call_graph( + { + "url": api + "/lookup?title={{ recording.title | urlencode }}", + "headers": {"X-Api-Key": "key-1", "X-Asset": "{{ recording.id }}"}, + } + ) + wid = workflows.create(db, "lookup", graph) + kept = workflows.get(db, wid)["graph"]["nodes"][1]["config"] + assert mcp_servers.is_sealed(kept["headers"]["X-Api-Key"]) and "key-1" not in json.dumps(kept) + activity.bind(db) # as the server and the worker do + workflows.run(db, cfg, rec, wid, say=lambda m: None) + out = answer(db, rec) + assert out["status"] == 200 and out["ok"] and out["untrusted"] + assert out["json"] == {"path": "/lookup?title=Courier%20call", "key": "key-1"} # opened only to send + assert _Api.got[-1]["headers"]["X-Asset"] == str(rec) + row = db.one("SELECT * OMIT id FROM activity WHERE action = 'workflow.http' ORDER BY at DESC LIMIT 1") + assert row["detail"] == {"host": "127.0.0.1", "method": "GET"} and f"recording:{rec}" in row["resources"] + + # a 4xx answer is passed on, to branch on; a sealed key moved to another host has to be entered again + workflows.save_version(db, wid, call_graph({**kept, "url": api + "/missing"})) + workflows.run(db, cfg, rec, wid, say=lambda m: None) + assert answer(db, rec)["status"] == 404 and answer(db, rec)["ok"] is False + with pytest.raises(ValueError, match="enter the X-Api-Key header again"): + workflows.save_version(db, wid, call_graph({**kept, "url": "https://elsewhere.example/"})) + + +def test_where_it_goes_is_written_out_and_guarded(api, db, cfg, rec): + def problem(http): + with pytest.raises(ValueError) as e: + workflows.validate_graph(db, call_graph(http)) + return str(e.value) + + assert "host written out" in problem({"url": "https://{{ input.host }}/x"}) + assert "host written out" in problem({"url": "https://api.example.com{{ input }}"}) + assert "sends no body" in problem({"url": "https://api.example.com/", "body": "{{ input }}"}) + assert "method is one of" in problem({"url": "https://api.example.com/", "method": "TRACE"}) + assert "isn't a header it can set" in problem({"url": "https://api.example.com/", "headers": {"Host": "x"}}) + assert "1 to 60 seconds" in problem({"url": "https://api.example.com/", "seconds": 0}) + assert "http" not in workflows.SCOPES["event"] and "http" not in workflows.SCOPES["tool"] # automations ask first + + # what flows through can't move it, and the guard keeps it to public addresses (or networks an admin allowed) + wid = workflows.create(db, "away", call_graph({"url": api + "/{{ recording.title }}"})) + with pytest.raises(ValueError, match="can't change where"): + workflows.run(db, cfg, rec, wid, say=lambda m: None) + cfg["documents"]["web_networks"] = [] + workflows.save_version(db, wid, call_graph({"url": api + "/lookup"})) + with pytest.raises(ValueError, match="public address"): + workflows.run(db, cfg, rec, wid, say=lambda m: None) + cfg["documents"]["web_networks"] = ["127.0.0.0/8"] + + # a credential isn't taken to another server by a redirect + workflows.save_version(db, wid, call_graph({"url": api + "/away", "headers": {"Authorization": "Bearer t"}})) + with pytest.raises(ValueError, match="redirects to another server"): + workflows.run(db, cfg, rec, wid, say=lambda m: None) + assert [g["path"] for g in _Api.got] == ["/away"] + + +def test_a_try_only_reads(api, client, db, cfg, rec): + make_user(db, "root@x.io", "root password 1", admin=True) + h = login(client, "root@x.io", "root password 1") + post = call_graph({"method": "POST", "url": api + "/notes", "body": '{"title": {{ recording.title | json }}}'}) + t = client.post("/api/v1/workflows/test", headers=h, json={"graph": post, "recording": rec}).json() + assert t["error"] is None and '"would": "POST' in t["trace"]["call"]["value"] and _Api.got == [] + get = call_graph({"url": api + "/lookup"}) + t = client.post("/api/v1/workflows/test", headers=h, json={"graph": get, "recording": rec}).json() + assert t["error"] is None and [g["method"] for g in _Api.got] == ["GET"] + + # run for real, a POST sends its body as JSON + wid = workflows.create(db, "post", post) + workflows.run(db, cfg, rec, wid, say=lambda m: None) + sent = _Api.got[-1] + assert sent["method"] == "POST" and json.loads(sent["body"]) == {"title": "Courier call"} + assert sent["headers"]["Content-Type"] == "application/json" and answer(db, rec)["body"] == "made" + types = {n["type"]: n for n in client.get("/api/v1/workflows", headers=h).json()["node_types"]} + assert types["http"]["scopes"] == ["recording", "graph"] + assert http_nodes.shown({"Authorization": "Bearer t", "Accept": "x"}) == {"Authorization": "••••", "Accept": "x"} diff --git a/nextjs-frontend/__tests__/workflow-flow.test.ts b/nextjs-frontend/__tests__/workflow-flow.test.ts index cb59f7d2f..8e545297b 100644 --- a/nextjs-frontend/__tests__/workflow-flow.test.ts +++ b/nextjs-frontend/__tests__/workflow-flow.test.ts @@ -182,3 +182,27 @@ describe("entity filter", () => { expect(nodeSummary({ id: "f", type: "entity_filter", config: {} }, names)).toBe("all types"); }); }); + +describe("HTTP call", () => { + const names = { template: () => undefined, field: () => undefined }; + const wf = (config: Record): WfGraph => ({ + nodes: [ + { id: "in", type: "input", config: {} }, + { id: "h", type: "http", config }, + { id: "o", type: "output", config: { key: "answer" } }, + ], + edges: [ + { source: "in", target: "h" }, + { source: "h", target: "o" }, + ], + }); + + it("an HTTP call node's host is written out", () => { + expect(inScope("http", "recording") && inScope("http", "graph")).toBe(true); + expect(inScope("http", "tool")).toBe(false); + expect(problems(wf({ method: "GET", url: "https://api.example.com/q?x={{ input }}" }), kind)).toEqual({}); + expect(problems(wf({ url: "https://{{ input }}/x" }), kind).h).toMatch(/host written out/); + expect(problems(wf({ method: "GET", url: "https://a.example/", body: "x" }), kind).h).toMatch(/send a body/); + expect(nodeSummary({ id: "h", type: "http", config: defaultConfig("http") }, names)).toBe("GET https://"); + }); +}); diff --git a/nextjs-frontend/components/costs/costs.tsx b/nextjs-frontend/components/costs/costs.tsx index b4e708792..79de4d4d2 100644 --- a/nextjs-frontend/components/costs/costs.tsx +++ b/nextjs-frontend/components/costs/costs.tsx @@ -132,6 +132,10 @@ export function actionText(e: ActivityEntry): string { if (a === "decision") return `Decision model${e.model ? ` · ${e.model}` : ""}`; if (a.startsWith("notify.")) return `Notification · ${a.slice(7)}`; if (a === "tool.http") return "Web tool"; + if (a === "workflow.http") { + const host = ((e.detail ?? {}) as { host?: string }).host; + return `Workflow web call${host ? ` · ${host}` : ""}`; + } if (a === "web.search" || a === "web.read") { const d = (e.detail ?? {}) as { provider?: string; host?: string }; const where = a === "web.search" ? d.provider : d.host; diff --git a/nextjs-frontend/components/workflows/node-settings.tsx b/nextjs-frontend/components/workflows/node-settings.tsx index a1220b56c..d494e7a62 100644 --- a/nextjs-frontend/components/workflows/node-settings.tsx +++ b/nextjs-frontend/components/workflows/node-settings.tsx @@ -60,6 +60,10 @@ function Lines({ ); } +/** How a sealed value starts (mcp_servers.SEALED), and how it is shown. */ +const SEALED = "lens-sealed:"; +const HIDDEN = "••••"; + const typesText = (v: unknown) => ((v as string[] | undefined) ?? []).join(", "); const typesOf = (s: string) => s @@ -458,6 +462,91 @@ export function NodeSettings({ )} + {node.type === "http" && ( + <> + + {({ id }) => ( + set({ url: e.target.value })} + /> + )} + + | undefined) ?? {})} + show={(v) => { + const [k, val] = v as [string, string]; + return `${k}: ${val.startsWith(SEALED) ? HIDDEN : val}`; + }} + parse={(lines) => { + const old = (c.headers as Record | undefined) ?? {}; + const headers: Record = {}; + for (const l of lines) { + const i = l.indexOf(":"); + const k = (i < 0 ? l : l.slice(0, i)).trim(); + const val = i < 0 ? "" : l.slice(i + 1).trim(); + if (k) headers[k] = val === HIDDEN && old[k]?.startsWith(SEALED) ? old[k] : val; + } + set({ headers: Object.keys(headers).length ? headers : undefined }); + }} + /> + {["POST", "PUT", "PATCH"].includes(String(c.method)) && ( + + {({ id, describedBy }) => ( +