From 00c753dbb252dd0d2e77a811e79e287da1f070a6 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 08:11:25 +0000 Subject: [PATCH 1/2] feat(scaling): Linux hosts over SSH as a provider Each unit is the first-boot script started in its own process group on one of the provider's hosts, up to per_host each; letting it go stops the group. Host keys are trusted on first use when an admin saves or checks the provider, kept when a later check fails, and held to on every call. Paramiko is the ssh extra, fetched when the first SSH provider is saved. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ALjNbLobvPe7VwvDL8j7V3 --- README.md | 8 +- docs/requirements.md | 1 + docs/scaling.md | 19 +- fastapi_backend/app/domain/scaling.py | 4 +- fastapi_backend/app/domain/scaling_drivers.py | 1 + fastapi_backend/app/domain/scaling_ssh.py | 198 ++++++++++++++ fastapi_backend/pyproject.toml | 3 + fastapi_backend/tests/api/test_scaling_ssh.py | 243 ++++++++++++++++++ fastapi_backend/uv.lock | 106 +++++++- threat-model.md | 2 +- threat-model.yaml | 2 +- 11 files changed, 575 insertions(+), 12 deletions(-) create mode 100644 fastapi_backend/app/domain/scaling_ssh.py create mode 100644 fastapi_backend/tests/api/test_scaling_ssh.py diff --git a/README.md b/README.md index a5eb642ca..a0794712b 100644 --- a/README.md +++ b/README.md @@ -186,8 +186,8 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p schedule rules, within min and max and a monthly budget, with every launch, join and hour of cost in the activity ledger. A group launches nothing until you approve its terms in Settings › Auto scaling; agents propose and manage groups over the API, MCP and the assistant, and never approve one. It runs more workers on this machine today; - DigitalOcean, Hetzner Cloud and Proxmox VE providers are in progress (built, launching once the mesh's install - line lands), and AWS, Linux and model hosts are planned. ([Auto scaling](docs/scaling.md); + DigitalOcean, Hetzner Cloud, Proxmox VE and Linux hosts over SSH are in progress (built, launching once the + mesh's install line lands), and AWS and model hosts are planned. ([Auto scaling](docs/scaling.md); [checks](docs/requirements.md#scale-out)) - **Keep it up to date and backed up.** Lens checks for a new release once a day and tells admins by email or a notification. One click in Settings › Updates (or one API call) upgrades it through a small agent on the machine, @@ -264,8 +264,8 @@ Next, and not in a pull request yet; each links to where it is tracked: topics, recordings and the other things that have pages ([notes](docs/notes.md)). - **Local models:** GPU builds of llama.cpp fetched by Lens, and local embedding models ([local models](docs/local-models.md)). -- **Auto scaling:** providers for AWS and Linux hosts over SSH, joining over the Nebula mesh, model hosts - (RunPod, Lambda, Hugging Face, SageMaker, Bedrock), and making providers and groups in Settings +- **Auto scaling:** a provider for AWS, joining over the Nebula mesh, model hosts (RunPod, Lambda, Hugging Face, + SageMaker, Bedrock), and making providers and groups in Settings ([auto scaling](docs/scaling.md)). - **Speech providers:** their summaries, chapters and entities as optional imports, and per-minute prices in the activity ledger ([speech providers](docs/speech-providers.md)). diff --git a/docs/requirements.md b/docs/requirements.md index 6fa07c2e5..f4a0d5a10 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -228,6 +228,7 @@ request, to another check that proves the same thing; loosening a requirement or | This machine runs more workers at no cost | [test_scaling.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling.py) `test_this_machine_runs_more_workers_at_no_cost` | | DigitalOcean and Hetzner Cloud: credentials checked, list prices, machines launched with a one-use join token, watched, found by tag and deleted; a group waits for the mesh's install line before launching | [test_scaling_clouds.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling_clouds.py) `test_a_digitalocean_group_waits_for_the_install_line_then_launches_droplets`, `test_hetzner_prices_in_euros_and_its_errors_read_as_hetzners`, `test_the_cloud_drivers_are_listed_and_need_the_mesh` | | Proxmox VE: clones of a VM template, started and booted through the guest agent, found by tag, stopped and deleted; a clone whose agent never answers says so | [test_scaling_proxmox.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling_proxmox.py) `test_proxmox_clones_boots_through_the_guest_agent_and_deletes`, `test_a_clone_whose_agent_never_answers_says_so`, `test_a_clone_still_booting_is_pending` | +| Linux hosts over SSH: host keys trusted on first use and held to, workers started per host with a script that removes itself, found, counted and stopped | [test_scaling_ssh.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling_ssh.py) `test_host_specs`, `test_ssh_trusts_host_keys_on_first_use_and_holds_to_them`, `test_ssh_starts_counts_and_stops_workers` | | Agents manage scaling in the assistant and over MCP: proposals wait for the person, actions within the terms happen at once | [test_scaling_agents.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_scaling_agents.py) `test_the_assistant_proposes_and_the_person_approves_in_the_web_app`, `test_a_setup_conversation_makes_a_group_that_still_waits_for_the_person`, `test_only_admins_get_the_scaling_tools`, `test_mcp_clients_manage_scaling_but_never_approve` | ## Keep it up to date and backed up diff --git a/docs/scaling.md b/docs/scaling.md index ed8072449..75fcdc0ba 100644 --- a/docs/scaling.md +++ b/docs/scaling.md @@ -10,9 +10,10 @@ Status: in progress. Built: groups, rules, approvals, budgets, the scaler, the A Settings › Auto scaling (the switch, and each group's terms to approve or pause), and **This machine**, a provider that runs more `lens worker` processes where Lens runs. In progress: **DigitalOcean** and **Hetzner Cloud** providers are built (credentials checked, list prices, launching, watching and deleting machines), and launch once the mesh's -install line, which another machine joins Lens with, lands ([the mesh](mesh.md)); so is **Proxmox VE**, clones of a -VM template on your own server. Planned: providers for AWS and any Linux host over SSH; model hosts (RunPod, Lambda, Hugging Face, SageMaker, Bedrock) for models; and making -providers and groups in Settings. +install line, which another machine joins Lens with, lands ([the mesh](mesh.md)); so are **Proxmox VE**, clones of a +VM template on your own server, and **Linux hosts over SSH**, workers started on machines you have. Planned: a +provider for AWS; model hosts (RunPod, Lambda, Hugging Face, SageMaker, Bedrock) for models; and making providers and +groups in Settings. Auto scaling is **off** until an admin turns it on, and a group launches nothing until a person approves its terms signed in to the web app. Agents can propose and manage groups, but never approve one. @@ -57,6 +58,7 @@ carries none of them. | `local` (This machine) | more `lens worker` processes where the scaler runs | nothing | `steps`: the steps they run (none: every step) | | `digitalocean` | droplets in `region`, with `token` (sealed), optional `ssh_keys` and `vpc_uuid` | DigitalOcean's hourly list price | `size` (`s-1vcpu-2gb`), `image` (`ubuntu-24-04-x64`), `steps` | | `proxmox` (Proxmox VE) | clones of a VM template on `node`, at `url` with an API token (`token_id`, `token_secret` sealed); `verify_tls` (on) and `ca_cert` for a private CA | nothing, unless you give it a price | `vm_template` (its id), `full_clone` (off: linked), `storage`, `pool`, `steps` | +| `ssh` (Linux hosts over SSH) | `lens worker` started on `hosts` (`user@host` or `user@host:port`), signed in with `private_key` (sealed; `passphrase` if it has one) | nothing, unless you give it a price | `per_host` (1): workers on each host; `steps` | | `hetzner` (Hetzner Cloud) | servers in `location`, with `token` (sealed), optional `ssh_keys` | Hetzner's hourly gross price in euros, times `usd_per_eur` (1.1; set your rate) | `server_type` (`cx22`), `image` (`ubuntu-24.04`), `steps` | A worker on This machine stops with the process that started it, and the scaler that takes over starts them again where @@ -79,6 +81,15 @@ deleted. Letting a unit go stops the clone and deletes it with its disks. The AP start, stop and delete VMs and use their guest agent (the `PVEVMAdmin` role on the template and the clones' pool is enough). +An SSH unit is the first-boot script started on one of the provider's hosts, the first with fewer than `per_host` of +Lens's workers running, in its own process group (`setsid`) under `~/.lens-scaling/`: the script is written there +readable only by that user and deletes itself as it starts, since it holds the unit's join token; the unit's process +id and log stay beside it. Letting a unit go stops its process group and removes its files. Host keys are trusted on +first use: saving or checking the provider records each host's key on it (`check.host_keys`, kept when a later check +fails), and every other call refuses a host whose key differs. A host whose key changed for a good reason is taken off +the provider and added back. The SSH client is Paramiko (LGPL-2.1, used unmodified), the `ssh` extra, fetched the +first time an SSH provider is saved. + Until the mesh is on and has its install line, a group on these providers waits (`waiting_for` says so) and launches nothing, since its machines couldn't join. They call the providers' REST APIs with httpx, so they install nothing. @@ -170,7 +181,7 @@ Turning scaling on (`change_settings scaling`) always asks. ## Refine later -Providers for SSH hosts and model hosts; Proxmox LXC containers; the AWS provider (in its own pull request); revoking a unit's mesh +Providers for model hosts; Proxmox LXC containers; the AWS provider (in its own pull request); revoking a unit's mesh certificate when it goes; reporting machines with Lens's tag that Lens has no unit for (`listing`); a live euro rate; spot and preemptible instances; warm pools; predictive scaling; health checks beyond joining; prices fetched from providers' pricing APIs. diff --git a/fastapi_backend/app/domain/scaling.py b/fastapi_backend/app/domain/scaling.py index dd4928626..738ad5ebb 100644 --- a/fastapi_backend/app/domain/scaling.py +++ b/fastapi_backend/app/domain/scaling.py @@ -44,7 +44,7 @@ import time from . import activity, budgets, jobs, schedule, settings, store -from . import scaling_clouds, scaling_proxmox # noqa: F401 - registers their drivers +from . import scaling_clouds, scaling_proxmox, scaling_ssh # noqa: F401 - registers their drivers from . import scaling_drivers as drivers from .store import R @@ -224,6 +224,8 @@ def _check_call(db, cfg, p, drv): result = {"ok": True, **(out if isinstance(out, dict) else {})} except Exception as e: # noqa: BLE001 - what the provider said, for the person result = {"ok": False, "error": f"{type(e).__name__}: {e}"[:500] if not isinstance(e, drivers.DriverError) else str(e)[:500]} + before = p.get("check") or {} + result |= {k: before[k] for k in drv.kept if k in before} # what a failed check mustn't forget (SSH host keys) db.q("UPDATE $r SET checked_at = $t, check = $c", r=R("scale_provider", p["id"]), t=store.now(), c=result) return result diff --git a/fastapi_backend/app/domain/scaling_drivers.py b/fastapi_backend/app/domain/scaling_drivers.py index e9e1835ed..64beb9e7b 100644 --- a/fastapi_backend/app/domain/scaling_drivers.py +++ b/fastapi_backend/app/domain/scaling_drivers.py @@ -58,6 +58,7 @@ class Driver: label = "" about = "" priced = True # it charges money: its groups need a budget + kept: tuple[str, ...] = () # what a failed check keeps of the last one's answer needs_join = False # its units are other machines, which join over the mesh's install line (scaling.bootstrap) extra = None # the optional dependency (pyproject.toml) it needs, fetched when a provider of it is saved modules = () # what that extra installs, to see whether it's here diff --git a/fastapi_backend/app/domain/scaling_ssh.py b/fastapi_backend/app/domain/scaling_ssh.py new file mode 100644 index 000000000..ffd0861d2 --- /dev/null +++ b/fastapi_backend/app/domain/scaling_ssh.py @@ -0,0 +1,198 @@ +"""Linux machines you have, reached over SSH, as an auto scaling provider (scaling_drivers.py, docs/scaling.md): each +unit is a `lens worker` started on one of the provider's hosts, which joins Lens over the mesh; letting it go stops it. +Nothing is created or deleted, so it costs nothing unless you give the provider a price. + +On each host a unit is a few files in `~/.lens-scaling/`: `.pid` (its process group, started with setsid) and +`.log`. The first-boot script (scaling_drivers.user_data) is written there with only the user able to read it, +and the unit deletes it as it starts, since it holds the unit's join token. + +Host keys are trusted on first use, when the provider is saved or checked: the check records each host's key on the +provider (`check.host_keys`, kept when a later check fails), and every other call refuses a host whose key isn't the +one recorded. A host whose key changed for a good reason is taken off the provider and added back. + +It uses Paramiko (LGPL-2.1, unmodified), the `ssh` extra, fetched the first time an SSH provider is saved. +""" + +from __future__ import annotations + +import base64 +import io +import re +import shlex + +from . import jobs +from .scaling_drivers import Driver, DriverError, _field, register, user_data + +TIMEOUT = 20.0 +DIR = ".lens-scaling" +HOST_RX = re.compile(r"^(?P[a-z_][a-z0-9_.-]{0,31})@(?P\[[0-9a-fA-F:.]+\]|[A-Za-z0-9.-]{1,253})(?::(?P\d{1,5}))?$") + + +def parse_host(spec): + """`user@host`, `user@host:port` or `user@[v6]:port` as (user, host, port); ValueError otherwise.""" + m = HOST_RX.match(spec or "") + if not m or not 0 < int(m["port"] or 22) < 65536: + raise ValueError(f"an SSH host is user@host or user@host:port, not {spec!r}") + return m["user"], m["host"].strip("[]"), int(m["port"] or 22) + + +def known_name(host, port): + """How known_hosts names a host: `host`, or `[host]:port` off port 22.""" + return host if port == 22 else f"[{host}]:{port}" + + +class Ssh(Driver): + id = "ssh" + label = "Linux hosts over SSH" + about = "Starts Lens workers on Linux machines you have, over SSH; they join Lens over the mesh. Costs nothing." + priced = False + needs_join = True + extra = "ssh" + modules = ("paramiko",) + license = "LGPL-2.1 (Paramiko)" + kept = ("host_keys",) + fields = { + "hosts": _field("Hosts", "list", required=True, help="user@host or user@host:port, one per machine"), + "private_key": _field("Private key", required=True, secret=True, help="an OpenSSH or PEM private key those users accept"), + "passphrase": _field("Key passphrase", secret=True, help="when the key has one"), + } + template = { + "per_host": _field("Workers on each host", "integer", default=1, help="how many of the group's units a host runs at once"), + "steps": _field( + "Steps", + "list", + help="the steps these workers run (transcribe, analyze, embed...); none: every step", + options=sorted(jobs.STEPS), + ), + } + + # ---------- SSH ---------- + def _key(self, p): + import paramiko + + text, phrase = p["secrets"].get("private_key") or "", p["secrets"].get("passphrase") or None + for kind in (paramiko.Ed25519Key, paramiko.ECDSAKey, paramiko.RSAKey): + try: + return kind.from_private_key(io.StringIO(text), password=phrase) + except paramiko.PasswordRequiredException: + raise DriverError("the private key needs its passphrase") from None + except (paramiko.SSHException, ValueError): + continue + raise DriverError("the private key can't be read: give an OpenSSH or PEM private key") + + def _known(self, p): + return dict(((p.get("check") or {}).get("host_keys")) or {}) + + def run(self, p, spec, command, stdin=None, learn=None): + """Run `command` on the host `spec` and return (exit code, stdout). With `learn` (a dict), a host with no + recorded key is trusted and its key recorded there; without it, only the recorded key is accepted.""" + import paramiko + + user, host, port = parse_host(spec) + name = known_name(host, port) + known = self._known(p) + client = paramiko.SSHClient() + if known.get(spec): + kind, _, b64 = known[spec].partition(" ") + client.get_host_keys().add(name, kind, paramiko.PKey.from_type_string(kind, base64.b64decode(b64))) + + class Learn(paramiko.MissingHostKeyPolicy): + def missing_host_key(self, c, hostname, key): + if learn is None: + raise DriverError(f"{spec}'s host key isn't one Lens has seen: check the provider to trust it") + learn[spec] = f"{key.get_name()} {key.get_base64()}" + + client.set_missing_host_key_policy(Learn()) + try: + client.connect( + host, + port=port, + username=user, + pkey=self._key(p), + timeout=TIMEOUT, + auth_timeout=TIMEOUT, + banner_timeout=TIMEOUT, + allow_agent=False, + look_for_keys=False, + ) + if learn is not None and known.get(spec): + learn[spec] = known[spec] + stdin_, stdout, _ = client.exec_command(command, timeout=TIMEOUT) + if stdin is not None: + stdin_.write(stdin) + stdin_.channel.shutdown_write() + out = stdout.read().decode(errors="replace") + return stdout.channel.recv_exit_status(), out + except paramiko.BadHostKeyException: + raise DriverError(f"{spec}'s host key changed: take the host off the provider and add it back if that's expected") from None + except paramiko.AuthenticationException: + raise DriverError(f"{spec} refused the key") from None + except (OSError, paramiko.SSHException) as e: + raise DriverError(f"couldn't reach {spec}: {type(e).__name__}: {e}"[:300]) from None + finally: + client.close() + + # ---------- the driver ---------- + def check(self, p): + seen: dict[str, str] = {} + for spec in p["settings"]["hosts"]: + code, out = self.run(p, spec, "uname -sm", learn=seen) + if code != 0 or not out.startswith("Linux"): + raise DriverError(f"{spec} isn't a Linux machine Lens can run on ({out.strip() or code})") + return {"hosts": len(seen), "host_keys": seen} + + def _units(self, p, spec): + code, out = self.run( + p, + spec, + f'cd ~/{DIR} 2>/dev/null && for f in *.pid; do [ -f "$f" ] && kill -0 "$(cat "$f")" 2>/dev/null && echo "${{f%.pid}}"; done; true', + ) + return [x for x in out.split() if x] + + def launch(self, p, template, name, boot): + per = template.get("per_host") or 1 + for spec in p["settings"]["hosts"]: + if len(self._units(p, spec)) >= per: + continue + q = shlex.quote(name) + self.run(p, spec, f"umask 077 && mkdir -p ~/{DIR} && cat > ~/{DIR}/{q}.sh", stdin=user_data(boot, template)) + start = ( + f'cd ~/{DIR} && nohup setsid sh -c \'f="$1"; s=$(cat "$f"); rm -f "$f"; exec sh -c "$s"\' sh {q}.sh' + f" > {q}.log 2>&1 < /dev/null & echo $! > ~/{DIR}/{q}.pid" + ) + code, _ = self.run(p, spec, start) + if code != 0: + raise DriverError(f"couldn't start a worker on {spec} (exit {code})") + return {"id": f"{spec}#{name}"} + raise DriverError(f"every host runs its {per} worker(s): add a host or lower the group's max") + + def _where(self, node): + spec, _, name = (node.get("external_id") or "").partition("#") + return spec, name + + def describe(self, p, node): + spec, name = self._where(node) + if not name: + return {"state": "gone"} + return {"state": "running" if name in self._units(p, spec) else "gone"} + + def terminate(self, p, node): + spec, name = self._where(node) + if not name or spec not in p["settings"]["hosts"]: + return + q = shlex.quote(name) + self.run( + p, + spec, + f'cd ~/{DIR} 2>/dev/null || exit 0; if [ -f {q}.pid ]; then g=$(cat {q}.pid); kill -TERM -- -"$g" 2>/dev/null;' + f' for i in 1 2 3 4 5 6 7 8 9 10; do kill -0 "$g" 2>/dev/null || break; sleep 1; done;' + f' kill -KILL -- -"$g" 2>/dev/null; fi; rm -f {q}.pid {q}.sh {q}.log; true', + ) + + def listing(self, p, tag): + return [ + {"id": f"{spec}#{n}", "name": n} for spec in p["settings"]["hosts"] for n in self._units(p, spec) if n.startswith(f"{tag}-") + ] + + +register(Ssh()) diff --git a/fastapi_backend/pyproject.toml b/fastapi_backend/pyproject.toml index 35255e11d..d0b0b718c 100644 --- a/fastapi_backend/pyproject.toml +++ b/fastapi_backend/pyproject.toml @@ -74,6 +74,9 @@ msg = ["extract-msg"] # secrets read from a Vaultwarden or Bitwarden vault (vaultwarden_secrets.py): python-vaultwarden (Apache-2.0); fetched # when Settings › Secrets has a Vaultwarden address vaultwarden = ["python-vaultwarden>=1.1.2,<2"] +# auto scaling on Linux hosts over SSH (scaling_ssh.py): Paramiko (LGPL-2.1, used unmodified as a library); fetched +# when the first SSH provider is saved +ssh = ["paramiko>=3.5,<5"] [project.scripts] lens = "app.cli:main" diff --git a/fastapi_backend/tests/api/test_scaling_ssh.py b/fastapi_backend/tests/api/test_scaling_ssh.py new file mode 100644 index 000000000..f5eef0b95 --- /dev/null +++ b/fastapi_backend/tests/api/test_scaling_ssh.py @@ -0,0 +1,243 @@ +"""Auto scaling on Linux hosts over SSH (docs/scaling.md "Providers"): host keys trusted on first use and held to, +workers started, found, counted per host and stopped. Paramiko is an extra, so a stand-in for it here runs each +command with this machine's `sh` in a home folder of its own: the real shell the driver sends.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time +import types + +import pytest + +from app.domain import scaling, scaling_ssh +from app.domain import scaling_drivers as drivers +from tests.helpers import login, make_user + + +class Key: + def __init__(self, name="ssh-ed25519", b64="QUFBQQ=="): + self.name, self.b64 = name, b64 + + def get_name(self): + return self.name + + def get_base64(self): + return self.b64 + + +def fake_paramiko(hosts): + """`hosts`: {(host, port): {"home": path, "key": Key}}.""" + m = types.ModuleType("paramiko") + + class SSHException(Exception): + pass + + class AuthenticationException(SSHException): + pass + + class PasswordRequiredException(AuthenticationException): + pass + + class BadHostKeyException(SSHException): + pass + + class MissingHostKeyPolicy: + pass + + class Out: + def __init__(self, data, code): + self.data, self.channel = data, types.SimpleNamespace(recv_exit_status=lambda: code) + + def read(self): + return self.data + + class In: + def __init__(self, run): + self.buf, self.run = [], run + self.channel = types.SimpleNamespace(shutdown_write=lambda: None) + + def write(self, s): + self.buf.append(s) + + class Keys(dict): + def add(self, name, kind, key): + self[name] = key + + class SSHClient: + def __init__(self): + self.keys, self.policy, self.host = Keys(), None, None + + def get_host_keys(self): + return self.keys + + def set_missing_host_key_policy(self, policy): + self.policy = policy + + def connect(self, host, port=22, username=None, pkey=None, **kw): + h = hosts.get((host, port)) + if h is None: + raise OSError("no route to host") + name = host if port == 22 else f"[{host}]:{port}" + if name in self.keys: + if self.keys[name].get_base64() != h["key"].get_base64(): + raise BadHostKeyException() + else: + self.policy.missing_host_key(self, name, h["key"]) + if pkey != "the key": + raise AuthenticationException() + self.host = h + + def exec_command(self, command, timeout=None): + client = self + + class Stdin(In): + pass + + stdin = Stdin(None) + + class Lazy(Out): + def __init__(self): + self._done = None + self.channel = types.SimpleNamespace(recv_exit_status=lambda: self._run()[0]) + + def _run(self): + if self._done is None: + env = {"HOME": client.host["home"], "PATH": os.environ["PATH"]} + r = subprocess.run( + ["sh", "-c", command], input="".join(stdin.buf).encode(), capture_output=True, env=env, timeout=30 + ) + self._done = (r.returncode, r.stdout) + return self._done + + def read(self): + return self._run()[1] + + return stdin, Lazy(), None + + def close(self): + pass + + class PKey: + @staticmethod + def from_type_string(kind, data): + import base64 + + return Key(kind, base64.b64encode(data).decode()) + + class Ed25519Key: + @staticmethod + def from_private_key(f, password=None): + if f.read() != "PRIVATE": + raise SSHException("not a key") + return "the key" + + class Other: + @staticmethod + def from_private_key(f, password=None): + raise SSHException("not a key") + + m.__dict__.update( + SSHException=SSHException, + AuthenticationException=AuthenticationException, + PasswordRequiredException=PasswordRequiredException, + BadHostKeyException=BadHostKeyException, + MissingHostKeyPolicy=MissingHostKeyPolicy, + SSHClient=SSHClient, + PKey=PKey, + Ed25519Key=Ed25519Key, + ECDSAKey=Other, + RSAKey=Other, + ) + return m + + +@pytest.fixture +def hosts(tmp_path, monkeypatch): + if not sys.platform.startswith("linux") and sys.platform != "darwin": + pytest.skip("needs a POSIX shell") + hs = { + ("box1", 22): {"home": str(tmp_path / "box1"), "key": Key(b64="Ym94MQ==")}, + ("10.0.0.5", 2222): {"home": str(tmp_path / "box2"), "key": Key(b64="Ym94Mg==")}, + } + for h in hs.values(): + os.makedirs(h["home"]) + monkeypatch.setitem(sys.modules, "paramiko", fake_paramiko(hs)) + return hs + + +@pytest.fixture +def h(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +def add(client, h, **over): + body = { + "name": "Boxes", + "driver": "ssh", + "settings": {"hosts": ["lens@box1", "lens@10.0.0.5:2222"]}, + "secrets": {"private_key": "PRIVATE"}, + **over, + } + r = client.post("/api/v1/scaling/providers", headers=h, json=body) + assert r.status_code == 201, r.text + return r.json() + + +def test_host_specs(): + assert scaling_ssh.parse_host("lens@box1") == ("lens", "box1", 22) + assert scaling_ssh.parse_host("ops@[fd00::5]:2200") == ("ops", "fd00::5", 2200) + assert scaling_ssh.known_name("fd00::5", 2200) == "[fd00::5]:2200" + for bad in ("box1", "lens@", "lens@box1:0", "lens@box1:99999", "Lens;rm@box", "lens@bo x"): + with pytest.raises(ValueError): + scaling_ssh.parse_host(bad) + + +def test_ssh_trusts_host_keys_on_first_use_and_holds_to_them(client, db, cfg, h, hosts, monkeypatch): + monkeypatch.setattr(drivers.Driver, "ready", lambda self: True) + p = add(client, h) + assert p["check"]["ok"] is True and p["check"]["host_keys"] == { + "lens@box1": "ssh-ed25519 Ym94MQ==", + "lens@10.0.0.5:2222": "ssh-ed25519 Ym94Mg==", + } + assert add(client, h, name="Bad key", secrets={"private_key": "nope"})["check"]["error"].startswith("the private key can't be read") + + hosts[("box1", 22)]["key"] = Key(b64="ZXZpbA==") # someone else answers at box1 now + r = client.post(f"/api/v1/scaling/providers/{p['id']}/check", headers=h).json() + assert r["ok"] is False and "host key changed" in r["error"] + assert r["host_keys"]["lens@box1"] == "ssh-ed25519 Ym94MQ==" # a failed check doesn't forget the keys it trusts + drv = drivers.get("ssh") + with pytest.raises(drivers.DriverError, match="host key changed"): + drv.launch(scaling.opened(db, cfg, p["id"]), {"per_host": 1}, "lens-w-1", {"env": {}, "line": "sleep 30", "tag": "lens"}) + + +def test_ssh_starts_counts_and_stops_workers(client, db, cfg, h, hosts, monkeypatch): + monkeypatch.setattr(drivers.Driver, "ready", lambda self: True) + p = add(client, h) + drv = drivers.get("ssh") + opened = scaling.opened(db, cfg, p["id"]) + boot = {"env": {"LENS_SCALING_UNIT": "x"}, "line": "sleep 30", "tag": "lens"} + a = drv.launch(opened, {"per_host": 1, "steps": ["ocr"]}, "lens-w-1", {**boot, "env": {"LENS_SCALING_UNIT": "lens-w-1"}}) + b = drv.launch(opened, {"per_host": 1}, "lens-w-2", {**boot, "env": {"LENS_SCALING_UNIT": "lens-w-2"}}) + assert a == {"id": "lens@box1#lens-w-1"} and b == {"id": "lens@10.0.0.5:2222#lens-w-2"} # one each: per_host 1 + with pytest.raises(drivers.DriverError, match="every host runs its 1 worker"): + drv.launch(opened, {"per_host": 1}, "lens-w-3", boot) + + box1 = os.path.join(hosts[("box1", 22)]["home"], ".lens-scaling") + for _ in range(50): # the script removes itself as it starts, since it holds the join token + if not os.path.exists(os.path.join(box1, "lens-w-1.sh")): + break + time.sleep(0.1) + assert sorted(os.listdir(box1)) == ["lens-w-1.log", "lens-w-1.pid"] + + n1 = {"name": "lens-w-1", "external_id": a["id"]} + assert drv.describe(opened, n1) == {"state": "running"} + assert {u["name"] for u in drv.listing(opened, "lens")} == {"lens-w-1", "lens-w-2"} + drv.terminate(opened, n1) + assert drv.describe(opened, n1) == {"state": "gone"} and os.listdir(box1) == [] + drv.terminate(opened, n1) # already gone: no error + drv.terminate(opened, {"name": "lens-w-2", "external_id": b["id"]}) + assert drv.listing(opened, "lens") == [] diff --git a/fastapi_backend/uv.lock b/fastapi_backend/uv.lock index bb992139f..d88c87c72 100644 --- a/fastapi_backend/uv.lock +++ b/fastapi_backend/uv.lock @@ -332,6 +332,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/69/47a3dc20abc4fa5486655fde681bd55e63211b46c886d8c02223d6468431/backrefs-8.0-py313-none-any.whl", hash = "sha256:601ce68ca12385dbda06ce264406b4c4210cf5b79fd0fd627592365c92f29a88", size = 400040, upload-time = "2026-07-26T19:54:21.194Z" }, ] +[[package]] +name = "bcrypt" +version = "5.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/36/3329e2518d70ad8e2e5817d5a4cac6bba05a47767ec416c7d020a965f408/bcrypt-5.0.0.tar.gz", hash = "sha256:f748f7c2d6fd375cc93d3fba7ef4a9e3a092421b8dbf34d8d4dc06be9492dfdd", size = 25386, upload-time = "2025-09-25T19:50:47.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/85/3e65e01985fddf25b64ca67275bb5bdb4040bd1a53b66d355c6c37c8a680/bcrypt-5.0.0-cp313-cp313t-macosx_10_12_universal2.whl", hash = "sha256:f3c08197f3039bec79cee59a606d62b96b16669cff3949f21e74796b6e3cd2be", size = 481806, upload-time = "2025-09-25T19:49:05.102Z" }, + { url = "https://files.pythonhosted.org/packages/44/dc/01eb79f12b177017a726cbf78330eb0eb442fae0e7b3dfd84ea2849552f3/bcrypt-5.0.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:200af71bc25f22006f4069060c88ed36f8aa4ff7f53e67ff04d2ab3f1e79a5b2", size = 268626, upload-time = "2025-09-25T19:49:06.723Z" }, + { url = "https://files.pythonhosted.org/packages/8c/cf/e82388ad5959c40d6afd94fb4743cc077129d45b952d46bdc3180310e2df/bcrypt-5.0.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:baade0a5657654c2984468efb7d6c110db87ea63ef5a4b54732e7e337253e44f", size = 271853, upload-time = "2025-09-25T19:49:08.028Z" }, + { url = "https://files.pythonhosted.org/packages/ec/86/7134b9dae7cf0efa85671651341f6afa695857fae172615e960fb6a466fa/bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:c58b56cdfb03202b3bcc9fd8daee8e8e9b6d7e3163aa97c631dfcfcc24d36c86", size = 269793, upload-time = "2025-09-25T19:49:09.727Z" }, + { url = "https://files.pythonhosted.org/packages/cc/82/6296688ac1b9e503d034e7d0614d56e80c5d1a08402ff856a4549cb59207/bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4bfd2a34de661f34d0bda43c3e4e79df586e4716ef401fe31ea39d69d581ef23", size = 289930, upload-time = "2025-09-25T19:49:11.204Z" }, + { url = "https://files.pythonhosted.org/packages/d1/18/884a44aa47f2a3b88dd09bc05a1e40b57878ecd111d17e5bba6f09f8bb77/bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:ed2e1365e31fc73f1825fa830f1c8f8917ca1b3ca6185773b349c20fd606cec2", size = 272194, upload-time = "2025-09-25T19:49:12.524Z" }, + { url = "https://files.pythonhosted.org/packages/0e/8f/371a3ab33c6982070b674f1788e05b656cfbf5685894acbfef0c65483a59/bcrypt-5.0.0-cp313-cp313t-manylinux_2_34_aarch64.whl", hash = "sha256:83e787d7a84dbbfba6f250dd7a5efd689e935f03dd83b0f919d39349e1f23f83", size = 269381, upload-time = "2025-09-25T19:49:14.308Z" }, + { url = "https://files.pythonhosted.org/packages/b1/34/7e4e6abb7a8778db6422e88b1f06eb07c47682313997ee8a8f9352e5a6f1/bcrypt-5.0.0-cp313-cp313t-manylinux_2_34_x86_64.whl", hash = "sha256:137c5156524328a24b9fac1cb5db0ba618bc97d11970b39184c1d87dc4bf1746", size = 271750, upload-time = "2025-09-25T19:49:15.584Z" }, + { url = "https://files.pythonhosted.org/packages/c0/1b/54f416be2499bd72123c70d98d36c6cd61a4e33d9b89562c22481c81bb30/bcrypt-5.0.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:38cac74101777a6a7d3b3e3cfefa57089b5ada650dce2baf0cbdd9d65db22a9e", size = 303757, upload-time = "2025-09-25T19:49:17.244Z" }, + { url = "https://files.pythonhosted.org/packages/13/62/062c24c7bcf9d2826a1a843d0d605c65a755bc98002923d01fd61270705a/bcrypt-5.0.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:d8d65b564ec849643d9f7ea05c6d9f0cd7ca23bdd4ac0c2dbef1104ab504543d", size = 306740, upload-time = "2025-09-25T19:49:18.693Z" }, + { url = "https://files.pythonhosted.org/packages/d5/c8/1fdbfc8c0f20875b6b4020f3c7dc447b8de60aa0be5faaf009d24242aec9/bcrypt-5.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:741449132f64b3524e95cd30e5cd3343006ce146088f074f31ab26b94e6c75ba", size = 334197, upload-time = "2025-09-25T19:49:20.523Z" }, + { url = "https://files.pythonhosted.org/packages/a6/c1/8b84545382d75bef226fbc6588af0f7b7d095f7cd6a670b42a86243183cd/bcrypt-5.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:212139484ab3207b1f0c00633d3be92fef3c5f0af17cad155679d03ff2ee1e41", size = 352974, upload-time = "2025-09-25T19:49:22.254Z" }, + { url = "https://files.pythonhosted.org/packages/10/a6/ffb49d4254ed085e62e3e5dd05982b4393e32fe1e49bb1130186617c29cd/bcrypt-5.0.0-cp313-cp313t-win32.whl", hash = "sha256:9d52ed507c2488eddd6a95bccee4e808d3234fa78dd370e24bac65a21212b861", size = 148498, upload-time = "2025-09-25T19:49:24.134Z" }, + { url = "https://files.pythonhosted.org/packages/48/a9/259559edc85258b6d5fc5471a62a3299a6aa37a6611a169756bf4689323c/bcrypt-5.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f6984a24db30548fd39a44360532898c33528b74aedf81c26cf29c51ee47057e", size = 145853, upload-time = "2025-09-25T19:49:25.702Z" }, + { url = "https://files.pythonhosted.org/packages/2d/df/9714173403c7e8b245acf8e4be8876aac64a209d1b392af457c79e60492e/bcrypt-5.0.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9fffdb387abe6aa775af36ef16f55e318dcda4194ddbf82007a6f21da29de8f5", size = 139626, upload-time = "2025-09-25T19:49:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/84/29/6237f151fbfe295fe3e074ecc6d44228faa1e842a81f6d34a02937ee1736/bcrypt-5.0.0-cp38-abi3-macosx_10_12_universal2.whl", hash = "sha256:fc746432b951e92b58317af8e0ca746efe93e66555f1b40888865ef5bf56446b", size = 494553, upload-time = "2025-09-25T19:49:49.006Z" }, + { url = "https://files.pythonhosted.org/packages/45/b6/4c1205dde5e464ea3bd88e8742e19f899c16fa8916fb8510a851fae985b5/bcrypt-5.0.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c2388ca94ffee269b6038d48747f4ce8df0ffbea43f31abfa18ac72f0218effb", size = 275009, upload-time = "2025-09-25T19:49:50.581Z" }, + { url = "https://files.pythonhosted.org/packages/3b/71/427945e6ead72ccffe77894b2655b695ccf14ae1866cd977e185d606dd2f/bcrypt-5.0.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:560ddb6ec730386e7b3b26b8b4c88197aaed924430e7b74666a586ac997249ef", size = 278029, upload-time = "2025-09-25T19:49:52.533Z" }, + { url = "https://files.pythonhosted.org/packages/17/72/c344825e3b83c5389a369c8a8e58ffe1480b8a699f46c127c34580c4666b/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d79e5c65dcc9af213594d6f7f1fa2c98ad3fc10431e7aa53c176b441943efbdd", size = 275907, upload-time = "2025-09-25T19:49:54.709Z" }, + { url = "https://files.pythonhosted.org/packages/0b/7e/d4e47d2df1641a36d1212e5c0514f5291e1a956a7749f1e595c07a972038/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2b732e7d388fa22d48920baa267ba5d97cca38070b69c0e2d37087b381c681fd", size = 296500, upload-time = "2025-09-25T19:49:56.013Z" }, + { url = "https://files.pythonhosted.org/packages/0f/c3/0ae57a68be2039287ec28bc463b82e4b8dc23f9d12c0be331f4782e19108/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0c8e093ea2532601a6f686edbc2c6b2ec24131ff5c52f7610dd64fa4553b5464", size = 278412, upload-time = "2025-09-25T19:49:57.356Z" }, + { url = "https://files.pythonhosted.org/packages/45/2b/77424511adb11e6a99e3a00dcc7745034bee89036ad7d7e255a7e47be7d8/bcrypt-5.0.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5b1589f4839a0899c146e8892efe320c0fa096568abd9b95593efac50a87cb75", size = 275486, upload-time = "2025-09-25T19:49:59.116Z" }, + { url = "https://files.pythonhosted.org/packages/43/0a/405c753f6158e0f3f14b00b462d8bca31296f7ecfc8fc8bc7919c0c7d73a/bcrypt-5.0.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:89042e61b5e808b67daf24a434d89bab164d4de1746b37a8d173b6b14f3db9ff", size = 277940, upload-time = "2025-09-25T19:50:00.869Z" }, + { url = "https://files.pythonhosted.org/packages/62/83/b3efc285d4aadc1fa83db385ec64dcfa1707e890eb42f03b127d66ac1b7b/bcrypt-5.0.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:e3cf5b2560c7b5a142286f69bde914494b6d8f901aaa71e453078388a50881c4", size = 310776, upload-time = "2025-09-25T19:50:02.393Z" }, + { url = "https://files.pythonhosted.org/packages/95/7d/47ee337dacecde6d234890fe929936cb03ebc4c3a7460854bbd9c97780b8/bcrypt-5.0.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f632fd56fc4e61564f78b46a2269153122db34988e78b6be8b32d28507b7eaeb", size = 312922, upload-time = "2025-09-25T19:50:04.232Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3a/43d494dfb728f55f4e1cf8fd435d50c16a2d75493225b54c8d06122523c6/bcrypt-5.0.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:801cad5ccb6b87d1b430f183269b94c24f248dddbbc5c1f78b6ed231743e001c", size = 341367, upload-time = "2025-09-25T19:50:05.559Z" }, + { url = "https://files.pythonhosted.org/packages/55/ab/a0727a4547e383e2e22a630e0f908113db37904f58719dc48d4622139b5c/bcrypt-5.0.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3cf67a804fc66fc217e6914a5635000259fbbbb12e78a99488e4d5ba445a71eb", size = 359187, upload-time = "2025-09-25T19:50:06.916Z" }, + { url = "https://files.pythonhosted.org/packages/1b/bb/461f352fdca663524b4643d8b09e8435b4990f17fbf4fea6bc2a90aa0cc7/bcrypt-5.0.0-cp38-abi3-win32.whl", hash = "sha256:3abeb543874b2c0524ff40c57a4e14e5d3a66ff33fb423529c88f180fd756538", size = 153752, upload-time = "2025-09-25T19:50:08.515Z" }, + { url = "https://files.pythonhosted.org/packages/41/aa/4190e60921927b7056820291f56fc57d00d04757c8b316b2d3c0d1d6da2c/bcrypt-5.0.0-cp38-abi3-win_amd64.whl", hash = "sha256:35a77ec55b541e5e583eb3436ffbbf53b0ffa1fa16ca6782279daf95d146dcd9", size = 150881, upload-time = "2025-09-25T19:50:09.742Z" }, + { url = "https://files.pythonhosted.org/packages/54/12/cd77221719d0b39ac0b55dbd39358db1cd1246e0282e104366ebbfb8266a/bcrypt-5.0.0-cp38-abi3-win_arm64.whl", hash = "sha256:cde08734f12c6a4e28dc6755cd11d3bdfea608d93d958fffbe95a7026ebe4980", size = 144931, upload-time = "2025-09-25T19:50:11.016Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ba/2af136406e1c3839aea9ecadc2f6be2bcd1eff255bd451dd39bcf302c47a/bcrypt-5.0.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:0c418ca99fd47e9c59a301744d63328f17798b5947b0f791e9af3c1c499c2d0a", size = 495313, upload-time = "2025-09-25T19:50:12.309Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ee/2f4985dbad090ace5ad1f7dd8ff94477fe089b5fab2040bd784a3d5f187b/bcrypt-5.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddb4e1500f6efdd402218ffe34d040a1196c072e07929b9820f363a1fd1f4191", size = 275290, upload-time = "2025-09-25T19:50:13.673Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6e/b77ade812672d15cf50842e167eead80ac3514f3beacac8902915417f8b7/bcrypt-5.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7aeef54b60ceddb6f30ee3db090351ecf0d40ec6e2abf41430997407a46d2254", size = 278253, upload-time = "2025-09-25T19:50:15.089Z" }, + { url = "https://files.pythonhosted.org/packages/36/c4/ed00ed32f1040f7990dac7115f82273e3c03da1e1a1587a778d8cea496d8/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f0ce778135f60799d89c9693b9b398819d15f1921ba15fe719acb3178215a7db", size = 276084, upload-time = "2025-09-25T19:50:16.699Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c4/fa6e16145e145e87f1fa351bbd54b429354fd72145cd3d4e0c5157cf4c70/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a71f70ee269671460b37a449f5ff26982a6f2ba493b3eabdd687b4bf35f875ac", size = 297185, upload-time = "2025-09-25T19:50:18.525Z" }, + { url = "https://files.pythonhosted.org/packages/24/b4/11f8a31d8b67cca3371e046db49baa7c0594d71eb40ac8121e2fc0888db0/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f8429e1c410b4073944f03bd778a9e066e7fad723564a52ff91841d278dfc822", size = 278656, upload-time = "2025-09-25T19:50:19.809Z" }, + { url = "https://files.pythonhosted.org/packages/ac/31/79f11865f8078e192847d2cb526e3fa27c200933c982c5b2869720fa5fce/bcrypt-5.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:edfcdcedd0d0f05850c52ba3127b1fce70b9f89e0fe5ff16517df7e81fa3cbb8", size = 275662, upload-time = "2025-09-25T19:50:21.567Z" }, + { url = "https://files.pythonhosted.org/packages/d4/8d/5e43d9584b3b3591a6f9b68f755a4da879a59712981ef5ad2a0ac1379f7a/bcrypt-5.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:611f0a17aa4a25a69362dcc299fda5c8a3d4f160e2abb3831041feb77393a14a", size = 278240, upload-time = "2025-09-25T19:50:23.305Z" }, + { url = "https://files.pythonhosted.org/packages/89/48/44590e3fc158620f680a978aafe8f87a4c4320da81ed11552f0323aa9a57/bcrypt-5.0.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:db99dca3b1fdc3db87d7c57eac0c82281242d1eabf19dcb8a6b10eb29a2e72d1", size = 311152, upload-time = "2025-09-25T19:50:24.597Z" }, + { url = "https://files.pythonhosted.org/packages/5f/85/e4fbfc46f14f47b0d20493669a625da5827d07e8a88ee460af6cd9768b44/bcrypt-5.0.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:5feebf85a9cefda32966d8171f5db7e3ba964b77fdfe31919622256f80f9cf42", size = 313284, upload-time = "2025-09-25T19:50:26.268Z" }, + { url = "https://files.pythonhosted.org/packages/25/ae/479f81d3f4594456a01ea2f05b132a519eff9ab5768a70430fa1132384b1/bcrypt-5.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3ca8a166b1140436e058298a34d88032ab62f15aae1c598580333dc21d27ef10", size = 341643, upload-time = "2025-09-25T19:50:28.02Z" }, + { url = "https://files.pythonhosted.org/packages/df/d2/36a086dee1473b14276cd6ea7f61aef3b2648710b5d7f1c9e032c29b859f/bcrypt-5.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:61afc381250c3182d9078551e3ac3a41da14154fbff647ddf52a769f588c4172", size = 359698, upload-time = "2025-09-25T19:50:31.347Z" }, + { url = "https://files.pythonhosted.org/packages/c0/f6/688d2cd64bfd0b14d805ddb8a565e11ca1fb0fd6817175d58b10052b6d88/bcrypt-5.0.0-cp39-abi3-win32.whl", hash = "sha256:64d7ce196203e468c457c37ec22390f1a61c85c6f0b8160fd752940ccfb3a683", size = 153725, upload-time = "2025-09-25T19:50:34.384Z" }, + { url = "https://files.pythonhosted.org/packages/9f/b9/9d9a641194a730bda138b3dfe53f584d61c58cd5230e37566e83ec2ffa0d/bcrypt-5.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:64ee8434b0da054d830fa8e89e1c8bf30061d539044a39524ff7dec90481e5c2", size = 150912, upload-time = "2025-09-25T19:50:35.69Z" }, + { url = "https://files.pythonhosted.org/packages/27/44/d2ef5e87509158ad2187f4dd0852df80695bb1ee0cfe0a684727b01a69e0/bcrypt-5.0.0-cp39-abi3-win_arm64.whl", hash = "sha256:f2347d3534e76bf50bca5500989d6c1d05ed64b440408057a37673282c654927", size = 144953, upload-time = "2025-09-25T19:50:37.32Z" }, +] + [[package]] name = "beautifulsoup4" version = "4.15.0" @@ -1729,6 +1782,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/d4/7c030e73d3064e63fc7927fc9c0cb5b8c52e12a52442317e9d0ad3d7bb37/insightface-2.1-py3-none-any.whl", hash = "sha256:dcd58ee79d99d689572477315f3149353afd65a36dc575f597cbb217a7bde399", size = 1169452, upload-time = "2026-10-03T16:52:15.821Z" }, ] +[[package]] +name = "invoke" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/f6/227c48c5fe47fa178ccf1fda8f047d16c97ba926567b661e9ce2045c600c/invoke-3.0.3.tar.gz", hash = "sha256:437b6a622223824380bfb4e64f612711a6b648c795f565efc8625af66fb57f0c", size = 343419, upload-time = "2026-04-07T15:17:48.307Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/de/bbc12563bbf979618d17625a4e753ff7a078523e28d870d3626daa97261a/invoke-3.0.3-py3-none-any.whl", hash = "sha256:f11327165e5cbb89b2ad1d88d3292b5113332c43b8553b494da435d6ec6f5053", size = 160958, upload-time = "2026-04-07T15:17:46.875Z" }, +] + [[package]] name = "itsdangerous" version = "2.2.0" @@ -2051,6 +2113,9 @@ smb = [ spacy = [ { name = "spacy" }, ] +ssh = [ + { name = "paramiko" }, +] validate = [ { name = "jsonschema" }, ] @@ -2124,6 +2189,7 @@ requires-dist = [ { name = "opentelemetry-api", specifier = ">=1.45.1,<2" }, { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.45.1,<2" }, { name = "opentelemetry-sdk", specifier = ">=1.27,<2" }, + { name = "paramiko", marker = "extra == 'ssh'", specifier = ">=3.5,<5" }, { name = "pillow", specifier = ">=10" }, { name = "pyannote-audio", marker = "extra == 'pyannote'" }, { name = "pydantic-settings", specifier = ">=2.5.2,<3" }, @@ -2153,7 +2219,7 @@ requires-dist = [ { name = "transformers", marker = "extra == 'punctuation'" }, { name = "webauthn", specifier = ">=3.0.1,<4" }, ] -provides-extras = ["sensevoice", "whisper", "mlx", "laya", "voices", "pyannote", "faces", "mac-ocr", "rapidocr", "doctr", "insightface", "objects", "punctuation", "spacy", "injection", "florence", "moondream", "validate", "smb", "sandbox", "web", "msg", "vaultwarden"] +provides-extras = ["sensevoice", "whisper", "mlx", "laya", "voices", "pyannote", "faces", "mac-ocr", "rapidocr", "doctr", "insightface", "objects", "punctuation", "spacy", "injection", "florence", "moondream", "validate", "smb", "sandbox", "web", "msg", "vaultwarden", "ssh"] [package.metadata.requires-dev] dev = [ @@ -3464,6 +3530,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/75/2e/e7b35b712edb068d382ddc8b2bea8a04974100515ba2daa22b478b265842/pandas-3.0.6-cp313-cp313-win_arm64.whl", hash = "sha256:cc39303913e2ea129915670de5d1c9fbd647f543bb72e5543bac8baa94e9e42f", size = 8952032, upload-time = "2026-09-17T23:21:37.729Z" }, ] +[[package]] +name = "paramiko" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bcrypt" }, + { name = "cryptography" }, + { name = "invoke" }, + { name = "pynacl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/e7/81fdcbc7f190cdb058cffc9431587eb289833bdd633e2002455ca9bb13d4/paramiko-4.0.0.tar.gz", hash = "sha256:6a25f07b380cc9c9a88d2b920ad37167ac4667f8d9886ccebd8f90f654b5d69f", size = 1630743, upload-time = "2025-08-04T01:02:03.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/90/a744336f5af32c433bd09af7854599682a383b37cfd78f7de263de6ad6cb/paramiko-4.0.0-py3-none-any.whl", hash = "sha256:0e20e00ac666503bf0b4eda3b6d833465a2b7aff2e2b3d79a8bba5ef144ee3b9", size = 223932, upload-time = "2025-08-04T01:02:02.029Z" }, +] + [[package]] name = "pathspec" version = "1.1.1" @@ -4021,6 +4102,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/d1/98313da89960a604402266a115311b510254900ff1295ea426403f9423cc/pymdown_extensions-12.1-py3-none-any.whl", hash = "sha256:4a254b771acfcddc6c110a7f4590d818f1f849e2d5aacf69b4b07c8dd2a9700d", size = 277069, upload-time = "2026-09-23T00:08:40.069Z" }, ] +[[package]] +name = "pynacl" +version = "1.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d9/9a/4019b524b03a13438637b11538c82781a5eda427394380381af8f04f467a/pynacl-1.6.2.tar.gz", hash = "sha256:018494d6d696ae03c7e656e5e74cdfd8ea1326962cc401bcf018f1ed8436811c", size = 3511692, upload-time = "2026-01-01T17:48:10.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/7b/4845bbf88e94586ec47a432da4e9107e3fc3ce37eb412b1398630a37f7dd/pynacl-1.6.2-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:c949ea47e4206af7c8f604b8278093b674f7c79ed0d4719cc836902bf4517465", size = 388458, upload-time = "2026-01-01T17:32:16.829Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b4/e927e0653ba63b02a4ca5b4d852a8d1d678afbf69b3dbf9c4d0785ac905c/pynacl-1.6.2-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8845c0631c0be43abdd865511c41eab235e0be69c81dc66a50911594198679b0", size = 800020, upload-time = "2026-01-01T17:32:18.34Z" }, + { url = "https://files.pythonhosted.org/packages/7f/81/d60984052df5c97b1d24365bc1e30024379b42c4edcd79d2436b1b9806f2/pynacl-1.6.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:22de65bb9010a725b0dac248f353bb072969c94fa8d6b1f34b87d7953cf7bbe4", size = 1399174, upload-time = "2026-01-01T17:32:20.239Z" }, + { url = "https://files.pythonhosted.org/packages/68/f7/322f2f9915c4ef27d140101dd0ed26b479f7e6f5f183590fd32dfc48c4d3/pynacl-1.6.2-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46065496ab748469cdd999246d17e301b2c24ae2fdf739132e580a0e94c94a87", size = 835085, upload-time = "2026-01-01T17:32:22.24Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d0/f301f83ac8dbe53442c5a43f6a39016f94f754d7a9815a875b65e218a307/pynacl-1.6.2-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a66d6fb6ae7661c58995f9c6435bda2b1e68b54b598a6a10247bfcdadac996c", size = 1437614, upload-time = "2026-01-01T17:32:23.766Z" }, + { url = "https://files.pythonhosted.org/packages/c4/58/fc6e649762b029315325ace1a8c6be66125e42f67416d3dbd47b69563d61/pynacl-1.6.2-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:26bfcd00dcf2cf160f122186af731ae30ab120c18e8375684ec2670dccd28130", size = 818251, upload-time = "2026-01-01T17:32:25.69Z" }, + { url = "https://files.pythonhosted.org/packages/c9/a8/b917096b1accc9acd878819a49d3d84875731a41eb665f6ebc826b1af99e/pynacl-1.6.2-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c8a231e36ec2cab018c4ad4358c386e36eede0319a0c41fed24f840b1dac59f6", size = 1402859, upload-time = "2026-01-01T17:32:27.215Z" }, + { url = "https://files.pythonhosted.org/packages/85/42/fe60b5f4473e12c72f977548e4028156f4d340b884c635ec6b063fe7e9a5/pynacl-1.6.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:68be3a09455743ff9505491220b64440ced8973fe930f270c8e07ccfa25b1f9e", size = 791926, upload-time = "2026-01-01T17:32:29.314Z" }, + { url = "https://files.pythonhosted.org/packages/fa/f9/e40e318c604259301cc091a2a63f237d9e7b424c4851cafaea4ea7c4834e/pynacl-1.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b097553b380236d51ed11356c953bf8ce36a29a3e596e934ecabe76c985a577", size = 1363101, upload-time = "2026-01-01T17:32:31.263Z" }, + { url = "https://files.pythonhosted.org/packages/48/47/e761c254f410c023a469284a9bc210933e18588ca87706ae93002c05114c/pynacl-1.6.2-cp38-abi3-win32.whl", hash = "sha256:5811c72b473b2f38f7e2a3dc4f8642e3a3e9b5e7317266e4ced1fba85cae41aa", size = 227421, upload-time = "2026-01-01T17:32:33.076Z" }, + { url = "https://files.pythonhosted.org/packages/41/ad/334600e8cacc7d86587fe5f565480fde569dfb487389c8e1be56ac21d8ac/pynacl-1.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:62985f233210dee6548c223301b6c25440852e13d59a8b81490203c3227c5ba0", size = 239754, upload-time = "2026-01-01T17:32:34.557Z" }, + { url = "https://files.pythonhosted.org/packages/29/7d/5945b5af29534641820d3bd7b00962abbbdfee84ec7e19f0d5b3175f9a31/pynacl-1.6.2-cp38-abi3-win_arm64.whl", hash = "sha256:834a43af110f743a754448463e8fd61259cd4ab5bbedcf70f9dabad1d28a394c", size = 184801, upload-time = "2026-01-01T17:32:36.309Z" }, +] + [[package]] name = "pynndescent" version = "0.6.0" diff --git a/threat-model.md b/threat-model.md index 8996b270d..888d8137a 100644 --- a/threat-model.md +++ b/threat-model.md @@ -539,7 +539,7 @@ Paths in the Voided by column are relative to `fastapi_backend/app/`. Searches c | P-MEDIA-LINK | Media, frames, documents and reports open without a session only through a link the server signed for that path, or a live share link. Text people or models write is never signed. | info-leak | security-critical | *(documented, docs/architecture.md "is never signed")* | Voided by: Knowing `ACCESS_SECRET_KEY` signs any path — `core/security.py:43`; dev compose key — `docker-compose.yml:24`. Lifetime `MEDIA_URL_EXPIRE_SECONDS` — `config.py:30`. Report pages sign every asset of the namespace — `api/pages.py:140-141`. Signing is limited to allow-listed fields — `api/media.py:25-30`. | | P-SHARE | A share link opens one asset, read-only. It expires and can be revoked. Only hashes are stored. A dead link answers the same 410 whatever went wrong. | integrity-bypass | security-critical | *(documented, docs/authentication.md "give read-only access to one asset's player and embed")* | Voided by: Liveness plus asset match only — `domain/auth.py:441-444`; `?s=` routes — `api/deps.py:252` (Q7). | | P-FEEDS | A published feed's address opens that feed alone, read-only, with no session: the feed's XML and the audio of items it lists. Only an editor of the namespace makes a feed, and what it lists is bounded by who already reads that namespace: an item's resource or activity is in the namespace, or in one every reader of the feed's namespace reads too, never a vault's, and a namespace made a vault stops its feeds. The token is found by its hash; rotating or deleting the feed stops the old address at once. Text in an item cannot add markup to a reader. | info-leak | security-critical | *(documented, docs/feeds.md "Private by default")* | Voided by: the token is sealed with the server key, not only hashed, so the namespace's readers (and whoever holds `secret.key`) see the address again — `domain/syndication.py` `token_of`; anyone given the address reads the feed until it is rotated; editors choose what goes in, as does a routine (admins make them) whose workflow nodes publish what they pick, bounded the same way; a resource's title and summary show as they are now, so an edit after publishing shows; the fit is by namespace roles (`domain/syndication.py` `Fit`), not per-asset grants, so an asset later closed by access level still shows while its namespace fits. | -| P-SCALING | Auto scaling launches nothing that costs money unless a person, signed in to the web app, approved the group's terms: its provider, what each unit is, its max, its budget and what happens at its cap. An API key, an app, MCP and the assistant acting on its own never approve, and a change that widens the terms takes the approval away. A provider's credentials are sealed per provider and field, opened only for a call to it, and never shown. A unit is handed no long-lived Lens secret to start. | integrity-bypass | security-critical | *(documented, docs/scaling.md "The person approves")* | Voided by: an admin's web session approves, and an admin is trusted; a yes on the Approvals page to the assistant's proposal approves the terms that proposal names; within the terms agents may raise the desired count up to max, spending up to the budget; costs are estimates from list prices (`domain/scaling.py` `accrue`), not bills, so a provider can charge more than the budget counts; a cloud unit's user-data holds a join token for that unit, usable once within an hour, which anyone with the cloud account or a process on that machine reading its metadata can see until it is spent (`domain/scaling_drivers.py` `user_data`); a cloud provider's API token can do whatever its account can, and Lens calls only the provider's fixed API host with it (`domain/scaling_clouds.py`); a Proxmox provider's address is the admin's to give, and turning `verify_tls` off trusts any server answering there with the API token; a Proxmox unit's first-boot script, with its join token, is passed to the clone through the guest agent (`domain/scaling_proxmox.py`). | +| P-SCALING | Auto scaling launches nothing that costs money unless a person, signed in to the web app, approved the group's terms: its provider, what each unit is, its max, its budget and what happens at its cap. An API key, an app, MCP and the assistant acting on its own never approve, and a change that widens the terms takes the approval away. A provider's credentials are sealed per provider and field, opened only for a call to it, and never shown. A unit is handed no long-lived Lens secret to start. | integrity-bypass | security-critical | *(documented, docs/scaling.md "The person approves")* | Voided by: an admin's web session approves, and an admin is trusted; a yes on the Approvals page to the assistant's proposal approves the terms that proposal names; within the terms agents may raise the desired count up to max, spending up to the budget; costs are estimates from list prices (`domain/scaling.py` `accrue`), not bills, so a provider can charge more than the budget counts; a cloud unit's user-data holds a join token for that unit, usable once within an hour, which anyone with the cloud account or a process on that machine reading its metadata can see until it is spent (`domain/scaling_drivers.py` `user_data`); a cloud provider's API token can do whatever its account can, and Lens calls only the provider's fixed API host with it (`domain/scaling_clouds.py`); a Proxmox provider's address is the admin's to give, and turning `verify_tls` off trusts any server answering there with the API token; a Proxmox unit's first-boot script, with its join token, is passed to the clone through the guest agent (`domain/scaling_proxmox.py`); an SSH provider's host keys are trusted on first use, when an admin saves or checks it, and held to after (a host that answered first with another key is trusted until taken off), and its private key signs in as the given users on the given hosts (`domain/scaling_ssh.py`). | | P-AUTH-REQ | A request is authenticated only if its bearer verifies and the account and session are live. Sign-out and disabling take effect at once, for every token kind. Revoking an app grant or API token ends its tokens at once. | integrity-bypass | security-critical | *(documented, docs/authentication.md "Every request checks that the account is still active")* | Voided by: Liveness checked per request — `api/deps.py:92`; JWT algorithm from `ALGORITHM` — `config.py:27`; dev secrets — `docker-compose.yml:24`. No bypass flag: `grep -rniE "disable_auth\|skip_auth\|no_auth\|auth_bypass\|AUTH_DISABLED\|ALLOW_ANON\|DEV_MODE" fastapi_backend/app nextjs-frontend/lib nextjs-frontend/auth.ts nextjs-frontend/proxy.ts docker-compose*.yml install.sh` — 0 hits. | | P-REMOVE | Removing an account ends every way it signs in at once (sessions, passkeys, sign-in links, connected accounts, API tokens, app grants, IIIF tokens) and takes every role, collection role and recording permission. Its email can make a new account, which inherits nothing. Admins only; never yourself, never the Matterbridge account, never the last passkey into a vault unless the admin says its files may be lost. | integrity-bypass | security-critical | *(documented, docs/authentication.md "Removing someone")* | Voided by: the account row stays as a marker with an `.invalid` email — `domain/auth.py` `remove_account`; shared notes, views and comments keep its id. | | P-BEARER-ONLY | `/api/v1` takes bearer tokens only, never cookies. The IIIF authorization flow and the web app's NextAuth handlers are the cookie surfaces. | integrity-bypass | security-critical | *(documented, docs/architecture.md "so there is no CSRF surface")* | Voided by: IIIF auth cookies — `api/iiif.py:232`, `api/iiif.py:333`; external-login state cookie — `api/v1/routes/external.py:34`. | diff --git a/threat-model.yaml b/threat-model.yaml index b053c0410..067e98fc9 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -1556,7 +1556,7 @@ properties_claimed: tier: security-critical conditions: "auto scaling launches nothing unless a person signed in to the web app approved the group's terms (its provider, what each unit is, its max, its budget, what happens at its cap); an API key, an app, MCP and the assistant acting on its own never approve, and a change that widens the terms takes the approval away; a provider's credentials are sealed per provider and field, opened only for a call to it and never shown; a unit is handed no long-lived Lens secret" violation_symptoms: [integrity-bypass] - x-voided-by: "an admin's web session approves, and an admin is trusted; a yes on the Approvals page to the assistant's proposal approves the terms that proposal names; within the terms agents may raise the desired count up to max, spending up to the budget; costs are estimates from list prices (domain/scaling.py accrue), not bills, so a provider can charge more than the budget counts; a cloud unit's user-data holds a join token for that unit, usable once within an hour, which anyone with the cloud account or a process on that machine reading its metadata can see until it is spent (domain/scaling_drivers.py user_data); a cloud provider's API token can do whatever its account can, and Lens calls only the provider's fixed API host with it (domain/scaling_clouds.py); a Proxmox provider's address is the admin's to give, and turning `verify_tls` off trusts any server answering there with the API token; a Proxmox unit's first-boot script, with its join token, is passed to the clone through the guest agent (`domain/scaling_proxmox.py`)" + x-voided-by: "an admin's web session approves, and an admin is trusted; a yes on the Approvals page to the assistant's proposal approves the terms that proposal names; within the terms agents may raise the desired count up to max, spending up to the budget; costs are estimates from list prices (domain/scaling.py accrue), not bills, so a provider can charge more than the budget counts; a cloud unit's user-data holds a join token for that unit, usable once within an hour, which anyone with the cloud account or a process on that machine reading its metadata can see until it is spent (domain/scaling_drivers.py user_data); a cloud provider's API token can do whatever its account can, and Lens calls only the provider's fixed API host with it (domain/scaling_clouds.py); a Proxmox provider's address is the admin's to give, and turning `verify_tls` off trusts any server answering there with the API token; a Proxmox unit's first-boot script, with its join token, is passed to the clone through the guest agent (`domain/scaling_proxmox.py`); an SSH provider's host keys are trusted on first use, when an admin saves or checks it, and held to after (a host that answered first with another key is trusted until taken off), and its private key signs in as the given users on the given hosts (`domain/scaling_ssh.py`)" provenance: {kind: documented, source: 'docs/scaling.md "The person approves"'} - id: P-AUTH-REQ kind: authentication From ec15241f346e0c18d8bf83763987dcb980892482 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 12:31:46 +0000 Subject: [PATCH 2/2] fix(scaling): launch on other machines only once mesh nodes run workers The mesh's install line landed (#591), so a group on a cloud, Proxmox or SSH provider could launch, but a node runs no `lens worker` yet: its units would never count as joined and only cost until given up on. can_join now also needs the mesh to say its nodes run workers (WORKER_NODES), and waiting_for says which of the three it waits for. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ALjNbLobvPe7VwvDL8j7V3 --- README.md | 2 +- docs/scaling.md | 9 +++++--- fastapi_backend/app/domain/scaling.py | 23 ++++++++++++++----- .../tests/api/test_scaling_clouds.py | 13 +++++++++-- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e21380c07..c59ece193 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,7 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p ledger. A group launches nothing until you approve its terms in Settings › Auto scaling; agents propose and manage groups over the API, MCP and the assistant, and never approve one. It runs more workers on this machine today; AWS EC2 (spot or on demand), DigitalOcean, Hetzner Cloud, Proxmox VE and Linux hosts over SSH providers are in - progress (built, launching once the mesh's install line lands), and model hosts are planned. + progress (built, launching once machines that join the mesh run workers), and model hosts are planned. ([Auto scaling](docs/scaling.md); [checks](docs/requirements.md#scale-out)) - **Keep it up to date and backed up.** Lens checks for a new release once a day and tells admins by email or a diff --git a/docs/scaling.md b/docs/scaling.md index a842a658b..bf48eb0e7 100644 --- a/docs/scaling.md +++ b/docs/scaling.md @@ -10,7 +10,8 @@ Status: in progress. Built: groups, rules, approvals, budgets, the scaler, the A Settings › Auto scaling (the switch, and each group's terms to approve or pause), and **This machine**, a provider that runs more `lens worker` processes where Lens runs. In progress: **AWS EC2** (spot or on demand), **DigitalOcean** and **Hetzner Cloud** providers are built (credentials checked, prices, launching, watching and deleting machines), -and launch once the mesh's install line, which another machine joins Lens with, lands ([the mesh](mesh.md)); so are +and launch once machines that join over the [mesh](mesh.md) run Lens workers (they join today, but run no worker +yet); so are **Proxmox VE**, clones of a VM template on your own server, and **Linux hosts over SSH**, workers started on machines you have. Planned: model hosts (RunPod, Lambda, Hugging Face, SageMaker, Bedrock) for models; and making providers and groups in Settings. @@ -91,8 +92,10 @@ fails), and every other call refuses a host whose key differs. A host whose key the provider and added back. The SSH client is Paramiko (LGPL-2.1, used unmodified), the `ssh` extra, fetched the first time an SSH provider is saved. -Until the mesh is on and has its install line, a group on these providers waits (`waiting_for` says so) and launches -nothing, since its machines couldn't join. They call the providers' REST APIs with httpx, so they install nothing. +Until the mesh is on and the machines that join it run Lens workers, a group on these providers waits +(`waiting_for` says which) and launches nothing: a machine that joined without a worker would never count as joined, +and would only cost until it was given up on. The mesh says its nodes run workers once they do (`WORKER_NODES` in +`domain/mesh.py`). The cloud providers call their REST APIs with httpx, so they install nothing. ## AWS EC2 diff --git a/fastapi_backend/app/domain/scaling.py b/fastapi_backend/app/domain/scaling.py index ea369f380..2571ac734 100644 --- a/fastapi_backend/app/domain/scaling.py +++ b/fastapi_backend/app/domain/scaling.py @@ -599,7 +599,7 @@ def waiting_for(view, cfg=None): if view.get("capped"): return f"its budget: {view['capped']}" if cfg is not None and view.get("needs_join") and not can_join(cfg): - return "the mesh's install line, which other machines join Lens with (in progress, docs/mesh.md)" + return join_wait(cfg) return None @@ -966,12 +966,23 @@ def mesh_module(): return mesh -def can_join(cfg): - """Whether another machine can join Lens now: the mesh is on and has the install line machines join with.""" +def join_wait(cfg): + """What a group on other machines waits for before it launches, or None: the mesh's install line, the mesh turned + on, and nodes that run `lens worker` (the mesh says so with WORKER_NODES), since a machine that joins without a + worker never counts as joined and only costs.""" mesh = mesh_module() - if mesh is None or not (getattr(mesh, "conf", lambda c: {})(cfg) or {}).get("enabled"): - return False - return bool(getattr(mesh, "make_token", None) and getattr(mesh, "join_line", None)) + if mesh is None or not (getattr(mesh, "make_token", None) and getattr(mesh, "join_line", None)): + return "the mesh's install line, which other machines join Lens with (in progress, docs/mesh.md)" + if not (getattr(mesh, "conf", lambda c: {})(cfg) or {}).get("enabled"): + return "the mesh, which other machines join Lens over: turn it on (docs/mesh.md)" + if not getattr(mesh, "WORKER_NODES", False): + return "nodes that run Lens workers: the mesh's install line doesn't start one yet (in progress, docs/mesh.md)" + return None + + +def can_join(cfg): + """Whether another machine can join Lens now and work: see join_wait.""" + return join_wait(cfg) is None def bootstrap(db, cfg, g, name): diff --git a/fastapi_backend/tests/api/test_scaling_clouds.py b/fastapi_backend/tests/api/test_scaling_clouds.py index 0a85ff84e..5b58b99af 100644 --- a/fastapi_backend/tests/api/test_scaling_clouds.py +++ b/fastapi_backend/tests/api/test_scaling_clouds.py @@ -96,7 +96,9 @@ def h(client, db): class Mesh: - """The mesh with its install line, as mesh.py will have it.""" + """The mesh with its install line, and nodes that run workers, as mesh.py will have it.""" + + WORKER_NODES = True def __init__(self): self.tokens = [] @@ -136,7 +138,14 @@ def test_a_digitalocean_group_waits_for_the_install_line_then_launches_droplets( assert g["template"] == {"size": "s-1vcpu-2gb", "image": "ubuntu-24-04-x64", "steps": ["transcribe"]} assert tick(db, cfg)["launched"] == 0 and not do.machines # no way for a droplet to join yet: none is started (view,) = client.get("/api/v1/scaling", headers=h).json()["groups"] - assert "install line" in view["waiting_for"] + assert "turn it on" in view["waiting_for"] # the mesh is off + + from app.domain import mesh as real_mesh + + monkeypatch.setattr(real_mesh, "conf", lambda cfg: {"enabled": True}) + (view,) = client.get("/api/v1/scaling", headers=h).json()["groups"] + assert "nodes that run Lens workers" in view["waiting_for"] # the mesh joins machines, but they don't work yet + assert tick(db, cfg)["launched"] == 0 and not do.machines mesh = Mesh() monkeypatch.setattr(scaling, "mesh_module", lambda: mesh)