From 1850d905bd8522875b661fd663365918e29c4b8e Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:53:37 +0000 Subject: [PATCH 01/18] feat(minvmd): control socket registers a box and hands its addresses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host-side box table (NET-138) had no way to be filled from a creator: the activating client had no door to minvmd, so an own-address box's row never landed in the registry the egress gate decides by, and its addresses were left to the daemon to draw itself. Add the VM host daemon's box-registration control socket, beside the ssh socket in the provider-instance dir, speaking one JSON line each way over the wire types in minimald-rpc (RegisterBoxRequest / RegisterBoxReply). A registration allocates the box's switch address from the plan's lease run and its loopback address from the slice the host switch publishes at — shared cursors across registry clones, so no address is ever handed twice — fills the row from the client's expanded spec, and answers with the addresses; one info line per registration names the box and both of them. run.rs binds the socket at boot and degrades to a warn, not a failed boot, when it cannot. --- crates/minimald-rpc/src/lib.rs | 63 ++++ crates/minvmd/src/box_registry.rs | 140 ++++++++- crates/minvmd/src/cmd/run.rs | 23 ++ crates/minvmd/src/control.rs | 486 ++++++++++++++++++++++++++++++ crates/minvmd/src/lib.rs | 1 + 5 files changed, 705 insertions(+), 8 deletions(-) create mode 100644 crates/minvmd/src/control.rs diff --git a/crates/minimald-rpc/src/lib.rs b/crates/minimald-rpc/src/lib.rs index 1b7c69fb29..fb78e073ad 100644 --- a/crates/minimald-rpc/src/lib.rs +++ b/crates/minimald-rpc/src/lib.rs @@ -9,6 +9,8 @@ //! //! The server-side serving glue lives in the `minimald` crate. +use std::net::Ipv4Addr; + use chrono::Utc; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use sessions::SessionId; @@ -405,6 +407,67 @@ fn default_hooks_enabled() -> bool { true } +/// The wire types of the VM host daemon's box-registration control socket +/// (T66): the one door a client has to the host-side box table (NET-138). +/// +/// On a minvmd-backed host, `min session activate` opens this socket — a +/// UDS beside the daemon's ssh socket in the provider's state dir — writes +/// one [`RegisterBoxRequest`] as a line of JSON, and reads one +/// [`RegisterBoxReply`] line back. Nothing else crosses it: the session +/// RPCs are the daemon crate's HTTP-shaped socket, and the box's switch +/// and loopback addresses come from here so the daemon's create request +/// can carry them (see [`SessionConfig`]). +/// +/// These types live here rather than in `minvmd` because both ends depend +/// on this crate — the activating client and the host daemon — and the +/// protocol must not drift between them. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct RegisterBoxRequest { + /// The box's name — the session name the following create request + /// carries, so the host-side row and the session identify the same box. + /// A box's id on the host is its name. + pub name: String, + /// The external ports the box's ingress rules admit, as the client + /// expanded them. The host row holds the expanded ports rather than the + /// rules so the rule grammar stays the client's: the attaching side + /// reaches ports without re-parsing declarations. + #[serde(default)] + pub ingress_ports: Vec, + /// The box's egress policy, as the client expanded it. Absent means the + /// allow-all default — the same meaning the create request's policy + /// carries. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub egress: Option, +} + +/// The addresses a successful registration hands back: the box's switch +/// address and its published loopback address, both allocated on the host +/// from the address plan the host switch serves. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct RegisterBoxAddresses { + /// The box's address on the switch. The create request carries it so + /// the in-VM daemon attaches with this address instead of drawing its + /// own — the row the host's egress gate decides by is keyed by it. + pub switch_address: Ipv4Addr, + /// The box's address on the guest's loopback, from the slice the host + /// switch publishes at. + pub loopback_address: Ipv4Addr, +} + +/// The one reply line the control socket answers a registration with: the +/// handed addresses, or the reason the registration did not happen. Untagged +/// so the reply stays one flat JSON object either way. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum RegisterBoxReply { + /// Registration succeeded: both addresses are allocated and the host + /// table's row is filled. + Addresses(RegisterBoxAddresses), + /// Registration failed: `error` is a sentence naming why, for the + /// activating client to warn with. + Error { error: String }, +} + /// The request for a [`CreateSession`] RPC. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct CreateSessionRequest { diff --git a/crates/minvmd/src/box_registry.rs b/crates/minvmd/src/box_registry.rs index 97da910ec5..f2f74b65be 100644 --- a/crates/minvmd/src/box_registry.rs +++ b/crates/minvmd/src/box_registry.rs @@ -21,11 +21,13 @@ //! read-only view whose only operations are lookups, so the one component //! that reads guest frames cannot add, replace, or withdraw a row. The //! registration path that carries a client's box declarations into this -//! registry is T66's; what lands here is the table's shape, the gate that -//! reads it, and the rows the host itself can name. +//! registry is [`crate::control`] — the host daemon's control socket, over +//! which the activating client registers a box and reads the addresses the +//! allocation hands back. use std::collections::BTreeMap; use std::net::Ipv4Addr; +use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, RwLock}; use sessions::EgressPolicy; @@ -151,21 +153,77 @@ impl BoxRegistration { } } +/// A box declaration as the activating client carries it over the host's +/// control socket: the facts a row is compiled from, without the addresses — +/// a client-driven registration allocates those on the host, from the +/// address plan this registry's switch serves, and hands them back with the +/// row ([`BoxRegistry::register_client_box`]). +#[derive(Debug, Clone)] +pub struct ClientBoxSpec { + /// The box's name: what the row is named by, and the name the create + /// request will carry — a box's id on the host is its name. + pub name: String, + /// The external ports the box's ingress rules admit, as the client + /// expanded them. + pub ingress_ports: Vec, + /// The box's egress policy, as the client declared it. Absent compiles + /// the allow-all default, the same meaning the create request's absent + /// policy carries. + pub egress: Option, +} + +/// Why a client-driven registration could not be allocated. Both runs a +/// box address comes from — the plan's switch lease run and the published +/// loopback slice — are finite; exhausting one is an answer to hand back +/// over the control socket, not a panic. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum AllocationError { + /// Every switch address in the plan's lease run is published. + #[error("the switch's address plan is exhausted; no box address remains")] + SwitchExhausted, + /// Every address in the loopback slice this subnet's switch publishes + /// at is handed out. + #[error("the host's loopback slice is exhausted; no published address remains")] + LoopbackExhausted, + /// The address plan does not serve this registry's subnet, so no box + /// address can be allocated against it. An explicit registration + /// ([`BoxRegistry::register`]) still works: it brings its own + /// addresses. + #[error("the address plan does not serve subnet {0}; no box address can be allocated")] + UnplannedSubnet(SwitchSubnet), +} + /// The writable half of the host-side table, held by the host process: the /// registration surface — the host's own node-namespace row, and T66's /// client-driven path — and the source of the read-only [`BoxTable`] the /// gate reads. /// -/// Cheap to clone, and every clone shares the rows: the table the gate holds -/// is the same one every later registration lands in, which is how a box -/// published after the gate started is decided by its rules from that moment -/// on. The subnet a registry is built with fixes the address plan its rows -/// compile against — the resolver the carve-out is keyed to, and the node -/// address — so it must be the same subnet the gate's switch serves. +/// Cheap to clone, and every clone shares the rows and the allocation +/// cursors: the table the gate holds is the same one every later registration +/// lands in, which is how a box published after the gate started is decided +/// by its rules from that moment on, and an address a registration takes on +/// one clone is never handed twice. The subnet a registry is built with +/// fixes the address plan its rows compile against — the resolver the +/// carve-out is keyed to, the node address, and the runs the client-driven +/// allocation draws from — so it must be the same subnet the gate's switch +/// serves. #[derive(Debug, Clone)] pub struct BoxRegistry { subnet: SwitchSubnet, rows: Arc>, + /// The next switch address the client-driven allocation hands out, + /// shared by every clone of this registry. + next_switch_addr: Arc, + /// The next published loopback address the client-driven allocation + /// hands out, shared the same way. + next_loopback_addr: Arc, + /// The loopback slice this subnet's switch publishes at, when the + /// address plan serves it: the run [`Self::register_client_box`] + /// allocates published addresses from. `None` for a subnet the plan + /// does not serve — such a registry still holds explicit registrations + /// (the node's own row among them), it just cannot allocate for a + /// client box. + loopback_slice: Option, } impl BoxRegistry { @@ -175,9 +233,15 @@ impl BoxRegistry { /// configured with. #[must_use] pub fn new(subnet: SwitchSubnet) -> Self { + let loopback_slice = switch::AddressPlan::default().loopback_slice_for_switch(subnet); Self { subnet, rows: Arc::new(RwLock::new(BTreeMap::new())), + next_switch_addr: Arc::new(AtomicU32::new(subnet.first_ptask())), + next_loopback_addr: Arc::new(AtomicU32::new( + loopback_slice.map_or(0, |slice| u32::from(slice.first())), + )), + loopback_slice, } } @@ -249,6 +313,56 @@ impl BoxRegistry { .remove(&switch_addr.octets()) } + /// Registers a client box: allocates its switch address from the plan's + /// lease run and its published loopback address from the slice this + /// subnet's switch serves at, then fills the row from `spec` — the same + /// compile [`Self::register`] does, addressed at the allocation — and + /// returns the row, whose addresses are the ones to hand back over the + /// control socket. + /// + /// Addresses are handed out in plan order from shared cursors: each + /// registration takes the next address the plan has not spent, and no + /// address is ever handed to two rows (clones of this registry share the + /// cursors, so that holds across every clone). The runs are finite — the + /// lease run ends below the daemon's reserved address, the slice ends + /// where the plan's next switch begins — and exhausting one is the + /// [`AllocationError`] the control socket hands back as the + /// registration's failure. + /// + /// An address is spent for good: withdrawing its row does not return it + /// to the cursor, and neither does an allocation whose other run is + /// exhausted. A spent address's host-side state — the gate's + /// rate-limit slots, the switch's static lease table — is keyed by it, + /// and re-issuing it to a new box would inherit all of that; a fresh + /// address starts clean. + pub fn register_client_box( + &self, + spec: ClientBoxSpec, + ) -> Result, AllocationError> { + let slice = self + .loopback_slice + .ok_or(AllocationError::UnplannedSubnet(self.subnet))?; + let switch_addr = take_next( + &self.next_switch_addr, + self.subnet.first_ptask(), + self.subnet.last_ptask(), + ) + .ok_or(AllocationError::SwitchExhausted)?; + let loopback_addr = take_next( + &self.next_loopback_addr, + u32::from(slice.first()), + u32::from(slice.last()), + ) + .ok_or(AllocationError::LoopbackExhausted)?; + let mut registration = + BoxRegistration::new(spec.name, switch_addr, loopback_addr) + .with_admitted_ports(spec.ingress_ports); + if let Some(policy) = spec.egress { + registration = registration.with_egress_policy(policy); + } + Ok(self.register(registration)) + } + /// Publishes the guest **node's** own namespace: the in-VM daemon's /// root-netns tap, the plane design §5.1 names node-plane. The address is /// the host's own derivation from the subnet it configured the switch @@ -281,6 +395,16 @@ impl BoxRegistry { } } +/// Takes the next unspent address from `cursor`, when `first..=last` still +/// holds one. Relaxed ordering: a cursor's only invariant is that no two +/// takes return the same address, which an atomic add gives on every +/// ordering; the run's bounds are checked on the taken value, so even a +/// cursor advanced past its run's end (or wrapped) hands out nothing. +fn take_next(cursor: &AtomicU32, first: u32, last: u32) -> Option { + let next = cursor.fetch_add(1, Ordering::Relaxed); + (first <= next && next <= last).then(|| Ipv4Addr::from(next)) +} + /// The read-only view of the published rows the egress gate decides by: the /// lookup a frame's source address resolves through, and nothing else. The /// registry hands the gate this view precisely because it has no mutation diff --git a/crates/minvmd/src/cmd/run.rs b/crates/minvmd/src/cmd/run.rs index e283090737..acc9a3ce6a 100644 --- a/crates/minvmd/src/cmd/run.rs +++ b/crates/minvmd/src/cmd/run.rs @@ -447,6 +447,29 @@ fn run_foreground() -> Result<()> { let boxes = crate::box_registry::BoxRegistry::new(switch::DEFAULT_SUBNET); boxes.register_node_namespace(); + // The host-side door to the box table (T66): the control socket the + // activating client registers an own-address box on and reads its + // allocated switch and loopback addresses from — the addresses the + // create request then carries, so the in-VM daemon attaches with the + // handed one. Bound before the guest boots, so a session activated + // against this VM can only ever be handed an address this table holds. + // Best-effort at startup, like the switch above: a bind failure is + // warned and the VM still boots — a registration then degrades to the + // gate's announced interim, exactly as against a supervisor predating + // the socket — rather than failing a boot the client could still + // activate against. + let _control = crate::control::resolve_control_sock() + .and_then(|sock_path| crate::control::spawn(sock_path, boxes.clone())) + .inspect_err(|error| { + tracing::warn!( + %error, + "failed to bind the box-registration control socket; own-address \ + activations will not be handed addresses (the egress gate's \ + announced interim applies)" + ); + }) + .ok(); + // Spawn + supervise the host gvproxy switch before the VMM child boots, so // its `-listen` switch socket exists when the gate relays into it for the // guest shuttle. The switch runtime starts the gate on the socket beside diff --git a/crates/minvmd/src/control.rs b/crates/minvmd/src/control.rs new file mode 100644 index 0000000000..14fd7d3566 --- /dev/null +++ b/crates/minvmd/src/control.rs @@ -0,0 +1,486 @@ +//! The VM host daemon's box-registration control socket (T66) — the one +//! host-side door a client has to the box table ([`crate::box_registry`], +//! NET-138). +//! +//! On a minvmd-backed host, `min session activate` registers every +//! own-address box here before it creates the session: it writes one +//! [`minimald_rpc::RegisterBoxRequest`] line, the daemon allocates the box's +//! switch and loopback addresses into the table — the row the host-side +//! egress gate decides every frame by — and answers with the +//! [`minimald_rpc::RegisterBoxAddresses`] the create request then carries, +//! so the in-VM daemon attaches with the handed address instead of drawing +//! its own. One connection, one request line in, one reply line out. +//! +//! The socket lives beside the daemon's ssh socket in the provider-instance +//! dir and is created with the same 0700-dir / 0600-socket posture the +//! bridge socket gets ([`crate::sock`]): only the same user may reach the +//! box table. The serving shape mirrors [`crate::net::HostGvproxy::spawn`] +//! — a dedicated OS thread off the supervisor's runtime — because both are +//! long-lived host services whose failures must not take the daemon down +//! with them: a failed registration answers the client with the reason and +//! keeps serving, and a client that never reaches the socket (a supervisor +//! predating this module) still activates, handed no addresses, under the +//! egress gate's announced interim. + +use std::io::{Read, Write}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::PathBuf; +use std::thread::JoinHandle; +use std::time::Duration; + +use minimald_rpc::{RegisterBoxAddresses, RegisterBoxReply, RegisterBoxRequest}; + +use crate::box_registry::{BoxRegistry, ClientBoxSpec}; + +/// The control socket's file name inside the provider-instance dir, beside +/// `paths::SSH_SOCK_FILE`. Deliberately not in the `paths` crate: that crate +/// is shared with consumers that have no box table, and this name only +/// means something where `minvmd` supervises one. +pub const CONTROL_SOCK_FILE: &str = "control.sock"; + +/// How long the server waits for a registration's one request line before +/// dropping the connection. Generous against a slow starter; a hung client +/// must not pin the serving thread — connections are served one at a +/// time — forever. +const REGISTER_READ_TIMEOUT: Duration = Duration::from_secs(30); + +/// The largest request line the server will read. A registration carries a +/// name, a port list, and a policy; anything past this bound is not one. +const MAX_REQUEST_LINE: usize = 64 * 1024; + +/// Resolve the control socket's path: `/control.sock`, the +/// dir the CLI resolves the ssh socket under, so the client finds both by +/// the same rule. +pub fn resolve_control_sock() -> std::io::Result { + Ok(crate::state::provider_dir().join(CONTROL_SOCK_FILE)) +} + +/// Bind the control socket at `sock_path` and serve box registrations +/// against `boxes` on a dedicated thread, whose handle the caller holds for +/// as long as the daemon lives. +/// +/// The bind happens on the calling thread so its failure surfaces to the +/// supervisor's own startup error handling; only the accept loop moves to +/// the thread. The socket gets the bridge socket's posture: path-length +/// check (libkrun aborts on over-long socket paths), a 0700 parent dir, a +/// stale socket removed, and 0600 on the socket itself. +pub fn spawn(sock_path: PathBuf, boxes: BoxRegistry) -> std::io::Result> { + crate::sock::check_uds_path_len(&sock_path)?; + crate::sock::prepare_socket_dir(&sock_path)?; + crate::sock::remove_stale_socket(&sock_path)?; + let listener = UnixListener::bind(&sock_path)?; + crate::sock::enforce_socket_permissions(&sock_path)?; + std::thread::Builder::new() + .name("minvmd-control".to_string()) + .spawn(move || accept_loop(listener, boxes)) +} + +/// Accept and serve registrations until the daemon exits. One connection at +/// a time: a registration is two map writes and one allocation, served +/// serially so the table sees its registrations in arrival order. +fn accept_loop(listener: UnixListener, boxes: BoxRegistry) { + for stream in listener.incoming() { + match stream { + Ok(stream) => { + if let Err(error) = serve_connection(stream, &boxes) { + tracing::debug!(error = %error, "box registration connection failed"); + } + } + Err(error) => tracing::debug!(error = %error, "control socket accept failed"), + } + } +} + +/// Serve one registration: read the request line, allocate, answer. +fn serve_connection(stream: UnixStream, boxes: &BoxRegistry) -> std::io::Result<()> { + let mut stream = stream; + stream.set_read_timeout(Some(REGISTER_READ_TIMEOUT))?; + let Some(line) = read_request_line(&mut stream)? else { + // EOF before any byte: a client that connected and left. Nothing to + // answer, nothing to log past the connection level. + return Ok(()); + }; + let request = match parse_request(&line) { + Ok(request) => request, + Err(error) => { + let error = error.to_string(); + tracing::debug!(error = %error, "box registration request did not parse"); + return write_reply(&mut stream, &RegisterBoxReply::Error { error }); + } + }; + register_and_reply(&mut stream, boxes, request) +} + +/// Read one line (terminated by `\n`) of the registration request. A +/// connection that closes before sending a line reads as no request; a line +/// past [`MAX_REQUEST_LINE`] is refused. +fn read_request_line(stream: &mut UnixStream) -> std::io::Result> { + let mut line = Vec::new(); + let mut buf = [0u8; 1024]; + loop { + let read = stream.read(&mut buf)?; + if read == 0 { + return if line.is_empty() { + Ok(None) + } else { + // A partial line at EOF — a client that died mid-write. + // Still answerable with a parse refusal, so hand what + // arrived back rather than hanging the slot on a timeout. + Ok(Some(String::from_utf8_lossy(&line).into_owned())) + }; + } + if let Some(newline) = buf[..read].iter().position(|byte| *byte == b'\n') { + line.extend_from_slice(&buf[..newline]); + return Ok(Some(String::from_utf8_lossy(&line).into_owned())); + } + line.extend_from_slice(&buf[..read]); + if line.len() > MAX_REQUEST_LINE { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "box registration request exceeded {MAX_REQUEST_LINE} bytes without a newline" + ), + )); + } + } +} + +fn parse_request(line: &str) -> Result { + serde_json_lenient::from_str(line) +} + +/// Allocate the box into the table and write the reply — the addresses on +/// success, the reason on a refusal. One info line per registration names +/// the box and both addresses: the diagnostic a bundle's VM host daemon log +/// is read for. +fn register_and_reply( + stream: &mut UnixStream, + boxes: &BoxRegistry, + request: RegisterBoxRequest, +) -> std::io::Result<()> { + let spec = ClientBoxSpec { + name: request.name.clone(), + ingress_ports: request.ingress_ports, + egress: request.egress, + }; + let reply = match boxes.register_client_box(spec) { + Ok(record) => { + tracing::info!( + box = %record.name(), + switch_address = %record.switch_addr(), + loopback_address = %record.loopback_addr(), + "registered box with the VM host daemon; addresses allocated" + ); + RegisterBoxReply::Addresses(RegisterBoxAddresses { + switch_address: record.switch_addr(), + loopback_address: record.loopback_addr(), + }) + } + Err(error) => { + tracing::debug!( + box = %request.name, + error = %error, + "box registration refused" + ); + RegisterBoxReply::Error { + error: error.to_string(), + } + } + }; + write_reply(stream, &reply) +} + +fn write_reply(stream: &mut UnixStream, reply: &RegisterBoxReply) -> std::io::Result<()> { + let mut line = serde_json_lenient::to_string(reply).map_err(|error| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("box registration reply did not serialize: {error}"), + ) + })?; + line.push('\n'); + stream.write_all(line.as_bytes()) +} + +#[cfg(test)] +mod tests { + use std::io::{BufRead, BufReader, Write}; + use std::net::Ipv4Addr; + use std::os::unix::net::UnixStream as TestStream; + use std::sync::OnceLock; + use std::time::Duration; + + use minimald_rpc::RegisterBoxRequest; + use switch::SwitchSubnet; + + use crate::box_registry::AllocationError; + use crate::net::egress_gate::test_support::CaptureWriter; + + use super::*; + + /// The default switch subnet, the plan the registries below are built + /// for. + const SUBNET: SwitchSubnet = switch::DEFAULT_SUBNET; + + /// A process-global log capture. The server serves on its own OS + /// thread, so the thread-local captures the crate's other tests install + /// do not see its lines; the global default is installed once per + /// process and shadowed by those thread-local ones wherever they exist. + static SERVER_LOG: OnceLock = OnceLock::new(); + + fn server_capture() -> &'static CaptureWriter { + SERVER_LOG.get_or_init(|| { + let capture = CaptureWriter::default(); + tracing::subscriber::set_global_default( + tracing_subscriber::fmt() + .with_writer(capture.clone()) + .with_ansi(false) + .finish(), + ) + .expect("no other test in this binary installs a process-wide subscriber"); + capture + }) + } + + /// Spawns the control server on a temp path and returns (path, handle + /// keeping the server thread identified). The thread outlives the test + /// the way a daemon's does; the temp dir's drop after the test closes + /// the test's view of the socket. + fn spawn_server(dir: &std::path::Path) -> std::io::Result<(PathBuf, JoinHandle<()>)> { + let sock_path = dir.join(CONTROL_SOCK_FILE); + let handle = spawn(sock_path.clone(), BoxRegistry::new(SUBNET))?; + // Wait until the socket accepts rather than racing the bind. + for _ in 0..500 { + if TestStream::connect(&sock_path).is_ok() { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + Ok((sock_path, handle)) + } + + /// A client that writes the request and reads the reply line back, + /// mirroring the CLI's registration helper. + fn register( + sock_path: &std::path::Path, + request: &RegisterBoxRequest, + ) -> std::io::Result { + let mut stream = TestStream::connect(sock_path)?; + let mut line = serde_json_lenient::to_string(request).expect("request serializes"); + line.push('\n'); + stream.write_all(line.as_bytes())?; + let mut reader = BufReader::new(stream); + let mut reply = String::new(); + reader.read_line(&mut reply)?; + serde_json_lenient::from_str(reply.trim()) + .map_err(|error| std::io::Error::other(format!("reply did not parse: {error}"))) + } + + /// The registered box's two addresses, asserted as the reply the daemon + /// hands back. + fn handed(reply: RegisterBoxReply) -> RegisterBoxAddresses { + match reply { + RegisterBoxReply::Addresses(addresses) => addresses, + RegisterBoxReply::Error { error } => { + panic!("a valid registration is answered with addresses, refused with {error}") + } + } + } + + /// The end-to-end shape of the layer: the activating client registers + /// over the control socket, the host allocates the box's switch and + /// loopback addresses into its table and hands them back, allocations + /// advance without reuse, a refusal names its reason, and the server + /// keeps serving past one. The registration's one info line names the + /// box and both addresses — the line a bundle's VM host daemon log is + /// read for. + #[test] + fn box_addresses_allocated_on_host_and_handed_to_daemon() { + let capture = server_capture(); + let dir = tempfile::TempDir::new().expect("temp dir"); + let (sock_path, _server) = spawn_server(dir.path()).expect("server binds"); + + // The first registration is handed the plan's first lease address + // and the first published loopback address. + let web = handed( + register( + &sock_path, + &RegisterBoxRequest { + name: "web".to_string(), + ingress_ports: vec![8080, 9090], + egress: Some(sessions::EgressPolicy { + allow_protocols: Some(vec![sessions::IpProto::Tcp]), + allow_subnets: Some(vec!["10.0.0.0/8".to_string()]), + allow_dns_hosts: None, + deny_subnets: None, + }), + }, + ) + .expect("first registration is answered"), + ); + assert_eq!( + web.switch_address, + Ipv4Addr::from(SUBNET.first_ptask()), + "the first box takes the plan's first lease address" + ); + assert_eq!( + web.loopback_address, + switch::AddressPlan::default() + .loopback_slice_for_switch(SUBNET) + .expect("the default subnet is planned") + .first(), + "the first box takes the first address of the slice the host switch publishes at" + ); + + // The registration's one info line names the box and both addresses + // it handed back. + let log = capture.contents(); + assert!( + log.contains("registered box with the VM host daemon; addresses allocated"), + "one info line per registration names the event: {log}" + ); + assert!( + log.contains("box=web") + && log.contains(&format!("switch_address={}", web.switch_address)) + && log.contains(&format!("loopback_address={}", web.loopback_address)), + "the info line names the box and the addresses it handed back: {log}" + ); + + // The second registration takes the next address on both runs — + // sequential allocation, no reuse. + let db = handed( + register( + &sock_path, + &RegisterBoxRequest { + name: "db".to_string(), + ingress_ports: vec![5432], + egress: None, + }, + ) + .expect("second registration is answered"), + ); + assert_eq!( + db.switch_address, + Ipv4Addr::from(SUBNET.first_ptask() + 1), + "the second box takes the plan's next lease address, never the first again" + ); + assert_eq!( + db.loopback_address, + Ipv4Addr::from(u32::from( + switch::AddressPlan::default() + .loopback_slice_for_switch(SUBNET) + .expect("the default subnet is planned") + .first() + ) + 1), + "the second box takes the next published loopback address" + ); + + // A malformed request is answered with the reason, not a hang. + let mut stream = TestStream::connect(&sock_path).expect("socket accepts"); + stream.write_all(b"this is not json\n").expect("write"); + let mut reader = BufReader::new(stream); + let mut reply = String::new(); + reader.read_line(&mut reply).expect("reply read"); + let refused: RegisterBoxReply = + serde_json_lenient::from_str(reply.trim()).expect("error reply parses"); + assert!( + matches!(refused, RegisterBoxReply::Error { .. }), + "a malformed request is answered with the reason, got {refused:?}" + ); + + // And the server keeps serving after a refusal. + handed( + register( + &sock_path, + &RegisterBoxRequest { + name: "cache".to_string(), + ingress_ports: Vec::new(), + egress: None, + }, + ) + .expect("server still serves after a refused request"), + ); + } + + /// The registry-level behaviour the socket fronts: a registration fills + /// the row the gate reads from the client's expanded spec, at the + /// addresses the reply would carry — asserted here at the verdict level + /// the gate decides by, not the field level. + #[test] + fn client_registration_fills_the_row_the_gate_reads() { + let registry = BoxRegistry::new(SUBNET); + let table = registry.table(); + let record = registry + .register_client_box(ClientBoxSpec { + name: "web".to_string(), + ingress_ports: vec![8080], + egress: Some(sessions::EgressPolicy { + allow_protocols: Some(vec![sessions::IpProto::Tcp]), + allow_subnets: Some(vec!["10.0.0.0/8".to_string()]), + allow_dns_hosts: None, + deny_subnets: None, + }), + }) + .expect("the default plan has addresses to allocate"); + + let row = table + .by_source(record.switch_addr().octets()) + .expect("the registration filled the row the gate reads"); + assert_eq!(row.name(), "web"); + assert_eq!(row.switch_addr(), record.switch_addr()); + assert_eq!(row.loopback_addr(), record.loopback_addr()); + assert_eq!(row.admitted_ports(), [8080]); + // The compiled rules are the gate's decision surface: a TCP frame to + // an allowed subnet is admitted, and one to a disallowed one is not. + let frame = crate::net::egress_gate::test_support::ipv4_frame( + record.switch_addr().octets(), + 6, + [10, 0, 0, 1], + 80, + ); + let summary = sessions::core::egress::summarize(&frame); + let verdict = sessions::core::egress::verdict(&summary, row.egress()); + assert!( + matches!(verdict, sessions::core::egress::FrameVerdict::Admit), + "the compiled policy admits what the declaration allows, got {verdict:?}" + ); + let outside = crate::net::egress_gate::test_support::ipv4_frame( + record.switch_addr().octets(), + 6, + [203, 0, 113, 7], + 443, + ); + let outside_summary = sessions::core::egress::summarize(&outside); + assert!( + matches!( + sessions::core::egress::verdict(&outside_summary, row.egress()), + sessions::core::egress::FrameVerdict::Drop(_) + ), + "the compiled policy drops what the declaration does not allow" + ); + } + + /// An unplanned subnet serves no loopback slice: an explicit + /// registration still works there (it brings its own addresses), a + /// client-driven one is refused with the reason — never a panic, never + /// a fabricated address. + #[test] + fn client_registration_on_an_unplanned_subnet_is_refused() { + let unplanned = SwitchSubnet::new(Ipv4Addr::new(10, 0, 1, 0), 24).expect("valid"); + let registry = BoxRegistry::new(unplanned); + let error = registry + .register_client_box(ClientBoxSpec { + name: "web".to_string(), + ingress_ports: Vec::new(), + egress: None, + }) + .expect_err("an unplanned subnet has no slice to allocate from"); + assert!( + matches!(error, AllocationError::UnplannedSubnet(_)), + "the refusal names the cause, got {error:?}" + ); + assert!( + registry.table().is_empty(), + "a refused registration publishes no row" + ); + } +} diff --git a/crates/minvmd/src/lib.rs b/crates/minvmd/src/lib.rs index 4f60a8d2da..bdb28e4435 100644 --- a/crates/minvmd/src/lib.rs +++ b/crates/minvmd/src/lib.rs @@ -12,6 +12,7 @@ pub mod box_registry; pub mod cmd; +pub mod control; pub mod config; pub mod error; pub mod image; From 93248d72c641e2c6932491e1f1bf7acf2bdb0531 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:16:08 +0000 Subject: [PATCH 02/18] feat(minimal): activate registers an own-address box with the VM host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An own-address activation on a minvmd-backed host had no way to put its box into the host-side table the egress gate decides by: it created the session without telling the VM host daemon anything, so the box drew its own switch address and its declared rules reached no host-side gate. Before the create, activate registers the box over the VM host daemon's control socket — the box's name, its expanded ingress ports, its egress policy — and carries the addresses the daemon hands back in the create request, so the in-VM daemon attaches with the handed switch address. A host-ip box shares the node's own row, a none box has no switch address, and a native daemon has no box table: all three register nothing. A registration that cannot be made degrades to a warning, never a failed activation, the same way a supervisor predating the socket behaves. Carries the wire groundwork with it: sessions::BoxAddresses (the pair handed together) rides the SessionConfig and the session Record, and the struct literals across the workspace gain the new field. --- crates/minimal-tui/src/rpc.rs | 1 + crates/minimal-tui/tests/snapshots.rs | 1 + crates/minimal/src/cmd/session.rs | 350 +++++++++++++++++- crates/minimal/src/task.rs | 1 + crates/minimal/tests/cli.rs | 2 + crates/minimald-rpc/src/lib.rs | 44 ++- crates/minimald/src/exec.rs | 1 + crates/minimald/src/rpc.rs | 3 + crates/minimald/src/session/tests.rs | 1 + crates/minimald/src/sessions.rs | 6 + crates/minimald/src/store.rs | 1 + crates/minimald/src/test_harness.rs | 1 + crates/minvmd/examples/exec.rs | 1 + crates/minvmd/src/control.rs | 6 +- .../tests/minimald_session_integration.rs | 1 + crates/sessions/src/lib.rs | 33 ++ crates/sessions/src/store.rs | 9 +- 17 files changed, 437 insertions(+), 25 deletions(-) diff --git a/crates/minimal-tui/src/rpc.rs b/crates/minimal-tui/src/rpc.rs index 6b6e5f7a52..9ba0633315 100644 --- a/crates/minimal-tui/src/rpc.rs +++ b/crates/minimal-tui/src/rpc.rs @@ -274,6 +274,7 @@ pub async fn activate( project_path: project_path.clone(), network, policy: SessionPolicy::default(), + box_addresses: None, // Same default as an activate with no flags: the dashboard // has no `--no-hooks` of its own, and a session created here // is attachable later like any other. diff --git a/crates/minimal-tui/tests/snapshots.rs b/crates/minimal-tui/tests/snapshots.rs index 25547f3e38..844969ee17 100644 --- a/crates/minimal-tui/tests/snapshots.rs +++ b/crates/minimal-tui/tests/snapshots.rs @@ -72,6 +72,7 @@ fn record(name: Option<&str>, network: NetworkMode) -> sessions::Record { project_path: paths::HostAbsPath::try_new("/src/api").unwrap(), network, policy: sessions::SessionPolicy::default(), + box_addresses: None, status: sessions::SessionStatus::Active, hooks_enabled: true, attrs: Default::default(), diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index 7d2930fafb..6df356fa18 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -27,6 +27,171 @@ pub async fn cmd_activate(global: &GlobalArgs, args: ActivateArgs) -> Result<(), activate_session(global, args, true).await } +/// How long a box registration may take before this activation gives up on +/// it. The control socket answers with two map writes and an allocation, so +/// healthy is milliseconds; the bound exists so a hung VM host daemon cannot +/// hang the activation the user asked for. +const BOX_REGISTRATION_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// Registers an own-address box with the VM host daemon over its control +/// socket (T66), returning the addresses it handed back. +/// +/// The control socket lives beside the ssh socket in the minvmd +/// provider-instance dir — the dir the daemon connection resolves through — +/// and speaks one JSON line each way: the box's expanded declaration in, the +/// allocated switch and loopback addresses out +/// ([`minimald_rpc::RegisterBoxReply`]). A refused registration (an +/// exhausted address plan, a malformed declaration) surfaces here as an +/// error naming the reason; the caller degrades it to a warning rather than +/// failing the activation. +async fn register_box_with_vm_host( + sock_path: &std::path::Path, + request: minimald_rpc::RegisterBoxRequest, +) -> anyhow::Result { + use tokio::io::AsyncBufReadExt as _; + use tokio::io::AsyncWriteExt as _; + + let mut stream = tokio::net::UnixStream::connect(sock_path) + .await + .with_context(|| { + format!( + "connecting to the VM host daemon's box-registration control socket at {}", + sock_path.display() + ) + })?; + let mut line = serde_json_lenient::to_string(&request) + .context("serializing the box registration request")?; + line.push('\n'); + stream + .write_all(line.as_bytes()) + .await + .context("writing the box registration request")?; + let mut reply = String::new(); + tokio::io::BufReader::new(stream) + .read_line(&mut reply) + .await + .context("reading the box registration reply")?; + if reply.trim().is_empty() { + anyhow::bail!( + "the VM host daemon closed its control socket without answering the box registration" + ); + } + let reply: minimald_rpc::RegisterBoxReply = serde_json_lenient::from_str(reply.trim()) + .with_context(|| { + format!("the VM host daemon's box registration reply did not parse: {reply}") + })?; + match reply { + minimald_rpc::RegisterBoxReply::Addresses(addresses) => Ok(addresses), + minimald_rpc::RegisterBoxReply::Error { error } => { + anyhow::bail!("the VM host daemon refused the box registration: {error}") + } + } +} + +/// Registers this activation's box with the VM host daemon, when there is +/// one to register with (T66), returning the addresses it handed back — +/// `None` when there is nothing to register or the registration could not +/// be made. +/// +/// The box this activation creates is registerable only on a minvmd-backed +/// host, and only when it is an own-address box: a `host_ip` box shares the +/// node's own row in the host table, and a `none` box has no switch address +/// at all — both register nothing and attach exactly as they always have. +/// +/// A registration that cannot be made — no control socket (a supervisor +/// predating it), a refusal, the deadline — degrades to a warning and +/// `None`, never to a failed activation: the box still creates, attaching +/// under the egress gate's announced interim the same way a pre-T66 box +/// does. Its egress is then what the interim admits until the daemon-side +/// default flips, not what this activation declared — the warning says so. +async fn register_box_for_activation( + global: &GlobalArgs, + network: sessions::NetworkMode, + name: &str, + policy: &sessions::SessionPolicy, +) -> Option { + if !global.use_minvmd() || network != sessions::NetworkMode::OwnIp { + return None; + } + // The control socket sits beside the ssh socket in the provider dir the + // daemon connection resolves through, so the client finds both by the + // same rule — named VMs included, since the resolution reads the same + // process-global VM name. + let ssh_sock = match client::resolve_socket_path(global.minimal_dir.as_deref(), true) { + Ok(path) => path, + Err(error) => { + tracing::warn!( + %error, + "cannot resolve the VM host provider dir; the box will not be \ + registered with the VM host daemon (its declared egress is not \ + enforced host-side)" + ); + return None; + } + }; + let Some(sock_path) = ssh_sock.parent().map(|dir| { + dir.join(minvmd::control::CONTROL_SOCK_FILE) + }) else { + tracing::warn!( + "no provider dir resolved for the ssh socket; the box will not be \ + registered with the VM host daemon (its declared egress is not \ + enforced host-side)" + ); + return None; + }; + // The declaration, as this activation expanded it: the ingress rules + // reduced to the external ports they admit — the shape the host row + // holds — and the egress policy verbatim. + let request = minimald_rpc::RegisterBoxRequest { + name: name.to_string(), + ingress_ports: policy + .ingress + .as_ref() + .map(|ingress| { + ingress + .port_mappings + .iter() + .map(|mapping| mapping.external_port) + .collect() + }) + .unwrap_or_default(), + egress: policy.egress.clone(), + }; + let registration = tokio::time::timeout( + BOX_REGISTRATION_TIMEOUT, + register_box_with_vm_host(&sock_path, request), + ) + .await; + match registration { + Ok(Ok(addresses)) => { + tracing::debug!( + box = %name, + switch_address = %addresses.switch_address, + loopback_address = %addresses.loopback_address, + "VM host daemon handed the box its addresses" + ); + Some(addresses) + } + Ok(Err(error)) => { + tracing::warn!( + %error, + "the box will not be registered with the VM host daemon; its \ + declared egress is not enforced host-side" + ); + None + } + Err(_) => { + tracing::warn!( + after = ?BOX_REGISTRATION_TIMEOUT, + "the VM host daemon did not answer the box registration in time; \ + the box will not be registered (its declared egress is not \ + enforced host-side)" + ); + None + } + } +} + /// The activation flow shared by [`cmd_activate`] and the bare-`min` router. /// `offer_scaffold` gates the `minimal.toml` scaffold offer: `cmd_activate` /// keeps it (its long-standing behavior, unchanged), while bare `min` @@ -96,13 +261,27 @@ pub(crate) async fn activate_session( .clone() .unwrap_or_else(|| autogen_session_name(&utf8_path, &random_hex4())); + let network: sessions::NetworkMode = args.network.into(); + + // T66: an own-address box on a VM-backed host registers with the VM host + // daemon before the create — the host allocates its switch and loopback + // addresses into the table its egress gate decides by and hands them + // back, and the create request carries them so the in-VM daemon attaches + // with the handed switch address instead of drawing its own. Every + // other shape of activation — a host-ip box sharing the node's own row, + // a none box with no switch address, a native daemon with no box table + // — registers nothing and attaches as it always has. + let box_addresses = + register_box_for_activation(global, network, &session_name, &policy).await; + // The daemon sources `username` from the authenticated SSH // connection context; the client doesn't send it. let config = minimald_rpc::SessionConfig { name: Some(session_name), project_path: abs_path.clone(), - network: args.network.into(), + network, policy, + box_addresses, hooks_enabled: !args.no_hooks, attrs: Default::default(), }; @@ -263,6 +442,19 @@ pub(crate) async fn activate_session( if should_retry_autogen(autogen, attempts, &error) { attempts += 1; config.name = Some(autogen_session_name(&utf8_path, &random_hex4())); + // A registered box's row carries the name it was + // registered under (T66), so the re-mint re-registers — + // the row the first attempt made stays published behind + // it: the control socket has no withdraw verb yet, its + // addresses are spent for good by design, and the rare + // colliding autogen retry leaves at most one such row. + config.box_addresses = register_box_for_activation( + global, + config.network, + config.name.as_deref().expect("just re-minted"), + &config.policy, + ) + .await; continue; } bail!("CreateSession failed: {error}"); @@ -1832,4 +2024,160 @@ mod tests { "an opted-out bare box must show allow-all: {rendered}" ); } + + /// Spawns a stand-in VM host control server at `sock_path`: answers + /// every registration with `reply` (one JSON line, newline appended), + /// recording each request line it received. Mirrors the line protocol + /// [`minvmd::control`](minvmd::control) serves, from the server side. + async fn fake_vm_host( + sock_path: std::path::PathBuf, + reply: String, + ) -> std::sync::Arc>> { + use tokio::io::{AsyncBufReadExt as _, AsyncWriteExt as _}; + let requests = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let listener = tokio::net::UnixListener::bind(&sock_path).unwrap(); + let seen = std::sync::Arc::clone(&requests); + tokio::spawn(async move { + loop { + let (stream, _) = match listener.accept().await { + Ok(accepted) => accepted, + Err(_) => return, + }; + let mut lines = tokio::io::BufReader::new(stream); + let mut line = String::new(); + if lines.read_line(&mut line).await.is_err() { + return; + } + seen.lock().unwrap().push(line.trim().to_string()); + let mut writer = lines.into_inner(); + let _ = writer.write_all(reply.as_bytes()).await; + let _ = writer.write_all(b"\n").await; + } + }); + requests + } + + /// T66: an own-address box on a VM-backed host registers with the VM + /// host daemon over its control socket — the request carries the box's + /// name, its expanded ingress ports, and its egress policy; the reply + /// hands back the two addresses the create request then carries — and + /// the decision about whether to register at all is the activation's: + /// only a minvmd-backed host with an own-address box registers, and a + /// refusal surfaces as a sentence, not a failed activation. + #[tokio::test] + async fn activate_registers_box_with_vm_host() { + let policy = sessions::SessionPolicy { + egress: Some(sessions::EgressPolicy { + allow_subnets: Some(vec!["10.0.0.0/8".to_string()]), + allow_dns_hosts: None, + allow_protocols: Some(vec![IpProto::Tcp]), + deny_subnets: None, + }), + ingress: Some(IngressPolicy { + port_mappings: vec![ + PortMapping { + external_port: 8080, + internal_port: 80, + proto: IpProto::Tcp, + }, + PortMapping { + external_port: 5432, + internal_port: 5432, + proto: IpProto::Tcp, + }, + ], + dynamic_allowed_range: None, + dynamic_ingress: None, + }), + }; + + // The successful shape, driven the way the activation drives it: + // through `register_box_for_activation`, whose socket resolution + // lands on the provider dir the daemon connection uses — here the + // stand-in bound where a real minvmd's control socket would be. + let dir = tempfile::TempDir::new().unwrap(); + let provider_dir = dir.path().join("providers").join("local-minvmd0"); + std::fs::create_dir_all(&provider_dir).unwrap(); + let sock_path = provider_dir.join("control.sock"); + let requests = fake_vm_host( + sock_path.clone(), + r#"{"switch_address":"100.64.0.2","loopback_address":"127.0.64.0"}"#.to_string(), + ) + .await; + let global = GlobalArgs { + provider: Some(Provider::LocalMinvmd), + minimal_dir: Some(dir.path().to_path_buf()), + ..Default::default() + }; + let handed = register_box_for_activation(&global, NetworkMode::OwnIp, "web", &policy) + .await + .expect("the registration is answered with addresses"); + assert_eq!( + handed.switch_address, + std::net::Ipv4Addr::new(100, 64, 0, 2) + ); + assert_eq!( + handed.loopback_address, + std::net::Ipv4Addr::new(127, 0, 64, 0) + ); + { + // The lock is scoped: holding a std MutexGuard across the awaits + // below is exactly what the await-holding-lock lint names. + let seen = requests.lock().unwrap(); + assert_eq!(seen.len(), 1, "one registration, one request"); + let request: minimald_rpc::RegisterBoxRequest = + serde_json_lenient::from_str(&seen[0]).expect("the request is the wire type"); + assert_eq!(request.name, "web"); + assert_eq!(request.ingress_ports, vec![8080, 5432]); + assert_eq!(request.egress.as_ref(), policy.egress.as_ref()); + } + + // Every other shape of activation registers nothing: a native + // daemon has no box table, a host-ip box shares the node's own row. + // Neither touches a socket, so a plain global (no provider, no + // dir) suffices. + let native = GlobalArgs::default(); + assert!( + register_box_for_activation(&native, NetworkMode::OwnIp, "web", &policy) + .await + .is_none(), + "a native daemon hosts no box table to register on" + ); + let vm_host = GlobalArgs { + provider: Some(Provider::LocalMinvmd), + ..Default::default() + }; + assert!( + register_box_for_activation(&vm_host, NetworkMode::HostNet, "web", &policy) + .await + .is_none(), + "a host-ip box shares the node's own row and registers nothing" + ); + + // The refusal shape: a daemon that answers with a reason produces an + // error naming it — what the activation warns on and continues + // from, never a failed create. + let refused_dir = tempfile::TempDir::new().unwrap(); + let refused_path = refused_dir.path().join("control.sock"); + let _refused_requests = fake_vm_host( + refused_path.clone(), + r#"{"error":"the switch's address plan is exhausted; no box address remains"}"# + .to_string(), + ) + .await; + let refused = register_box_with_vm_host( + &refused_path, + minimald_rpc::RegisterBoxRequest { + name: "db".to_string(), + ingress_ports: Vec::new(), + egress: None, + }, + ) + .await + .expect_err("a refused registration is an error"); + assert!( + refused.to_string().contains("address plan is exhausted"), + "the refusal surfaces with its reason: {refused}" + ); + } } diff --git a/crates/minimal/src/task.rs b/crates/minimal/src/task.rs index e21b814fd8..490678268b 100644 --- a/crates/minimal/src/task.rs +++ b/crates/minimal/src/task.rs @@ -620,6 +620,7 @@ pub async fn cmd_task_run(global: &GlobalArgs, args: TaskRunArgs) -> Result<(), project_path: abs_path.clone(), network: sessions::NetworkMode::HostNet, policy: sessions::SessionPolicy::default(), + box_addresses: None, // Same default as an activate with no flags, matching the // loadout handling below. `min task run` has no `--no-hooks` of // its own; a `--keep` session is attachable later, so its hooks diff --git a/crates/minimal/tests/cli.rs b/crates/minimal/tests/cli.rs index 8f321fedac..6edb92a12a 100644 --- a/crates/minimal/tests/cli.rs +++ b/crates/minimal/tests/cli.rs @@ -1628,6 +1628,7 @@ async fn create_pending_session(daemon: &common::TestDaemon, name: &str) -> Sess project_path: paths::HostAbsPath::try_new(project_path).unwrap(), network: sessions::NetworkMode::NoNet, policy: Default::default(), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }; @@ -1701,6 +1702,7 @@ async fn create_session_with( project_path, network, policy, + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }; diff --git a/crates/minimald-rpc/src/lib.rs b/crates/minimald-rpc/src/lib.rs index fb78e073ad..1f3e1d0b9c 100644 --- a/crates/minimald-rpc/src/lib.rs +++ b/crates/minimald-rpc/src/lib.rs @@ -9,8 +9,6 @@ //! //! The server-side serving glue lives in the `minimald` crate. -use std::net::Ipv4Addr; - use chrono::Utc; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use sessions::SessionId; @@ -20,8 +18,8 @@ pub mod taskenv; pub mod trace; pub use sessions::{ - DynamicIngress, EffectiveEgress, EffectiveSessionPolicy, EgressPolicy, IngressPolicy, IpProto, - NetworkMode, PortMapping, SessionPolicy, + BoxAddresses, DynamicIngress, EffectiveEgress, EffectiveSessionPolicy, EgressPolicy, + IngressPolicy, IpProto, NetworkMode, PortMapping, SessionPolicy, }; pub const RPC_SUBSYSTEM_PREFIX: &str = "minimald-v1-"; @@ -387,6 +385,15 @@ pub struct SessionConfig { /// Per-session networking policy (egress + ingress). #[serde(default)] pub policy: SessionPolicy, + /// The addresses the VM host daemon handed this box's registration + /// (T66), when the activating client registered one: its switch + /// address, which the in-VM daemon attaches with instead of drawing + /// its own, and its published loopback address. `None` for every other + /// activation — a host that is not minvmd-backed, a box that is not + /// own-address — and the daemon then attaches exactly as it always + /// has. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub box_addresses: Option, /// Whether the session runs the lifecycle hooks composed into it. /// Cleared by `min session activate --no-hooks`, and persisted onto /// the session record so the later attach/detach/destroy @@ -440,20 +447,12 @@ pub struct RegisterBoxRequest { pub egress: Option, } -/// The addresses a successful registration hands back: the box's switch -/// address and its published loopback address, both allocated on the host -/// from the address plan the host switch serves. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct RegisterBoxAddresses { - /// The box's address on the switch. The create request carries it so - /// the in-VM daemon attaches with this address instead of drawing its - /// own — the row the host's egress gate decides by is keyed by it. - pub switch_address: Ipv4Addr, - /// The box's address on the guest's loopback, from the slice the host - /// switch publishes at. - pub loopback_address: Ipv4Addr, -} - +/// The addresses a successful registration hands back +/// ([`sessions::BoxAddresses`]): the box's switch address and its published +/// loopback address, both allocated on the host from the address plan the +/// host switch serves — the same pair the create request carries so the +/// in-VM daemon attaches with it. +/// /// The one reply line the control socket answers a registration with: the /// handed addresses, or the reason the registration did not happen. Untagged /// so the reply stays one flat JSON object either way. @@ -462,7 +461,7 @@ pub struct RegisterBoxAddresses { pub enum RegisterBoxReply { /// Registration succeeded: both addresses are allocated and the host /// table's row is filled. - Addresses(RegisterBoxAddresses), + Addresses(BoxAddresses), /// Registration failed: `error` is a sentence naming why, for the /// activating client to warn with. Error { error: String }, @@ -1468,6 +1467,13 @@ mod tests { project_path: paths::HostAbsPath::try_new("/home/u/proj").unwrap(), network: NetworkMode::OwnIp, policy: SessionPolicy::default(), + // The non-`None` shape of the handed addresses: a fixture + // leaving it `None` would round-trip green even if the + // field never reached the wire. + box_addresses: Some(BoxAddresses { + switch_address: std::net::Ipv4Addr::new(100, 64, 0, 2), + loopback_address: std::net::Ipv4Addr::new(127, 0, 64, 0), + }), // The non-default (`--no-hooks`): `true` is the serde // default, so a fixture using it would round-trip green // even if the field never reached the wire. diff --git a/crates/minimald/src/exec.rs b/crates/minimald/src/exec.rs index 84e99b7901..e564311ecc 100644 --- a/crates/minimald/src/exec.rs +++ b/crates/minimald/src/exec.rs @@ -2040,6 +2040,7 @@ mod tests { project_path: paths::HostAbsPath::try_new("/tmp/project").unwrap(), network: mode, policy: sessions::SessionPolicy::default(), + box_addresses: None, status: sessions::SessionStatus::Active, hooks_enabled: true, attrs: Default::default(), diff --git a/crates/minimald/src/rpc.rs b/crates/minimald/src/rpc.rs index 09891c7930..2f5cd6be3a 100644 --- a/crates/minimald/src/rpc.rs +++ b/crates/minimald/src/rpc.rs @@ -2957,6 +2957,7 @@ mod tests { project_path: HostAbsPath::try_new("/uwu").unwrap(), network: NetworkMode::OwnIp, policy: SessionPolicy::new(Some(egress.clone()), None), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }, @@ -2992,6 +2993,7 @@ mod tests { project_path: HostAbsPath::try_new("/uwu").unwrap(), network: NetworkMode::OwnIp, policy, + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }, @@ -3307,6 +3309,7 @@ mod tests { project_path: HostAbsPath::try_new("/uwu").unwrap(), network: NetworkMode::NoNet, policy: SessionPolicy::new(Some(egress), None), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }, diff --git a/crates/minimald/src/session/tests.rs b/crates/minimald/src/session/tests.rs index 3e2cc88108..345aeba3ff 100644 --- a/crates/minimald/src/session/tests.rs +++ b/crates/minimald/src/session/tests.rs @@ -2619,6 +2619,7 @@ async fn own_ip_default_deny_all() { project_path: paths::HostAbsPath::try_new("/uwu").unwrap(), network: sessions::NetworkMode::OwnIp, policy: sessions::SessionPolicy::default(), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }, diff --git a/crates/minimald/src/sessions.rs b/crates/minimald/src/sessions.rs index 8f334de9a6..7f9ca0b8a5 100644 --- a/crates/minimald/src/sessions.rs +++ b/crates/minimald/src/sessions.rs @@ -78,6 +78,11 @@ fn build_record( project_path: config.project_path, network: config.network, policy: config.policy, + // The handed addresses ride the record: the attach path reads the + // switch address back out of it, so a re-attach — after a restart + // included — reuses the address the host table still holds instead + // of drawing a new one the row would not match. + box_addresses: config.box_addresses, status, hooks_enabled: config.hooks_enabled, attrs: config.attrs, @@ -1248,6 +1253,7 @@ pub(crate) mod tests { project_path: HostAbsPath::try_new("/proj").unwrap(), network: sessions::NetworkMode::default(), policy: Default::default(), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), } diff --git a/crates/minimald/src/store.rs b/crates/minimald/src/store.rs index 52bb51c70b..fb333de54a 100644 --- a/crates/minimald/src/store.rs +++ b/crates/minimald/src/store.rs @@ -391,6 +391,7 @@ mod tests { project_path: HostAbsPath::try_new("/home/alice/proj").unwrap(), network: NetworkMode::default(), policy: Default::default(), + box_addresses: None, hooks_enabled: true, status: SessionStatus::default(), attrs: Default::default(), diff --git a/crates/minimald/src/test_harness.rs b/crates/minimald/src/test_harness.rs index b6d282836d..72d2d4e2f2 100644 --- a/crates/minimald/src/test_harness.rs +++ b/crates/minimald/src/test_harness.rs @@ -564,6 +564,7 @@ pub fn create_session_req(name: &str, project: &str) -> minimald_rpc::CreateSess project_path: paths::HostAbsPath::try_new(project).unwrap(), network: sessions::NetworkMode::default(), policy: Default::default(), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }, diff --git a/crates/minvmd/examples/exec.rs b/crates/minvmd/examples/exec.rs index 9b968dbb87..71e033dd6b 100644 --- a/crates/minvmd/examples/exec.rs +++ b/crates/minvmd/examples/exec.rs @@ -171,6 +171,7 @@ async fn run_session_exec( .map_err(|e| format!("project_path: {e}"))?, network: sessions::NetworkMode::default(), policy: Default::default(), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }, diff --git a/crates/minvmd/src/control.rs b/crates/minvmd/src/control.rs index 14fd7d3566..9268619f43 100644 --- a/crates/minvmd/src/control.rs +++ b/crates/minvmd/src/control.rs @@ -28,7 +28,7 @@ use std::path::PathBuf; use std::thread::JoinHandle; use std::time::Duration; -use minimald_rpc::{RegisterBoxAddresses, RegisterBoxReply, RegisterBoxRequest}; +use minimald_rpc::{BoxAddresses, RegisterBoxReply, RegisterBoxRequest}; use crate::box_registry::{BoxRegistry, ClientBoxSpec}; @@ -171,7 +171,7 @@ fn register_and_reply( loopback_address = %record.loopback_addr(), "registered box with the VM host daemon; addresses allocated" ); - RegisterBoxReply::Addresses(RegisterBoxAddresses { + RegisterBoxReply::Addresses(BoxAddresses { switch_address: record.switch_addr(), loopback_address: record.loopback_addr(), }) @@ -277,7 +277,7 @@ mod tests { /// The registered box's two addresses, asserted as the reply the daemon /// hands back. - fn handed(reply: RegisterBoxReply) -> RegisterBoxAddresses { + fn handed(reply: RegisterBoxReply) -> BoxAddresses { match reply { RegisterBoxReply::Addresses(addresses) => addresses, RegisterBoxReply::Error { error } => { diff --git a/crates/minvmd/tests/minimald_session_integration.rs b/crates/minvmd/tests/minimald_session_integration.rs index 3daf48cd20..30208cf8b3 100644 --- a/crates/minvmd/tests/minimald_session_integration.rs +++ b/crates/minvmd/tests/minimald_session_integration.rs @@ -276,6 +276,7 @@ async fn run_session_exec( .map_err(|e| format!("project_path: {e}"))?, network: sessions::NetworkMode::default(), policy: Default::default(), + box_addresses: None, // The serde default, and what every non-`--no-hooks` // activation sends. This session only runs an exec, so // it declares no hooks either way. diff --git a/crates/sessions/src/lib.rs b/crates/sessions/src/lib.rs index c9c8a75da3..d5eb7455c9 100644 --- a/crates/sessions/src/lib.rs +++ b/crates/sessions/src/lib.rs @@ -3,6 +3,7 @@ use std::collections::BTreeMap; use std::fmt; +use std::net::Ipv4Addr; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -554,6 +555,24 @@ where Ok(name.filter(|name| !name.is_empty())) } +/// The pair of addresses a VM host daemon allocates for a box and hands +/// back to its creator (T66): where the box lives on the switch, and where +/// it is published on the guest's loopback. +/// +/// One type because the pair is one fact — handed together over the +/// registration, carried together in the create request, recorded together +/// on the session record — and every consumer of one half needs to be able +/// to say it came from a host allocation, not a local draw. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub struct BoxAddresses { + /// The box's address on the switch: the lease its frames must carry + /// (NET-084) and the key the host-side box table's row is filed under. + pub switch_address: Ipv4Addr, + /// The box's address on the guest's loopback, from the slice the host + /// switch publishes at. + pub loopback_address: Ipv4Addr, +} + /// The on-disk row/record pertaining to a session. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Record { @@ -605,6 +624,19 @@ pub struct Record { #[serde(default = "hooks_enabled_default")] pub hooks_enabled: bool, + /// The addresses the VM host daemon handed this box's registration at + /// activation (T66): the switch address the attach path configures the + /// tap with instead of drawing its own, and the published loopback + /// address the host side names the box by. Persisted so a re-attach — + /// including after a daemon restart — attaches with the same address + /// the host table still holds a row for; drawing a fresh one would + /// silently orphan the row and drop the box to the egress gate's + /// unregistered-source interim. Defaults to `None` for records that + /// predate the field (every pre-T66 session): those boxes self-allocate + /// exactly as they always have. + #[serde(default)] + pub box_addresses: Option, + /// Free-form attributes. pub attrs: BTreeMap, } @@ -757,6 +789,7 @@ mod tests { policy, status: SessionStatus::default(), hooks_enabled: true, + box_addresses: None, attrs: BTreeMap::new(), } } diff --git a/crates/sessions/src/store.rs b/crates/sessions/src/store.rs index 3892b76ad1..d2961d13f2 100644 --- a/crates/sessions/src/store.rs +++ b/crates/sessions/src/store.rs @@ -1115,10 +1115,15 @@ mod tests { None, ), status: SessionStatus::default(), - // Deliberately the non-default (`--no-hooks`): `true` is the - // serde default, so a fixture using it would round-trip + // Deliberately the non-default shapes: `hooks_enabled: false` + // and a present `box_addresses` are both serde defaults to + // `None`/`true`, so a fixture using those would round-trip // green even if the field were dropped on write. hooks_enabled: false, + box_addresses: Some(crate::BoxAddresses { + switch_address: std::net::Ipv4Addr::new(100, 64, 0, 2), + loopback_address: std::net::Ipv4Addr::new(127, 0, 64, 0), + }), attrs: [("color".to_string(), "blue".to_string())] .into_iter() .collect(), From fed4b2fe1dfd16e39526e188dba4f15ba1ca67fa Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:35:35 +0000 Subject: [PATCH 03/18] feat(minimald): attach an own-address box at its handed address An own-address session now attaches at the switch and loopback addresses the VM host daemon allocated for its box registration and handed back in the create request, instead of drawing one from the switch allocator itself. The handed address still lands in the lease table, so the never-reuse rule and every downstream consumer of that table see it the same way as a self-allocated one. - Record carries the handed pair from the create request into the session launcher, and the task path passes none deliberately: a task sandbox is not the box the registration named. - SwitchClient::attach_handed validates the handed address against the plan's PTask run and records the lease without a draw; a handed address outside the plan is NetError::HandedAddressOutsidePlan. - The one debug line per attach names the handed address and whether it was honored from the host. The switch allocator's next draw skips past a handed lease, so an interim self-allocating session cannot take the box's address. --- crates/minimald/src/exec.rs | 5 + crates/minimald/src/net/gvproxy_network.rs | 165 ++++++++++++++++++++- crates/minimald/src/net/mod.rs | 92 ++++++++++++ crates/minimald/src/net/provider.rs | 48 ++++-- crates/minimald/src/session.rs | 6 + crates/minimald/src/session_host.rs | 9 ++ 6 files changed, 314 insertions(+), 11 deletions(-) diff --git a/crates/minimald/src/exec.rs b/crates/minimald/src/exec.rs index e564311ecc..09885e140d 100644 --- a/crates/minimald/src/exec.rs +++ b/crates/minimald/src/exec.rs @@ -188,6 +188,11 @@ pub(crate) fn task_network( &format!("{session}-task"), egress.map(|section| sessions::SessionPolicy::new(Some(section), None)), None, + // Deliberately not the record's handed addresses (T66): the task's + // sandbox is not the box the registration named — attaching it at the + // session box's address would key its frames to the session's row. + // A task's sandbox self-allocates, as it always has. + None, ) } diff --git a/crates/minimald/src/net/gvproxy_network.rs b/crates/minimald/src/net/gvproxy_network.rs index 1a3ddc43ac..096c287d90 100644 --- a/crates/minimald/src/net/gvproxy_network.rs +++ b/crates/minimald/src/net/gvproxy_network.rs @@ -87,6 +87,15 @@ impl NetGuard for OwnIpGuard { /// here just propagates: the release of that lease stays with the launch /// (`sandbox2::PlannedLaunch`), and detaching as well would double-decrement /// gvproxy's attach count. +/// +/// `handed_from_host` says whether `lease_ip` is the address the VM host +/// daemon allocated for this box's registration and handed back (T66), or +/// one this daemon drew itself — the one debug line per attach names the +/// address and that distinction, so a log tail can tell an attach that +/// honored its handed address from one that did not. +// Left positional: a single call site that has just planned the launch, so a +// struct would be single-use ceremony. +#[allow(clippy::too_many_arguments)] pub(crate) async fn complete_own_ip_attach( switch: &Arc>, tap_fd: OwnedFd, @@ -95,7 +104,14 @@ pub(crate) async fn complete_own_ip_attach( session_name: &str, policy: Option<&sessions::SessionPolicy>, own_address: Option<&crate::net::provider::OwnAddressReporter>, + handed_from_host: bool, ) -> io::Result { + tracing::debug!( + session = session_name, + switch_address = %lease_ip, + handed_from_host, + "own-address box attached to the switch" + ); // The relay's egress carve-out and deprecation notice (NET-004) derive // their addresses from the subnet of the switch this box attaches to, so // a custom-subnet switch is keyed to its own resolver and watched at its @@ -253,14 +269,23 @@ async fn finish_own_ip_attach( #[cfg(test)] mod tests { use std::net::Ipv4Addr; + use std::os::fd::FromRawFd as _; use std::path::PathBuf; + use std::sync::Arc; + use switch::MacAddr; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use tokio::net::{UnixListener, UnixStream}; - use tokio::sync::mpsc; + use tokio::sync::{Mutex, mpsc}; use crate::net::dns; use crate::net::policy::{ControlChannel, register_dns_name}; + use crate::net::provider::network_for; + use crate::net::PtaskLease; + use crate::net::SwitchClient; + use crate::net::SwitchTransport; + use crate::test_harness::CaptureWriter; + use sessions::BoxAddresses; /// Reads one control request off `sock` — its head up to the /// end-of-head marker, then exactly its `Content-Length` body — and @@ -442,4 +467,142 @@ mod tests { shared name both daemons publish, got: {bodies:?}" ); } + + /// A switch whose attach/detach are pure bookkeeping: `HostShuttle` + /// leaves the gvproxy process to `minvmd`, so only the count moves — + /// the shape a VM-backed host's daemon carries. + fn vm_host_switch() -> Arc> { + Arc::new(Mutex::new( + SwitchClient::new("/usr/bin/gvproxy", "/run/minimal/gvproxy") + .with_transport(SwitchTransport::HostShuttle { + cid: crate::net::VSOCK_HOST_CID, + port: crate::net::VSOCK_GVPROXY_SHUTTLE_PORT, + }), + )) + } + + /// T66, both halves of the daemon-side promise, on a VM host's switch: + /// + /// An own-address attach whose launch carries the addresses the VM host + /// daemon handed its registration attaches **at the handed address** — + /// the address the host-side table's row is keyed by — and **selects + /// none**: the allocator's run shows exactly the handed lease, no + /// self-allocation spent beside it, and the next unregistered box still + /// draws sequentially from the run, never re-issuing the handed one. + /// And the attach names the handed address in one debug line, with the + /// handed/self-drawn distinction a log tail reads. + #[tokio::test] + async fn own_ip_attach_uses_handed_address() { + let switch = vm_host_switch(); + let handed_switch = Ipv4Addr::new(100, 64, 0, 9); + let handed_loopback = Ipv4Addr::new(127, 0, 64, 9); + let net = network_for( + sessions::NetworkMode::OwnIp, + &switch, + "web", + None, + None, + Some(BoxAddresses { + switch_address: handed_switch, + loopback_address: handed_loopback, + }), + ); + + // The plan's lease IS the handed address — the address the tap is + // configured with, whatever mechanism builds it — and the switch's + // table carries exactly that one lease: nothing was drawn beside it. + let plan = net.plan().await.expect("the handed attach plans"); + assert_eq!( + switch.lock().await.attached(), + 1, + "the handed attach counts like any other" + ); + assert_eq!( + switch.lock().await.leases(), + &[PtaskLease { + ip: handed_switch, + mac: MacAddr::for_switch_ip(handed_switch), + }], + "the handed attach records the handed lease and selects nothing else" + ); + let _ = plan; // the plan's resolver/tap carry the same lease; asserted above + + // Releasing it detaches like any other, and the next box — one the + // activating client did not register — draws sequentially from the + // run, past the handed address, never re-issuing it. + net.abandon().await; + assert_eq!(switch.lock().await.attached(), 0, "the abandon detaches"); + let unregistered = network_for( + sessions::NetworkMode::OwnIp, + &switch, + "other", + None, + None, + None, + ); + let _ = unregistered.plan().await.expect("a self-allocated attach plans"); + let leases = switch.lock().await.leases().to_vec(); + assert_eq!( + leases.len(), + 2, + "one handed lease, one self-allocated: the handed attach spent no \ + allocation, got {leases:?}" + ); + assert_eq!( + leases[1].ip, + Ipv4Addr::from(u32::from(Ipv4Addr::new(100, 64, 0, 9)) + 1), + "the self-allocation draws past the handed address — the run never \ + re-issues it, got {leases:?}" + ); + + // The attach's one debug line names the address the box carries and + // says it was handed. Driven through the completion the provider's + // attach calls, against a stand-in control channel, with a socketpair + // standing in for the tap. The capture reads at DEBUG — the + // harness's global capture reads at INFO and would never see it. + let capture = CaptureWriter::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(capture.clone()) + .with_max_level(tracing::Level::DEBUG) + .with_ansi(false) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + let dir = tempfile::TempDir::new().unwrap(); + let control_path = dir.path().join("gvproxy.sock"); + // One `/connect` upgrade plus one DNS publish per host id. + let _bodies = spawn_control_channel_capturing(control_path.clone(), 8); + let mut fds = [0 as libc::c_int; 2]; + // SAFETY: `socketpair` with a valid domain/type either returns -1 + // (checked) or fills `fds` with two fresh descriptors. + let rc = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_DGRAM, 0, fds.as_mut_ptr()) }; + assert_eq!(rc, 0, "socketpair: {}", std::io::Error::last_os_error()); + // SAFETY: each fd in `fds` is a fresh, valid, owned descriptor just + // returned by socketpair. + let tap_fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(fds[0]) }; + let _box_end = unsafe { std::fs::File::from_raw_fd(fds[1]) }; + let guard = crate::net::gvproxy_network::complete_own_ip_attach( + &switch, + tap_fd, + ControlChannel::Unix(control_path), + handed_switch, + "web", + None, + None, + true, + ) + .await + .expect("the attach completes against the stand-in"); + drop(guard); + let log = capture.contents(); + assert!( + log.contains(&format!("switch_address={handed_switch}")) + && log.contains("handed_from_host=true"), + "the attach's debug line names the handed address and says it was \ + handed: {log}" + ); + assert!( + !log.contains("handed_from_host=false"), + "this run handed every address; no self-drawn attach was logged: {log}" + ); + } } diff --git a/crates/minimald/src/net/mod.rs b/crates/minimald/src/net/mod.rs index 5f315cdc57..1ed55d8e12 100644 --- a/crates/minimald/src/net/mod.rs +++ b/crates/minimald/src/net/mod.rs @@ -97,6 +97,15 @@ pub enum NetError { /// The configured subnet has no remaining host address to hand out. #[error("gvproxy subnet {0} is exhausted; no free PTask address remains")] SubnetExhausted(SwitchSubnet), + /// The host handed the box a switch address outside the plan's PTask run + /// (T66): the daemon's subnet cannot honor it, and attaching with it + /// would source frames from an address no row of the host-side table is + /// keyed by. + #[error("handed switch address {address} is outside the plan's PTask run on subnet {subnet}")] + HandedAddressOutsidePlan { + address: Ipv4Addr, + subnet: SwitchSubnet, + }, /// Spawning the gvproxy binary failed. #[error("spawning gvproxy at {path:?}: {source}")] Spawn { @@ -207,6 +216,42 @@ impl IpAllocator { pub fn leases(&self) -> &[PtaskLease] { &self.leases } + + /// Records `ip` as handed out by the host (T66) rather than drawn here, + /// returning its lease. The lease joins [`Self::leases`] — the + /// static-lease table the switch is configured from — and the cursor + /// advances past it, so a later [`Self::allocate`] never re-issues an + /// address the host has (or had) someone else's box on. The + /// never-reuse rule is one rule across both allocation shapes. + /// + /// A re-attach of a box the daemon already knows — the shape a daemon + /// restart produces, every session re-attaching with the address its + /// record carries — finds the lease already recorded and adds no + /// duplicate to the table. + /// + /// # Errors + /// + /// [`NetError::HandedAddressOutsidePlan`] when `ip` is outside the + /// plan's PTask run: the address cannot be honored on this subnet. + pub fn hand(&mut self, ip: Ipv4Addr) -> Result { + let ip_num = u32::from(ip); + if ip_num < self.subnet.first_ptask() || ip_num > self.subnet.last_ptask() { + return Err(NetError::HandedAddressOutsidePlan { + address: ip, + subnet: self.subnet, + }); + } + self.next = self.next.max(ip_num + 1); + if let Some(lease) = self.leases.iter().find(|lease| lease.ip == ip) { + return Ok(*lease); + } + let lease = PtaskLease { + ip, + mac: MacAddr::for_switch_ip(ip), + }; + self.leases.push(lease); + Ok(lease) + } } impl Default for IpAllocator { @@ -330,6 +375,17 @@ impl SwitchClient { self.allocator.subnet() } + /// Every lease this switch knows — self-allocated and handed (T66) + /// alike, oldest first — the table the generated config is written + /// from. A diagnostic surface for the addresses the switch has handed + /// out and which shape of attach took them, without reaching into the + /// allocator. + #[cfg(test)] + #[must_use] + pub(crate) fn leases(&self) -> &[PtaskLease] { + self.allocator.leases() + } + fn config_path(&self) -> PathBuf { self.state_dir.join("gvproxy.yaml") } @@ -368,6 +424,42 @@ impl SwitchClient { Ok(AttachResult { lease, exit_signal }) } + /// Attaches a PTask with the switch address the host handed it (T66) + /// instead of drawing one: the box was registered on the host-side table + /// keyed by this address, so the daemon reuses it and mints nothing. + /// + /// The handed address is recorded like an allocated one — it joins the + /// static-lease table the switch is configured from, and the allocator's + /// cursor advances past it ([`IpAllocator::hand`]) — so the + /// never-reuse rule holds across both allocation shapes. Everything + /// else matches [`Self::attach`], including the DM2 config/spawn steps + /// and the attach count. + /// + /// # Errors + /// + /// [`NetError::HandedAddressOutsidePlan`] when the host handed an + /// address this daemon's subnet cannot honor; propagates config-write, + /// spawn, and socket-readiness failures as [`Self::attach`] does. + pub async fn attach_handed(&mut self, handed: Ipv4Addr) -> Result { + let lease = self.allocator.hand(handed)?; + // DM2 spawns + configures gvproxy locally; DM1/3/4 (HostShuttle) leaves + // gvproxy to `minvmd` on the host, so skip the spawn/config steps and + // only track the attach count — same shape as [`Self::attach`]. + if matches!(self.transport, SwitchTransport::LocalSpawn) { + self.write_config().await?; + self.ensure_running().await?; + } + self.attached += 1; + tracing::info!( + ip = %lease.ip, + mac = %lease.mac, + attached = self.attached, + "attached OwnIp PTask to gvproxy switch at its handed address" + ); + let exit_signal = self.exit_tx.subscribe(); + Ok(AttachResult { lease, exit_signal }) + } + /// Records that a PTask detached. When the last one leaves, the switch is /// stopped. /// diff --git a/crates/minimald/src/net/provider.rs b/crates/minimald/src/net/provider.rs index 8b79bd743e..025a667131 100644 --- a/crates/minimald/src/net/provider.rs +++ b/crates/minimald/src/net/provider.rs @@ -76,13 +76,20 @@ impl OwnAddressReporter { /// ingress, the session's own PTask being attached at the same time. /// `own_address` carries the registry handle an own-address launch reports its /// lease through once the box attaches (NET-001); a task launch passes `None` -/// for that too. +/// for that too. `box_addresses` carries the switch and loopback addresses +/// the VM host daemon handed the box's registration (T66): its switch +/// address is what this `OwnIp` PTask attaches with instead of drawing one, +/// because the host-side table's row is keyed by it. A task launch passes +/// `None` deliberately — the task's sandbox is not the box the registration +/// named, and attaching it at the box's address would key its frames to the +/// session's row. pub(crate) fn network_for( mode: NetworkMode, switch: &Arc>, identity: &str, policy: Option, own_address: Option, + box_addresses: Option, ) -> Arc { match mode { NetworkMode::HostNet => Arc::new(HostIpAddressNetwork { @@ -93,6 +100,7 @@ pub(crate) fn network_for( identity: identity.to_string(), policy, own_address, + box_addresses, reserved: std::sync::Mutex::new(None), }), _ => Arc::new(sandbox2::NoNet), @@ -253,6 +261,14 @@ struct OwnIpNetwork { /// box's proxy route exists exactly while the lease does (NET-001). `None` /// for a task launch, which owns no proxy route. own_address: Option, + /// The addresses the VM host daemon handed this box's registration + /// (T66), when it was registered: the switch address this PTask attaches + /// with instead of drawing one — the host-side table's row is keyed by + /// it, so a self-allocated lease would never match — and the published + /// loopback address the host side names the box by. `None` for a launch + /// the activating client did not register, which draws as it always + /// has. + box_addresses: Option, /// Taken by `plan`, taken back out by `attach` or `abandon`. A `std` mutex, /// never held across an await, so a cancelled launch cannot leak it. reserved: std::sync::Mutex>, @@ -263,6 +279,7 @@ impl std::fmt::Debug for OwnIpNetwork { f.debug_struct("OwnIpNetwork") .field("identity", &self.identity) .field("has_policy", &self.policy.is_some()) + .field("handed_addresses", &self.box_addresses.is_some()) .finish_non_exhaustive() } } @@ -271,12 +288,22 @@ impl Network for OwnIpNetwork { /// Lease an address and make sure the switch is up, then describe the tap /// the sandbox layer should build. Before the process starts, because the /// sandbox layer assigns the address as it creates the tap. + /// + /// The lease is the host's handed address when the box was registered + /// (T66) — the address the host-side table's row is keyed by — and a + /// self-allocated one otherwise, exactly as before this existed. fn plan(&self) -> PlanFuture<'_> { Box::pin(async move { let (lease, control, subnet) = { let mut s = self.switch.lock().await; let subnet = s.subnet(); - let attach = s.attach().await.map_err(NetworkError::new)?; + let attach = match self.box_addresses.as_ref() { + Some(handed) => s + .attach_handed(handed.switch_address) + .await + .map_err(NetworkError::new)?, + None => s.attach().await.map_err(NetworkError::new)?, + }; let control = match s.transport() { crate::net::SwitchTransport::LocalSpawn => { ControlChannel::Unix(s.control_socket()) @@ -342,6 +369,7 @@ impl Network for OwnIpNetwork { &self.identity, self.policy.as_ref(), self.own_address.as_ref(), + self.box_addresses.is_some(), ) .await .map_err(NetworkError::new)?; @@ -391,7 +419,7 @@ mod tests { async fn every_mode_gets_its_provider() { let switch = counting_switch(); - let host = network_for(NetworkMode::HostNet, &switch, "s", None, None) + let host = network_for(NetworkMode::HostNet, &switch, "s", None, None, None) .plan() .await .unwrap(); @@ -405,14 +433,14 @@ mod tests { &Resolver::Nameservers(vec![crate::net::SwitchSubnet::default().dns_server()]) ); - let no_net = network_for(NetworkMode::NoNet, &switch, "s", None, None) + let no_net = network_for(NetworkMode::NoNet, &switch, "s", None, None, None) .plan() .await .unwrap(); assert!(no_net.isolates_netns() && no_net.tap().is_none()); assert_eq!(no_net.resolver(), &Resolver::None); - let own_ip = network_for(NetworkMode::OwnIp, &switch, "s", None, None); + let own_ip = network_for(NetworkMode::OwnIp, &switch, "s", None, None, None); assert!(own_ip.plan().await.unwrap().isolates_netns()); assert_eq!(switch.lock().await.attached(), 1, "own-IP takes a lease"); own_ip.abandon().await; @@ -483,7 +511,7 @@ mod tests { let switch = counting_switch(); let before = switch.lock().await.attached(); - let net = network_for(NetworkMode::OwnIp, &switch, "s", None, None); + let net = network_for(NetworkMode::OwnIp, &switch, "s", None, None, None); net.plan().await.expect("planning leases an address"); assert_eq!(switch.lock().await.attached(), before + 1); @@ -514,7 +542,7 @@ mod tests { async fn concurrent_own_ip_launches_do_not_serialize() { let switch = counting_switch(); let launches: Vec<_> = (0..4) - .map(|i| network_for(NetworkMode::OwnIp, &switch, &format!("p{i}"), None, None)) + .map(|i| network_for(NetworkMode::OwnIp, &switch, &format!("p{i}"), None, None, None)) .collect(); for net in &launches { net.plan().await.expect("planning leases an address"); @@ -538,7 +566,7 @@ mod tests { async fn host_ip_box_resolves_through_node_dns_layer() { let subnet = crate::net::SwitchSubnet::default(); let switch = counting_switch(); - let plan = network_for(NetworkMode::HostNet, &switch, "s", None, None) + let plan = network_for(NetworkMode::HostNet, &switch, "s", None, None, None) .plan() .await .expect("host-address plans do not fail"); @@ -577,7 +605,7 @@ mod tests { "/usr/bin/gvproxy", "/run/minimal/gvproxy", ))); - let plan = network_for(NetworkMode::HostNet, &native, "s", None, None) + let plan = network_for(NetworkMode::HostNet, &native, "s", None, None, None) .plan() .await .unwrap(); @@ -593,7 +621,7 @@ mod tests { // VM-host host-address: the node's DNS layer answers the same name. let vm = counting_switch(); - let plan = network_for(NetworkMode::HostNet, &vm, "s", None, None) + let plan = network_for(NetworkMode::HostNet, &vm, "s", None, None, None) .plan() .await .unwrap(); diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index 442536aa0a..328aa71b8f 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -2384,6 +2384,12 @@ impl Session { Arc::clone(&self.hostnames), record.id, )), + // The addresses the VM host daemon handed this box's registration + // (T66), persisted on the record: the `OwnIp` attach reuses the + // handed switch address instead of drawing one the host-side row + // would not match — including on a re-attach after a restart, + // where the record is the only thing that still knows it. + box_addresses: record.box_addresses, composition: self.composition(), // A weak handle so in-sandbox `min build` can drive session // side-ops without keeping the actor alive past teardown. diff --git a/crates/minimald/src/session_host.rs b/crates/minimald/src/session_host.rs index ecc5ced214..d0d23b74e4 100644 --- a/crates/minimald/src/session_host.rs +++ b/crates/minimald/src/session_host.rs @@ -2147,6 +2147,13 @@ pub(crate) struct SandboxLauncher { /// on attach, so the box's `.min.internal` route exists exactly /// while the box does (NET-001). Ignored by every other network mode. pub(crate) own_address: Option, + /// The addresses the VM host daemon handed this box's registration + /// (T66), when it was registered: the switch address this `OwnIp` PTask + /// attaches with instead of drawing one, and the published loopback + /// address the host side names the box by. Ignored by every other + /// network mode; `None` for a session the activating client did not + /// register, which self-allocates as it always has. + pub(crate) box_addresses: Option, /// Composition to merge into the launcher's baseline packages and /// vars. Patches and lifecycle hooks are ignored today. pub(crate) composition: Option>, @@ -2242,6 +2249,7 @@ impl SessionLauncher for SandboxLauncher { let network_mode = self.network_mode; let net_switch = self.net_switch; let own_address = self.own_address; + let box_addresses = self.box_addresses; // The session name, registered as this PTask's `*.min.internal` hostname on // an own-IP attach (finding #3 / UC6); cloned because `name` is consumed by // the sandbox env below. @@ -2271,6 +2279,7 @@ impl SessionLauncher for SandboxLauncher { &session_name, Some(policy), own_address, + box_addresses, )) .await .map_err(|e| io::Error::other(format!("planning the session network: {e}")))?; From 86d4ca3a4dcf60c7383a4d6a07f0711f14cd035f Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:39:11 +0000 Subject: [PATCH 04/18] style: rustfmt the box-registration layers cargo fmt across the three box-registration commits; no code-meaning change. --- crates/minimal/src/cmd/session.rs | 10 +++++----- crates/minimald/src/net/gvproxy_network.rs | 18 +++++++++++------- crates/minimald/src/net/provider.rs | 11 ++++++++++- crates/minvmd/src/box_registry.rs | 5 ++--- crates/minvmd/src/control.rs | 14 ++++++++------ crates/minvmd/src/lib.rs | 2 +- 6 files changed, 37 insertions(+), 23 deletions(-) diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index 6df356fa18..547c1caa7b 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -129,9 +129,10 @@ async fn register_box_for_activation( return None; } }; - let Some(sock_path) = ssh_sock.parent().map(|dir| { - dir.join(minvmd::control::CONTROL_SOCK_FILE) - }) else { + let Some(sock_path) = ssh_sock + .parent() + .map(|dir| dir.join(minvmd::control::CONTROL_SOCK_FILE)) + else { tracing::warn!( "no provider dir resolved for the ssh socket; the box will not be \ registered with the VM host daemon (its declared egress is not \ @@ -271,8 +272,7 @@ pub(crate) async fn activate_session( // other shape of activation — a host-ip box sharing the node's own row, // a none box with no switch address, a native daemon with no box table // — registers nothing and attaches as it always has. - let box_addresses = - register_box_for_activation(global, network, &session_name, &policy).await; + let box_addresses = register_box_for_activation(global, network, &session_name, &policy).await; // The daemon sources `username` from the authenticated SSH // connection context; the client doesn't send it. diff --git a/crates/minimald/src/net/gvproxy_network.rs b/crates/minimald/src/net/gvproxy_network.rs index 096c287d90..f6b147487b 100644 --- a/crates/minimald/src/net/gvproxy_network.rs +++ b/crates/minimald/src/net/gvproxy_network.rs @@ -278,12 +278,12 @@ mod tests { use tokio::net::{UnixListener, UnixStream}; use tokio::sync::{Mutex, mpsc}; - use crate::net::dns; - use crate::net::policy::{ControlChannel, register_dns_name}; - use crate::net::provider::network_for; use crate::net::PtaskLease; use crate::net::SwitchClient; use crate::net::SwitchTransport; + use crate::net::dns; + use crate::net::policy::{ControlChannel, register_dns_name}; + use crate::net::provider::network_for; use crate::test_harness::CaptureWriter; use sessions::BoxAddresses; @@ -473,11 +473,12 @@ mod tests { /// the shape a VM-backed host's daemon carries. fn vm_host_switch() -> Arc> { Arc::new(Mutex::new( - SwitchClient::new("/usr/bin/gvproxy", "/run/minimal/gvproxy") - .with_transport(SwitchTransport::HostShuttle { + SwitchClient::new("/usr/bin/gvproxy", "/run/minimal/gvproxy").with_transport( + SwitchTransport::HostShuttle { cid: crate::net::VSOCK_HOST_CID, port: crate::net::VSOCK_GVPROXY_SHUTTLE_PORT, - }), + }, + ), )) } @@ -540,7 +541,10 @@ mod tests { None, None, ); - let _ = unregistered.plan().await.expect("a self-allocated attach plans"); + let _ = unregistered + .plan() + .await + .expect("a self-allocated attach plans"); let leases = switch.lock().await.leases().to_vec(); assert_eq!( leases.len(), diff --git a/crates/minimald/src/net/provider.rs b/crates/minimald/src/net/provider.rs index 025a667131..f2eef50ff8 100644 --- a/crates/minimald/src/net/provider.rs +++ b/crates/minimald/src/net/provider.rs @@ -542,7 +542,16 @@ mod tests { async fn concurrent_own_ip_launches_do_not_serialize() { let switch = counting_switch(); let launches: Vec<_> = (0..4) - .map(|i| network_for(NetworkMode::OwnIp, &switch, &format!("p{i}"), None, None, None)) + .map(|i| { + network_for( + NetworkMode::OwnIp, + &switch, + &format!("p{i}"), + None, + None, + None, + ) + }) .collect(); for net in &launches { net.plan().await.expect("planning leases an address"); diff --git a/crates/minvmd/src/box_registry.rs b/crates/minvmd/src/box_registry.rs index f2f74b65be..10bfd4b3e7 100644 --- a/crates/minvmd/src/box_registry.rs +++ b/crates/minvmd/src/box_registry.rs @@ -354,9 +354,8 @@ impl BoxRegistry { u32::from(slice.last()), ) .ok_or(AllocationError::LoopbackExhausted)?; - let mut registration = - BoxRegistration::new(spec.name, switch_addr, loopback_addr) - .with_admitted_ports(spec.ingress_ports); + let mut registration = BoxRegistration::new(spec.name, switch_addr, loopback_addr) + .with_admitted_ports(spec.ingress_ports); if let Some(policy) = spec.egress { registration = registration.with_egress_policy(policy); } diff --git a/crates/minvmd/src/control.rs b/crates/minvmd/src/control.rs index 9268619f43..ab3e06efa8 100644 --- a/crates/minvmd/src/control.rs +++ b/crates/minvmd/src/control.rs @@ -365,12 +365,14 @@ mod tests { ); assert_eq!( db.loopback_address, - Ipv4Addr::from(u32::from( - switch::AddressPlan::default() - .loopback_slice_for_switch(SUBNET) - .expect("the default subnet is planned") - .first() - ) + 1), + Ipv4Addr::from( + u32::from( + switch::AddressPlan::default() + .loopback_slice_for_switch(SUBNET) + .expect("the default subnet is planned") + .first() + ) + 1 + ), "the second box takes the next published loopback address" ); diff --git a/crates/minvmd/src/lib.rs b/crates/minvmd/src/lib.rs index bdb28e4435..eb2d6f7696 100644 --- a/crates/minvmd/src/lib.rs +++ b/crates/minvmd/src/lib.rs @@ -12,8 +12,8 @@ pub mod box_registry; pub mod cmd; -pub mod control; pub mod config; +pub mod control; pub mod error; pub mod image; pub mod lifecycle; From 2be91e7c7dc3b35e62ad11fa534bc27754b9bf8c Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:47:12 +0000 Subject: [PATCH 05/18] style: satisfy the strict clippy gate on the registration layers - read_request_line carries the slice invariant as an expect with a reason, matching the header-parts precedent. - the test register helper maps a serialization failure into its io::Result instead of expecting. - complete_own_ip_attach switches its too_many_arguments allow to an expect with a reason. - the socketpair's second descriptor gets its own SAFETY comment. - the fake VM host's reply writes propagate a client that is already gone instead of discarding the must-use result. --- crates/minimal/src/cmd/session.rs | 8 ++++++-- crates/minimald/src/net/gvproxy_network.rs | 7 ++++++- crates/minvmd/src/control.rs | 12 +++++++++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index 547c1caa7b..51e419445e 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -2050,8 +2050,12 @@ mod tests { } seen.lock().unwrap().push(line.trim().to_string()); let mut writer = lines.into_inner(); - let _ = writer.write_all(reply.as_bytes()).await; - let _ = writer.write_all(b"\n").await; + if writer.write_all(reply.as_bytes()).await.is_err() { + return; + } + if writer.write_all(b"\n").await.is_err() { + return; + } } }); requests diff --git a/crates/minimald/src/net/gvproxy_network.rs b/crates/minimald/src/net/gvproxy_network.rs index f6b147487b..f81a8c79f6 100644 --- a/crates/minimald/src/net/gvproxy_network.rs +++ b/crates/minimald/src/net/gvproxy_network.rs @@ -95,7 +95,10 @@ impl NetGuard for OwnIpGuard { /// honored its handed address from one that did not. // Left positional: a single call site that has just planned the launch, so a // struct would be single-use ceremony. -#[allow(clippy::too_many_arguments)] +#[expect( + clippy::too_many_arguments, + reason = "left positional: a single call site that has just planned the launch, so a struct would be single-use ceremony" +)] pub(crate) async fn complete_own_ip_attach( switch: &Arc>, tap_fd: OwnedFd, @@ -583,6 +586,8 @@ mod tests { // SAFETY: each fd in `fds` is a fresh, valid, owned descriptor just // returned by socketpair. let tap_fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(fds[0]) }; + // SAFETY: each fd in `fds` is a fresh, valid, owned descriptor just + // returned by socketpair. let _box_end = unsafe { std::fs::File::from_raw_fd(fds[1]) }; let guard = crate::net::gvproxy_network::complete_own_ip_attach( &switch, diff --git a/crates/minvmd/src/control.rs b/crates/minvmd/src/control.rs index ab3e06efa8..c886db446e 100644 --- a/crates/minvmd/src/control.rs +++ b/crates/minvmd/src/control.rs @@ -114,6 +114,14 @@ fn serve_connection(stream: UnixStream, boxes: &BoxRegistry) -> std::io::Result< /// Read one line (terminated by `\n`) of the registration request. A /// connection that closes before sending a line reads as no request; a line /// past [`MAX_REQUEST_LINE`] is refused. +/// +/// The slices are cut at `read`, the byte count `read()` reported filling +/// `buf` with, or at `newline`, an index `position` found inside +/// `buf[..read]` — so neither range can be out of bounds. +#[expect( + clippy::indexing_slicing, + reason = "cut at `read`, the byte count `read()` reported, or at `newline`, an index `position` found inside `buf[..read]`" +)] fn read_request_line(stream: &mut UnixStream) -> std::io::Result> { let mut line = Vec::new(); let mut buf = [0u8; 1024]; @@ -265,7 +273,9 @@ mod tests { request: &RegisterBoxRequest, ) -> std::io::Result { let mut stream = TestStream::connect(sock_path)?; - let mut line = serde_json_lenient::to_string(request).expect("request serializes"); + let mut line = serde_json_lenient::to_string(request).map_err(|error| { + std::io::Error::other(format!("request did not serialize: {error}")) + })?; line.push('\n'); stream.write_all(line.as_bytes())?; let mut reader = BufReader::new(stream); From 4cff8561ae4bb0da9a8a4450c81529ec5b863008 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:10:06 +0000 Subject: [PATCH 06/18] feat(minimald): the coder's uncommitted work as it stood when its 7200s ran out the coder's uncommitted work as it stood when its 7200s ran out The coder was stopped at the end of its time with these changes not committed. The runtime committed its working tree as it stood so the work is kept: this commit is a snapshot, not a finished step, and may not build or pass its tests. --- crates/minimald/src/exec.rs | 13 +++ crates/minimald/src/net/gvproxy_network.rs | 41 +++++++++- crates/minimald/src/net/mod.rs | 92 ++++++++++++++++++++-- 3 files changed, 140 insertions(+), 6 deletions(-) diff --git a/crates/minimald/src/exec.rs b/crates/minimald/src/exec.rs index 09885e140d..2a849d938f 100644 --- a/crates/minimald/src/exec.rs +++ b/crates/minimald/src/exec.rs @@ -192,6 +192,19 @@ pub(crate) fn task_network( // sandbox is not the box the registration named — attaching it at the // session box's address would key its frames to the session's row. // A task's sandbox self-allocates, as it always has. + // + // What the interim costs, stated: the host's registration cursor and + // this daemon's allocation cursor draw from the same address run + // without seeing each other's takes, so the host can hand a box an + // address a task's sandbox has just drawn. Task sandboxes keep + // self-allocating from that shared run until the host hands them + // addresses too, and a collision is refused daemon-side rather than + // shared — `IpAllocator::hand` errors on a handed address a locally + // drawn lease holds, so the box's attach fails instead of a second + // tap landing on the task's address. Reserving a disjoint sub-run + // for daemon self-allocation would close the window instead; that + // shape is still being decided, so the refusal is this build's + // guard. None, ) } diff --git a/crates/minimald/src/net/gvproxy_network.rs b/crates/minimald/src/net/gvproxy_network.rs index f81a8c79f6..12298124c0 100644 --- a/crates/minimald/src/net/gvproxy_network.rs +++ b/crates/minimald/src/net/gvproxy_network.rs @@ -494,7 +494,9 @@ mod tests { /// self-allocation spent beside it, and the next unregistered box still /// draws sequentially from the run, never re-issuing the handed one. /// And the attach names the handed address in one debug line, with the - /// handed/self-drawn distinction a log tail reads. + /// handed/self-drawn distinction a log tail reads. A handed address a + /// locally-drawn lease already holds is refused at the attach path + /// rather than shared with it. #[tokio::test] async fn own_ip_attach_uses_handed_address() { let switch = vm_host_switch(); @@ -562,6 +564,43 @@ mod tests { re-issues it, got {leases:?}" ); + // And the collision is refused at the attach path, never shared: the + // host's registration cursor and this daemon's allocation cursor run + // over the same address run without seeing each other's takes — the + // task sandbox above just drew .10 — so a box handed that address + // must fail its plan rather than put a second tap on it. The refusal + // spends nothing: no lease joins the table, no attach is counted. + let colliding = network_for( + sessions::NetworkMode::OwnIp, + &switch, + "third", + None, + None, + Some(BoxAddresses { + switch_address: leases[1].ip, + loopback_address: Ipv4Addr::LOCALHOST, + }), + ); + let err = colliding + .plan() + .await + .expect_err("a handed address a locally-drawn lease holds refuses"); + assert!( + err.to_string() + .contains("already held by a locally-drawn lease"), + "the refusal names the locally-drawn holder: {err}" + ); + assert_eq!( + switch.lock().await.leases().len(), + 2, + "the refusal added no lease to the table" + ); + assert_eq!( + switch.lock().await.attached(), + 1, + "only the self-allocated attach counts; the refused one never did" + ); + // The attach's one debug line names the address the box carries and // says it was handed. Driven through the completion the provider's // attach calls, against a stand-in control channel, with a socketpair diff --git a/crates/minimald/src/net/mod.rs b/crates/minimald/src/net/mod.rs index 1ed55d8e12..39c812998f 100644 --- a/crates/minimald/src/net/mod.rs +++ b/crates/minimald/src/net/mod.rs @@ -106,6 +106,18 @@ pub enum NetError { address: Ipv4Addr, subnet: SwitchSubnet, }, + /// The host handed the box a switch address this daemon has already + /// drawn a lease for itself (T66). The host's registration cursor and + /// this daemon's allocation cursor run over the same address run without + /// seeing each other's draws — task sandboxes still self-allocate, and + /// so does any box the control socket could not register — so the same + /// address can be spent on both sides. The collision is refused, never + /// shared: attaching the box at a lease another PTask already holds + /// would key that PTask's frames to the box's host-side row. + #[error( + "handed switch address {address} is already held by a locally-drawn lease (MAC {holder})" + )] + HandedAddressCollision { address: Ipv4Addr, holder: MacAddr }, /// Spawning the gvproxy binary failed. #[error("spawning gvproxy at {path:?}: {source}")] Spawn { @@ -157,6 +169,14 @@ pub struct IpAllocator { /// Every address handed out, in allocation order. Doubles as the /// static-lease table written into gvproxy's config. leases: Vec, + /// The addresses of [`Self::leases`] the host handed in (T66) rather than + /// ones [`Self::allocate`] drew. Membership decides what [`Self::hand`] + /// may match: an address whose lease is already recorded **and handed** + /// is the legitimate re-attach (a session restarting against the same + /// daemon, or re-attaching after a daemon restart), while a lease the + /// daemon drew itself at a handed address is two cursors over one run — + /// refused, never shared. + handed: Vec, } impl IpAllocator { @@ -168,6 +188,7 @@ impl IpAllocator { next: subnet.first_ptask(), subnet, leases: Vec::new(), + handed: Vec::new(), } } @@ -226,13 +247,24 @@ impl IpAllocator { /// /// A re-attach of a box the daemon already knows — the shape a daemon /// restart produces, every session re-attaching with the address its - /// record carries — finds the lease already recorded and adds no - /// duplicate to the table. + /// record carries — finds the lease already recorded, marked handed, + /// and adds no duplicate to the table. + /// + /// A lease this daemon **drew itself** at the handed address is the one + /// match this method refuses ([`NetError::HandedAddressCollision`]): + /// the host's registration cursor and this daemon's allocation cursor + /// run over the same address run without seeing each other's draws, so + /// the host can hand an address a task sandbox (or an unregistered box) + /// already holds. Two taps on one address would key one of their frames + /// to the other's host-side row, so the collision is an error and the + /// address stays with its original holder. /// /// # Errors /// /// [`NetError::HandedAddressOutsidePlan`] when `ip` is outside the /// plan's PTask run: the address cannot be honored on this subnet. + /// [`NetError::HandedAddressCollision`] when a locally-drawn lease + /// already holds `ip`. pub fn hand(&mut self, ip: Ipv4Addr) -> Result { let ip_num = u32::from(ip); if ip_num < self.subnet.first_ptask() || ip_num > self.subnet.last_ptask() { @@ -243,8 +275,15 @@ impl IpAllocator { } self.next = self.next.max(ip_num + 1); if let Some(lease) = self.leases.iter().find(|lease| lease.ip == ip) { - return Ok(*lease); + if self.handed.contains(&ip) { + return Ok(*lease); + } + return Err(NetError::HandedAddressCollision { + address: ip, + holder: lease.mac, + }); } + self.handed.push(ip); let lease = PtaskLease { ip, mac: MacAddr::for_switch_ip(ip), @@ -438,8 +477,10 @@ impl SwitchClient { /// # Errors /// /// [`NetError::HandedAddressOutsidePlan`] when the host handed an - /// address this daemon's subnet cannot honor; propagates config-write, - /// spawn, and socket-readiness failures as [`Self::attach`] does. + /// address this daemon's subnet cannot honor, and + /// [`NetError::HandedAddressCollision`] when a locally-drawn lease + /// already holds the handed address; propagates config-write, spawn, + /// and socket-readiness failures as [`Self::attach`] does. pub async fn attach_handed(&mut self, handed: Ipv4Addr) -> Result { let lease = self.allocator.hand(handed)?; // DM2 spawns + configures gvproxy locally; DM1/3/4 (HostShuttle) leaves @@ -753,6 +794,47 @@ mod tests { assert!(matches!(a.allocate(), Err(NetError::SubnetExhausted(_)))); } + #[test] + fn hand_refuses_an_address_a_local_lease_holds() { + // The host's registration cursor and the daemon's allocation cursor + // run over the same address run without seeing each other's draws — + // task sandboxes still self-allocate beside the handed boxes — so + // the host can hand an address a locally-drawn lease already holds. + // The collision is refused, never shared: two taps on one address + // would key one PTask's frames to the other's host-side row. + let mut a = IpAllocator::new(SwitchSubnet::default()); + let handed_box = a.hand(Ipv4Addr::new(100, 64, 0, 2)).unwrap(); + assert_eq!(handed_box.ip, Ipv4Addr::new(100, 64, 0, 2)); + // The next self-allocation draws past the handed address... + let task = a.allocate().unwrap(); + assert_eq!(task.ip, Ipv4Addr::new(100, 64, 0, 3)); + // ...so a box handed the task's address is a collision, not a match + // on the task's lease. + let err = a.hand(task.ip).unwrap_err(); + assert!( + matches!( + err, + NetError::HandedAddressCollision { address, holder } + if address == task.ip && holder == task.mac + ), + "handing a locally-drawn address is refused with the holder named" + ); + // The refusal changes nothing: no duplicate joined the table, and + // handing the same address again refuses just the same — the + // address stays with its original holder for the allocator's life. + assert_eq!(a.leases().len(), 2); + assert!(matches!( + a.hand(task.ip), + Err(NetError::HandedAddressCollision { .. }) + )); + // A re-attach of a *handed* lease stays legitimate — the only match + // this method makes: the box re-attaching with the address its + // record carries (same daemon lifetime or after a restart) finds + // the lease marked handed and takes it back without a duplicate. + assert_eq!(a.hand(Ipv4Addr::new(100, 64, 0, 2)).unwrap(), handed_box); + assert_eq!(a.leases().len(), 2); + } + #[test] fn subnet_rejects_overly_narrow_prefix() { assert!(matches!( From c797826179cd4fb4f13d447e905464087966629b Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:21:33 +0000 Subject: [PATCH 07/18] fix(minimald,minvmd): split the PTask run between the two allocators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plan's PTask run is now two disjoint sub-runs: the daemon self-allocates task sandboxes and unregistered boxes from the lower half (net::self_allocation_run), and the VM host daemon hands registered boxes only from the upper half (box_registry::hand_out_run, the mirrored midpoint rule). Each side's tests pin the default plan's split literally, so the two statements cannot drift silently. IpAllocator::hand now refuses any recorded lease whatever drew it (a box attaches once per daemon life; a restart starts from an empty allocator, so the re-hand allowance and its handed list are gone) and refuses an address inside the reserve, where a host that disagrees about the split would put a box beside the daemon's own draws. Exhaustion stays an explicit error with no wrap or reuse on either side; the interim is stated in the IpAllocator and task_network docs — task sandboxes self-allocate from the reserve until the task registering every live box host-side retires daemon-side allocation — and the control socket's module doc states its v1 posture: the same-uid 0600 socket is the trust, requests are served serially with the 30 s read timeout. --- crates/minimald/src/exec.rs | 21 +- crates/minimald/src/net/gvproxy_network.rs | 82 ++++-- crates/minimald/src/net/mod.rs | 312 +++++++++++++++------ crates/minvmd/src/box_registry.rs | 141 ++++++++-- crates/minvmd/src/control.rs | 21 +- 5 files changed, 426 insertions(+), 151 deletions(-) diff --git a/crates/minimald/src/exec.rs b/crates/minimald/src/exec.rs index 2a849d938f..307812bf6d 100644 --- a/crates/minimald/src/exec.rs +++ b/crates/minimald/src/exec.rs @@ -193,18 +193,15 @@ pub(crate) fn task_network( // session box's address would key its frames to the session's row. // A task's sandbox self-allocates, as it always has. // - // What the interim costs, stated: the host's registration cursor and - // this daemon's allocation cursor draw from the same address run - // without seeing each other's takes, so the host can hand a box an - // address a task's sandbox has just drawn. Task sandboxes keep - // self-allocating from that shared run until the host hands them - // addresses too, and a collision is refused daemon-side rather than - // shared — `IpAllocator::hand` errors on a handed address a locally - // drawn lease holds, so the box's attach fails instead of a second - // tap landing on the task's address. Reserving a disjoint sub-run - // for daemon self-allocation would close the window instead; that - // shape is still being decided, so the refusal is this build's - // guard. + // The interim, stated plainly: task sandboxes self-allocate from the + // daemon's reserve — the plan run's lower half + // (`crate::net::self_allocation_run`) — until the task registering + // every live box host-side (NET-138) retires self-allocation, after + // which no daemon-side draw happens once a control socket exists. + // The VM host daemon hands registered boxes only from the run above + // the reserve, so the two allocators cannot meet; the daemon-side + // refusals (`IpAllocator::hand`) stay the guard against a pair that + // disagrees about the split. None, ) } diff --git a/crates/minimald/src/net/gvproxy_network.rs b/crates/minimald/src/net/gvproxy_network.rs index 12298124c0..16ce6da9c8 100644 --- a/crates/minimald/src/net/gvproxy_network.rs +++ b/crates/minimald/src/net/gvproxy_network.rs @@ -491,16 +491,21 @@ mod tests { /// daemon handed its registration attaches **at the handed address** — /// the address the host-side table's row is keyed by — and **selects /// none**: the allocator's run shows exactly the handed lease, no - /// self-allocation spent beside it, and the next unregistered box still - /// draws sequentially from the run, never re-issuing the handed one. + /// self-allocation spent beside it. The next unregistered box + /// self-allocates from the daemon's reserve — the plan run's lower half, + /// disjoint from the hand-out run the handed address came from — so the + /// two allocators cannot meet and the handed address is never drawn. /// And the attach names the handed address in one debug line, with the - /// handed/self-drawn distinction a log tail reads. A handed address a - /// locally-drawn lease already holds is refused at the attach path - /// rather than shared with it. + /// handed/self-drawn distinction a log tail reads. A handed address the + /// daemon already spends is refused at the attach path rather than + /// shared with it — a reserve address, and a lease's own address alike. #[tokio::test] async fn own_ip_attach_uses_handed_address() { let switch = vm_host_switch(); - let handed_switch = Ipv4Addr::new(100, 64, 0, 9); + // The handed address comes from the hand-out run — the plan run's + // upper half, above the daemon's self-allocation reserve, which is + // where a real host's registrations hand from. + let handed_switch = Ipv4Addr::new(100, 64, 128, 9); let handed_loopback = Ipv4Addr::new(127, 0, 64, 9); let net = network_for( sessions::NetworkMode::OwnIp, @@ -534,8 +539,10 @@ mod tests { let _ = plan; // the plan's resolver/tap carry the same lease; asserted above // Releasing it detaches like any other, and the next box — one the - // activating client did not register — draws sequentially from the - // run, past the handed address, never re-issuing it. + // activating client did not register — self-allocates from the + // daemon's reserve: the plan run's lower half, a sub-run disjoint + // from the hand-out run the handed address came from, so the two + // allocators cannot meet. net.abandon().await; assert_eq!(switch.lock().await.attached(), 0, "the abandon detaches"); let unregistered = network_for( @@ -559,18 +566,19 @@ mod tests { ); assert_eq!( leases[1].ip, - Ipv4Addr::from(u32::from(Ipv4Addr::new(100, 64, 0, 9)) + 1), - "the self-allocation draws past the handed address — the run never \ - re-issues it, got {leases:?}" + Ipv4Addr::new(100, 64, 0, 2), + "the self-allocation draws the reserve's first address — the \ + sub-run is disjoint from the hand-out run, so the handed \ + address is never drawn, got {leases:?}" ); - // And the collision is refused at the attach path, never shared: the - // host's registration cursor and this daemon's allocation cursor run - // over the same address run without seeing each other's takes — the - // task sandbox above just drew .10 — so a box handed that address - // must fail its plan rather than put a second tap on it. The refusal - // spends nothing: no lease joins the table, no attach is counted. - let colliding = network_for( + // And a handed address this daemon already spends is refused at the + // attach path, never shared. Two shapes, both refused: the task + // sandbox's just-drawn reserve address — the shape a host that + // hands from the run's start would produce — and the box's own + // handed address, whose lease is still recorded. The refusals spend + // nothing: no lease joins the table, no attach is counted. + let colliding_reserve = network_for( sessions::NetworkMode::OwnIp, &switch, "third", @@ -581,14 +589,42 @@ mod tests { loopback_address: Ipv4Addr::LOCALHOST, }), ); - let err = colliding + let err = colliding_reserve .plan() .await - .expect_err("a handed address a locally-drawn lease holds refuses"); + .expect_err("a handed address inside the reserve refuses"); assert!( - err.to_string() - .contains("already held by a locally-drawn lease"), - "the refusal names the locally-drawn holder: {err}" + err.to_string().contains("self-allocation reserve"), + "the refusal names the reserve: {err}" + ); + assert_eq!( + switch.lock().await.leases().len(), + 2, + "the refusal added no lease to the table" + ); + assert_eq!( + switch.lock().await.attached(), + 1, + "only the self-allocated attach counts; the refused one never did" + ); + let colliding_held = network_for( + sessions::NetworkMode::OwnIp, + &switch, + "fourth", + None, + None, + Some(BoxAddresses { + switch_address: leases[0].ip, + loopback_address: Ipv4Addr::LOCALHOST, + }), + ); + let err = colliding_held + .plan() + .await + .expect_err("a handed address a lease holds refuses"); + assert!( + err.to_string().contains("already held by a lease"), + "the refusal names the held lease: {err}" ); assert_eq!( switch.lock().await.leases().len(), diff --git a/crates/minimald/src/net/mod.rs b/crates/minimald/src/net/mod.rs index 39c812998f..11c3fde990 100644 --- a/crates/minimald/src/net/mod.rs +++ b/crates/minimald/src/net/mod.rs @@ -94,8 +94,12 @@ const TERM_GRACE: Duration = Duration::from_secs(5); #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum NetError { - /// The configured subnet has no remaining host address to hand out. - #[error("gvproxy subnet {0} is exhausted; no free PTask address remains")] + /// The daemon's self-allocation reserve — the sub-run of the plan's PTask + /// run it draws task sandboxes and unregistered boxes from — has no + /// address left. Explicit, never wrapped: a spent address stays spent. + #[error( + "the self-allocation reserve on subnet {0} is exhausted; no daemon-side address remains" + )] SubnetExhausted(SwitchSubnet), /// The host handed the box a switch address outside the plan's PTask run /// (T66): the daemon's subnet cannot honor it, and attaching with it @@ -106,18 +110,30 @@ pub enum NetError { address: Ipv4Addr, subnet: SwitchSubnet, }, - /// The host handed the box a switch address this daemon has already - /// drawn a lease for itself (T66). The host's registration cursor and - /// this daemon's allocation cursor run over the same address run without - /// seeing each other's draws — task sandboxes still self-allocate, and - /// so does any box the control socket could not register — so the same - /// address can be spent on both sides. The collision is refused, never - /// shared: attaching the box at a lease another PTask already holds - /// would key that PTask's frames to the box's host-side row. + /// The host handed the box a switch address a recorded lease already + /// holds, whatever drew that lease (T66). A box attaches once per daemon + /// life and a restart starts from an empty allocator, so no legitimate + /// attach re-uses a lease; and two taps on one address would key one + /// PTask's frames to the other's host-side row. The collision is + /// refused, never shared. + #[error("handed switch address {address} is already held by a lease (MAC {holder})")] + HandedAddressCollision { address: Ipv4Addr, holder: MacAddr }, + /// The host handed the box a switch address inside this daemon's + /// self-allocation reserve (T66) — the sub-run task sandboxes and + /// unregistered boxes draw from. Host and daemon keep disjoint sub-runs + /// of the plan's PTask run (the daemon's half is [`self_allocation_run`], + /// the host's mirror `minvmd`'s `box_registry::hand_out_run`), so a + /// handed address inside the reserve means the two sides disagree about + /// the split — a skewed pair, a misconfigured host. Attaching with it + /// would put the box's tap beside the daemon's own draws; refused, never + /// shared. #[error( - "handed switch address {address} is already held by a locally-drawn lease (MAC {holder})" + "handed switch address {address} is inside this daemon's self-allocation reserve on subnet {subnet}" )] - HandedAddressCollision { address: Ipv4Addr, holder: MacAddr }, + HandedAddressInReserve { + address: Ipv4Addr, + subnet: SwitchSubnet, + }, /// Spawning the gvproxy binary failed. #[error("spawning gvproxy at {path:?}: {source}")] Spawn { @@ -159,24 +175,58 @@ pub struct AttachResult { pub exit_signal: watch::Receiver, } +/// The sub-run of `subnet`'s PTask run this daemon **self-allocates** from: +/// the run's lower half, `[first_ptask, midpoint]`, as an inclusive +/// `(first, last)` pair. +/// +/// The plan's PTask run is split into two disjoint sub-runs so the two +/// allocators that draw on it cannot meet: this daemon self-allocates task +/// sandboxes and unregistered boxes from the lower half, and the VM host +/// daemon hands registered boxes only from the upper half — the run above +/// this reserve. The host's half is the same midpoint rule mirrored in +/// `minvmd`'s `box_registry::hand_out_run`; one rule, two statements, so +/// change both together — each side's tests pin the default plan's split +/// literally. +/// +/// This is the interim allocation shape (NET-138): task sandboxes +/// self-allocate from the reserve until the task registering every live box +/// host-side retires daemon-side allocation, after which a daemon with a +/// control socket draws nothing — every own-address box arrives handed. +#[must_use] +pub fn self_allocation_run(subnet: SwitchSubnet) -> (u32, u32) { + let first = subnet.first_ptask(); + let last = subnet.last_ptask(); + let reserve_len = (last - first + 1) / 2; + (first, first + reserve_len - 1) +} + /// Hands out unique switch addresses, never reusing one for the lifetime of /// the allocator (R1.6). +/// +/// Two allocation shapes share the never-reuse rule: [`Self::allocate`] +/// draws a self-allocation from the plan run's lower half — the sub-run +/// [`self_allocation_run`] reserves for this daemon (task sandboxes, +/// unregistered boxes) — and [`Self::hand`] records the address the VM host +/// daemon handed a registered box, from the upper half. The two sub-runs are +/// disjoint, so a self-allocation can never spend a handed address, nor the +/// reverse. +/// +/// The interim this shape ships in: task sandboxes self-allocate from the +/// reserve until the task registering every live box host-side (NET-138) +/// retires self-allocation, after which no daemon-side draw happens once a +/// control socket exists — every own-address box arrives handed. #[derive(Debug)] pub struct IpAllocator { subnet: SwitchSubnet, - /// The next host offset to try; only ever advances. + /// This daemon's self-allocation reserve, inclusive — both bounds from + /// [`self_allocation_run`]. `allocate` draws inside it and nothing else. + reserve: (u32, u32), + /// The next host offset to draw for a self-allocation; only ever + /// advances, within the reserve. next: u32, /// Every address handed out, in allocation order. Doubles as the /// static-lease table written into gvproxy's config. leases: Vec, - /// The addresses of [`Self::leases`] the host handed in (T66) rather than - /// ones [`Self::allocate`] drew. Membership decides what [`Self::hand`] - /// may match: an address whose lease is already recorded **and handed** - /// is the legitimate re-attach (a session restarting against the same - /// daemon, or re-attaching after a daemon restart), while a lease the - /// daemon drew itself at a handed address is two cursors over one run — - /// refused, never shared. - handed: Vec, } impl IpAllocator { @@ -186,9 +236,9 @@ impl IpAllocator { pub fn new(subnet: SwitchSubnet) -> Self { Self { next: subnet.first_ptask(), + reserve: self_allocation_run(subnet), subnet, leases: Vec::new(), - handed: Vec::new(), } } @@ -208,7 +258,8 @@ impl IpAllocator { self.subnet } - /// Allocates the next free address and its derived MAC. + /// Allocates the next free address and its derived MAC, drawing from the + /// daemon's self-allocation reserve — the plan run's lower half. /// /// Addresses are sequential and never reused, even after the PTask /// detaches, so a stale frame from a torn-down PTask can never be @@ -216,10 +267,10 @@ impl IpAllocator { /// /// # Errors /// - /// Returns [`NetError::SubnetExhausted`] once the subnet's host range is - /// used up. + /// Returns [`NetError::SubnetExhausted`] once the reserve is used up — + /// explicitly, with no wrap and no reuse; a spent address stays spent. pub fn allocate(&mut self) -> Result { - if self.next > self.subnet.last_ptask() { + if self.next > self.reserve.1 { return Err(NetError::SubnetExhausted(self.subnet)); } let ip = Ipv4Addr::from(self.next); @@ -238,33 +289,36 @@ impl IpAllocator { &self.leases } - /// Records `ip` as handed out by the host (T66) rather than drawn here, - /// returning its lease. The lease joins [`Self::leases`] — the - /// static-lease table the switch is configured from — and the cursor - /// advances past it, so a later [`Self::allocate`] never re-issues an - /// address the host has (or had) someone else's box on. The - /// never-reuse rule is one rule across both allocation shapes. + /// Records `ip` as the switch address a registered box attaches at (T66): + /// the VM host daemon allocated it into its table and handed it back, so + /// the daemon mints nothing. The lease joins [`Self::leases`] — the + /// static-lease table the switch is configured from — and is spent for + /// good, as a drawn one is: the never-reuse rule is one rule across both + /// allocation shapes. /// - /// A re-attach of a box the daemon already knows — the shape a daemon - /// restart produces, every session re-attaching with the address its - /// record carries — finds the lease already recorded, marked handed, - /// and adds no duplicate to the table. + /// Three refusals, never a share: /// - /// A lease this daemon **drew itself** at the handed address is the one - /// match this method refuses ([`NetError::HandedAddressCollision`]): - /// the host's registration cursor and this daemon's allocation cursor - /// run over the same address run without seeing each other's draws, so - /// the host can hand an address a task sandbox (or an unregistered box) - /// already holds. Two taps on one address would key one of their frames - /// to the other's host-side row, so the collision is an error and the - /// address stays with its original holder. + /// * an address outside the plan's PTask run + /// ([`NetError::HandedAddressOutsidePlan`]) — the subnet cannot honor + /// it; + /// * an address inside this daemon's self-allocation reserve + /// ([`NetError::HandedAddressInReserve`]) — the host hands only from + /// the run above the reserve, so one there means the two sides + /// disagree about the split; + /// * an address any recorded lease holds, whatever drew it + /// ([`NetError::HandedAddressCollision`]) — a box attaches once per + /// daemon life and a restart starts from an empty allocator (the + /// re-attach a restart produces re-hands into a fresh allocator, so it + /// needs no allowance), and two taps on one address would key one + /// PTask's frames to the other's host-side row. /// /// # Errors /// /// [`NetError::HandedAddressOutsidePlan`] when `ip` is outside the /// plan's PTask run: the address cannot be honored on this subnet. - /// [`NetError::HandedAddressCollision`] when a locally-drawn lease - /// already holds `ip`. + /// [`NetError::HandedAddressInReserve`] when `ip` is inside the daemon's + /// self-allocation reserve. [`NetError::HandedAddressCollision`] when a + /// recorded lease already holds `ip`. pub fn hand(&mut self, ip: Ipv4Addr) -> Result { let ip_num = u32::from(ip); if ip_num < self.subnet.first_ptask() || ip_num > self.subnet.last_ptask() { @@ -273,17 +327,19 @@ impl IpAllocator { subnet: self.subnet, }); } - self.next = self.next.max(ip_num + 1); + let (reserve_first, reserve_last) = self.reserve; + if (reserve_first..=reserve_last).contains(&ip_num) { + return Err(NetError::HandedAddressInReserve { + address: ip, + subnet: self.subnet, + }); + } if let Some(lease) = self.leases.iter().find(|lease| lease.ip == ip) { - if self.handed.contains(&ip) { - return Ok(*lease); - } return Err(NetError::HandedAddressCollision { address: ip, holder: lease.mac, }); } - self.handed.push(ip); let lease = PtaskLease { ip, mac: MacAddr::for_switch_ip(ip), @@ -468,19 +524,21 @@ impl SwitchClient { /// keyed by this address, so the daemon reuses it and mints nothing. /// /// The handed address is recorded like an allocated one — it joins the - /// static-lease table the switch is configured from, and the allocator's - /// cursor advances past it ([`IpAllocator::hand`]) — so the - /// never-reuse rule holds across both allocation shapes. Everything + /// static-lease table the switch is configured from + /// ([`IpAllocator::hand`]) — and is spent for good, as a drawn one is: + /// the never-reuse rule holds across both allocation shapes. Everything /// else matches [`Self::attach`], including the DM2 config/spawn steps /// and the attach count. /// /// # Errors /// /// [`NetError::HandedAddressOutsidePlan`] when the host handed an - /// address this daemon's subnet cannot honor, and - /// [`NetError::HandedAddressCollision`] when a locally-drawn lease - /// already holds the handed address; propagates config-write, spawn, - /// and socket-readiness failures as [`Self::attach`] does. + /// address this daemon's subnet cannot honor, + /// [`NetError::HandedAddressInReserve`] when it handed one from the + /// daemon's own self-allocation reserve, and + /// [`NetError::HandedAddressCollision`] when a lease already holds the + /// handed address; propagates config-write, spawn, and socket-readiness + /// failures as [`Self::attach`] does. pub async fn attach_handed(&mut self, handed: Ipv4Addr) -> Result { let lease = self.allocator.hand(handed)?; // DM2 spawns + configures gvproxy locally; DM1/3/4 (HostShuttle) leaves @@ -778,61 +836,129 @@ mod tests { fn allocator_exhausts_a_tiny_subnet() { // /29 => 8 addresses; network, gateway, daemon, host-alias, broadcast // reserved, leaving exactly three allocatable hosts (.2 through .4). + // The run splits at its midpoint, so this daemon's reserve is .2 + // alone and the host's hand-out run is .3 and .4 — exhaustion is + // explicit on each side, with no wrap and no reuse. let subnet = SwitchSubnet::new(Ipv4Addr::new(10, 0, 0, 0), 29).unwrap(); let mut a = IpAllocator::new(subnet); - let got: Vec<_> = std::iter::from_fn(|| a.allocate().ok()) - .map(|l| l.ip) - .collect(); + assert_eq!(a.allocate().unwrap().ip, Ipv4Addr::new(10, 0, 0, 2)); + assert!(matches!(a.allocate(), Err(NetError::SubnetExhausted(_)))); + // The reserve bounds only the daemon's draws: the run's remainder is + // still there for the host's hand-outs, each spent for good. + a.hand(Ipv4Addr::new(10, 0, 0, 3)).unwrap(); + a.hand(Ipv4Addr::new(10, 0, 0, 4)).unwrap(); + assert!(matches!( + a.hand(Ipv4Addr::new(10, 0, 0, 3)), + Err(NetError::HandedAddressCollision { .. }) + )); + } + + #[test] + fn self_allocation_run_splits_the_plan_ptask_run() { + // The default plan's split, pinned literally: the daemon's reserve + // is the PTask run's lower half and the host hands registered boxes + // from the run above it. The mirror is `minvmd`'s + // `box_registry::hand_out_run` — one rule, two statements; change + // both together. + let subnet = SwitchSubnet::default(); + let (reserve_first, reserve_last) = self_allocation_run(subnet); assert_eq!( - got, - vec![ - Ipv4Addr::new(10, 0, 0, 2), - Ipv4Addr::new(10, 0, 0, 3), - Ipv4Addr::new(10, 0, 0, 4), - ] + (reserve_first, reserve_last), + ( + u32::from(Ipv4Addr::new(100, 64, 0, 2)), + u32::from(Ipv4Addr::new(100, 64, 127, 254)), + ), + "the reserve is the run's lower half, starting at the first PTask address" + ); + assert_eq!( + reserve_last + 1, + u32::from(Ipv4Addr::new(100, 64, 127, 255)), + "the host's hand-out run starts immediately above the reserve" + ); + assert!( + subnet.first_ptask() <= reserve_first && reserve_last < subnet.last_ptask(), + "both sub-runs are inside the plan's PTask run, and neither is empty" + ); + // A tiny subnet splits in half, so both sides keep at least one + // address: a /29's three PTask addresses give the reserve .2 and the + // hand-out run .3 and .4. + let tiny = SwitchSubnet::new(Ipv4Addr::new(10, 0, 0, 0), 29).unwrap(); + assert_eq!( + self_allocation_run(tiny), + ( + u32::from(Ipv4Addr::new(10, 0, 0, 2)), + u32::from(Ipv4Addr::new(10, 0, 0, 2)), + ) ); - assert!(matches!(a.allocate(), Err(NetError::SubnetExhausted(_)))); } #[test] - fn hand_refuses_an_address_a_local_lease_holds() { - // The host's registration cursor and the daemon's allocation cursor - // run over the same address run without seeing each other's draws — - // task sandboxes still self-allocate beside the handed boxes — so - // the host can hand an address a locally-drawn lease already holds. - // The collision is refused, never shared: two taps on one address - // would key one PTask's frames to the other's host-side row. + fn hand_refuses_an_address_in_the_self_allocation_reserve() { + // The host hands registered boxes only from the run above this + // daemon's self-allocation reserve, so an address inside the reserve + // means the two sides disagree about the split — a skewed pair, a + // misconfigured host. Refused, never attached beside the daemon's + // own draws: the sequence a shared run once allowed, a task sandbox + // self-allocating an address a box is then handed, ends at the + // refusal. let mut a = IpAllocator::new(SwitchSubnet::default()); - let handed_box = a.hand(Ipv4Addr::new(100, 64, 0, 2)).unwrap(); - assert_eq!(handed_box.ip, Ipv4Addr::new(100, 64, 0, 2)); - // The next self-allocation draws past the handed address... let task = a.allocate().unwrap(); - assert_eq!(task.ip, Ipv4Addr::new(100, 64, 0, 3)); - // ...so a box handed the task's address is a collision, not a match - // on the task's lease. + assert_eq!(task.ip, Ipv4Addr::new(100, 64, 0, 2)); let err = a.hand(task.ip).unwrap_err(); + assert!( + matches!( + err, + NetError::HandedAddressInReserve { address, .. } if address == task.ip + ), + "handing a reserve address is refused, held or not" + ); + // The refusal is positional, not lease-dependent: an unheld reserve + // address refuses the same way, and a refusal records nothing — the + // table holds the task's draw alone. + assert!(matches!( + a.hand(Ipv4Addr::new(100, 64, 64, 128)), + Err(NetError::HandedAddressInReserve { .. }) + )); + assert_eq!(a.leases().len(), 1, "a refusal adds no lease"); + // The task's address stays the task's: the next draw takes the + // reserve's next address, never the refused one again. + assert_eq!(a.allocate().unwrap().ip, Ipv4Addr::new(100, 64, 0, 3)); + } + + #[test] + fn hand_refuses_an_address_a_local_lease_holds() { + // Any recorded lease holds its address for the allocator's life, + // whatever drew it: a box attaches once per daemon life and a + // restart starts from an empty allocator, so no attach legitimately + // re-uses a lease — and two taps on one address would key one + // PTask's frames to the other's host-side row. With the run split, a + // locally-drawn lease always sits in the daemon's reserve + // (`hand_refuses_an_address_in_the_self_allocation_reserve` refuses + // that shape outright), so the lease this one exercises is a handed + // one — the same refusal rule. + let mut a = IpAllocator::new(SwitchSubnet::default()); + let box_a = a.hand(Ipv4Addr::new(100, 64, 127, 255)).unwrap(); + // A task's sandbox self-allocates beside it, from the reserve... + let task = a.allocate().unwrap(); + assert_eq!(task.ip, Ipv4Addr::new(100, 64, 0, 2)); + // ...and handing the box's address again is refused with the holder + // named. The refusal changes nothing: no duplicate joined the table, + // and handing the same address again refuses just the same — the + // address stays with its holder for the allocator's life. + let err = a.hand(box_a.ip).unwrap_err(); assert!( matches!( err, NetError::HandedAddressCollision { address, holder } - if address == task.ip && holder == task.mac + if address == box_a.ip && holder == box_a.mac ), - "handing a locally-drawn address is refused with the holder named" + "handing a lease's address is refused with the holder named" ); - // The refusal changes nothing: no duplicate joined the table, and - // handing the same address again refuses just the same — the - // address stays with its original holder for the allocator's life. assert_eq!(a.leases().len(), 2); assert!(matches!( - a.hand(task.ip), + a.hand(box_a.ip), Err(NetError::HandedAddressCollision { .. }) )); - // A re-attach of a *handed* lease stays legitimate — the only match - // this method makes: the box re-attaching with the address its - // record carries (same daemon lifetime or after a restart) finds - // the lease marked handed and takes it back without a duplicate. - assert_eq!(a.hand(Ipv4Addr::new(100, 64, 0, 2)).unwrap(), handed_box); - assert_eq!(a.leases().len(), 2); } #[test] diff --git a/crates/minvmd/src/box_registry.rs b/crates/minvmd/src/box_registry.rs index 10bfd4b3e7..1989f1618b 100644 --- a/crates/minvmd/src/box_registry.rs +++ b/crates/minvmd/src/box_registry.rs @@ -172,6 +172,30 @@ pub struct ClientBoxSpec { pub egress: Option, } +/// The run of `subnet`'s address plan the host hands registered boxes from: +/// the PTask run's upper half, `[midpoint + 1, last_ptask]`, as an inclusive +/// `(first, last)` pair. +/// +/// The plan's PTask run is split into two disjoint sub-runs so the two +/// allocators that draw on it cannot meet: this host hands a registered box +/// only from the upper half, and the in-VM daemon self-allocates task +/// sandboxes and unregistered boxes from the lower half — the run below this +/// hand-out run. The daemon's half is the same midpoint rule mirrored in +/// `minimald::net::self_allocation_run`; one rule, two statements, so change +/// both together — each side's tests pin the default plan's split literally. +/// +/// This is the interim allocation shape (NET-138): the daemon keeps +/// self-allocating from the reserve until the task registering every live +/// box host-side retires daemon-side allocation, after which a daemon with a +/// control socket draws nothing — every own-address box arrives handed. +#[must_use] +fn hand_out_run(subnet: SwitchSubnet) -> (u32, u32) { + let first = subnet.first_ptask(); + let last = subnet.last_ptask(); + let reserve_len = (last - first + 1) / 2; + (first + reserve_len, last) +} + /// Why a client-driven registration could not be allocated. Both runs a /// box address comes from — the plan's switch lease run and the published /// loopback slice — are finite; exhausting one is an answer to hand back @@ -212,7 +236,9 @@ pub struct BoxRegistry { subnet: SwitchSubnet, rows: Arc>, /// The next switch address the client-driven allocation hands out, - /// shared by every clone of this registry. + /// shared by every clone of this registry. Draws from the hand-out run + /// — the plan run's upper half, above the daemon's self-allocation + /// reserve ([`hand_out_run`]) — never from the reserve itself. next_switch_addr: Arc, /// The next published loopback address the client-driven allocation /// hands out, shared the same way. @@ -237,7 +263,7 @@ impl BoxRegistry { Self { subnet, rows: Arc::new(RwLock::new(BTreeMap::new())), - next_switch_addr: Arc::new(AtomicU32::new(subnet.first_ptask())), + next_switch_addr: Arc::new(AtomicU32::new(hand_out_run(subnet).0)), next_loopback_addr: Arc::new(AtomicU32::new( loopback_slice.map_or(0, |slice| u32::from(slice.first())), )), @@ -323,11 +349,13 @@ impl BoxRegistry { /// Addresses are handed out in plan order from shared cursors: each /// registration takes the next address the plan has not spent, and no /// address is ever handed to two rows (clones of this registry share the - /// cursors, so that holds across every clone). The runs are finite — the - /// lease run ends below the daemon's reserved address, the slice ends - /// where the plan's next switch begins — and exhausting one is the - /// [`AllocationError`] the control socket hands back as the - /// registration's failure. + /// cursors, so that holds across every clone). The switch cursor draws + /// only from the hand-out run — the plan run's upper half, above the + /// daemon's self-allocation reserve ([`hand_out_run`]) — so the two + /// allocators cannot meet. The runs are finite — the hand-out run ends + /// at the plan's last PTask address, the slice ends where the plan's + /// next switch begins — and exhausting one is the [`AllocationError`] + /// the control socket hands back as the registration's failure. /// /// An address is spent for good: withdrawing its row does not return it /// to the cursor, and neither does an allocation whose other run is @@ -342,10 +370,11 @@ impl BoxRegistry { let slice = self .loopback_slice .ok_or(AllocationError::UnplannedSubnet(self.subnet))?; + let (hand_out_first, hand_out_last) = hand_out_run(self.subnet); let switch_addr = take_next( &self.next_switch_addr, - self.subnet.first_ptask(), - self.subnet.last_ptask(), + hand_out_first, + hand_out_last, ) .ok_or(AllocationError::SwitchExhausted)?; let loopback_addr = take_next( @@ -439,14 +468,18 @@ impl BoxTable { /// ([`SwitchSubnet::first_ptask`] through [`SwitchSubnet::last_ptask`]) — /// the one set of addresses a published row is ever keyed by, and so the /// one set whose rows the host-side creator will supply (T66's registration - /// path). The subnet's own infrastructure sits outside that run: the - /// gateway the resolver carve-out is keyed to, the host alias, and the - /// guest daemon's own tap, which the registry publishes a row for itself. - /// The announced interim the gate ships admits an unregistered source only - /// here, so no amount of it can borrow the plan's infrastructure as a - /// source; the per-box default that replaces the interim admits nothing, - /// and this predicate is what keeps the difference between them one - /// address range wide. + /// path). The run spans both of the plan's sub-runs — the daemon's + /// self-allocation reserve included, because a task sandbox holds a + /// reserve address and stays an unregistered source the interim admits; + /// the host hands registered boxes only from the upper half + /// ([`hand_out_run`]). The subnet's own infrastructure sits outside that + /// run: the gateway the resolver carve-out is keyed to, the host alias, + /// and the guest daemon's own tap, which the registry publishes a row for + /// itself. The announced interim the gate ships admits an unregistered + /// source only here, so no amount of it can borrow the plan's + /// infrastructure as a source; the per-box default that replaces the + /// interim admits nothing, and this predicate is what keeps the + /// difference between them one address range wide. #[must_use] pub fn is_allocatable(&self, src: [u8; 4]) -> bool { let addr = u32::from(Ipv4Addr::from(src)); @@ -686,6 +719,80 @@ mod tests { ); } + /// The host hands registered boxes only from the hand-out run — the plan + /// run's upper half, above the daemon's self-allocation reserve — and + /// the loopback run's exhaustion stays an explicit refusal, with no + /// wrap. Driven on a planned carved /24, whose runs are small enough to + /// see both edges of. + #[test] + fn client_boxes_hand_out_from_the_run_above_the_reserve() { + // The plan's default subnet splits at 100.64.127.255 — pinned + // literally, mirrored from `minimald::net::self_allocation_run`: + // the first box takes the hand-out run's first address, never the + // PTask run's first (that is the daemon's reserve). + let registry = BoxRegistry::new(SUBNET); + let first = registry + .register_client_box(ClientBoxSpec { + name: "web".to_string(), + ingress_ports: Vec::new(), + egress: None, + }) + .expect("the default plan has hand-out addresses"); + assert_eq!( + first.switch_addr(), + Ipv4Addr::new(100, 64, 127, 255), + "the first box takes the hand-out run's first address, above the \ + daemon's reserve" + ); + + // A planned carved /24: its PTask run is 100.64.1.2 through + // 100.64.1.252, so its hand-out run starts at .127 — and its + // loopback slice holds 32 addresses, which run out first. The + // refusal is explicit and repeats: no wrap, no reuse. + let carved = SwitchSubnet::new(Ipv4Addr::new(100, 64, 1, 0), 24).expect("valid"); + assert!( + switch::AddressPlan::default() + .loopback_slice_for_switch(carved) + .is_some(), + "the carved subnet is planned, so a refusal is a run's exhaustion, not the plan's absence" + ); + let registry = BoxRegistry::new(carved); + let first = registry + .register_client_box(ClientBoxSpec { + name: "web".to_string(), + ingress_ports: Vec::new(), + egress: None, + }) + .expect("the carved subnet has hand-out addresses"); + assert_eq!( + first.switch_addr(), + Ipv4Addr::new(100, 64, 1, 127), + "the carved /24's hand-out run also starts above its reserve" + ); + for index in 1..32 { + registry + .register_client_box(ClientBoxSpec { + name: format!("box{index}"), + ingress_ports: Vec::new(), + egress: None, + }) + .expect("the slice holds 32 published addresses"); + } + for _ in 0..2 { + assert!( + matches!( + registry.register_client_box(ClientBoxSpec { + name: "late".to_string(), + ingress_ports: Vec::new(), + egress: None, + }), + Err(AllocationError::LoopbackExhausted) + ), + "exhaustion is explicit and never wraps" + ); + } + } + /// NET-138's trust boundary: the guest never sources a row. The table the /// gate holds is read-only by construction — `BoxTable`'s only operations /// are lookups — and what that means behaviourally is that no amount of diff --git a/crates/minvmd/src/control.rs b/crates/minvmd/src/control.rs index c886db446e..dd6f283539 100644 --- a/crates/minvmd/src/control.rs +++ b/crates/minvmd/src/control.rs @@ -21,6 +21,12 @@ //! keeps serving, and a client that never reaches the socket (a supervisor //! predating this module) still activates, handed no addresses, under the //! egress gate's announced interim. +//! +//! The v1 posture, stated: the trust is the socket itself — the same-uid +//! 0600 the bind enforces, with no peer-credential check on a connection — +//! and requests are served serially, one connection at a time, each read +//! bounded by a 30-second timeout. That is what v1 ships, not a design +//! endpoint. use std::io::{Read, Write}; use std::os::unix::net::{UnixListener, UnixStream}; @@ -309,8 +315,10 @@ mod tests { let dir = tempfile::TempDir::new().expect("temp dir"); let (sock_path, _server) = spawn_server(dir.path()).expect("server binds"); - // The first registration is handed the plan's first lease address - // and the first published loopback address. + // The first registration is handed the hand-out run's first switch + // address — the plan's PTask run above the daemon's self-allocation + // reserve, mirrored from `minimald::net::self_allocation_run` — and + // the first published loopback address. let web = handed( register( &sock_path, @@ -329,8 +337,9 @@ mod tests { ); assert_eq!( web.switch_address, - Ipv4Addr::from(SUBNET.first_ptask()), - "the first box takes the plan's first lease address" + Ipv4Addr::new(100, 64, 127, 255), + "the first box takes the hand-out run's first address, above the \ + daemon's reserve" ); assert_eq!( web.loopback_address, @@ -370,8 +379,8 @@ mod tests { ); assert_eq!( db.switch_address, - Ipv4Addr::from(SUBNET.first_ptask() + 1), - "the second box takes the plan's next lease address, never the first again" + Ipv4Addr::new(100, 64, 128, 0), + "the second box takes the hand-out run's next address, never the first again" ); assert_eq!( db.loopback_address, From d6622689e10076671f19d93912a26e18ca0c0f5e Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:24:18 +0000 Subject: [PATCH 08/18] fix(minimal): warn that an unregistered box runs unattributed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A box whose VM host registration cannot be made runs with no host-side row: its frames are unattributed under the egress gate's announced interim, and the gate's per-box default fails closed for it when it binds (#1790). The warnings on the registration failure path — an unresolvable provider dir, a refusal, the deadline — now name the missing registration and that posture instead of the old egress-not-enforced phrasing. --- crates/minimal/src/cmd/session.rs | 33 ++++++++++++++++++------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index d0cfddeb9e..04b04456ec 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -100,10 +100,11 @@ async fn register_box_with_vm_host( /// /// A registration that cannot be made — no control socket (a supervisor /// predating it), a refusal, the deadline — degrades to a warning and -/// `None`, never to a failed activation: the box still creates, attaching -/// under the egress gate's announced interim the same way a pre-T66 box -/// does. Its egress is then what the interim admits until the daemon-side -/// default flips, not what this activation declared — the warning says so. +/// `None`, never to a failed activation: the box still creates, but +/// unregistered. No host-side row holds it, so its frames run unattributed +/// — the egress gate's announced interim admits what it admits — until the +/// gate's per-box default is in force, which then fails closed for the box +/// (#1790). The warning says so. async fn register_box_for_activation( global: &GlobalArgs, network: sessions::NetworkMode, @@ -122,9 +123,10 @@ async fn register_box_for_activation( Err(error) => { tracing::warn!( %error, - "cannot resolve the VM host provider dir; the box will not be \ - registered with the VM host daemon (its declared egress is not \ - enforced host-side)" + "cannot resolve the VM host provider dir; the box registration \ + is skipped and the box runs unregistered — its frames \ + unattributed — until the egress gate's per-box default is in \ + force (fail-closed then; #1790)" ); return None; } @@ -134,9 +136,10 @@ async fn register_box_for_activation( .map(|dir| dir.join(minvmd::control::CONTROL_SOCK_FILE)) else { tracing::warn!( - "no provider dir resolved for the ssh socket; the box will not be \ - registered with the VM host daemon (its declared egress is not \ - enforced host-side)" + "no provider dir resolved for the ssh socket; the box registration \ + is skipped and the box runs unregistered — its frames \ + unattributed — until the egress gate's per-box default is in \ + force (fail-closed then; #1790)" ); return None; }; @@ -176,8 +179,9 @@ async fn register_box_for_activation( Ok(Err(error)) => { tracing::warn!( %error, - "the box will not be registered with the VM host daemon; its \ - declared egress is not enforced host-side" + "the box registration failed; the box runs unregistered — its \ + frames unattributed — until the egress gate's per-box default \ + is in force (fail-closed then; #1790)" ); None } @@ -185,8 +189,9 @@ async fn register_box_for_activation( tracing::warn!( after = ?BOX_REGISTRATION_TIMEOUT, "the VM host daemon did not answer the box registration in time; \ - the box will not be registered (its declared egress is not \ - enforced host-side)" + the box runs unregistered — its frames unattributed — until \ + the egress gate's per-box default is in force (fail-closed \ + then; #1790)" ); None } From a0e0f96a50cddc86776ae1fb6cc437f93ff3012a Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:39:54 +0000 Subject: [PATCH 09/18] style(minimald,minvmd): take the reserve length with div_ceil clippy's manual_div_ceil flags (last - first + 1) / 2 in both mirrored split functions; (last - first).div_ceil(2) is the same number, since a PTask run's address span is always even. Doc: state that the hand-out run keeps one address more than the reserve. --- crates/minimald/src/net/mod.rs | 5 +++-- crates/minvmd/src/box_registry.rs | 19 ++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/crates/minimald/src/net/mod.rs b/crates/minimald/src/net/mod.rs index 11c3fde990..6ae88cfc9e 100644 --- a/crates/minimald/src/net/mod.rs +++ b/crates/minimald/src/net/mod.rs @@ -177,7 +177,8 @@ pub struct AttachResult { /// The sub-run of `subnet`'s PTask run this daemon **self-allocates** from: /// the run's lower half, `[first_ptask, midpoint]`, as an inclusive -/// `(first, last)` pair. +/// `(first, last)` pair — the hand-out run above it keeps one address +/// more, a PTask run holding an odd number of addresses. /// /// The plan's PTask run is split into two disjoint sub-runs so the two /// allocators that draw on it cannot meet: this daemon self-allocates task @@ -196,7 +197,7 @@ pub struct AttachResult { pub fn self_allocation_run(subnet: SwitchSubnet) -> (u32, u32) { let first = subnet.first_ptask(); let last = subnet.last_ptask(); - let reserve_len = (last - first + 1) / 2; + let reserve_len = (last - first).div_ceil(2); (first, first + reserve_len - 1) } diff --git a/crates/minvmd/src/box_registry.rs b/crates/minvmd/src/box_registry.rs index 1989f1618b..0e81d27141 100644 --- a/crates/minvmd/src/box_registry.rs +++ b/crates/minvmd/src/box_registry.rs @@ -174,7 +174,8 @@ pub struct ClientBoxSpec { /// The run of `subnet`'s address plan the host hands registered boxes from: /// the PTask run's upper half, `[midpoint + 1, last_ptask]`, as an inclusive -/// `(first, last)` pair. +/// `(first, last)` pair — one address more than the daemon's reserve below +/// it, a PTask run holding an odd number of addresses. /// /// The plan's PTask run is split into two disjoint sub-runs so the two /// allocators that draw on it cannot meet: this host hands a registered box @@ -192,7 +193,7 @@ pub struct ClientBoxSpec { fn hand_out_run(subnet: SwitchSubnet) -> (u32, u32) { let first = subnet.first_ptask(); let last = subnet.last_ptask(); - let reserve_len = (last - first + 1) / 2; + let reserve_len = (last - first).div_ceil(2); (first + reserve_len, last) } @@ -238,7 +239,7 @@ pub struct BoxRegistry { /// The next switch address the client-driven allocation hands out, /// shared by every clone of this registry. Draws from the hand-out run /// — the plan run's upper half, above the daemon's self-allocation - /// reserve ([`hand_out_run`]) — never from the reserve itself. + /// reserve (`hand_out_run`) — never from the reserve itself. next_switch_addr: Arc, /// The next published loopback address the client-driven allocation /// hands out, shared the same way. @@ -351,7 +352,7 @@ impl BoxRegistry { /// address is ever handed to two rows (clones of this registry share the /// cursors, so that holds across every clone). The switch cursor draws /// only from the hand-out run — the plan run's upper half, above the - /// daemon's self-allocation reserve ([`hand_out_run`]) — so the two + /// daemon's self-allocation reserve (`hand_out_run`) — so the two /// allocators cannot meet. The runs are finite — the hand-out run ends /// at the plan's last PTask address, the slice ends where the plan's /// next switch begins — and exhausting one is the [`AllocationError`] @@ -371,12 +372,8 @@ impl BoxRegistry { .loopback_slice .ok_or(AllocationError::UnplannedSubnet(self.subnet))?; let (hand_out_first, hand_out_last) = hand_out_run(self.subnet); - let switch_addr = take_next( - &self.next_switch_addr, - hand_out_first, - hand_out_last, - ) - .ok_or(AllocationError::SwitchExhausted)?; + let switch_addr = take_next(&self.next_switch_addr, hand_out_first, hand_out_last) + .ok_or(AllocationError::SwitchExhausted)?; let loopback_addr = take_next( &self.next_loopback_addr, u32::from(slice.first()), @@ -472,7 +469,7 @@ impl BoxTable { /// self-allocation reserve included, because a task sandbox holds a /// reserve address and stays an unregistered source the interim admits; /// the host hands registered boxes only from the upper half - /// ([`hand_out_run`]). The subnet's own infrastructure sits outside that + /// (`hand_out_run`). The subnet's own infrastructure sits outside that /// run: the gateway the resolver carve-out is keyed to, the host alias, /// and the guest daemon's own tap, which the registry publishes a row for /// itself. The announced interim the gate ships admits an unregistered From d7491c05d03b97e772c52083dabd6f9b492bdad4 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:08:25 +0000 Subject: [PATCH 10/18] fix(minimald): release a handed lease when its attach ends Re-attaching a handed (own-ip VM) box after its in-process host was rebuilt met HandedAddressCollision: hand() records the handed address into the allocator's table, detach() never removed a lease, so the lease outlived the teardown and the re-attach looked like a live attach. The session launcher threads the persisted box addresses into the respawn, so the box stayed unreachable until daemon restart. Key the collision to a live attach instead: detach now releases the handed lease it is given (a drawn lease is never released), and the OwnIp guard and abandon path pass their lease through. The same box re-attaching re-hands its address; a genuinely colliding attach is still refused. Addresses the review finding on IpAllocator::hand. --- crates/minimald/src/net/gvproxy_network.rs | 138 +++++++++++++----- crates/minimald/src/net/mod.rs | 159 +++++++++++++++------ crates/minimald/src/net/provider.rs | 11 +- 3 files changed, 226 insertions(+), 82 deletions(-) diff --git a/crates/minimald/src/net/gvproxy_network.rs b/crates/minimald/src/net/gvproxy_network.rs index 16ce6da9c8..da0f68e63c 100644 --- a/crates/minimald/src/net/gvproxy_network.rs +++ b/crates/minimald/src/net/gvproxy_network.rs @@ -30,10 +30,12 @@ use crate::net::switch::SwitchRelay; /// down explicitly via [`NetGuard::teardown`] at the end of the sandbox's life. /// /// Teardown removes this PTask's ingress forwards then detaches it from the -/// switch (decrementing the switch refcount, which stops gvproxy once the last -/// `OwnIp` PTask leaves). It is **explicit** — driven on a live runtime by the -/// owner — rather than a `Drop` schedule, so it cannot be lost to a stopped -/// runtime. Dropping the held [`SwitchRelay`] aborts the frame relay either way. +/// switch (decrementing the switch refcount — which stops gvproxy once the last +/// `OwnIp` PTask leaves — and releasing the handed lease with the count, so +/// the same box's re-attach re-hands it, T66). It is **explicit** — driven on a +/// live runtime by the owner — rather than a `Drop` schedule, so it cannot be +/// lost to a stopped runtime. Dropping the held [`SwitchRelay`] aborts the +/// frame relay either way. pub(crate) struct OwnIpGuard { /// Held for its `Drop`, which aborts the relay tasks; never read. _relay: SwitchRelay, @@ -45,6 +47,9 @@ pub(crate) struct OwnIpGuard { /// The static ingress forwards exposed for this PTask (R2.3), removed on /// teardown. Empty when no ingress was configured. exposed: Vec, + /// The lease ip this guard's attach holds, passed to `detach` so the + /// handed lease among them is released with the count (T66). + lease_ip: Ipv4Addr, } impl NetGuard for OwnIpGuard { @@ -56,7 +61,7 @@ impl NetGuard for OwnIpGuard { if !self.exposed.is_empty() { crate::net::policy::remove_ingress(&self.control, &self.exposed).await; } - if let Err(e) = self.switch.lock().await.detach().await { + if let Err(e) = self.switch.lock().await.detach(self.lease_ip).await { tracing::warn!(error = %e, "detaching OwnIp PTask from switch on session end"); } // `_relay` drops here, aborting the relay tasks. @@ -266,6 +271,7 @@ async fn finish_own_ip_attach( switch: Arc::clone(switch), control, exposed, + lease_ip, }) } @@ -491,14 +497,17 @@ mod tests { /// daemon handed its registration attaches **at the handed address** — /// the address the host-side table's row is keyed by — and **selects /// none**: the allocator's run shows exactly the handed lease, no - /// self-allocation spent beside it. The next unregistered box - /// self-allocates from the daemon's reserve — the plan run's lower half, - /// disjoint from the hand-out run the handed address came from — so the - /// two allocators cannot meet and the handed address is never drawn. - /// And the attach names the handed address in one debug line, with the - /// handed/self-drawn distinction a log tail reads. A handed address the - /// daemon already spends is refused at the attach path rather than - /// shared with it — a reserve address, and a lease's own address alike. + /// self-allocation spent beside it. The lease releases with its attach, + /// so the same box re-attaching — the re-attach an in-process host + /// rebuild produces, where this allocator survives the rebuild — re-hands + /// its own address, and the next unregistered box self-allocates from + /// the daemon's reserve: the plan run's lower half, disjoint from the + /// hand-out run the handed address came from, so the two allocators + /// cannot meet. The attach names the handed address in one debug line, + /// with the handed/self-drawn distinction a log tail reads. A handed + /// address that would put a second tap beside a live one is refused at + /// the attach path rather than shared with it — a reserve address, and + /// an attach's own address alike. #[tokio::test] async fn own_ip_attach_uses_handed_address() { let switch = vm_host_switch(); @@ -538,13 +547,24 @@ mod tests { ); let _ = plan; // the plan's resolver/tap carry the same lease; asserted above - // Releasing it detaches like any other, and the next box — one the - // activating client did not register — self-allocates from the - // daemon's reserve: the plan run's lower half, a sub-run disjoint - // from the hand-out run the handed address came from, so the two - // allocators cannot meet. + // Releasing it detaches like any other — and the handed lease goes + // with the count: the address is the registered box's, keyed by the + // host-side row, so once its attach ends the same box re-attaching + // re-hands it instead of meeting the collision refusal. That is the + // re-attach an in-process host rebuild produces, where this + // allocator survives the rebuild. net.abandon().await; assert_eq!(switch.lock().await.attached(), 0, "the abandon detaches"); + assert!( + switch.lock().await.leases().is_empty(), + "the abandon releases the handed lease with the count" + ); + + // The next box — one the activating client did not register — + // self-allocates from the daemon's reserve: the plan run's lower + // half, a sub-run disjoint from the hand-out run the handed address + // came from, so the two allocators cannot meet and the handed + // address is never drawn. let unregistered = network_for( sessions::NetworkMode::OwnIp, &switch, @@ -560,24 +580,25 @@ mod tests { let leases = switch.lock().await.leases().to_vec(); assert_eq!( leases.len(), - 2, - "one handed lease, one self-allocated: the handed attach spent no \ - allocation, got {leases:?}" + 1, + "the released handed lease is out of the table; one draw beside \ + it: {leases:?}" ); + let drawn = *leases + .first() + .expect("the assert above pinned exactly one lease in the table"); assert_eq!( - leases[1].ip, + drawn.ip, Ipv4Addr::new(100, 64, 0, 2), "the self-allocation draws the reserve's first address — the \ sub-run is disjoint from the hand-out run, so the handed \ address is never drawn, got {leases:?}" ); - // And a handed address this daemon already spends is refused at the - // attach path, never shared. Two shapes, both refused: the task - // sandbox's just-drawn reserve address — the shape a host that - // hands from the run's start would produce — and the box's own - // handed address, whose lease is still recorded. The refusals spend - // nothing: no lease joins the table, no attach is counted. + // A handed address inside the reserve is refused at the attach path + // — the shape a host that hands from the run's start would produce — + // and the refusal spends nothing: no lease joins the table, no + // attach is counted. let colliding_reserve = network_for( sessions::NetworkMode::OwnIp, &switch, @@ -585,7 +606,7 @@ mod tests { None, None, Some(BoxAddresses { - switch_address: leases[1].ip, + switch_address: drawn.ip, loopback_address: Ipv4Addr::LOCALHOST, }), ); @@ -599,7 +620,7 @@ mod tests { ); assert_eq!( switch.lock().await.leases().len(), - 2, + 1, "the refusal added no lease to the table" ); assert_eq!( @@ -607,21 +628,66 @@ mod tests { 1, "only the self-allocated attach counts; the refused one never did" ); - let colliding_held = network_for( + + // The same box re-attaching re-hands its own address: the lease its + // previous attach released is back in the table, and the + // select-none half of the promise holds on the re-attach too — the + // table holds exactly the drawn lease and the handed one, nothing + // drawn beside it, and the attach counts like any other. + let reattached = network_for( + sessions::NetworkMode::OwnIp, + &switch, + "web", + None, + None, + Some(BoxAddresses { + switch_address: handed_switch, + loopback_address: handed_loopback, + }), + ); + let _ = reattached + .plan() + .await + .expect("the same box re-attaching re-hands its own address"); + let leases = switch.lock().await.leases().to_vec(); + assert_eq!( + leases.len(), + 2, + "the re-hand records the handed lease beside the drawn one: \ + {leases:?}" + ); + assert!( + leases.contains(&PtaskLease { + ip: handed_switch, + mac: MacAddr::for_switch_ip(handed_switch), + }), + "the re-hand carries the same lease the first attach recorded: \ + {leases:?}" + ); + assert_eq!( + switch.lock().await.attached(), + 2, + "the re-hand counts like any attach" + ); + + // While that attach is live, handing its address again is refused — + // two taps on one address would key one PTask's frames to the + // other's host-side row. The refusal spends nothing. + let colliding_live = network_for( sessions::NetworkMode::OwnIp, &switch, "fourth", None, None, Some(BoxAddresses { - switch_address: leases[0].ip, + switch_address: handed_switch, loopback_address: Ipv4Addr::LOCALHOST, }), ); - let err = colliding_held + let err = colliding_live .plan() .await - .expect_err("a handed address a lease holds refuses"); + .expect_err("a handed address a live attach holds refuses"); assert!( err.to_string().contains("already held by a lease"), "the refusal names the held lease: {err}" @@ -633,8 +699,8 @@ mod tests { ); assert_eq!( switch.lock().await.attached(), - 1, - "only the self-allocated attach counts; the refused one never did" + 2, + "the refused hand never counted" ); // The attach's one debug line names the address the box carries and diff --git a/crates/minimald/src/net/mod.rs b/crates/minimald/src/net/mod.rs index 6ae88cfc9e..ca6b6c014d 100644 --- a/crates/minimald/src/net/mod.rs +++ b/crates/minimald/src/net/mod.rs @@ -40,6 +40,7 @@ pub(crate) mod dns_gate; #[cfg(feature = "networking-wg")] pub mod wg; +use std::collections::HashSet; use std::io; use std::net::Ipv4Addr; use std::path::{Path, PathBuf}; @@ -110,12 +111,14 @@ pub enum NetError { address: Ipv4Addr, subnet: SwitchSubnet, }, - /// The host handed the box a switch address a recorded lease already - /// holds, whatever drew that lease (T66). A box attaches once per daemon - /// life and a restart starts from an empty allocator, so no legitimate - /// attach re-uses a lease; and two taps on one address would key one - /// PTask's frames to the other's host-side row. The collision is - /// refused, never shared. + /// The host handed the box a switch address an attach still holds (T66): + /// two taps on one address would key one PTask's frames to the other's + /// host-side row, so the collision is refused, never shared. A handed + /// lease is released when its attach ends ([`IpAllocator::release_handed`], + /// via [`SwitchClient::detach`]), so the same box re-attaching — the + /// re-attach an in-process host rebuild produces, where this allocator + /// survives the rebuild — re-hands its own address instead of meeting + /// this refusal. #[error("handed switch address {address} is already held by a lease (MAC {holder})")] HandedAddressCollision { address: Ipv4Addr, holder: MacAddr }, /// The host handed the box a switch address inside this daemon's @@ -201,16 +204,20 @@ pub fn self_allocation_run(subnet: SwitchSubnet) -> (u32, u32) { (first, first + reserve_len - 1) } -/// Hands out unique switch addresses, never reusing one for the lifetime of -/// the allocator (R1.6). +/// Hands out unique switch addresses, never reusing a drawn one for the +/// lifetime of the allocator (R1.6). /// -/// Two allocation shapes share the never-reuse rule: [`Self::allocate`] -/// draws a self-allocation from the plan run's lower half — the sub-run -/// [`self_allocation_run`] reserves for this daemon (task sandboxes, -/// unregistered boxes) — and [`Self::hand`] records the address the VM host -/// daemon handed a registered box, from the upper half. The two sub-runs are -/// disjoint, so a self-allocation can never spend a handed address, nor the -/// reverse. +/// Two allocation shapes: [`Self::allocate`] draws a self-allocation from +/// the plan run's lower half — the sub-run [`self_allocation_run`] reserves +/// for this daemon (task sandboxes, unregistered boxes) — and [`Self::hand`] +/// records the address the VM host daemon handed a registered box, from the +/// upper half. The two sub-runs are disjoint, so a self-allocation can never +/// spend a handed address, nor the reverse. A drawn lease is spent for good; +/// a handed one is the registered box's — the host-side row is keyed by it — +/// and leaves with its attach ([`Self::release_handed`]), so the same box +/// re-attaching re-hands it. That re-hand is not a reuse: it is the same row +/// coming back, and the collision refusal guards only an address an attach +/// still holds. /// /// The interim this shape ships in: task sandboxes self-allocate from the /// reserve until the task registering every live box host-side (NET-138) @@ -226,8 +233,13 @@ pub struct IpAllocator { /// advances, within the reserve. next: u32, /// Every address handed out, in allocation order. Doubles as the - /// static-lease table written into gvproxy's config. + /// static-lease table written into gvproxy's config. A handed lease + /// leaves with its attach ([`Self::release_handed`]); a drawn one stays. leases: Vec, + /// Which of the recorded leases were handed (T66) rather than drawn: + /// the set [`Self::release_handed`] consults, so a detach releases a + /// handed lease and never a drawn one. + handed: HashSet, } impl IpAllocator { @@ -240,6 +252,7 @@ impl IpAllocator { reserve: self_allocation_run(subnet), subnet, leases: Vec::new(), + handed: HashSet::new(), } } @@ -293,9 +306,11 @@ impl IpAllocator { /// Records `ip` as the switch address a registered box attaches at (T66): /// the VM host daemon allocated it into its table and handed it back, so /// the daemon mints nothing. The lease joins [`Self::leases`] — the - /// static-lease table the switch is configured from — and is spent for - /// good, as a drawn one is: the never-reuse rule is one rule across both - /// allocation shapes. + /// static-lease table the switch is configured from — and stays there + /// while the box's attach does: [`Self::release_handed`] takes it out + /// when the attach ends, so the same box re-attaching re-hands it. A + /// drawn lease ([`Self::allocate`]) is the opposite: spent for good, the + /// never-reuse rule (R1.6) keeps a spent address spent. /// /// Three refusals, never a share: /// @@ -306,12 +321,12 @@ impl IpAllocator { /// ([`NetError::HandedAddressInReserve`]) — the host hands only from /// the run above the reserve, so one there means the two sides /// disagree about the split; - /// * an address any recorded lease holds, whatever drew it - /// ([`NetError::HandedAddressCollision`]) — a box attaches once per - /// daemon life and a restart starts from an empty allocator (the - /// re-attach a restart produces re-hands into a fresh allocator, so it - /// needs no allowance), and two taps on one address would key one - /// PTask's frames to the other's host-side row. + /// * an address a recorded lease still holds + /// ([`NetError::HandedAddressCollision`]) — two taps on one address + /// would key one PTask's frames to the other's host-side row. What the + /// refusal guards is a live attach: the lease of an attach that ended + /// is released ([`Self::release_handed`]), so the same box's re-attach + /// re-hands its own address instead of meeting this refusal. /// /// # Errors /// @@ -319,7 +334,7 @@ impl IpAllocator { /// plan's PTask run: the address cannot be honored on this subnet. /// [`NetError::HandedAddressInReserve`] when `ip` is inside the daemon's /// self-allocation reserve. [`NetError::HandedAddressCollision`] when a - /// recorded lease already holds `ip`. + /// recorded lease still holds `ip`. pub fn hand(&mut self, ip: Ipv4Addr) -> Result { let ip_num = u32::from(ip); if ip_num < self.subnet.first_ptask() || ip_num > self.subnet.last_ptask() { @@ -346,8 +361,26 @@ impl IpAllocator { mac: MacAddr::for_switch_ip(ip), }; self.leases.push(lease); + self.handed.insert(ip); Ok(lease) } + + /// Releases the handed lease for `ip`, when the allocator holds one: the + /// box's attach ended, so its address is free for the same box to re-hand + /// on the re-attach an in-process host rebuild or a restart produces + /// (T66). A drawn lease is never released — the never-reuse rule (R1.6) + /// keeps a spent address spent, so a stale frame from a torn-down PTask + /// can never be misdelivered to a freshly-attached one — and an address + /// the allocator never recorded releases nothing. + /// + /// Returns whether a handed lease was released. + pub(crate) fn release_handed(&mut self, ip: Ipv4Addr) -> bool { + if !self.handed.remove(&ip) { + return false; + } + self.leases.retain(|lease| lease.ip != ip); + true + } } impl Default for IpAllocator { @@ -526,8 +559,9 @@ impl SwitchClient { /// /// The handed address is recorded like an allocated one — it joins the /// static-lease table the switch is configured from - /// ([`IpAllocator::hand`]) — and is spent for good, as a drawn one is: - /// the never-reuse rule holds across both allocation shapes. Everything + /// ([`IpAllocator::hand`]) — and leaves with the attach: [`Self::detach`] + /// releases it, so the same box's re-attach re-hands it. A drawn lease, + /// by contrast, is spent for good (the never-reuse rule). Everything /// else matches [`Self::attach`], including the DM2 config/spawn steps /// and the attach count. /// @@ -537,7 +571,7 @@ impl SwitchClient { /// address this daemon's subnet cannot honor, /// [`NetError::HandedAddressInReserve`] when it handed one from the /// daemon's own self-allocation reserve, and - /// [`NetError::HandedAddressCollision`] when a lease already holds the + /// [`NetError::HandedAddressCollision`] when an attach still holds the /// handed address; propagates config-write, spawn, and socket-readiness /// failures as [`Self::attach`] does. pub async fn attach_handed(&mut self, handed: Ipv4Addr) -> Result { @@ -560,14 +594,22 @@ impl SwitchClient { Ok(AttachResult { lease, exit_signal }) } - /// Records that a PTask detached. When the last one leaves, the switch is - /// stopped. + /// Records that a PTask detached, releasing its handed lease with it + /// (T66): a handed address is the registered box's, keyed by the + /// host-side row, so once the box's attach ends the same box re-attaching + /// re-hands it — the collision refusal then guards only an address an + /// attach still holds. A drawn lease is never released (R1.6); see + /// [`IpAllocator::release_handed`]. When the last one leaves, the switch + /// is stopped. /// /// # Errors /// /// Propagates teardown failures from [`stop`](Self::stop). - pub async fn detach(&mut self) -> Result<(), NetError> { + pub async fn detach(&mut self, released: Ipv4Addr) -> Result<(), NetError> { self.attached = self.attached.saturating_sub(1); + if self.allocator.release_handed(released) { + tracing::debug!(ip = %released, "released the handed lease with the attach"); + } tracing::info!(attached = self.attached, "detached OwnIp PTask from switch"); if self.attached == 0 { self.stop().await?; @@ -845,7 +887,7 @@ mod tests { assert_eq!(a.allocate().unwrap().ip, Ipv4Addr::new(10, 0, 0, 2)); assert!(matches!(a.allocate(), Err(NetError::SubnetExhausted(_)))); // The reserve bounds only the daemon's draws: the run's remainder is - // still there for the host's hand-outs, each spent for good. + // still there for the host's hand-outs, none of them ever drawn. a.hand(Ipv4Addr::new(10, 0, 0, 3)).unwrap(); a.hand(Ipv4Addr::new(10, 0, 0, 4)).unwrap(); assert!(matches!( @@ -928,11 +970,11 @@ mod tests { #[test] fn hand_refuses_an_address_a_local_lease_holds() { - // Any recorded lease holds its address for the allocator's life, - // whatever drew it: a box attaches once per daemon life and a - // restart starts from an empty allocator, so no attach legitimately - // re-uses a lease — and two taps on one address would key one - // PTask's frames to the other's host-side row. With the run split, a + // An address a recorded lease still holds refuses a second hand: + // two taps on one address would key one PTask's frames to the + // other's host-side row. What the refusal guards is a live attach — + // `handed_lease_releases_on_detach_and_the_box_rehands` covers the + // release that lets the same box re-attach. With the run split, a // locally-drawn lease always sits in the daemon's reserve // (`hand_refuses_an_address_in_the_self_allocation_reserve` refuses // that shape outright), so the lease this one exercises is a handed @@ -945,7 +987,7 @@ mod tests { // ...and handing the box's address again is refused with the holder // named. The refusal changes nothing: no duplicate joined the table, // and handing the same address again refuses just the same — the - // address stays with its holder for the allocator's life. + // address stays with its holder while its attach lives. let err = a.hand(box_a.ip).unwrap_err(); assert!( matches!( @@ -962,6 +1004,40 @@ mod tests { )); } + #[test] + fn handed_lease_releases_on_detach_and_the_box_rehands() { + // T66's re-attach: a handed lease leaves the table when its attach + // ends, so the same box re-attaching — the re-attach an in-process + // host rebuild produces, where this allocator survives the rebuild — + // re-hands its own address instead of meeting the collision refusal. + let mut a = IpAllocator::new(SwitchSubnet::default()); + let handed_ip = Ipv4Addr::new(100, 64, 128, 9); + let lease = a + .hand(handed_ip) + .expect("the hand-out run's address is handed"); + assert_eq!(a.leases().len(), 1); + + // A drawn lease is never released: the never-reuse rule keeps a + // spent address spent, whatever detach asked for. + let task = a.allocate().unwrap(); + assert!(!a.release_handed(task.ip), "a drawn lease is not released"); + assert_eq!(a.leases().len(), 2); + // An address the allocator never recorded releases nothing. + assert!(!a.release_handed(Ipv4Addr::new(100, 64, 200, 1))); + assert_eq!(a.leases().len(), 2); + + // The handed lease releases, and the re-hand records the same lease + // again — the same address, the same derived MAC, the same row the + // host-side table is keyed by. + assert!(a.release_handed(handed_ip)); + assert_eq!(a.leases(), &[task], "only the drawn lease remains"); + let rehanded = a + .hand(handed_ip) + .expect("the same box re-attaching re-hands its own address"); + assert_eq!(rehanded.ip, lease.ip); + assert_eq!(rehanded.mac, lease.mac); + } + #[test] fn subnet_rejects_overly_narrow_prefix() { assert!(matches!( @@ -1040,7 +1116,8 @@ mod tests { let result = switch.attach().await.expect("host-shuttle attach"); assert_eq!(result.lease.ip, Ipv4Addr::new(100, 64, 0, 2)); - // No gvproxy child was spawned; detach just decrements the count. - switch.detach().await.expect("detach"); + // No gvproxy child was spawned; detach decrements the count — and + // releases nothing here, the drawn lease stays spent. + switch.detach(result.lease.ip).await.expect("detach"); } } diff --git a/crates/minimald/src/net/provider.rs b/crates/minimald/src/net/provider.rs index f2eef50ff8..edafbbd6b0 100644 --- a/crates/minimald/src/net/provider.rs +++ b/crates/minimald/src/net/provider.rs @@ -380,16 +380,17 @@ impl Network for OwnIpNetwork { } /// Give the lease back; the sandbox layer runs this on every path out of a - /// launch that does not reach `attach`. + /// launch that does not reach `attach`. The detach releases the handed + /// lease with the count (T66), so an abandoned launch does not leave the + /// box's address spent behind a lease nothing holds. fn abandon(&self) -> AbandonFuture<'_> { Box::pin(async move { - let reserved = self.reserved.lock().unwrap().take(); - if reserved.is_none() { + let Some(reserved) = self.reserved.lock().unwrap().take() else { // The plan failed or an attach took it; detaching anyway would // decrement the switch's count below the truth. return; - } - if let Err(e) = self.switch.lock().await.detach().await { + }; + if let Err(e) = self.switch.lock().await.detach(reserved.lease.ip).await { tracing::warn!(error = %e, "detaching OwnIp PTask after an abandoned launch"); } }) From 078e69e61ed1ebf5922835e32abf8c53cdc5b171 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:15:59 +0000 Subject: [PATCH 11/18] style(minimald): use expect for the reserved mutex in abandon clippy-strict flags unwrap_used on changed lines; the lock is the one the abandon rewrite touched, so give it the same poisoned-mutex expect the rest of the strict-gated lines carry. --- crates/minimald/src/net/provider.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/minimald/src/net/provider.rs b/crates/minimald/src/net/provider.rs index edafbbd6b0..87ae6e131b 100644 --- a/crates/minimald/src/net/provider.rs +++ b/crates/minimald/src/net/provider.rs @@ -385,7 +385,12 @@ impl Network for OwnIpNetwork { /// box's address spent behind a lease nothing holds. fn abandon(&self) -> AbandonFuture<'_> { Box::pin(async move { - let Some(reserved) = self.reserved.lock().unwrap().take() else { + let Some(reserved) = self + .reserved + .lock() + .expect("reserved mutex poisoned") + .take() + else { // The plan failed or an attach took it; detaching anyway would // decrement the switch's count below the truth. return; From fb36cef5902dbc6d7c431cf037d63e7c847ccd4b Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:50:42 +0000 Subject: [PATCH 12/18] fix(minvmd): pass box_addresses in the vm-escape session config The merge of #1792 added the field to SessionConfig and every other initializer in the tree carries it; this one did not, so the macOS unit lane failed to compile the minvmd test target (E0063). --- crates/minvmd/tests/vm_escape_integration.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/minvmd/tests/vm_escape_integration.rs b/crates/minvmd/tests/vm_escape_integration.rs index d607cc5caa..20dc1e69e9 100644 --- a/crates/minvmd/tests/vm_escape_integration.rs +++ b/crates/minvmd/tests/vm_escape_integration.rs @@ -382,6 +382,9 @@ impl BoxSession { .map_err(|e| format!("project_path: {e}"))?, network, policy, + // No registration happened on this path: the box attaches + // as an unregistered one always has. + box_addresses: None, // The serde default, and what every non-`--no-hooks` // activation sends. This session only runs execs, so it // declares no hooks either way. From d0d7c1b70fb6c743b85c9bb08ce80129aabdbc6a Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:50:43 +0000 Subject: [PATCH 13/18] fix(minimald): release every lease with its attach Per the ARCH-EXPERT ruling the lease's life is the attachment's for every attachment, handed or drawn alike: a task sandbox's self-drawn address ends with the sandbox the way a registered box's handed one ends with the box's attach. SwitchClient::detach withdraws the lease on every detach and IpAllocator::release_handed is renamed release; the handed set is unneeded and goes. The never-reuse rule (R1.6) is carried by the draw cursor, not by a lease the static-lease table keeps for a tap that is gone: the cursor never regresses, so a withdrawn drawn address is never re-drawn, and it is never handed either (the reserve hand refuses). --- crates/minimald/src/net/gvproxy_network.rs | 8 +- crates/minimald/src/net/mod.rs | 166 +++++++++++++-------- crates/minimald/src/net/provider.rs | 7 +- 3 files changed, 112 insertions(+), 69 deletions(-) diff --git a/crates/minimald/src/net/gvproxy_network.rs b/crates/minimald/src/net/gvproxy_network.rs index da0f68e63c..2d2d38f676 100644 --- a/crates/minimald/src/net/gvproxy_network.rs +++ b/crates/minimald/src/net/gvproxy_network.rs @@ -31,8 +31,9 @@ use crate::net::switch::SwitchRelay; /// /// Teardown removes this PTask's ingress forwards then detaches it from the /// switch (decrementing the switch refcount — which stops gvproxy once the last -/// `OwnIp` PTask leaves — and releasing the handed lease with the count, so -/// the same box's re-attach re-hands it, T66). It is **explicit** — driven on a +/// `OwnIp` PTask leaves — and releasing the lease with the count, handed or +/// drawn alike; a handed lease released with it is what lets the same box's +/// re-attach re-hand its address, T66). It is **explicit** — driven on a /// live runtime by the owner — rather than a `Drop` schedule, so it cannot be /// lost to a stopped runtime. Dropping the held [`SwitchRelay`] aborts the /// frame relay either way. @@ -48,7 +49,8 @@ pub(crate) struct OwnIpGuard { /// teardown. Empty when no ingress was configured. exposed: Vec, /// The lease ip this guard's attach holds, passed to `detach` so the - /// handed lease among them is released with the count (T66). + /// lease is released with the count (T66) — handed or drawn alike, a + /// lease's life is its attachment's. lease_ip: Ipv4Addr, } diff --git a/crates/minimald/src/net/mod.rs b/crates/minimald/src/net/mod.rs index ca6b6c014d..49e0b0231e 100644 --- a/crates/minimald/src/net/mod.rs +++ b/crates/minimald/src/net/mod.rs @@ -40,7 +40,6 @@ pub(crate) mod dns_gate; #[cfg(feature = "networking-wg")] pub mod wg; -use std::collections::HashSet; use std::io; use std::net::Ipv4Addr; use std::path::{Path, PathBuf}; @@ -113,9 +112,9 @@ pub enum NetError { }, /// The host handed the box a switch address an attach still holds (T66): /// two taps on one address would key one PTask's frames to the other's - /// host-side row, so the collision is refused, never shared. A handed - /// lease is released when its attach ends ([`IpAllocator::release_handed`], - /// via [`SwitchClient::detach`]), so the same box re-attaching — the + /// host-side row, so the collision is refused, never shared. Every lease + /// is withdrawn when its attach ends ([`IpAllocator::release`], via + /// [`SwitchClient::detach`]), so the same box re-attaching — the /// re-attach an in-process host rebuild produces, where this allocator /// survives the rebuild — re-hands its own address instead of meeting /// this refusal. @@ -212,12 +211,17 @@ pub fn self_allocation_run(subnet: SwitchSubnet) -> (u32, u32) { /// for this daemon (task sandboxes, unregistered boxes) — and [`Self::hand`] /// records the address the VM host daemon handed a registered box, from the /// upper half. The two sub-runs are disjoint, so a self-allocation can never -/// spend a handed address, nor the reverse. A drawn lease is spent for good; -/// a handed one is the registered box's — the host-side row is keyed by it — -/// and leaves with its attach ([`Self::release_handed`]), so the same box -/// re-attaching re-hands it. That re-hand is not a reuse: it is the same row -/// coming back, and the collision refusal guards only an address an attach -/// still holds. +/// spend a handed address, nor the reverse. A lease's life is its +/// attachment's, handed or drawn alike: it joins the static-lease table the +/// switch is configured from when the tap arrives and leaves with the attach +/// ([`Self::release`], via [`SwitchClient::detach`]), so the table holds only +/// live taps. The never-reuse rule is carried by the draw cursor, not by the +/// table — `next` never regresses, so a withdrawn drawn address is never +/// re-drawn, and it is never handed either (it sits in the reserve, which +/// [`Self::hand`] refuses). A handed address, by contrast, is the registered +/// box's — the host-side row is keyed by it — and the same box re-attaching +/// re-hands it; that re-hand is not a reuse: it is the same row coming back, +/// and the collision refusal guards only an address an attach still holds. /// /// The interim this shape ships in: task sandboxes self-allocate from the /// reserve until the task registering every live box host-side (NET-138) @@ -233,13 +237,10 @@ pub struct IpAllocator { /// advances, within the reserve. next: u32, /// Every address handed out, in allocation order. Doubles as the - /// static-lease table written into gvproxy's config. A handed lease - /// leaves with its attach ([`Self::release_handed`]); a drawn one stays. + /// static-lease table written into gvproxy's config, which therefore + /// holds only live taps: a lease leaves with its attach + /// ([`Self::release`]), handed or drawn alike. leases: Vec, - /// Which of the recorded leases were handed (T66) rather than drawn: - /// the set [`Self::release_handed`] consults, so a detach releases a - /// handed lease and never a drawn one. - handed: HashSet, } impl IpAllocator { @@ -252,7 +253,6 @@ impl IpAllocator { reserve: self_allocation_run(subnet), subnet, leases: Vec::new(), - handed: HashSet::new(), } } @@ -306,11 +306,14 @@ impl IpAllocator { /// Records `ip` as the switch address a registered box attaches at (T66): /// the VM host daemon allocated it into its table and handed it back, so /// the daemon mints nothing. The lease joins [`Self::leases`] — the - /// static-lease table the switch is configured from — and stays there - /// while the box's attach does: [`Self::release_handed`] takes it out - /// when the attach ends, so the same box re-attaching re-hands it. A - /// drawn lease ([`Self::allocate`]) is the opposite: spent for good, the - /// never-reuse rule (R1.6) keeps a spent address spent. + /// static-lease table the switch is configured from — and leaves with + /// the box's attach ([`Self::release`]), so the same box re-attaching + /// re-hands it: the host's persisted row still holds the address, and + /// detach never touches that row. A drawn lease ([`Self::allocate`]) + /// leaves the same way, and its address is never seen again — the draw + /// cursor never regresses, and the reserve [`Self::hand`] refuses — so + /// the never-reuse rule (R1.6) is carried by the cursor, not by a lease + /// the table keeps for a tap that is gone. /// /// Three refusals, never a share: /// @@ -325,7 +328,7 @@ impl IpAllocator { /// ([`NetError::HandedAddressCollision`]) — two taps on one address /// would key one PTask's frames to the other's host-side row. What the /// refusal guards is a live attach: the lease of an attach that ended - /// is released ([`Self::release_handed`]), so the same box's re-attach + /// is withdrawn ([`Self::release`]), so the same box's re-attach /// re-hands its own address instead of meeting this refusal. /// /// # Errors @@ -361,25 +364,25 @@ impl IpAllocator { mac: MacAddr::for_switch_ip(ip), }; self.leases.push(lease); - self.handed.insert(ip); Ok(lease) } - /// Releases the handed lease for `ip`, when the allocator holds one: the - /// box's attach ended, so its address is free for the same box to re-hand - /// on the re-attach an in-process host rebuild or a restart produces - /// (T66). A drawn lease is never released — the never-reuse rule (R1.6) - /// keeps a spent address spent, so a stale frame from a torn-down PTask - /// can never be misdelivered to a freshly-attached one — and an address - /// the allocator never recorded releases nothing. + /// Withdraws the lease for `ip`, when the allocator holds one: the + /// attachment ended, so the lease ends with it — every attachment's, + /// handed or drawn alike. A handed address is free for the same box to + /// re-hand on the re-attach an in-process host rebuild or a restart + /// produces (T66): the host's persisted row still holds it, and nothing + /// here does. A drawn address is never re-drawn — the draw cursor never + /// regresses (R1.6), so a stale frame from a torn-down PTask can never be + /// misdelivered to a freshly-attached one — and never handed either: it + /// sits in the reserve [`Self::hand`] refuses. An address the allocator + /// never recorded withdraws nothing. /// - /// Returns whether a handed lease was released. - pub(crate) fn release_handed(&mut self, ip: Ipv4Addr) -> bool { - if !self.handed.remove(&ip) { - return false; - } + /// Returns whether a lease was withdrawn. + pub(crate) fn release(&mut self, ip: Ipv4Addr) -> bool { + let held = self.leases.iter().any(|lease| lease.ip == ip); self.leases.retain(|lease| lease.ip != ip); - true + held } } @@ -560,9 +563,19 @@ impl SwitchClient { /// The handed address is recorded like an allocated one — it joins the /// static-lease table the switch is configured from /// ([`IpAllocator::hand`]) — and leaves with the attach: [`Self::detach`] - /// releases it, so the same box's re-attach re-hands it. A drawn lease, - /// by contrast, is spent for good (the never-reuse rule). Everything - /// else matches [`Self::attach`], including the DM2 config/spawn steps + /// withdraws it, so the same box's re-attach re-hands it. That re-handed + /// address comes from the host's persisted row — the registration + /// allocated it into the host-side table and nothing on the host + /// withdraws it — and detach never touches that row: the daemon-side + /// lease ends with the attach, the host-side row does not. What this + /// daemon cannot yet check is that the address it is handed is still the + /// row's for this box: the host-row check by box id — the same address + /// for the same box, a refusal on mismatch — lands with the box id in + /// the registration (T44, #1660, NET-133), which names it as the gap; + /// until then this daemon attaches with whatever the create request + /// carries. A drawn lease, by contrast, is never re-drawn: the draw + /// cursor never regresses (the never-reuse rule). Everything else + /// matches [`Self::attach`], including the DM2 config/spawn steps /// and the attach count. /// /// # Errors @@ -594,21 +607,25 @@ impl SwitchClient { Ok(AttachResult { lease, exit_signal }) } - /// Records that a PTask detached, releasing its handed lease with it - /// (T66): a handed address is the registered box's, keyed by the - /// host-side row, so once the box's attach ends the same box re-attaching - /// re-hands it — the collision refusal then guards only an address an - /// attach still holds. A drawn lease is never released (R1.6); see - /// [`IpAllocator::release_handed`]. When the last one leaves, the switch - /// is stopped. + /// Records that a PTask detached, withdrawing its lease with it (T66): + /// a lease's life is its attachment's, handed or drawn alike — a task + /// sandbox's self-drawn address ends with the sandbox the way a + /// registered box's handed one ends with the box's attach. The handed + /// address is the registered box's, keyed by the host-side row, so once + /// the box's attach ends the same box re-attaching re-hands it — the + /// collision refusal then guards only an address an attach still holds. + /// A drawn address is never re-drawn: the allocator's draw cursor never + /// regresses (R1.6), so the rule holds without the table keeping a lease + /// for a tap that is gone; see [`IpAllocator::release`]. When the last + /// one leaves, the switch is stopped. /// /// # Errors /// /// Propagates teardown failures from [`stop`](Self::stop). pub async fn detach(&mut self, released: Ipv4Addr) -> Result<(), NetError> { self.attached = self.attached.saturating_sub(1); - if self.allocator.release_handed(released) { - tracing::debug!(ip = %released, "released the handed lease with the attach"); + if self.allocator.release(released) { + tracing::debug!(ip = %released, "released the lease with the attach"); } tracing::info!(attached = self.attached, "detached OwnIp PTask from switch"); if self.attached == 0 { @@ -1006,10 +1023,11 @@ mod tests { #[test] fn handed_lease_releases_on_detach_and_the_box_rehands() { - // T66's re-attach: a handed lease leaves the table when its attach - // ends, so the same box re-attaching — the re-attach an in-process - // host rebuild produces, where this allocator survives the rebuild — - // re-hands its own address instead of meeting the collision refusal. + // T66's re-attach: a lease — handed or drawn alike — leaves the + // table when its attach ends, so the same box re-attaching — the + // re-attach an in-process host rebuild produces, where this + // allocator survives the rebuild — re-hands its own address instead + // of meeting the collision refusal. let mut a = IpAllocator::new(SwitchSubnet::default()); let handed_ip = Ipv4Addr::new(100, 64, 128, 9); let lease = a @@ -1017,20 +1035,41 @@ mod tests { .expect("the hand-out run's address is handed"); assert_eq!(a.leases().len(), 1); - // A drawn lease is never released: the never-reuse rule keeps a - // spent address spent, whatever detach asked for. + // A drawn lease leaves the same way — a task sandbox's self-drawn + // address ends with its sandbox — and the never-reuse rule (R1.6) is + // carried by the cursor, not by the table: the next draw is a higher + // address, never the withdrawn one. let task = a.allocate().unwrap(); - assert!(!a.release_handed(task.ip), "a drawn lease is not released"); - assert_eq!(a.leases().len(), 2); + assert!(a.release(task.ip), "the drawn lease is withdrawn"); + assert_eq!(a.leases(), &[lease], "only the handed lease remains"); + let redrawn = a.allocate().unwrap(); + assert_eq!( + redrawn.ip, + Ipv4Addr::new(100, 64, 0, 3), + "the withdrawn address is never re-drawn" + ); // An address the allocator never recorded releases nothing. - assert!(!a.release_handed(Ipv4Addr::new(100, 64, 200, 1))); + assert!(!a.release(Ipv4Addr::new(100, 64, 200, 1))); assert_eq!(a.leases().len(), 2); + // A handed address an attach still holds refuses a second hand: the + // refusal guards a live attach, and the lease's withdrawal is what + // lets the re-attach through. + assert!(matches!( + a.hand(handed_ip), + Err(NetError::HandedAddressCollision { .. }) + )); + // The handed lease releases, and the re-hand records the same lease // again — the same address, the same derived MAC, the same row the - // host-side table is keyed by. - assert!(a.release_handed(handed_ip)); - assert_eq!(a.leases(), &[task], "only the drawn lease remains"); + // host-side table is keyed by — beside the draw that replaced the + // withdrawn one. + assert!(a.release(handed_ip)); + assert_eq!( + a.leases(), + &[redrawn], + "the table holds the live draw alone" + ); let rehanded = a .hand(handed_ip) .expect("the same box re-attaching re-hands its own address"); @@ -1116,8 +1155,9 @@ mod tests { let result = switch.attach().await.expect("host-shuttle attach"); assert_eq!(result.lease.ip, Ipv4Addr::new(100, 64, 0, 2)); - // No gvproxy child was spawned; detach decrements the count — and - // releases nothing here, the drawn lease stays spent. + // No gvproxy child was spawned; detach decrements the count and + // withdraws the lease with the attach — the drawn address stays + // spent regardless, the draw cursor never regresses. switch.detach(result.lease.ip).await.expect("detach"); } } diff --git a/crates/minimald/src/net/provider.rs b/crates/minimald/src/net/provider.rs index 87ae6e131b..d60b1cff9d 100644 --- a/crates/minimald/src/net/provider.rs +++ b/crates/minimald/src/net/provider.rs @@ -380,9 +380,10 @@ impl Network for OwnIpNetwork { } /// Give the lease back; the sandbox layer runs this on every path out of a - /// launch that does not reach `attach`. The detach releases the handed - /// lease with the count (T66), so an abandoned launch does not leave the - /// box's address spent behind a lease nothing holds. + /// launch that does not reach `attach`. The detach releases the lease + /// with the count (T66), so an abandoned launch leaves no lease in the + /// static-lease table for a tap nothing holds — the handed one included, + /// which is what lets the same box's re-attach re-hand its address. fn abandon(&self) -> AbandonFuture<'_> { Box::pin(async move { let Some(reserved) = self From cf02a4ac60dbba9cf5e5c9799695b786dadcaae4 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:16:05 +0000 Subject: [PATCH 14/18] feat(minvmd,minimald-rpc): add the withdraw verb to box control MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The destroyed or failed activation's client withdraws the row its registration bought: a tagged request envelope (register | withdraw), a withdrawal answered with the pair echoed back, and the pair-proven removal in the registry — lookup, proof, and removal under one write of the row lock, no row at the address answered as the success it is. The withdrawn addresses are not returned to the cursors; what their frames do next stays the gate phase's to say, both arms pinned at relay level through the phase parameter in host_row_withdrawn_on_destroy. The unregistered-source phase flip is not this change: it lands with the last row source (T66's follow-up), and the docs on register_client_box and the control module say so. --- crates/minimald-rpc/src/lib.rs | 88 ++++-- crates/minvmd/src/box_registry.rs | 98 +++++++ crates/minvmd/src/control.rs | 426 ++++++++++++++++++++++++++---- 3 files changed, 552 insertions(+), 60 deletions(-) diff --git a/crates/minimald-rpc/src/lib.rs b/crates/minimald-rpc/src/lib.rs index 1f3e1d0b9c..666c4e724a 100644 --- a/crates/minimald-rpc/src/lib.rs +++ b/crates/minimald-rpc/src/lib.rs @@ -414,16 +414,18 @@ fn default_hooks_enabled() -> bool { true } -/// The wire types of the VM host daemon's box-registration control socket -/// (T66): the one door a client has to the host-side box table (NET-138). +/// The wire types of the VM host daemon's box control socket (T66): the +/// one door a client has to the host-side box table (NET-138). /// /// On a minvmd-backed host, `min session activate` opens this socket — a /// UDS beside the daemon's ssh socket in the provider's state dir — writes -/// one [`RegisterBoxRequest`] as a line of JSON, and reads one -/// [`RegisterBoxReply`] line back. Nothing else crosses it: the session -/// RPCs are the daemon crate's HTTP-shaped socket, and the box's switch -/// and loopback addresses come from here so the daemon's create request -/// can carry them (see [`SessionConfig`]). +/// one [`BoxControlRequest`] line of JSON — a registration when the box is +/// created, a withdrawal when the session that registered it is destroyed +/// or its activation fails — and reads one [`BoxControlReply`] line back. +/// Nothing else crosses it: the session RPCs are the daemon crate's +/// HTTP-shaped socket, and the box's switch and loopback addresses come +/// from here so the daemon's create request can carry them (see +/// [`SessionConfig`]). /// /// These types live here rather than in `minvmd` because both ends depend /// on this crate — the activating client and the host daemon — and the @@ -447,23 +449,79 @@ pub struct RegisterBoxRequest { pub egress: Option, } +/// The withdrawal a destroyed session's client sends for the row its +/// activation registered: the name the row went by and the pair the +/// registration handed back. The pair is the proof that the withdrawer is +/// the row's creator — a row is its registering side's to withdraw (NET-138), +/// and no other client holds the pair, which no session record but the +/// creator's carries. +/// +/// Withdrawal answers [`BoxControlReply::Addresses`] echoing the pair back +/// when the row is gone. The daemon answers a row already withdrawn — or a +/// daemon restarted since the registration — the same way: no row at the +/// named switch address is the goal state either way, so the withdrawal +/// succeeds. A row that *is* published there under a different name or pair +/// is refused with [`BoxControlReply::Error`]: the requesting client is not +/// its creator, and no client may remove another box's row. +/// +/// The addresses themselves are spent for good by design — the host's +/// allocation cursors never regress — so a withdrawal ends a row's +/// admissions without ever returning its addresses to the plan. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct WithdrawBoxRequest { + /// The name the row to withdraw was registered under. + pub name: String, + /// The switch address the registration handed back, which keys the row + /// in the host table. + pub switch_address: std::net::Ipv4Addr, + /// The loopback address the registration handed back, which the pair + /// proof checks against the row's own. + pub loopback_address: std::net::Ipv4Addr, +} + +/// The one request line the control socket takes: which verb the client +/// wants, `register` or `withdraw`, tagged in the line itself. +/// +/// The tag is the version corner, and it leans the safe way. A daemon that +/// predates a verb cannot parse the tagged line and refuses it — and an old +/// CLI's registration, untagged, fails the new daemon's parse the same way, +/// so the daemon is autospawned by the CLI from the same install and the +/// mixed-version pair is the corner, not the rule. Ordered dispatch would +/// parse an old daemon's withdraw line as a *spurious registration* — a row +/// with no ports and an allow-all policy, spending a finite hand-out +/// address — which is worse than the refusal: the registering side's row +/// stays published either way, but only the refusal leaves no new fact on +/// the host. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "verb", rename_all = "snake_case")] +pub enum BoxControlRequest { + /// Register the box's declaration: allocate the box's addresses on the + /// host and fill the row the egress gate decides by. + Register(RegisterBoxRequest), + /// Withdraw the row a registration filled: the destroyed or failed + /// session's creator presenting the pair the registration handed back. + Withdraw(WithdrawBoxRequest), +} + /// The addresses a successful registration hands back /// ([`sessions::BoxAddresses`]): the box's switch address and its published /// loopback address, both allocated on the host from the address plan the /// host switch serves — the same pair the create request carries so the /// in-VM daemon attaches with it. /// -/// The one reply line the control socket answers a registration with: the -/// handed addresses, or the reason the registration did not happen. Untagged -/// so the reply stays one flat JSON object either way. +/// The one reply line the control socket answers either verb with: the +/// handed addresses, or the reason the verb did not happen. A registration +/// hands the allocated pair back; a withdrawal echoes the pair it withdrew +/// by, so the client can check the daemon meant the row it asked about. +/// Untagged so the reply stays one flat JSON object either way. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(untagged)] -pub enum RegisterBoxReply { - /// Registration succeeded: both addresses are allocated and the host - /// table's row is filled. +pub enum BoxControlReply { + /// The verb succeeded: a registration's allocated addresses, or a + /// withdrawal's echo of the pair the row went by. Addresses(BoxAddresses), - /// Registration failed: `error` is a sentence naming why, for the - /// activating client to warn with. + /// The verb failed: `error` is a sentence naming why, for the client to + /// warn with. Error { error: String }, } diff --git a/crates/minvmd/src/box_registry.rs b/crates/minvmd/src/box_registry.rs index 0e81d27141..9fd9044a6c 100644 --- a/crates/minvmd/src/box_registry.rs +++ b/crates/minvmd/src/box_registry.rs @@ -218,6 +218,44 @@ pub enum AllocationError { UnplannedSubnet(SwitchSubnet), } +/// Why a client-driven withdrawal was refused. The pair a withdrawal +/// presents is the proof that its client is the row's creator (T66), so a +/// refusal is the daemon saying the proof does not match the row the +/// switch address publishes — never a failure of the goal state, which +/// [`BoxRegistry::withdraw_client_box`] reports as `Ok(None)`. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum WithdrawError { + /// A row is published at the switch address under another box's name: + /// the requesting client is not its creator. + #[error( + "the row at switch address {switch_addr} is held by box {held_name:?}, \ + not by the withdrawing {asked_name:?}" + )] + NotTheCreatorsRow { + /// The switch address the withdrawal named. + switch_addr: Ipv4Addr, + /// The name the published row carries. + held_name: String, + /// The name the withdrawal presented. + asked_name: String, + }, + /// The row at the switch address carries the requested name but another + /// loopback address than the pair presented: not the pair the + /// registration handed back. + #[error( + "the row at switch address {switch_addr} carries loopback address \ + {held_loopback}, not the {asked_loopback} the withdrawal presented" + )] + NotTheHandedPair { + /// The switch address the withdrawal named. + switch_addr: Ipv4Addr, + /// The loopback address the published row carries. + held_loopback: Ipv4Addr, + /// The loopback address the withdrawal presented. + asked_loopback: Ipv4Addr, + }, +} + /// The writable half of the host-side table, held by the host process: the /// registration surface — the host's own node-namespace row, and T66's /// client-driven path — and the source of the read-only [`BoxTable`] the @@ -364,6 +402,14 @@ impl BoxRegistry { /// rate-limit slots, the switch's static lease table — is keyed by it, /// and re-issuing it to a new box would inherit all of that; a fresh /// address starts clean. + /// + /// While a row this registration filled stands, the box's frames are + /// decided by its rules; what a box with **no** row runs as — one whose + /// registration never reached the daemon, or whose row was withdrawn — + /// is the gate's announced unregistered-source interim, and putting the + /// per-box default that eventually refuses it in force is the flip that + /// lands with the last row source (T66's follow-up), not a change this + /// registration makes. pub fn register_client_box( &self, spec: ClientBoxSpec, @@ -388,6 +434,58 @@ impl BoxRegistry { Ok(self.register(registration)) } + /// Withdraws the client box's row when the pair `(name, switch_addr, + /// loopback_addr)` proves its client is the row's creator, returning the + /// row removed — `Ok(None)` when no row is published at `switch_addr` at + /// all: the row is already withdrawn, or the daemon restarted since the + /// registration, and either way the goal state — nothing admits the + /// pair's addresses by a row — already holds. Lookup, proof, and removal + /// happen under one write of the row lock, so no registration can land + /// between the proof and the removal. + /// + /// The proof is the same pair the registration handed back + /// ([`ClientBoxSpec`]'s allocation), which only the registering session's + /// record carries; a row published under another name or another + /// loopback is not the requesting client's to remove and is refused with + /// [`WithdrawError`]. The withdrawn addresses are not returned to the + /// allocation cursors — spent for good, as [`Self::register_client_box`] + /// documents — and what their frames do next is the gate phase's to say + /// ([`Self::withdraw`]). + /// + /// This is the host-side half of the withdrawal a destroyed or failed + /// activation sends over the control socket + /// ([`crate::control::serve_request`]) — the guest daemon never asserts + /// or withdraws address→box facts (NET-138). + pub fn withdraw_client_box( + &self, + name: &str, + switch_addr: Ipv4Addr, + loopback_addr: Ipv4Addr, + ) -> Result>, WithdrawError> { + let mut rows = self + .rows + .write() + .expect("the row lock is never held across a panic, so it cannot be poisoned"); + let Some(record) = rows.get(&switch_addr.octets()) else { + return Ok(None); + }; + if record.name() != name { + return Err(WithdrawError::NotTheCreatorsRow { + switch_addr, + held_name: record.name().to_string(), + asked_name: name.to_string(), + }); + } + if record.loopback_addr() != loopback_addr { + return Err(WithdrawError::NotTheHandedPair { + switch_addr, + held_loopback: record.loopback_addr(), + asked_loopback: loopback_addr, + }); + } + Ok(rows.remove(&switch_addr.octets())) + } + /// Publishes the guest **node's** own namespace: the in-VM daemon's /// root-netns tap, the plane design §5.1 names node-plane. The address is /// the host's own derivation from the subnet it configured the switch diff --git a/crates/minvmd/src/control.rs b/crates/minvmd/src/control.rs index dd6f283539..5128a03b84 100644 --- a/crates/minvmd/src/control.rs +++ b/crates/minvmd/src/control.rs @@ -1,15 +1,17 @@ -//! The VM host daemon's box-registration control socket (T66) — the one -//! host-side door a client has to the box table ([`crate::box_registry`], -//! NET-138). +//! The VM host daemon's box control socket (T66) — the one host-side door a +//! client has to the box table ([`crate::box_registry`], NET-138). //! //! On a minvmd-backed host, `min session activate` registers every //! own-address box here before it creates the session: it writes one -//! [`minimald_rpc::RegisterBoxRequest`] line, the daemon allocates the box's -//! switch and loopback addresses into the table — the row the host-side -//! egress gate decides every frame by — and answers with the -//! [`minimald_rpc::RegisterBoxAddresses`] the create request then carries, -//! so the in-VM daemon attaches with the handed address instead of drawing -//! its own. One connection, one request line in, one reply line out. +//! [`minimald_rpc::BoxControlRequest`] line asking `register`, the daemon +//! allocates the box's switch and loopback addresses into the table — the +//! row the host-side egress gate decides every frame by — and answers with +//! the addresses the create request then carries, so the in-VM daemon +//! attaches with the handed address instead of drawing its own. The same +//! client withdraws the row when the session is destroyed or the activation +//! fails after registering: one line asking `withdraw`, answered with the +//! pair echoed back. One connection, one request line in, one reply line +//! out. //! //! The socket lives beside the daemon's ssh socket in the provider-instance //! dir and is created with the same 0700-dir / 0600-socket posture the @@ -27,6 +29,11 @@ //! and requests are served serially, one connection at a time, each read //! bounded by a 30-second timeout. That is what v1 ships, not a design //! endpoint. +//! +//! The interim is also where a box whose row is gone runs — never +//! registered, or withdrawn at destroy — and putting the per-box default +//! that eventually refuses it in force is the flip that lands with the last +//! row source (T66's follow-up), not a change this socket makes. use std::io::{Read, Write}; use std::os::unix::net::{UnixListener, UnixStream}; @@ -34,7 +41,7 @@ use std::path::PathBuf; use std::thread::JoinHandle; use std::time::Duration; -use minimald_rpc::{BoxAddresses, RegisterBoxReply, RegisterBoxRequest}; +use minimald_rpc::{BoxAddresses, BoxControlReply, BoxControlRequest, RegisterBoxRequest, WithdrawBoxRequest}; use crate::box_registry::{BoxRegistry, ClientBoxSpec}; @@ -61,9 +68,9 @@ pub fn resolve_control_sock() -> std::io::Result { Ok(crate::state::provider_dir().join(CONTROL_SOCK_FILE)) } -/// Bind the control socket at `sock_path` and serve box registrations -/// against `boxes` on a dedicated thread, whose handle the caller holds for -/// as long as the daemon lives. +/// Bind the control socket at `sock_path` and serve box control requests +/// — registrations and their withdrawals — against `boxes` on a dedicated +/// thread, whose handle the caller holds for as long as the daemon lives. /// /// The bind happens on the calling thread so its failure surfaces to the /// supervisor's own startup error handling; only the accept loop moves to @@ -81,15 +88,15 @@ pub fn spawn(sock_path: PathBuf, boxes: BoxRegistry) -> std::io::Result { if let Err(error) = serve_connection(stream, &boxes) { - tracing::debug!(error = %error, "box registration connection failed"); + tracing::debug!(error = %error, "box control connection failed"); } } Err(error) => tracing::debug!(error = %error, "control socket accept failed"), @@ -97,7 +104,7 @@ fn accept_loop(listener: UnixListener, boxes: BoxRegistry) { } } -/// Serve one registration: read the request line, allocate, answer. +/// Serve one request: read the line, dispatch the verb, answer. fn serve_connection(stream: UnixStream, boxes: &BoxRegistry) -> std::io::Result<()> { let mut stream = stream; stream.set_read_timeout(Some(REGISTER_READ_TIMEOUT))?; @@ -110,11 +117,28 @@ fn serve_connection(stream: UnixStream, boxes: &BoxRegistry) -> std::io::Result< Ok(request) => request, Err(error) => { let error = error.to_string(); - tracing::debug!(error = %error, "box registration request did not parse"); - return write_reply(&mut stream, &RegisterBoxReply::Error { error }); + tracing::debug!(error = %error, "box control request did not parse"); + return write_reply(&mut stream, &BoxControlReply::Error { error }); } }; - register_and_reply(&mut stream, boxes, request) + serve_request(&mut stream, boxes, request) +} + +/// Dispatch one parsed request to its verb and write its one reply line. +/// +/// The verb dispatch is where the wire's parse refusal pays off: a line +/// that names no verb this build knows never reaches the table at all, so +/// a skewed client cannot make a withdraw look like a register (the +/// [`minimald_rpc::BoxControlRequest`] docs carry that corner). +fn serve_request( + stream: &mut UnixStream, + boxes: &BoxRegistry, + request: BoxControlRequest, +) -> std::io::Result<()> { + match request { + BoxControlRequest::Register(request) => register_and_reply(stream, boxes, request), + BoxControlRequest::Withdraw(request) => withdraw_and_reply(stream, boxes, request), + } } /// Read one line (terminated by `\n`) of the registration request. A @@ -159,7 +183,7 @@ fn read_request_line(stream: &mut UnixStream) -> std::io::Result> } } -fn parse_request(line: &str) -> Result { +fn parse_request(line: &str) -> Result { serde_json_lenient::from_str(line) } @@ -185,7 +209,7 @@ fn register_and_reply( loopback_address = %record.loopback_addr(), "registered box with the VM host daemon; addresses allocated" ); - RegisterBoxReply::Addresses(BoxAddresses { + BoxControlReply::Addresses(BoxAddresses { switch_address: record.switch_addr(), loopback_address: record.loopback_addr(), }) @@ -196,7 +220,58 @@ fn register_and_reply( error = %error, "box registration refused" ); - RegisterBoxReply::Error { + BoxControlReply::Error { + error: error.to_string(), + } + } + }; + write_reply(stream, &reply) +} + +/// Remove the row the request's pair proves its client created, and write +/// the reply — the pair echoed back on success, the reason on a refusal. +/// One info line per withdrawal names the box and both addresses, mirroring +/// the registration's; a withdrawal that finds no row is the goal state +/// already holding (already withdrawn, or the daemon restarted since) and +/// is a debug line, not an error. +fn withdraw_and_reply( + stream: &mut UnixStream, + boxes: &BoxRegistry, + request: WithdrawBoxRequest, +) -> std::io::Result<()> { + let reply = match boxes.withdraw_client_box( + &request.name, + request.switch_address, + request.loopback_address, + ) { + Ok(withdrawn) => { + if withdrawn.is_some() { + tracing::info!( + box = %request.name, + switch_address = %request.switch_address, + loopback_address = %request.loopback_address, + "withdrew the box's host row; its addresses admit nothing" + ); + } else { + tracing::debug!( + box = %request.name, + switch_address = %request.switch_address, + "no host row held at the withdrawn switch address; already withdrawn" + ); + } + BoxControlReply::Addresses(BoxAddresses { + switch_address: request.switch_address, + loopback_address: request.loopback_address, + }) + } + Err(error) => { + tracing::debug!( + box = %request.name, + switch_address = %request.switch_address, + error = %error, + "box row withdrawal refused" + ); + BoxControlReply::Error { error: error.to_string(), } } @@ -204,11 +279,11 @@ fn register_and_reply( write_reply(stream, &reply) } -fn write_reply(stream: &mut UnixStream, reply: &RegisterBoxReply) -> std::io::Result<()> { +fn write_reply(stream: &mut UnixStream, reply: &BoxControlReply) -> std::io::Result<()> { let mut line = serde_json_lenient::to_string(reply).map_err(|error| { std::io::Error::new( std::io::ErrorKind::InvalidData, - format!("box registration reply did not serialize: {error}"), + format!("box control reply did not serialize: {error}"), ) })?; line.push('\n'); @@ -223,7 +298,7 @@ mod tests { use std::sync::OnceLock; use std::time::Duration; - use minimald_rpc::RegisterBoxRequest; + use minimald_rpc::{BoxControlReply, BoxControlRequest, RegisterBoxRequest, WithdrawBoxRequest}; use switch::SwitchSubnet; use crate::box_registry::AllocationError; @@ -255,13 +330,18 @@ mod tests { }) } - /// Spawns the control server on a temp path and returns (path, handle - /// keeping the server thread identified). The thread outlives the test - /// the way a daemon's does; the temp dir's drop after the test closes - /// the test's view of the socket. - fn spawn_server(dir: &std::path::Path) -> std::io::Result<(PathBuf, JoinHandle<()>)> { + /// Spawns the control server on a temp path over a fresh registry and + /// returns (path, handle keeping the server thread identified, the + /// registry the server serves — the same rows a gate the test brings up + /// later shares). The thread outlives the test the way a daemon's does; + /// the temp dir's drop after the test closes the test's view of the + /// socket. + fn spawn_server( + dir: &std::path::Path, + ) -> std::io::Result<(PathBuf, JoinHandle<()>, BoxRegistry)> { let sock_path = dir.join(CONTROL_SOCK_FILE); - let handle = spawn(sock_path.clone(), BoxRegistry::new(SUBNET))?; + let boxes = BoxRegistry::new(SUBNET); + let handle = spawn(sock_path.clone(), boxes.clone())?; // Wait until the socket accepts rather than racing the bind. for _ in 0..500 { if TestStream::connect(&sock_path).is_ok() { @@ -269,15 +349,15 @@ mod tests { } std::thread::sleep(Duration::from_millis(2)); } - Ok((sock_path, handle)) + Ok((sock_path, handle, boxes)) } /// A client that writes the request and reads the reply line back, - /// mirroring the CLI's registration helper. - fn register( + /// mirroring the CLI's control helper. + fn control( sock_path: &std::path::Path, - request: &RegisterBoxRequest, - ) -> std::io::Result { + request: &BoxControlRequest, + ) -> std::io::Result { let mut stream = TestStream::connect(sock_path)?; let mut line = serde_json_lenient::to_string(request).map_err(|error| { std::io::Error::other(format!("request did not serialize: {error}")) @@ -291,13 +371,22 @@ mod tests { .map_err(|error| std::io::Error::other(format!("reply did not parse: {error}"))) } + /// A client that registers a box, wrapping the registration into the + /// wire envelope. + fn register( + sock_path: &std::path::Path, + request: &RegisterBoxRequest, + ) -> std::io::Result { + control(sock_path, &BoxControlRequest::Register(request.clone())) + } + /// The registered box's two addresses, asserted as the reply the daemon /// hands back. - fn handed(reply: RegisterBoxReply) -> BoxAddresses { + fn handed(reply: BoxControlReply) -> BoxAddresses { match reply { - RegisterBoxReply::Addresses(addresses) => addresses, - RegisterBoxReply::Error { error } => { - panic!("a valid registration is answered with addresses, refused with {error}") + BoxControlReply::Addresses(addresses) => addresses, + BoxControlReply::Error { error } => { + panic!("a valid request is answered with addresses, refused with {error}") } } } @@ -313,7 +402,7 @@ mod tests { fn box_addresses_allocated_on_host_and_handed_to_daemon() { let capture = server_capture(); let dir = tempfile::TempDir::new().expect("temp dir"); - let (sock_path, _server) = spawn_server(dir.path()).expect("server binds"); + let (sock_path, _server, _boxes) = spawn_server(dir.path()).expect("server binds"); // The first registration is handed the hand-out run's first switch // address — the plan's PTask run above the daemon's self-allocation @@ -401,10 +490,10 @@ mod tests { let mut reader = BufReader::new(stream); let mut reply = String::new(); reader.read_line(&mut reply).expect("reply read"); - let refused: RegisterBoxReply = + let refused: BoxControlReply = serde_json_lenient::from_str(reply.trim()).expect("error reply parses"); assert!( - matches!(refused, RegisterBoxReply::Error { .. }), + matches!(refused, BoxControlReply::Error { .. }), "a malformed request is answered with the reason, got {refused:?}" ); @@ -504,4 +593,251 @@ mod tests { "a refused registration publishes no row" ); } + + /// The withdrawal round-trip the destroyed session's client drives + /// (T66): it registers the box, presents the pair the registration + /// handed back to prove it is the row's creator, and the daemon removes + /// the row from its table — at both addresses. What the gate then + /// decides at the withdrawn switch address is the phase's to say, and + /// both arms are pinned at relay level through the phase parameter (not + /// the shipped constant), so the flip that ends the interim breaks + /// nothing here: the announced interim admits the withdrawn address's + /// frame under its warn, the in-force per-box default drops it. + #[tokio::test] + async fn host_row_withdrawn_on_destroy() { + use crate::net::egress_gate::{UnregisteredSourcePhase, test_support}; + + let capture = server_capture(); + let dir = tempfile::TempDir::new().expect("temp dir"); + let (sock_path, _server, registry) = spawn_server(dir.path()).expect("server binds"); + + // The registering client holds the pair the registration hands + // back — the destroy-side proof it is the row's creator. + let web = handed( + register( + &sock_path, + &RegisterBoxRequest { + name: "web".to_string(), + ingress_ports: vec![8080], + egress: Some(sessions::EgressPolicy { + allow_protocols: Some(vec![sessions::IpProto::Tcp]), + allow_subnets: Some(vec!["10.0.0.0/8".to_string()]), + allow_dns_hosts: None, + deny_subnets: None, + }), + }, + ) + .expect("the registration is answered"), + ); + let row = registry + .table() + .by_source(web.switch_address.octets()) + .expect("the registration published the row the gate reads"); + assert_eq!(row.loopback_addr(), web.loopback_address); + + // The withdrawal round-trips and is answered with the pair echoed + // back, and the table holds no row at either address after it. + let reply = control( + &sock_path, + &BoxControlRequest::Withdraw(WithdrawBoxRequest { + name: "web".to_string(), + switch_address: web.switch_address, + loopback_address: web.loopback_address, + }), + ) + .expect("the withdrawal is answered"); + match reply { + BoxControlReply::Addresses(echoed) => { + assert_eq!(echoed, web, "the withdrawal echoes the pair it went by"); + } + BoxControlReply::Error { error } => { + panic!("the creator's withdrawal is answered with the pair, refused with {error}") + } + } + assert!( + registry + .table() + .by_source(web.switch_address.octets()) + .is_none(), + "the withdrawn switch address publishes no row" + ); + assert!( + registry + .table() + .rows() + .iter() + .all(|row| row.loopback_addr() != web.loopback_address), + "the withdrawn loopback address publishes no row either" + ); + + // One info line per withdrawal, mirroring the registration's. + let log = capture.contents(); + assert!( + log.contains("withdrew the box's host row; its addresses admit nothing"), + "one info line names the withdrawal: {log}" + ); + assert!( + log.contains("box=web") + && log.contains(&format!("switch_address={}", web.switch_address)) + && log.contains(&format!("loopback_address={}", web.loopback_address)), + "the info line names the box and both addresses: {log}" + ); + + // A row is its creator's to withdraw, so a live row whose proof + // does not match is refused with the reason and stays published: + // another box's name at this row's address, and the right name with + // a loopback the registration did not hand back. + let db = handed( + register( + &sock_path, + &RegisterBoxRequest { + name: "db".to_string(), + ingress_ports: Vec::new(), + egress: None, + }, + ) + .expect("the marker box is registered"), + ); + for (name, loopback, needle) in [ + ( + "web", + db.loopback_address, + "held by box", + ), + ( + "db", + Ipv4Addr::LOCALHOST, + "carries loopback address", + ), + ] { + let refused = control( + &sock_path, + &BoxControlRequest::Withdraw(WithdrawBoxRequest { + name: name.to_string(), + switch_address: db.switch_address, + loopback_address: loopback, + }), + ) + .expect("the withdrawal is answered"); + match refused { + BoxControlReply::Error { error } => { + assert!( + error.contains(needle), + "the refusal names why the pair is not this row's ({needle}): {error}" + ); + } + BoxControlReply::Addresses(..) => { + panic!("a foreign pair's withdrawal must be refused, got addresses") + } + } + } + assert!( + registry + .table() + .by_source(db.switch_address.octets()) + .is_some(), + "a refused withdrawal leaves the row published" + ); + + // No row at the address is the goal state either way, so withdrawing + // the same pair again is answered the same way. + let again = control( + &sock_path, + &BoxControlRequest::Withdraw(WithdrawBoxRequest { + name: "web".to_string(), + switch_address: web.switch_address, + loopback_address: web.loopback_address, + }), + ) + .expect("the repeat withdrawal is answered"); + match again { + BoxControlReply::Addresses(echoed) => { + assert_eq!(echoed, web, "the repeat withdrawal echoes the pair too"); + } + BoxControlReply::Error { error } => { + panic!("no row at the address is success, refused with {error}") + } + } + + // A line that names no verb this build knows is refused at the + // parse, never reaching the table — the tagged wire's corner. + let mut stream = TestStream::connect(&sock_path).expect("socket accepts"); + stream + .write_all(b"{\"verb\":\"retire\",\"name\":\"web\"}\n") + .expect("write the unknown-verb line"); + let mut reply = String::new(); + BufReader::new(stream) + .read_line(&mut reply) + .expect("reply read"); + let refused: BoxControlReply = + serde_json_lenient::from_str(reply.trim()).expect("error reply parses"); + assert!( + matches!(refused, BoxControlReply::Error { .. }), + "an unknown verb is refused, got {refused:?}" + ); + assert!( + registry.table().by_source(db.switch_address.octets()).is_some(), + "a refused line publishes nothing and removes nothing" + ); + + // The gate shares the registry's table, as production does: the + // announced interim admits the withdrawn address's frame under its + // warn — the row's rules ended with the row — while a published + // box's frame is the marker that proves nothing else slipped. + let marker = test_support::ipv4_frame(db.switch_address.octets(), 6, [10, 1, 2, 3], 80); + let from_withdrawn = + test_support::ipv4_frame(web.switch_address.octets(), 6, [10, 1, 2, 3], 80); + let mut h = test_support::gate_over_with_phase( + registry.clone(), + UnregisteredSourcePhase::Announced, + ) + .await; + test_support::send_frame(&mut h.guest, &from_withdrawn).await; + test_support::send_frame(&mut h.guest, &marker).await; + let seen = test_support::expect_frame(&mut h.switch).await; + assert_eq!( + seen, from_withdrawn, + "the announced interim admits the withdrawn address's frame" + ); + let seen = test_support::expect_frame(&mut h.switch).await; + assert_eq!(seen, marker, "the published box's frame is the marker"); + test_support::expect_silence(&mut h.switch).await; + test_support::wait_for_log(&h.log, "egress-unregistered-source").await; + let logged = h.log.contents(); + assert!( + logged.contains(&format!("source={}", web.switch_address)), + "the interim's line names the withdrawn source, got: {logged}" + ); + assert!( + logged.contains("T66 (#1711)"), + "the interim's line names the path that ends it, got: {logged}" + ); + + // And the in-force arm drops the same frame — the per-box default + // the flip puts in force — while the marker still passes. + let mut h = test_support::gate_over_with_phase( + registry.clone(), + UnregisteredSourcePhase::InForce, + ) + .await; + test_support::send_frame(&mut h.guest, &from_withdrawn).await; + test_support::send_frame(&mut h.guest, &marker).await; + let seen = test_support::expect_frame(&mut h.switch).await; + assert_eq!( + seen, marker, + "the in-force default drops the withdrawn address's frame and \ + passes the published box's" + ); + test_support::expect_silence(&mut h.switch).await; + test_support::wait_for_log(&h.log, "egress-unknown-source").await; + let logged = h.log.contents(); + assert!( + logged.contains(&format!("source={}", web.switch_address)), + "the drop line names the withdrawn source, got: {logged}" + ); + assert!( + !logged.contains("egress-unregistered-source"), + "the in-force arm fires no interim line, got: {logged}" + ); + } } From 692ebe59e105c1ac8bf455eeb68519262f42b3ea Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:46:16 +0000 Subject: [PATCH 15/18] feat(minimal): withdraw the box's row on destroy and failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The session's creator now speaks the withdraw verb everywhere the registration bought a row it no longer holds: at destroy (single and --all, whose per-session record fetch carries the pair), at every activation failure after registering — create transport and daemon refusals, the autogen retry's abandoned row, the version-skew bail, compose and verdict failures, the finalize teardown, the Ctrl-C guard — and the registration itself moved to just before the create, the last fallible client-side step, so the checks between config and create can no longer strand a row. Withdrawals are best-effort and quiet when nothing is owed; the Ctrl-C guard fetches the record before aborting, which may take the pair with the session. cmd/mod.rs and task.rs are outside this task's layers; the guard's withdrawal required the former (the signature is unchanged, so task sessions need no edit) — noted in result.json. --- crates/minimal/src/cmd/mod.rs | 26 ++ crates/minimal/src/cmd/session.rs | 415 +++++++++++++++++++++++++----- 2 files changed, 375 insertions(+), 66 deletions(-) diff --git a/crates/minimal/src/cmd/mod.rs b/crates/minimal/src/cmd/mod.rs index 997db4c6dd..2ee3ab3f27 100644 --- a/crates/minimal/src/cmd/mod.rs +++ b/crates/minimal/src/cmd/mod.rs @@ -23,6 +23,11 @@ mod net; mod project; mod session; +// The Ctrl-C cleanup (`arm_activation_interrupt`) withdraws the row the +// activation registered with the VM host daemon; the withdrawal lives with +// the session commands. +use session::{vm_host_control_sock, withdraw_box_row}; + pub use admin::*; pub use list::*; pub use net::*; @@ -663,6 +668,9 @@ pub(crate) fn arm_activation_interrupt( session_id: sessions::SessionId, ) -> ActivationInterrupt { let sock = client::resolve_socket_path(global.minimal_dir.as_deref(), global.use_minvmd()); + // Resolved here, not inside the task: the withdrawal's socket is the + // same provider dir's, and the borrow must not cross the spawn. + let control_sock = vm_host_control_sock(global); let task = tokio::spawn(async move { // Only the first Ctrl-C is intercepted; a second falls through to // the default disposition so a wedged cleanup can still be killed. @@ -677,9 +685,27 @@ pub(crate) fn arm_activation_interrupt( && let Ok(mut client) = client::Client::connect(&sock).await { use minimald_rpc::{AbortSession, AbortSessionRequest}; + // The record is fetched **before** the abort, which may take it + // with the session: it carries the pair the box's registration + // handed back (T66), whose row is withdrawn after the abort — + // the creator's withdrawal, best-effort. A session that + // registered no box (a task session, a native host) holds no + // pair, and the withdrawal stays silent for it. + let row = get_session_record(&mut client, &session_id.to_string()) + .await + .ok() + .and_then(|resp| resp.record); let _ = client .oneshot_rpc::(AbortSessionRequest { id: session_id }) .await; + if let Some(record) = row { + withdraw_box_row( + control_sock, + record.name.as_deref(), + record.box_addresses, + ) + .await; + } } std::process::exit(130); }); diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index 04b04456ec..8a4354f87e 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -27,27 +27,28 @@ pub async fn cmd_activate(global: &GlobalArgs, args: ActivateArgs) -> Result<(), activate_session(global, args, true).await } -/// How long a box registration may take before this activation gives up on -/// it. The control socket answers with two map writes and an allocation, so -/// healthy is milliseconds; the bound exists so a hung VM host daemon cannot -/// hang the activation the user asked for. -const BOX_REGISTRATION_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); - -/// Registers an own-address box with the VM host daemon over its control -/// socket (T66), returning the addresses it handed back. +/// How long a box control request — a registration, or the withdrawal of +/// the row a registration bought — may take before this invocation gives +/// up on it. The control socket answers with map writes and an allocation, +/// so healthy is milliseconds; the bound exists so a hung VM host daemon +/// cannot hang the activation or destroy the user asked for. +const BOX_CONTROL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// One control-socket exchange with the VM host daemon (T66): one JSON +/// line in — the verb's request — and one line out — the untagged reply +/// the verb is answered with. /// /// The control socket lives beside the ssh socket in the minvmd -/// provider-instance dir — the dir the daemon connection resolves through — -/// and speaks one JSON line each way: the box's expanded declaration in, the -/// allocated switch and loopback addresses out -/// ([`minimald_rpc::RegisterBoxReply`]). A refused registration (an -/// exhausted address plan, a malformed declaration) surfaces here as an -/// error naming the reason; the caller degrades it to a warning rather than -/// failing the activation. -async fn register_box_with_vm_host( +/// provider-instance dir — the dir the daemon connection resolves through. +/// A refusal (an exhausted address plan, a malformed declaration, a pair +/// that is not the row's) surfaces here as an error naming the reason; the +/// caller degrades a failed registration to a warning rather than failing +/// the activation, and a failed withdrawal to one rather than failing the +/// destroy it rides on. +async fn control_request_with_vm_host( sock_path: &std::path::Path, - request: minimald_rpc::RegisterBoxRequest, -) -> anyhow::Result { + request: minimald_rpc::BoxControlRequest, +) -> anyhow::Result { use tokio::io::AsyncBufReadExt as _; use tokio::io::AsyncWriteExt as _; @@ -55,39 +56,156 @@ async fn register_box_with_vm_host( .await .with_context(|| { format!( - "connecting to the VM host daemon's box-registration control socket at {}", + "connecting to the VM host daemon's box control socket at {}", sock_path.display() ) })?; - let mut line = serde_json_lenient::to_string(&request) - .context("serializing the box registration request")?; + let mut line = + serde_json_lenient::to_string(&request).context("serializing the box control request")?; line.push('\n'); stream .write_all(line.as_bytes()) .await - .context("writing the box registration request")?; + .context("writing the box control request")?; let mut reply = String::new(); tokio::io::BufReader::new(stream) .read_line(&mut reply) .await - .context("reading the box registration reply")?; + .context("reading the box control reply")?; if reply.trim().is_empty() { anyhow::bail!( - "the VM host daemon closed its control socket without answering the box registration" + "the VM host daemon closed its control socket without answering the box control request" ); } - let reply: minimald_rpc::RegisterBoxReply = serde_json_lenient::from_str(reply.trim()) - .with_context(|| { - format!("the VM host daemon's box registration reply did not parse: {reply}") - })?; - match reply { - minimald_rpc::RegisterBoxReply::Addresses(addresses) => Ok(addresses), - minimald_rpc::RegisterBoxReply::Error { error } => { + let reply: minimald_rpc::BoxControlReply = serde_json_lenient::from_str(reply.trim()) + .with_context(|| format!("the VM host daemon's box control reply did not parse: {reply}"))?; + Ok(reply) +} + +/// Registers an own-address box with the VM host daemon over its control +/// socket (T66), returning the addresses it handed back. +async fn register_box_with_vm_host( + sock_path: &std::path::Path, + request: minimald_rpc::RegisterBoxRequest, +) -> anyhow::Result { + match control_request_with_vm_host( + sock_path, + minimald_rpc::BoxControlRequest::Register(request), + ) + .await? + { + minimald_rpc::BoxControlReply::Addresses(addresses) => Ok(addresses), + minimald_rpc::BoxControlReply::Error { error } => { anyhow::bail!("the VM host daemon refused the box registration: {error}") } } } +/// The VM host daemon's control socket, when this invocation can reach one +/// (T66): it sits beside the ssh socket in the provider dir the daemon +/// connection resolves through, so the client finds both by the same rule +/// — named VMs included, since the resolution reads the same process-global +/// VM name. +/// +/// Silent by design: an invocation that owes nothing — no VM host, an +/// unresolvable dir — asks for nothing, and the withdrawal's own warning is +/// the one that names the row left published. +pub(crate) fn vm_host_control_sock(global: &GlobalArgs) -> Option { + if !global.use_minvmd() { + return None; + } + let ssh_sock = client::resolve_socket_path(global.minimal_dir.as_deref(), true).ok()?; + ssh_sock + .parent() + .map(|dir| dir.join(minvmd::control::CONTROL_SOCK_FILE)) +} + +/// Withdraws the box's host row (T66) when the session that registered it +/// is gone — destroyed, or an activation that failed after registering: +/// the row has no holder left, so its creator presents the pair the +/// registration handed back and the daemon stops the pair's addresses +/// admitting anything. +/// +/// Always quiet when it owes nothing: a session that registered no box — a +/// native host, a host-ip box sharing the node's own row, a refused +/// registration — withdraws nothing and says nothing. A withdrawal that +/// cannot be made — no control socket, a daemon that predates the verb and +/// refuses the line, the deadline — leaves the row published and warns +/// rather than failing the destroy or the activation error it rides on; +/// the daemon restarting between registration and withdrawal answers the +/// withdrawal as already gone, which is the goal state either way. +pub(crate) async fn withdraw_box_row( + control_sock: Option, + name: Option<&str>, + box_addresses: Option, +) { + let Some(name) = name else { return }; + let Some(addresses) = box_addresses else { return }; + let Some(sock_path) = control_sock else { + tracing::warn!( + box = %name, + "cannot resolve the VM host daemon's control socket; the box's \ + host row stays published until the daemon next restarts" + ); + return; + }; + let request = minimald_rpc::WithdrawBoxRequest { + name: name.to_string(), + switch_address: addresses.switch_address, + loopback_address: addresses.loopback_address, + }; + let withdrawn = tokio::time::timeout( + BOX_CONTROL_TIMEOUT, + control_request_with_vm_host( + &sock_path, + minimald_rpc::BoxControlRequest::Withdraw(request), + ), + ) + .await; + match withdrawn { + Ok(Ok(reply)) => match reply { + // The daemon answers the withdrawal with the pair it went by; a + // pair back that is not the pair asked is a daemon speaking + // another protocol's answer — the row asked for is not + // withdrawn, so say so. + minimald_rpc::BoxControlReply::Addresses(handed) if handed == addresses => { + tracing::debug!(box = %name, "withdrew the box's host row"); + } + minimald_rpc::BoxControlReply::Addresses(handed) => { + tracing::warn!( + box = %name, + switch_address = %handed.switch_address, + "the VM host daemon answered the withdrawal with a different \ + address pair; the row asked for stays published" + ); + } + minimald_rpc::BoxControlReply::Error { error } => { + tracing::warn!( + box = %name, + %error, + "the VM host daemon refused the box row withdrawal; the row \ + stays published" + ); + } + }, + Ok(Err(error)) => { + tracing::warn!( + box = %name, + %error, + "the box row withdrawal failed; the row stays published" + ); + } + Err(_) => { + tracing::warn!( + after = ?BOX_CONTROL_TIMEOUT, + box = %name, + "the VM host daemon did not answer the box row withdrawal in \ + time; the row stays published" + ); + } + } +} + /// Registers this activation's box with the VM host daemon, when there is /// one to register with (T66), returning the addresses it handed back — /// `None` when there is nothing to register or the registration could not @@ -162,7 +280,7 @@ async fn register_box_for_activation( egress: policy.egress.clone(), }; let registration = tokio::time::timeout( - BOX_REGISTRATION_TIMEOUT, + BOX_CONTROL_TIMEOUT, register_box_with_vm_host(&sock_path, request), ) .await; @@ -187,7 +305,7 @@ async fn register_box_for_activation( } Err(_) => { tracing::warn!( - after = ?BOX_REGISTRATION_TIMEOUT, + after = ?BOX_CONTROL_TIMEOUT, "the VM host daemon did not answer the box registration in time; \ the box runs unregistered — its frames unattributed — until \ the egress gate's per-box default is in force (fail-closed \ @@ -269,24 +387,17 @@ pub(crate) async fn activate_session( let network: sessions::NetworkMode = args.network.into(); - // T66: an own-address box on a VM-backed host registers with the VM host - // daemon before the create — the host allocates its switch and loopback - // addresses into the table its egress gate decides by and hands them - // back, and the create request carries them so the in-VM daemon attaches - // with the handed switch address instead of drawing its own. Every - // other shape of activation — a host-ip box sharing the node's own row, - // a none box with no switch address, a native daemon with no box table - // — registers nothing and attaches as it always has. - let box_addresses = register_box_for_activation(global, network, &session_name, &policy).await; - // The daemon sources `username` from the authenticated SSH - // connection context; the client doesn't send it. - let config = minimald_rpc::SessionConfig { + // connection context; the client doesn't send it. `box_addresses` + // lands just before the create (T66): the registration is the last + // fallible client-side step, so a loadout, policy, or hook failure + // between here and there never strands a row on the host. + let mut config = minimald_rpc::SessionConfig { name: Some(session_name), project_path: abs_path.clone(), network, policy, - box_addresses, + box_addresses: None, hooks_enabled: !args.no_hooks, attrs: Default::default(), }; @@ -419,6 +530,27 @@ pub(crate) async fn activate_session( } } + // T66: an own-address box on a VM-backed host registers with the VM host + // daemon just before the create — the host allocates the box's switch + // and loopback addresses into the table its egress gate decides by and + // hands them back, and the create request carries them so the in-VM + // daemon attaches with the handed switch address instead of drawing its + // own. Every other shape of activation — a host-ip box sharing the + // node's own row, a none box with no switch address, a native daemon + // with no box table — registers nothing and attaches as it always has. + // This is the last fallible client-side step, so a failure from here on + // owes the row its creator's withdrawal (T66): the sites below send it. + // The control socket is resolved once, here, and the withdrawals ride + // on it. + let control_sock = vm_host_control_sock(global); + config.box_addresses = register_box_for_activation( + global, + config.network, + config.name.as_deref().expect("the session name is minted before the create"), + &config.policy, + ) + .await; + use minimald_rpc::{ ConfigureLoadout, ConfigureLoadoutRequest, CreateSession, CreateSessionRequest, }; @@ -427,7 +559,6 @@ pub(crate) async fn activate_session( // re-mint the hex suffix and retry a bounded number of times. A // user-supplied name never retries — its collision, and any other failure // (e.g. a policy/network-mode validation error), surfaces unchanged. - let mut config = config; let mut attempts = 0u32; let created = loop { let resp = client @@ -439,20 +570,34 @@ pub(crate) async fn activate_session( // operator proceed. must_match_version: version_assertion(), }) - .await - .context("CreateSession RPC failed")?; + .await; + let resp = match resp { + Ok(resp) => resp, + // A transport failure abandons the activation and the row its + // registration bought: the create never happened, so no session + // holds the pair (T66). + Err(error) => { + withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + return Err(error.context("CreateSession RPC failed")); + } + }; match resp { minimald_rpc::Errorable::Ok(r) => break r, minimald_rpc::Errorable::Err { error } => { if should_retry_autogen(autogen, attempts, &error) { attempts += 1; + // The row the first attempt bought is this attempt's to + // leave behind: its creator withdraws it before + // re-registering under the re-minted name (T66) — it was + // bought for a session this create never made. + withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; config.name = Some(autogen_session_name(&utf8_path, &random_hex4())); // A registered box's row carries the name it was - // registered under (T66), so the re-mint re-registers — - // the row the first attempt made stays published behind - // it: the control socket has no withdraw verb yet, its - // addresses are spent for good by design, and the rare - // colliding autogen retry leaves at most one such row. + // registered under (T66), so the re-mint re-registers; + // the abandoned row's addresses stay spent by design + // (the host's cursors never regress), but its + // admissions are withdrawn above, and the retry leaves + // no row behind. config.box_addresses = register_box_for_activation( global, config.network, @@ -462,6 +607,9 @@ pub(crate) async fn activate_session( .await; continue; } + // A create failure that is not a retryable autogen collision + // abandons the activation and its row the same way. + withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; bail!("CreateSession failed: {error}"); } } @@ -472,7 +620,12 @@ pub(crate) async fn activate_session( // itself proof of a skew, so refuse here — still before the upload, the // loadout, and the finalize that #1251 died at, and with the session left // unfinalized for the daemon to reap when this connection drops. - ensure_version_reported(created.daemon_version.as_deref())?; + if let Err(error) = ensure_version_reported(created.daemon_version.as_deref()) { + // A skewed daemon's refusal abandons the unfinalized session and its + // row; the withdrawal rides out with the error (T66). + withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + return Err(error); + } warn_if_hostname_routing_down( created.hostname_routing_unavailable.as_deref(), "min session activate", @@ -596,10 +749,15 @@ pub(crate) async fn activate_session( )?, }; if should_upload { - client + let uploaded = client .upload_workspace_files(id, upload_root.as_std_path()) - .await - .context("Failed to upload project files")?; + .await; + if let Err(error) = uploaded { + // The upload failed: the activation is abandoned, and the + // row its registration bought goes with it (T66). + withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + return Err(error.context("Failed to upload project files")); + } } else if !headless { eprintln!( "Skipping file upload; the session will start with an \ @@ -636,13 +794,22 @@ pub(crate) async fn activate_session( session_id: id, contribution, }) - .await - .context("ConfigureLoadout RPC failed")?; + .await; + let configured = match configured { + // A transport failure abandons the activation and its row (T66). + Err(error) => { + withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + return Err(error.context("ConfigureLoadout RPC failed")); + } + Ok(configured) => configured, + }; let configured = match configured { minimald_rpc::Errorable::Ok(r) => r, // Bails before the `println!("{id}")` below: a session that cannot // compose never puts an id on stdout for a script to capture. minimald_rpc::Errorable::Err { error } => { + // The session cannot compose and is abandoned with its row (T66). + withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; bail!(composition_failure_message(&utf8_path, &error)); } }; @@ -674,6 +841,9 @@ pub(crate) async fn activate_session( Ok((verdict, _final_policy)) => verdict, Err(e) => { send_abort(&mut client, session_id).await; + // The session is aborted and its row withdrawn (T66): + // the path failed after registering. + withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; // The route an activation actually reaches today: the // daemon routes project config back for gating, so a // project it cannot compose surfaces here rather than as @@ -684,6 +854,9 @@ pub(crate) async fn activate_session( let summary = hooks.into_summary(); if summary.count() > 0 { send_abort(&mut client, session_id).await; + // The session is aborted and its row withdrawn (T66): the + // path failed after registering. + withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; let count = summary.count(); let snippet = summary.as_toml_snippet(); bail!( @@ -696,7 +869,13 @@ pub(crate) async fn activate_session( ); } collected_patches.extend(approved_patches_from_verdict(&verdict)); - submit_verdict_and_wait(&mut client, session_id, verdict).await?; + let submitted = submit_verdict_and_wait(&mut client, session_id, verdict).await; + if let Err(error) = submitted { + // The verdict never landed: the activation is abandoned with + // its row (T66). + withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + return Err(error); + } } else { // The hook stashes policy mutations in interior // `RefCell`s so a `DenyPermanent` (which returns @@ -734,6 +913,11 @@ pub(crate) async fn activate_session( Err(e) => eprintln!("warning: failed to update {}: {e}", policy_path.display()), } } + if result.is_err() { + // The activation failed after registering: its row is + // withdrawn with it (T66). + withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + } result?; } } @@ -763,8 +947,10 @@ pub(crate) async fn activate_session( { // Best-effort teardown: the session is stuck in // Materializing on the daemon. Destroy it so the operator's - // `min ls` doesn't fill with half-finalized sessions. + // `min ls` doesn't fill with half-finalized sessions, and + // withdraw the row the registration bought with it (T66). best_effort_destroy(&mut client, id).await; + withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; return Err(e); } @@ -1561,7 +1747,7 @@ pub async fn cmd_destroy(global: &GlobalArgs, args: DestroyArgs) -> Result<(), a let mut client = connect_daemon(global).await?; if args.all { - return destroy_all_sessions(&mut client, args.force).await; + return destroy_all_sessions(global, &mut client, args.force).await; } let session = args @@ -1601,7 +1787,14 @@ pub async fn cmd_destroy(global: &GlobalArgs, args: DestroyArgs) -> Result<(), a } } - destroy_session(&mut client, record.id, record.name.as_deref()).await + destroy_session( + &mut client, + global, + record.id, + record.name.as_deref(), + record.box_addresses, + ) + .await } /// What the daemon's at-risk report means for the destroy gate. @@ -1737,6 +1930,7 @@ pub(crate) async fn session_delta( } pub(crate) async fn destroy_all_sessions( + global: &GlobalArgs, client: &mut client::Client, force: bool, ) -> Result<(), anyhow::Error> { @@ -1766,7 +1960,20 @@ pub(crate) async fn destroy_all_sessions( let session_count = sessions.len(); let mut failures = 0; for session in sessions { - if let Err(error) = destroy_session(client, session.id, session.name.as_deref()).await { + // The record carries the pair the box's registration handed back + // (T66), which the destroy-side withdrawal presents; a session that + // registered no box holds none, and the withdrawal stays silent for + // it. A record the daemon cannot answer destroys the session all the + // same — nothing is left on this path to withdraw the row by. + let box_addresses = get_session_record(client, &session.id.to_string()) + .await + .ok() + .and_then(|resp| resp.record) + .and_then(|record| record.box_addresses); + if let Err(error) = + destroy_session(client, global, session.id, session.name.as_deref(), box_addresses) + .await + { failures += 1; eprintln!( "Failed to destroy session {} ({}): {error:#}", @@ -1785,8 +1992,10 @@ pub(crate) async fn destroy_all_sessions( pub(crate) async fn destroy_session( client: &mut client::Client, + global: &GlobalArgs, id: sessions::SessionId, name: Option<&str>, + box_addresses: Option, ) -> Result<(), anyhow::Error> { use minimald_rpc::{DestroySession, DestroySessionRequest}; @@ -1797,6 +2006,12 @@ pub(crate) async fn destroy_session( if resp.ok().is_some() { println!("Destroyed session {} ({})", id, name.unwrap_or("-")); + // The session is gone; the row its activation bought outlives it on + // the VM host daemon, and its creator withdraws it here (T66) — + // presenting the pair the registration handed back. Best-effort: a + // withdrawal that cannot be made leaves the row published and warns + // rather than failing a destroy that already succeeded. + withdraw_box_row(vm_host_control_sock(global), name, box_addresses).await; } else { bail!("DestroySession returned an error from the daemon"); } @@ -2198,8 +2413,11 @@ mod tests { // below is exactly what the await-holding-lock lint names. let seen = requests.lock().unwrap(); assert_eq!(seen.len(), 1, "one registration, one request"); - let request: minimald_rpc::RegisterBoxRequest = + let request: minimald_rpc::BoxControlRequest = serde_json_lenient::from_str(&seen[0]).expect("the request is the wire type"); + let minimald_rpc::BoxControlRequest::Register(request) = request else { + panic!("a registration is carried by the register verb"); + }; assert_eq!(request.name, "web"); assert_eq!(request.ingress_ports, vec![8080, 5432]); assert_eq!(request.egress.as_ref(), policy.egress.as_ref()); @@ -2253,4 +2471,69 @@ mod tests { "the refusal surfaces with its reason: {refused}" ); } + + /// T66's destroy side: the client holding a destroyed session's record + /// withdraws the row its activation registered — one withdraw request + /// naming the pair the registration handed back — and a session that + /// registered no box (no pair, no name, no VM host) sends nothing at + /// all. + #[tokio::test] + async fn destroy_sends_the_withdraw() { + let dir = tempfile::TempDir::new().unwrap(); + let provider_dir = dir.path().join("providers").join("local-minvmd0"); + std::fs::create_dir_all(&provider_dir).unwrap(); + let sock_path = provider_dir.join("control.sock"); + let requests = fake_vm_host( + sock_path.clone(), + r#"{"switch_address":"100.64.0.2","loopback_address":"127.0.64.0"}"#.to_string(), + ) + .await; + let global = GlobalArgs { + provider: Some(Provider::LocalMinvmd), + minimal_dir: Some(dir.path().to_path_buf()), + ..Default::default() + }; + let handed = sessions::BoxAddresses { + switch_address: std::net::Ipv4Addr::new(100, 64, 0, 2), + loopback_address: std::net::Ipv4Addr::new(127, 0, 64, 0), + }; + // The withdrawal the destroy side owes: the pair comes off the + // session record, the control socket off the provider dir the + // daemon connection resolves through. + withdraw_box_row(vm_host_control_sock(&global), Some("web"), Some(handed)).await; + { + let seen = requests.lock().unwrap(); + assert_eq!(seen.len(), 1, "one withdrawal, one request"); + let request: minimald_rpc::BoxControlRequest = + serde_json_lenient::from_str(&seen[0]).expect("the request is the wire type"); + let minimald_rpc::BoxControlRequest::Withdraw(request) = request else { + panic!("a withdrawal is carried by the withdraw verb"); + }; + assert_eq!(request.name, "web"); + assert_eq!(request.switch_address, handed.switch_address); + assert_eq!(request.loopback_address, handed.loopback_address); + } + + // And a session that registered nothing owes nothing: no pair, no + // name, no VM host — each sends nothing, on the same socket. + withdraw_box_row(vm_host_control_sock(&global), Some("web"), None).await; + assert_eq!( + requests.lock().unwrap().len(), + 1, + "no pair on the record, no withdrawal" + ); + withdraw_box_row(vm_host_control_sock(&global), None, Some(handed)).await; + assert_eq!( + requests.lock().unwrap().len(), + 1, + "no name on the record, no withdrawal" + ); + let native = GlobalArgs::default(); + withdraw_box_row(vm_host_control_sock(&native), Some("web"), Some(handed)).await; + assert_eq!( + requests.lock().unwrap().len(), + 1, + "no VM host to withdraw from, no withdrawal" + ); + } } From ef485daaf52603ceb3369b725688bf0527da622c Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:46:40 +0000 Subject: [PATCH 16/18] style(minimal,minvmd): cargo fmt the withdraw changes --- crates/minimal/src/cmd/mod.rs | 7 +- crates/minimal/src/cmd/session.rs | 108 +++++++++++++++++++++++++----- crates/minvmd/src/control.rs | 33 +++++---- 3 files changed, 106 insertions(+), 42 deletions(-) diff --git a/crates/minimal/src/cmd/mod.rs b/crates/minimal/src/cmd/mod.rs index 2ee3ab3f27..d106666502 100644 --- a/crates/minimal/src/cmd/mod.rs +++ b/crates/minimal/src/cmd/mod.rs @@ -699,12 +699,7 @@ pub(crate) fn arm_activation_interrupt( .oneshot_rpc::(AbortSessionRequest { id: session_id }) .await; if let Some(record) = row { - withdraw_box_row( - control_sock, - record.name.as_deref(), - record.box_addresses, - ) - .await; + withdraw_box_row(control_sock, record.name.as_deref(), record.box_addresses).await; } } std::process::exit(130); diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index 8a4354f87e..cbe4728802 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -78,7 +78,9 @@ async fn control_request_with_vm_host( ); } let reply: minimald_rpc::BoxControlReply = serde_json_lenient::from_str(reply.trim()) - .with_context(|| format!("the VM host daemon's box control reply did not parse: {reply}"))?; + .with_context(|| { + format!("the VM host daemon's box control reply did not parse: {reply}") + })?; Ok(reply) } @@ -140,7 +142,9 @@ pub(crate) async fn withdraw_box_row( box_addresses: Option, ) { let Some(name) = name else { return }; - let Some(addresses) = box_addresses else { return }; + let Some(addresses) = box_addresses else { + return; + }; let Some(sock_path) = control_sock else { tracing::warn!( box = %name, @@ -546,7 +550,10 @@ pub(crate) async fn activate_session( config.box_addresses = register_box_for_activation( global, config.network, - config.name.as_deref().expect("the session name is minted before the create"), + config + .name + .as_deref() + .expect("the session name is minted before the create"), &config.policy, ) .await; @@ -577,7 +584,12 @@ pub(crate) async fn activate_session( // registration bought: the create never happened, so no session // holds the pair (T66). Err(error) => { - withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; return Err(error.context("CreateSession RPC failed")); } }; @@ -590,7 +602,12 @@ pub(crate) async fn activate_session( // leave behind: its creator withdraws it before // re-registering under the re-minted name (T66) — it was // bought for a session this create never made. - withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; config.name = Some(autogen_session_name(&utf8_path, &random_hex4())); // A registered box's row carries the name it was // registered under (T66), so the re-mint re-registers; @@ -609,7 +626,12 @@ pub(crate) async fn activate_session( } // A create failure that is not a retryable autogen collision // abandons the activation and its row the same way. - withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; bail!("CreateSession failed: {error}"); } } @@ -623,7 +645,12 @@ pub(crate) async fn activate_session( if let Err(error) = ensure_version_reported(created.daemon_version.as_deref()) { // A skewed daemon's refusal abandons the unfinalized session and its // row; the withdrawal rides out with the error (T66). - withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; return Err(error); } warn_if_hostname_routing_down( @@ -755,7 +782,12 @@ pub(crate) async fn activate_session( if let Err(error) = uploaded { // The upload failed: the activation is abandoned, and the // row its registration bought goes with it (T66). - withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; return Err(error.context("Failed to upload project files")); } } else if !headless { @@ -798,7 +830,12 @@ pub(crate) async fn activate_session( let configured = match configured { // A transport failure abandons the activation and its row (T66). Err(error) => { - withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; return Err(error.context("ConfigureLoadout RPC failed")); } Ok(configured) => configured, @@ -809,7 +846,12 @@ pub(crate) async fn activate_session( // compose never puts an id on stdout for a script to capture. minimald_rpc::Errorable::Err { error } => { // The session cannot compose and is abandoned with its row (T66). - withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; bail!(composition_failure_message(&utf8_path, &error)); } }; @@ -843,7 +885,12 @@ pub(crate) async fn activate_session( send_abort(&mut client, session_id).await; // The session is aborted and its row withdrawn (T66): // the path failed after registering. - withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; // The route an activation actually reaches today: the // daemon routes project config back for gating, so a // project it cannot compose surfaces here rather than as @@ -856,7 +903,12 @@ pub(crate) async fn activate_session( send_abort(&mut client, session_id).await; // The session is aborted and its row withdrawn (T66): the // path failed after registering. - withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; let count = summary.count(); let snippet = summary.as_toml_snippet(); bail!( @@ -873,7 +925,12 @@ pub(crate) async fn activate_session( if let Err(error) = submitted { // The verdict never landed: the activation is abandoned with // its row (T66). - withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; return Err(error); } } else { @@ -916,7 +973,12 @@ pub(crate) async fn activate_session( if result.is_err() { // The activation failed after registering: its row is // withdrawn with it (T66). - withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; } result?; } @@ -950,7 +1012,12 @@ pub(crate) async fn activate_session( // `min ls` doesn't fill with half-finalized sessions, and // withdraw the row the registration bought with it (T66). best_effort_destroy(&mut client, id).await; - withdraw_box_row(control_sock.clone(), config.name.as_deref(), config.box_addresses).await; + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; return Err(e); } @@ -1970,9 +2037,14 @@ pub(crate) async fn destroy_all_sessions( .ok() .and_then(|resp| resp.record) .and_then(|record| record.box_addresses); - if let Err(error) = - destroy_session(client, global, session.id, session.name.as_deref(), box_addresses) - .await + if let Err(error) = destroy_session( + client, + global, + session.id, + session.name.as_deref(), + box_addresses, + ) + .await { failures += 1; eprintln!( diff --git a/crates/minvmd/src/control.rs b/crates/minvmd/src/control.rs index 5128a03b84..36dd07ac70 100644 --- a/crates/minvmd/src/control.rs +++ b/crates/minvmd/src/control.rs @@ -41,7 +41,9 @@ use std::path::PathBuf; use std::thread::JoinHandle; use std::time::Duration; -use minimald_rpc::{BoxAddresses, BoxControlReply, BoxControlRequest, RegisterBoxRequest, WithdrawBoxRequest}; +use minimald_rpc::{ + BoxAddresses, BoxControlReply, BoxControlRequest, RegisterBoxRequest, WithdrawBoxRequest, +}; use crate::box_registry::{BoxRegistry, ClientBoxSpec}; @@ -298,7 +300,9 @@ mod tests { use std::sync::OnceLock; use std::time::Duration; - use minimald_rpc::{BoxControlReply, BoxControlRequest, RegisterBoxRequest, WithdrawBoxRequest}; + use minimald_rpc::{ + BoxControlReply, BoxControlRequest, RegisterBoxRequest, WithdrawBoxRequest, + }; use switch::SwitchSubnet; use crate::box_registry::AllocationError; @@ -699,16 +703,8 @@ mod tests { .expect("the marker box is registered"), ); for (name, loopback, needle) in [ - ( - "web", - db.loopback_address, - "held by box", - ), - ( - "db", - Ipv4Addr::LOCALHOST, - "carries loopback address", - ), + ("web", db.loopback_address, "held by box"), + ("db", Ipv4Addr::LOCALHOST, "carries loopback address"), ] { let refused = control( &sock_path, @@ -776,7 +772,10 @@ mod tests { "an unknown verb is refused, got {refused:?}" ); assert!( - registry.table().by_source(db.switch_address.octets()).is_some(), + registry + .table() + .by_source(db.switch_address.octets()) + .is_some(), "a refused line publishes nothing and removes nothing" ); @@ -815,11 +814,9 @@ mod tests { // And the in-force arm drops the same frame — the per-box default // the flip puts in force — while the marker still passes. - let mut h = test_support::gate_over_with_phase( - registry.clone(), - UnregisteredSourcePhase::InForce, - ) - .await; + let mut h = + test_support::gate_over_with_phase(registry.clone(), UnregisteredSourcePhase::InForce) + .await; test_support::send_frame(&mut h.guest, &from_withdrawn).await; test_support::send_frame(&mut h.guest, &marker).await; let seen = test_support::expect_frame(&mut h.switch).await; From 142148e47782b12e4f300d1050f0f2245cdb1450 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:53:00 +0000 Subject: [PATCH 17/18] fix(minvmd): silence the strict gate's poison-guard expect --- crates/minvmd/src/box_registry.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/minvmd/src/box_registry.rs b/crates/minvmd/src/box_registry.rs index 9fd9044a6c..9e99844e67 100644 --- a/crates/minvmd/src/box_registry.rs +++ b/crates/minvmd/src/box_registry.rs @@ -462,6 +462,12 @@ impl BoxRegistry { switch_addr: Ipv4Addr, loopback_addr: Ipv4Addr, ) -> Result>, WithdrawError> { + #[expect( + clippy::unwrap_in_result, + reason = "the expect is the lock's poison guard, not this function's error \ + handling: the row lock is never held across a panic, so it cannot be \ + poisoned" + )] let mut rows = self .rows .write() From dbc7482b1dced10d16e80884b293065f07c65c7a Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:55:59 +0000 Subject: [PATCH 18/18] docs(minimald-rpc,minvmd): sharpen the verb-tag corner and the request wording --- crates/minimald-rpc/src/lib.rs | 13 +++++++------ crates/minvmd/src/control.rs | 6 ++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/crates/minimald-rpc/src/lib.rs b/crates/minimald-rpc/src/lib.rs index 666c4e724a..be3850f17c 100644 --- a/crates/minimald-rpc/src/lib.rs +++ b/crates/minimald-rpc/src/lib.rs @@ -486,12 +486,13 @@ pub struct WithdrawBoxRequest { /// predates a verb cannot parse the tagged line and refuses it — and an old /// CLI's registration, untagged, fails the new daemon's parse the same way, /// so the daemon is autospawned by the CLI from the same install and the -/// mixed-version pair is the corner, not the rule. Ordered dispatch would -/// parse an old daemon's withdraw line as a *spurious registration* — a row -/// with no ports and an allow-all policy, spending a finite hand-out -/// address — which is worse than the refusal: the registering side's row -/// stays published either way, but only the refusal leaves no new fact on -/// the host. +/// mixed-version pair is the corner, not the rule. Without the tag, dispatch +/// ordered register-then-withdraw would parse a withdraw line on a daemon +/// that predates the verb as a *spurious registration* — the line's extra +/// fields ignored, a row with no ports and an allow-all policy filling in, +/// spending a finite hand-out address — which is worse than the refusal: +/// the registering side's row stays published either way, but only the +/// refusal leaves no new fact on the host. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(tag = "verb", rename_all = "snake_case")] pub enum BoxControlRequest { diff --git a/crates/minvmd/src/control.rs b/crates/minvmd/src/control.rs index 36dd07ac70..097d30043a 100644 --- a/crates/minvmd/src/control.rs +++ b/crates/minvmd/src/control.rs @@ -143,7 +143,7 @@ fn serve_request( } } -/// Read one line (terminated by `\n`) of the registration request. A +/// Read one line (terminated by `\n`) of the box control request. A /// connection that closes before sending a line reads as no request; a line /// past [`MAX_REQUEST_LINE`] is refused. /// @@ -177,9 +177,7 @@ fn read_request_line(stream: &mut UnixStream) -> std::io::Result> if line.len() > MAX_REQUEST_LINE { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, - format!( - "box registration request exceeded {MAX_REQUEST_LINE} bytes without a newline" - ), + format!("box control request exceeded {MAX_REQUEST_LINE} bytes without a newline"), )); } }