diff --git a/crates/minimal-tui/src/rpc.rs b/crates/minimal-tui/src/rpc.rs index 6b6e5f7a52..9ba0633315 100644 --- a/crates/minimal-tui/src/rpc.rs +++ b/crates/minimal-tui/src/rpc.rs @@ -274,6 +274,7 @@ pub async fn activate( project_path: project_path.clone(), network, policy: SessionPolicy::default(), + box_addresses: None, // Same default as an activate with no flags: the dashboard // has no `--no-hooks` of its own, and a session created here // is attachable later like any other. diff --git a/crates/minimal-tui/tests/snapshots.rs b/crates/minimal-tui/tests/snapshots.rs index 25547f3e38..844969ee17 100644 --- a/crates/minimal-tui/tests/snapshots.rs +++ b/crates/minimal-tui/tests/snapshots.rs @@ -72,6 +72,7 @@ fn record(name: Option<&str>, network: NetworkMode) -> sessions::Record { project_path: paths::HostAbsPath::try_new("/src/api").unwrap(), network, policy: sessions::SessionPolicy::default(), + box_addresses: None, status: sessions::SessionStatus::Active, hooks_enabled: true, attrs: Default::default(), diff --git a/crates/minimal/src/cmd/mod.rs b/crates/minimal/src/cmd/mod.rs index 997db4c6dd..d106666502 100644 --- a/crates/minimal/src/cmd/mod.rs +++ b/crates/minimal/src/cmd/mod.rs @@ -23,6 +23,11 @@ mod net; mod project; mod session; +// The Ctrl-C cleanup (`arm_activation_interrupt`) withdraws the row the +// activation registered with the VM host daemon; the withdrawal lives with +// the session commands. +use session::{vm_host_control_sock, withdraw_box_row}; + pub use admin::*; pub use list::*; pub use net::*; @@ -663,6 +668,9 @@ pub(crate) fn arm_activation_interrupt( session_id: sessions::SessionId, ) -> ActivationInterrupt { let sock = client::resolve_socket_path(global.minimal_dir.as_deref(), global.use_minvmd()); + // Resolved here, not inside the task: the withdrawal's socket is the + // same provider dir's, and the borrow must not cross the spawn. + let control_sock = vm_host_control_sock(global); let task = tokio::spawn(async move { // Only the first Ctrl-C is intercepted; a second falls through to // the default disposition so a wedged cleanup can still be killed. @@ -677,9 +685,22 @@ pub(crate) fn arm_activation_interrupt( && let Ok(mut client) = client::Client::connect(&sock).await { use minimald_rpc::{AbortSession, AbortSessionRequest}; + // The record is fetched **before** the abort, which may take it + // with the session: it carries the pair the box's registration + // handed back (T66), whose row is withdrawn after the abort — + // the creator's withdrawal, best-effort. A session that + // registered no box (a task session, a native host) holds no + // pair, and the withdrawal stays silent for it. + let row = get_session_record(&mut client, &session_id.to_string()) + .await + .ok() + .and_then(|resp| resp.record); let _ = client .oneshot_rpc::(AbortSessionRequest { id: session_id }) .await; + if let Some(record) = row { + withdraw_box_row(control_sock, record.name.as_deref(), record.box_addresses).await; + } } std::process::exit(130); }); diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index cd1908912e..cbe4728802 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -27,6 +27,299 @@ pub async fn cmd_activate(global: &GlobalArgs, args: ActivateArgs) -> Result<(), activate_session(global, args, true).await } +/// How long a box control request — a registration, or the withdrawal of +/// the row a registration bought — may take before this invocation gives +/// up on it. The control socket answers with map writes and an allocation, +/// so healthy is milliseconds; the bound exists so a hung VM host daemon +/// cannot hang the activation or destroy the user asked for. +const BOX_CONTROL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// One control-socket exchange with the VM host daemon (T66): one JSON +/// line in — the verb's request — and one line out — the untagged reply +/// the verb is answered with. +/// +/// The control socket lives beside the ssh socket in the minvmd +/// provider-instance dir — the dir the daemon connection resolves through. +/// A refusal (an exhausted address plan, a malformed declaration, a pair +/// that is not the row's) surfaces here as an error naming the reason; the +/// caller degrades a failed registration to a warning rather than failing +/// the activation, and a failed withdrawal to one rather than failing the +/// destroy it rides on. +async fn control_request_with_vm_host( + sock_path: &std::path::Path, + request: minimald_rpc::BoxControlRequest, +) -> anyhow::Result { + use tokio::io::AsyncBufReadExt as _; + use tokio::io::AsyncWriteExt as _; + + let mut stream = tokio::net::UnixStream::connect(sock_path) + .await + .with_context(|| { + format!( + "connecting to the VM host daemon's box control socket at {}", + sock_path.display() + ) + })?; + let mut line = + serde_json_lenient::to_string(&request).context("serializing the box control request")?; + line.push('\n'); + stream + .write_all(line.as_bytes()) + .await + .context("writing the box control request")?; + let mut reply = String::new(); + tokio::io::BufReader::new(stream) + .read_line(&mut reply) + .await + .context("reading the box control reply")?; + if reply.trim().is_empty() { + anyhow::bail!( + "the VM host daemon closed its control socket without answering the box control request" + ); + } + let reply: minimald_rpc::BoxControlReply = serde_json_lenient::from_str(reply.trim()) + .with_context(|| { + format!("the VM host daemon's box control reply did not parse: {reply}") + })?; + Ok(reply) +} + +/// Registers an own-address box with the VM host daemon over its control +/// socket (T66), returning the addresses it handed back. +async fn register_box_with_vm_host( + sock_path: &std::path::Path, + request: minimald_rpc::RegisterBoxRequest, +) -> anyhow::Result { + match control_request_with_vm_host( + sock_path, + minimald_rpc::BoxControlRequest::Register(request), + ) + .await? + { + minimald_rpc::BoxControlReply::Addresses(addresses) => Ok(addresses), + minimald_rpc::BoxControlReply::Error { error } => { + anyhow::bail!("the VM host daemon refused the box registration: {error}") + } + } +} + +/// The VM host daemon's control socket, when this invocation can reach one +/// (T66): it sits beside the ssh socket in the provider dir the daemon +/// connection resolves through, so the client finds both by the same rule +/// — named VMs included, since the resolution reads the same process-global +/// VM name. +/// +/// Silent by design: an invocation that owes nothing — no VM host, an +/// unresolvable dir — asks for nothing, and the withdrawal's own warning is +/// the one that names the row left published. +pub(crate) fn vm_host_control_sock(global: &GlobalArgs) -> Option { + if !global.use_minvmd() { + return None; + } + let ssh_sock = client::resolve_socket_path(global.minimal_dir.as_deref(), true).ok()?; + ssh_sock + .parent() + .map(|dir| dir.join(minvmd::control::CONTROL_SOCK_FILE)) +} + +/// Withdraws the box's host row (T66) when the session that registered it +/// is gone — destroyed, or an activation that failed after registering: +/// the row has no holder left, so its creator presents the pair the +/// registration handed back and the daemon stops the pair's addresses +/// admitting anything. +/// +/// Always quiet when it owes nothing: a session that registered no box — a +/// native host, a host-ip box sharing the node's own row, a refused +/// registration — withdraws nothing and says nothing. A withdrawal that +/// cannot be made — no control socket, a daemon that predates the verb and +/// refuses the line, the deadline — leaves the row published and warns +/// rather than failing the destroy or the activation error it rides on; +/// the daemon restarting between registration and withdrawal answers the +/// withdrawal as already gone, which is the goal state either way. +pub(crate) async fn withdraw_box_row( + control_sock: Option, + name: Option<&str>, + box_addresses: Option, +) { + let Some(name) = name else { return }; + let Some(addresses) = box_addresses else { + return; + }; + let Some(sock_path) = control_sock else { + tracing::warn!( + box = %name, + "cannot resolve the VM host daemon's control socket; the box's \ + host row stays published until the daemon next restarts" + ); + return; + }; + let request = minimald_rpc::WithdrawBoxRequest { + name: name.to_string(), + switch_address: addresses.switch_address, + loopback_address: addresses.loopback_address, + }; + let withdrawn = tokio::time::timeout( + BOX_CONTROL_TIMEOUT, + control_request_with_vm_host( + &sock_path, + minimald_rpc::BoxControlRequest::Withdraw(request), + ), + ) + .await; + match withdrawn { + Ok(Ok(reply)) => match reply { + // The daemon answers the withdrawal with the pair it went by; a + // pair back that is not the pair asked is a daemon speaking + // another protocol's answer — the row asked for is not + // withdrawn, so say so. + minimald_rpc::BoxControlReply::Addresses(handed) if handed == addresses => { + tracing::debug!(box = %name, "withdrew the box's host row"); + } + minimald_rpc::BoxControlReply::Addresses(handed) => { + tracing::warn!( + box = %name, + switch_address = %handed.switch_address, + "the VM host daemon answered the withdrawal with a different \ + address pair; the row asked for stays published" + ); + } + minimald_rpc::BoxControlReply::Error { error } => { + tracing::warn!( + box = %name, + %error, + "the VM host daemon refused the box row withdrawal; the row \ + stays published" + ); + } + }, + Ok(Err(error)) => { + tracing::warn!( + box = %name, + %error, + "the box row withdrawal failed; the row stays published" + ); + } + Err(_) => { + tracing::warn!( + after = ?BOX_CONTROL_TIMEOUT, + box = %name, + "the VM host daemon did not answer the box row withdrawal in \ + time; the row stays published" + ); + } + } +} + +/// Registers this activation's box with the VM host daemon, when there is +/// one to register with (T66), returning the addresses it handed back — +/// `None` when there is nothing to register or the registration could not +/// be made. +/// +/// The box this activation creates is registerable only on a minvmd-backed +/// host, and only when it is an own-address box: a `host_ip` box shares the +/// node's own row in the host table, and a `none` box has no switch address +/// at all — both register nothing and attach exactly as they always have. +/// +/// A registration that cannot be made — no control socket (a supervisor +/// predating it), a refusal, the deadline — degrades to a warning and +/// `None`, never to a failed activation: the box still creates, but +/// unregistered. No host-side row holds it, so its frames run unattributed +/// — the egress gate's announced interim admits what it admits — until the +/// gate's per-box default is in force, which then fails closed for the box +/// (#1790). The warning says so. +async fn register_box_for_activation( + global: &GlobalArgs, + network: sessions::NetworkMode, + name: &str, + policy: &sessions::SessionPolicy, +) -> Option { + if !global.use_minvmd() || network != sessions::NetworkMode::OwnIp { + return None; + } + // The control socket sits beside the ssh socket in the provider dir the + // daemon connection resolves through, so the client finds both by the + // same rule — named VMs included, since the resolution reads the same + // process-global VM name. + let ssh_sock = match client::resolve_socket_path(global.minimal_dir.as_deref(), true) { + Ok(path) => path, + Err(error) => { + tracing::warn!( + %error, + "cannot resolve the VM host provider dir; the box registration \ + is skipped and the box runs unregistered — its frames \ + unattributed — until the egress gate's per-box default is in \ + force (fail-closed then; #1790)" + ); + return None; + } + }; + let Some(sock_path) = ssh_sock + .parent() + .map(|dir| dir.join(minvmd::control::CONTROL_SOCK_FILE)) + else { + tracing::warn!( + "no provider dir resolved for the ssh socket; the box registration \ + is skipped and the box runs unregistered — its frames \ + unattributed — until the egress gate's per-box default is in \ + force (fail-closed then; #1790)" + ); + return None; + }; + // The declaration, as this activation expanded it: the ingress rules + // reduced to the external ports they admit — the shape the host row + // holds — and the egress policy verbatim. + let request = minimald_rpc::RegisterBoxRequest { + name: name.to_string(), + ingress_ports: policy + .ingress + .as_ref() + .map(|ingress| { + ingress + .port_mappings + .iter() + .map(|mapping| mapping.external_port) + .collect() + }) + .unwrap_or_default(), + egress: policy.egress.clone(), + }; + let registration = tokio::time::timeout( + BOX_CONTROL_TIMEOUT, + register_box_with_vm_host(&sock_path, request), + ) + .await; + match registration { + Ok(Ok(addresses)) => { + tracing::debug!( + box = %name, + switch_address = %addresses.switch_address, + loopback_address = %addresses.loopback_address, + "VM host daemon handed the box its addresses" + ); + Some(addresses) + } + Ok(Err(error)) => { + tracing::warn!( + %error, + "the box registration failed; the box runs unregistered — its \ + frames unattributed — until the egress gate's per-box default \ + is in force (fail-closed then; #1790)" + ); + None + } + Err(_) => { + tracing::warn!( + after = ?BOX_CONTROL_TIMEOUT, + "the VM host daemon did not answer the box registration in time; \ + the box runs unregistered — its frames unattributed — until \ + the egress gate's per-box default is in force (fail-closed \ + then; #1790)" + ); + None + } + } +} + /// The activation flow shared by [`cmd_activate`] and the bare-`min` router. /// `offer_scaffold` gates the `minimal.toml` scaffold offer: `cmd_activate` /// keeps it (its long-standing behavior, unchanged), while bare `min` @@ -96,13 +389,19 @@ pub(crate) async fn activate_session( .clone() .unwrap_or_else(|| autogen_session_name(&utf8_path, &random_hex4())); + let network: sessions::NetworkMode = args.network.into(); + // The daemon sources `username` from the authenticated SSH - // connection context; the client doesn't send it. - let config = minimald_rpc::SessionConfig { + // connection context; the client doesn't send it. `box_addresses` + // lands just before the create (T66): the registration is the last + // fallible client-side step, so a loadout, policy, or hook failure + // between here and there never strands a row on the host. + let mut config = minimald_rpc::SessionConfig { name: Some(session_name), project_path: abs_path.clone(), - network: args.network.into(), + network, policy, + box_addresses: None, hooks_enabled: !args.no_hooks, attrs: Default::default(), }; @@ -235,6 +534,30 @@ pub(crate) async fn activate_session( } } + // T66: an own-address box on a VM-backed host registers with the VM host + // daemon just before the create — the host allocates the box's switch + // and loopback addresses into the table its egress gate decides by and + // hands them back, and the create request carries them so the in-VM + // daemon attaches with the handed switch address instead of drawing its + // own. Every other shape of activation — a host-ip box sharing the + // node's own row, a none box with no switch address, a native daemon + // with no box table — registers nothing and attaches as it always has. + // This is the last fallible client-side step, so a failure from here on + // owes the row its creator's withdrawal (T66): the sites below send it. + // The control socket is resolved once, here, and the withdrawals ride + // on it. + let control_sock = vm_host_control_sock(global); + config.box_addresses = register_box_for_activation( + global, + config.network, + config + .name + .as_deref() + .expect("the session name is minted before the create"), + &config.policy, + ) + .await; + use minimald_rpc::{ ConfigureLoadout, ConfigureLoadoutRequest, CreateSession, CreateSessionRequest, }; @@ -243,7 +566,6 @@ pub(crate) async fn activate_session( // re-mint the hex suffix and retry a bounded number of times. A // user-supplied name never retries — its collision, and any other failure // (e.g. a policy/network-mode validation error), surfaces unchanged. - let mut config = config; let mut attempts = 0u32; let created = loop { let resp = client @@ -255,16 +577,61 @@ pub(crate) async fn activate_session( // operator proceed. must_match_version: version_assertion(), }) - .await - .context("CreateSession RPC failed")?; + .await; + let resp = match resp { + Ok(resp) => resp, + // A transport failure abandons the activation and the row its + // registration bought: the create never happened, so no session + // holds the pair (T66). + Err(error) => { + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; + return Err(error.context("CreateSession RPC failed")); + } + }; match resp { minimald_rpc::Errorable::Ok(r) => break r, minimald_rpc::Errorable::Err { error } => { if should_retry_autogen(autogen, attempts, &error) { attempts += 1; + // The row the first attempt bought is this attempt's to + // leave behind: its creator withdraws it before + // re-registering under the re-minted name (T66) — it was + // bought for a session this create never made. + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; config.name = Some(autogen_session_name(&utf8_path, &random_hex4())); + // A registered box's row carries the name it was + // registered under (T66), so the re-mint re-registers; + // the abandoned row's addresses stay spent by design + // (the host's cursors never regress), but its + // admissions are withdrawn above, and the retry leaves + // no row behind. + config.box_addresses = register_box_for_activation( + global, + config.network, + config.name.as_deref().expect("just re-minted"), + &config.policy, + ) + .await; continue; } + // A create failure that is not a retryable autogen collision + // abandons the activation and its row the same way. + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; bail!("CreateSession failed: {error}"); } } @@ -275,7 +642,17 @@ pub(crate) async fn activate_session( // itself proof of a skew, so refuse here — still before the upload, the // loadout, and the finalize that #1251 died at, and with the session left // unfinalized for the daemon to reap when this connection drops. - ensure_version_reported(created.daemon_version.as_deref())?; + if let Err(error) = ensure_version_reported(created.daemon_version.as_deref()) { + // A skewed daemon's refusal abandons the unfinalized session and its + // row; the withdrawal rides out with the error (T66). + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; + return Err(error); + } warn_if_hostname_routing_down( created.hostname_routing_unavailable.as_deref(), "min session activate", @@ -399,10 +776,20 @@ pub(crate) async fn activate_session( )?, }; if should_upload { - client + let uploaded = client .upload_workspace_files(id, upload_root.as_std_path()) - .await - .context("Failed to upload project files")?; + .await; + if let Err(error) = uploaded { + // The upload failed: the activation is abandoned, and the + // row its registration bought goes with it (T66). + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; + return Err(error.context("Failed to upload project files")); + } } else if !headless { eprintln!( "Skipping file upload; the session will start with an \ @@ -439,13 +826,32 @@ pub(crate) async fn activate_session( session_id: id, contribution, }) - .await - .context("ConfigureLoadout RPC failed")?; + .await; + let configured = match configured { + // A transport failure abandons the activation and its row (T66). + Err(error) => { + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; + return Err(error.context("ConfigureLoadout RPC failed")); + } + Ok(configured) => configured, + }; let configured = match configured { minimald_rpc::Errorable::Ok(r) => r, // Bails before the `println!("{id}")` below: a session that cannot // compose never puts an id on stdout for a script to capture. minimald_rpc::Errorable::Err { error } => { + // The session cannot compose and is abandoned with its row (T66). + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; bail!(composition_failure_message(&utf8_path, &error)); } }; @@ -477,6 +883,14 @@ pub(crate) async fn activate_session( Ok((verdict, _final_policy)) => verdict, Err(e) => { send_abort(&mut client, session_id).await; + // The session is aborted and its row withdrawn (T66): + // the path failed after registering. + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; // The route an activation actually reaches today: the // daemon routes project config back for gating, so a // project it cannot compose surfaces here rather than as @@ -487,6 +901,14 @@ pub(crate) async fn activate_session( let summary = hooks.into_summary(); if summary.count() > 0 { send_abort(&mut client, session_id).await; + // The session is aborted and its row withdrawn (T66): the + // path failed after registering. + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; let count = summary.count(); let snippet = summary.as_toml_snippet(); bail!( @@ -499,7 +921,18 @@ pub(crate) async fn activate_session( ); } collected_patches.extend(approved_patches_from_verdict(&verdict)); - submit_verdict_and_wait(&mut client, session_id, verdict).await?; + let submitted = submit_verdict_and_wait(&mut client, session_id, verdict).await; + if let Err(error) = submitted { + // The verdict never landed: the activation is abandoned with + // its row (T66). + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; + return Err(error); + } } else { // The hook stashes policy mutations in interior // `RefCell`s so a `DenyPermanent` (which returns @@ -537,6 +970,16 @@ pub(crate) async fn activate_session( Err(e) => eprintln!("warning: failed to update {}: {e}", policy_path.display()), } } + if result.is_err() { + // The activation failed after registering: its row is + // withdrawn with it (T66). + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; + } result?; } } @@ -566,8 +1009,15 @@ pub(crate) async fn activate_session( { // Best-effort teardown: the session is stuck in // Materializing on the daemon. Destroy it so the operator's - // `min ls` doesn't fill with half-finalized sessions. + // `min ls` doesn't fill with half-finalized sessions, and + // withdraw the row the registration bought with it (T66). best_effort_destroy(&mut client, id).await; + withdraw_box_row( + control_sock.clone(), + config.name.as_deref(), + config.box_addresses, + ) + .await; return Err(e); } @@ -1364,7 +1814,7 @@ pub async fn cmd_destroy(global: &GlobalArgs, args: DestroyArgs) -> Result<(), a let mut client = connect_daemon(global).await?; if args.all { - return destroy_all_sessions(&mut client, args.force).await; + return destroy_all_sessions(global, &mut client, args.force).await; } let session = args @@ -1404,7 +1854,14 @@ pub async fn cmd_destroy(global: &GlobalArgs, args: DestroyArgs) -> Result<(), a } } - destroy_session(&mut client, record.id, record.name.as_deref()).await + destroy_session( + &mut client, + global, + record.id, + record.name.as_deref(), + record.box_addresses, + ) + .await } /// What the daemon's at-risk report means for the destroy gate. @@ -1540,6 +1997,7 @@ pub(crate) async fn session_delta( } pub(crate) async fn destroy_all_sessions( + global: &GlobalArgs, client: &mut client::Client, force: bool, ) -> Result<(), anyhow::Error> { @@ -1569,7 +2027,25 @@ pub(crate) async fn destroy_all_sessions( let session_count = sessions.len(); let mut failures = 0; for session in sessions { - if let Err(error) = destroy_session(client, session.id, session.name.as_deref()).await { + // The record carries the pair the box's registration handed back + // (T66), which the destroy-side withdrawal presents; a session that + // registered no box holds none, and the withdrawal stays silent for + // it. A record the daemon cannot answer destroys the session all the + // same — nothing is left on this path to withdraw the row by. + let box_addresses = get_session_record(client, &session.id.to_string()) + .await + .ok() + .and_then(|resp| resp.record) + .and_then(|record| record.box_addresses); + if let Err(error) = destroy_session( + client, + global, + session.id, + session.name.as_deref(), + box_addresses, + ) + .await + { failures += 1; eprintln!( "Failed to destroy session {} ({}): {error:#}", @@ -1588,8 +2064,10 @@ pub(crate) async fn destroy_all_sessions( pub(crate) async fn destroy_session( client: &mut client::Client, + global: &GlobalArgs, id: sessions::SessionId, name: Option<&str>, + box_addresses: Option, ) -> Result<(), anyhow::Error> { use minimald_rpc::{DestroySession, DestroySessionRequest}; @@ -1600,6 +2078,12 @@ pub(crate) async fn destroy_session( if resp.ok().is_some() { println!("Destroyed session {} ({})", id, name.unwrap_or("-")); + // The session is gone; the row its activation bought outlives it on + // the VM host daemon, and its creator withdraws it here (T66) — + // presenting the pair the registration handed back. Best-effort: a + // withdrawal that cannot be made leaves the row published and warns + // rather than failing a destroy that already succeeded. + withdraw_box_row(vm_host_control_sock(global), name, box_addresses).await; } else { bail!("DestroySession returned an error from the daemon"); } @@ -1896,4 +2380,232 @@ mod tests { "an opted-out bare box must show allow-all: {rendered}" ); } + + /// Spawns a stand-in VM host control server at `sock_path`: answers + /// every registration with `reply` (one JSON line, newline appended), + /// recording each request line it received. Mirrors the line protocol + /// [`minvmd::control`](minvmd::control) serves, from the server side. + async fn fake_vm_host( + sock_path: std::path::PathBuf, + reply: String, + ) -> std::sync::Arc>> { + use tokio::io::{AsyncBufReadExt as _, AsyncWriteExt as _}; + let requests = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let listener = tokio::net::UnixListener::bind(&sock_path).unwrap(); + let seen = std::sync::Arc::clone(&requests); + tokio::spawn(async move { + loop { + let (stream, _) = match listener.accept().await { + Ok(accepted) => accepted, + Err(_) => return, + }; + let mut lines = tokio::io::BufReader::new(stream); + let mut line = String::new(); + if lines.read_line(&mut line).await.is_err() { + return; + } + seen.lock().unwrap().push(line.trim().to_string()); + let mut writer = lines.into_inner(); + if writer.write_all(reply.as_bytes()).await.is_err() { + return; + } + if writer.write_all(b"\n").await.is_err() { + return; + } + } + }); + requests + } + + /// T66: an own-address box on a VM-backed host registers with the VM + /// host daemon over its control socket — the request carries the box's + /// name, its expanded ingress ports, and its egress policy; the reply + /// hands back the two addresses the create request then carries — and + /// the decision about whether to register at all is the activation's: + /// only a minvmd-backed host with an own-address box registers, and a + /// refusal surfaces as a sentence, not a failed activation. + #[tokio::test] + async fn activate_registers_box_with_vm_host() { + let policy = sessions::SessionPolicy { + egress: Some(sessions::EgressPolicy { + allow_subnets: Some(vec!["10.0.0.0/8".to_string()]), + allow_dns_hosts: None, + allow_protocols: Some(vec![IpProto::Tcp]), + deny_subnets: None, + }), + ingress: Some(IngressPolicy { + port_mappings: vec![ + PortMapping { + external_port: 8080, + internal_port: 80, + proto: IpProto::Tcp, + }, + PortMapping { + external_port: 5432, + internal_port: 5432, + proto: IpProto::Tcp, + }, + ], + dynamic_allowed_range: None, + dynamic_ingress: None, + }), + }; + + // The successful shape, driven the way the activation drives it: + // through `register_box_for_activation`, whose socket resolution + // lands on the provider dir the daemon connection uses — here the + // stand-in bound where a real minvmd's control socket would be. + let dir = tempfile::TempDir::new().unwrap(); + let provider_dir = dir.path().join("providers").join("local-minvmd0"); + std::fs::create_dir_all(&provider_dir).unwrap(); + let sock_path = provider_dir.join("control.sock"); + let requests = fake_vm_host( + sock_path.clone(), + r#"{"switch_address":"100.64.0.2","loopback_address":"127.0.64.0"}"#.to_string(), + ) + .await; + let global = GlobalArgs { + provider: Some(Provider::LocalMinvmd), + minimal_dir: Some(dir.path().to_path_buf()), + ..Default::default() + }; + let handed = register_box_for_activation(&global, NetworkMode::OwnIp, "web", &policy) + .await + .expect("the registration is answered with addresses"); + assert_eq!( + handed.switch_address, + std::net::Ipv4Addr::new(100, 64, 0, 2) + ); + assert_eq!( + handed.loopback_address, + std::net::Ipv4Addr::new(127, 0, 64, 0) + ); + { + // The lock is scoped: holding a std MutexGuard across the awaits + // below is exactly what the await-holding-lock lint names. + let seen = requests.lock().unwrap(); + assert_eq!(seen.len(), 1, "one registration, one request"); + let request: minimald_rpc::BoxControlRequest = + serde_json_lenient::from_str(&seen[0]).expect("the request is the wire type"); + let minimald_rpc::BoxControlRequest::Register(request) = request else { + panic!("a registration is carried by the register verb"); + }; + assert_eq!(request.name, "web"); + assert_eq!(request.ingress_ports, vec![8080, 5432]); + assert_eq!(request.egress.as_ref(), policy.egress.as_ref()); + } + + // Every other shape of activation registers nothing: a native + // daemon has no box table, a host-ip box shares the node's own row. + // Neither touches a socket, so a plain global (no provider, no + // dir) suffices. + let native = GlobalArgs::default(); + assert!( + register_box_for_activation(&native, NetworkMode::OwnIp, "web", &policy) + .await + .is_none(), + "a native daemon hosts no box table to register on" + ); + let vm_host = GlobalArgs { + provider: Some(Provider::LocalMinvmd), + ..Default::default() + }; + assert!( + register_box_for_activation(&vm_host, NetworkMode::HostNet, "web", &policy) + .await + .is_none(), + "a host-ip box shares the node's own row and registers nothing" + ); + + // The refusal shape: a daemon that answers with a reason produces an + // error naming it — what the activation warns on and continues + // from, never a failed create. + let refused_dir = tempfile::TempDir::new().unwrap(); + let refused_path = refused_dir.path().join("control.sock"); + let _refused_requests = fake_vm_host( + refused_path.clone(), + r#"{"error":"the switch's address plan is exhausted; no box address remains"}"# + .to_string(), + ) + .await; + let refused = register_box_with_vm_host( + &refused_path, + minimald_rpc::RegisterBoxRequest { + name: "db".to_string(), + ingress_ports: Vec::new(), + egress: None, + }, + ) + .await + .expect_err("a refused registration is an error"); + assert!( + refused.to_string().contains("address plan is exhausted"), + "the refusal surfaces with its reason: {refused}" + ); + } + + /// T66's destroy side: the client holding a destroyed session's record + /// withdraws the row its activation registered — one withdraw request + /// naming the pair the registration handed back — and a session that + /// registered no box (no pair, no name, no VM host) sends nothing at + /// all. + #[tokio::test] + async fn destroy_sends_the_withdraw() { + let dir = tempfile::TempDir::new().unwrap(); + let provider_dir = dir.path().join("providers").join("local-minvmd0"); + std::fs::create_dir_all(&provider_dir).unwrap(); + let sock_path = provider_dir.join("control.sock"); + let requests = fake_vm_host( + sock_path.clone(), + r#"{"switch_address":"100.64.0.2","loopback_address":"127.0.64.0"}"#.to_string(), + ) + .await; + let global = GlobalArgs { + provider: Some(Provider::LocalMinvmd), + minimal_dir: Some(dir.path().to_path_buf()), + ..Default::default() + }; + let handed = sessions::BoxAddresses { + switch_address: std::net::Ipv4Addr::new(100, 64, 0, 2), + loopback_address: std::net::Ipv4Addr::new(127, 0, 64, 0), + }; + // The withdrawal the destroy side owes: the pair comes off the + // session record, the control socket off the provider dir the + // daemon connection resolves through. + withdraw_box_row(vm_host_control_sock(&global), Some("web"), Some(handed)).await; + { + let seen = requests.lock().unwrap(); + assert_eq!(seen.len(), 1, "one withdrawal, one request"); + let request: minimald_rpc::BoxControlRequest = + serde_json_lenient::from_str(&seen[0]).expect("the request is the wire type"); + let minimald_rpc::BoxControlRequest::Withdraw(request) = request else { + panic!("a withdrawal is carried by the withdraw verb"); + }; + assert_eq!(request.name, "web"); + assert_eq!(request.switch_address, handed.switch_address); + assert_eq!(request.loopback_address, handed.loopback_address); + } + + // And a session that registered nothing owes nothing: no pair, no + // name, no VM host — each sends nothing, on the same socket. + withdraw_box_row(vm_host_control_sock(&global), Some("web"), None).await; + assert_eq!( + requests.lock().unwrap().len(), + 1, + "no pair on the record, no withdrawal" + ); + withdraw_box_row(vm_host_control_sock(&global), None, Some(handed)).await; + assert_eq!( + requests.lock().unwrap().len(), + 1, + "no name on the record, no withdrawal" + ); + let native = GlobalArgs::default(); + withdraw_box_row(vm_host_control_sock(&native), Some("web"), Some(handed)).await; + assert_eq!( + requests.lock().unwrap().len(), + 1, + "no VM host to withdraw from, no withdrawal" + ); + } } diff --git a/crates/minimal/src/task.rs b/crates/minimal/src/task.rs index e21b814fd8..490678268b 100644 --- a/crates/minimal/src/task.rs +++ b/crates/minimal/src/task.rs @@ -620,6 +620,7 @@ pub async fn cmd_task_run(global: &GlobalArgs, args: TaskRunArgs) -> Result<(), project_path: abs_path.clone(), network: sessions::NetworkMode::HostNet, policy: sessions::SessionPolicy::default(), + box_addresses: None, // Same default as an activate with no flags, matching the // loadout handling below. `min task run` has no `--no-hooks` of // its own; a `--keep` session is attachable later, so its hooks diff --git a/crates/minimal/tests/cli.rs b/crates/minimal/tests/cli.rs index 9e95bf1226..426f14c59f 100644 --- a/crates/minimal/tests/cli.rs +++ b/crates/minimal/tests/cli.rs @@ -1747,6 +1747,7 @@ async fn create_pending_session(daemon: &common::TestDaemon, name: &str) -> Sess project_path: paths::HostAbsPath::try_new(project_path).unwrap(), network: sessions::NetworkMode::NoNet, policy: Default::default(), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }; @@ -1820,6 +1821,7 @@ async fn create_session_with( project_path, network, policy, + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }; diff --git a/crates/minimald-rpc/src/lib.rs b/crates/minimald-rpc/src/lib.rs index 1b7c69fb29..be3850f17c 100644 --- a/crates/minimald-rpc/src/lib.rs +++ b/crates/minimald-rpc/src/lib.rs @@ -18,8 +18,8 @@ pub mod taskenv; pub mod trace; pub use sessions::{ - DynamicIngress, EffectiveEgress, EffectiveSessionPolicy, EgressPolicy, IngressPolicy, IpProto, - NetworkMode, PortMapping, SessionPolicy, + BoxAddresses, DynamicIngress, EffectiveEgress, EffectiveSessionPolicy, EgressPolicy, + IngressPolicy, IpProto, NetworkMode, PortMapping, SessionPolicy, }; pub const RPC_SUBSYSTEM_PREFIX: &str = "minimald-v1-"; @@ -385,6 +385,15 @@ pub struct SessionConfig { /// Per-session networking policy (egress + ingress). #[serde(default)] pub policy: SessionPolicy, + /// The addresses the VM host daemon handed this box's registration + /// (T66), when the activating client registered one: its switch + /// address, which the in-VM daemon attaches with instead of drawing + /// its own, and its published loopback address. `None` for every other + /// activation — a host that is not minvmd-backed, a box that is not + /// own-address — and the daemon then attaches exactly as it always + /// has. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub box_addresses: Option, /// Whether the session runs the lifecycle hooks composed into it. /// Cleared by `min session activate --no-hooks`, and persisted onto /// the session record so the later attach/detach/destroy @@ -405,6 +414,118 @@ fn default_hooks_enabled() -> bool { true } +/// The wire types of the VM host daemon's box control socket (T66): the +/// one door a client has to the host-side box table (NET-138). +/// +/// On a minvmd-backed host, `min session activate` opens this socket — a +/// UDS beside the daemon's ssh socket in the provider's state dir — writes +/// one [`BoxControlRequest`] line of JSON — a registration when the box is +/// created, a withdrawal when the session that registered it is destroyed +/// or its activation fails — and reads one [`BoxControlReply`] line back. +/// Nothing else crosses it: the session RPCs are the daemon crate's +/// HTTP-shaped socket, and the box's switch and loopback addresses come +/// from here so the daemon's create request can carry them (see +/// [`SessionConfig`]). +/// +/// These types live here rather than in `minvmd` because both ends depend +/// on this crate — the activating client and the host daemon — and the +/// protocol must not drift between them. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct RegisterBoxRequest { + /// The box's name — the session name the following create request + /// carries, so the host-side row and the session identify the same box. + /// A box's id on the host is its name. + pub name: String, + /// The external ports the box's ingress rules admit, as the client + /// expanded them. The host row holds the expanded ports rather than the + /// rules so the rule grammar stays the client's: the attaching side + /// reaches ports without re-parsing declarations. + #[serde(default)] + pub ingress_ports: Vec, + /// The box's egress policy, as the client expanded it. Absent means the + /// allow-all default — the same meaning the create request's policy + /// carries. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub egress: Option, +} + +/// The withdrawal a destroyed session's client sends for the row its +/// activation registered: the name the row went by and the pair the +/// registration handed back. The pair is the proof that the withdrawer is +/// the row's creator — a row is its registering side's to withdraw (NET-138), +/// and no other client holds the pair, which no session record but the +/// creator's carries. +/// +/// Withdrawal answers [`BoxControlReply::Addresses`] echoing the pair back +/// when the row is gone. The daemon answers a row already withdrawn — or a +/// daemon restarted since the registration — the same way: no row at the +/// named switch address is the goal state either way, so the withdrawal +/// succeeds. A row that *is* published there under a different name or pair +/// is refused with [`BoxControlReply::Error`]: the requesting client is not +/// its creator, and no client may remove another box's row. +/// +/// The addresses themselves are spent for good by design — the host's +/// allocation cursors never regress — so a withdrawal ends a row's +/// admissions without ever returning its addresses to the plan. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct WithdrawBoxRequest { + /// The name the row to withdraw was registered under. + pub name: String, + /// The switch address the registration handed back, which keys the row + /// in the host table. + pub switch_address: std::net::Ipv4Addr, + /// The loopback address the registration handed back, which the pair + /// proof checks against the row's own. + pub loopback_address: std::net::Ipv4Addr, +} + +/// The one request line the control socket takes: which verb the client +/// wants, `register` or `withdraw`, tagged in the line itself. +/// +/// The tag is the version corner, and it leans the safe way. A daemon that +/// predates a verb cannot parse the tagged line and refuses it — and an old +/// CLI's registration, untagged, fails the new daemon's parse the same way, +/// so the daemon is autospawned by the CLI from the same install and the +/// mixed-version pair is the corner, not the rule. Without the tag, dispatch +/// ordered register-then-withdraw would parse a withdraw line on a daemon +/// that predates the verb as a *spurious registration* — the line's extra +/// fields ignored, a row with no ports and an allow-all policy filling in, +/// spending a finite hand-out address — which is worse than the refusal: +/// the registering side's row stays published either way, but only the +/// refusal leaves no new fact on the host. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "verb", rename_all = "snake_case")] +pub enum BoxControlRequest { + /// Register the box's declaration: allocate the box's addresses on the + /// host and fill the row the egress gate decides by. + Register(RegisterBoxRequest), + /// Withdraw the row a registration filled: the destroyed or failed + /// session's creator presenting the pair the registration handed back. + Withdraw(WithdrawBoxRequest), +} + +/// The addresses a successful registration hands back +/// ([`sessions::BoxAddresses`]): the box's switch address and its published +/// loopback address, both allocated on the host from the address plan the +/// host switch serves — the same pair the create request carries so the +/// in-VM daemon attaches with it. +/// +/// The one reply line the control socket answers either verb with: the +/// handed addresses, or the reason the verb did not happen. A registration +/// hands the allocated pair back; a withdrawal echoes the pair it withdrew +/// by, so the client can check the daemon meant the row it asked about. +/// Untagged so the reply stays one flat JSON object either way. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum BoxControlReply { + /// The verb succeeded: a registration's allocated addresses, or a + /// withdrawal's echo of the pair the row went by. + Addresses(BoxAddresses), + /// The verb failed: `error` is a sentence naming why, for the client to + /// warn with. + Error { error: String }, +} + /// The request for a [`CreateSession`] RPC. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct CreateSessionRequest { @@ -1405,6 +1526,13 @@ mod tests { project_path: paths::HostAbsPath::try_new("/home/u/proj").unwrap(), network: NetworkMode::OwnIp, policy: SessionPolicy::default(), + // The non-`None` shape of the handed addresses: a fixture + // leaving it `None` would round-trip green even if the + // field never reached the wire. + box_addresses: Some(BoxAddresses { + switch_address: std::net::Ipv4Addr::new(100, 64, 0, 2), + loopback_address: std::net::Ipv4Addr::new(127, 0, 64, 0), + }), // The non-default (`--no-hooks`): `true` is the serde // default, so a fixture using it would round-trip green // even if the field never reached the wire. diff --git a/crates/minimald/src/exec.rs b/crates/minimald/src/exec.rs index c81b57cf1e..3740685b6a 100644 --- a/crates/minimald/src/exec.rs +++ b/crates/minimald/src/exec.rs @@ -188,6 +188,21 @@ pub(crate) fn task_network( &format!("{session}-task"), egress.map(|section| sessions::SessionPolicy::new(Some(section), None)), None, + // Deliberately not the record's handed addresses (T66): the task's + // sandbox is not the box the registration named — attaching it at the + // session box's address would key its frames to the session's row. + // A task's sandbox self-allocates, as it always has. + // + // The interim, stated plainly: task sandboxes self-allocate from the + // daemon's reserve — the plan run's lower half + // (`crate::net::self_allocation_run`) — until the task registering + // every live box host-side (NET-138) retires self-allocation, after + // which no daemon-side draw happens once a control socket exists. + // The VM host daemon hands registered boxes only from the run above + // the reserve, so the two allocators cannot meet; the daemon-side + // refusals (`IpAllocator::hand`) stay the guard against a pair that + // disagrees about the split. + None, ) } @@ -2169,6 +2184,7 @@ mod tests { project_path: paths::HostAbsPath::try_new("/tmp/project").unwrap(), network: mode, policy: sessions::SessionPolicy::default(), + box_addresses: None, status: sessions::SessionStatus::Active, hooks_enabled: true, attrs: Default::default(), diff --git a/crates/minimald/src/net/gvproxy_network.rs b/crates/minimald/src/net/gvproxy_network.rs index 1a3ddc43ac..2d2d38f676 100644 --- a/crates/minimald/src/net/gvproxy_network.rs +++ b/crates/minimald/src/net/gvproxy_network.rs @@ -30,10 +30,13 @@ use crate::net::switch::SwitchRelay; /// down explicitly via [`NetGuard::teardown`] at the end of the sandbox's life. /// /// Teardown removes this PTask's ingress forwards then detaches it from the -/// switch (decrementing the switch refcount, which stops gvproxy once the last -/// `OwnIp` PTask leaves). It is **explicit** — driven on a live runtime by the -/// owner — rather than a `Drop` schedule, so it cannot be lost to a stopped -/// runtime. Dropping the held [`SwitchRelay`] aborts the frame relay either way. +/// switch (decrementing the switch refcount — which stops gvproxy once the last +/// `OwnIp` PTask leaves — and releasing the lease with the count, handed or +/// drawn alike; a handed lease released with it is what lets the same box's +/// re-attach re-hand its address, T66). It is **explicit** — driven on a +/// live runtime by the owner — rather than a `Drop` schedule, so it cannot be +/// lost to a stopped runtime. Dropping the held [`SwitchRelay`] aborts the +/// frame relay either way. pub(crate) struct OwnIpGuard { /// Held for its `Drop`, which aborts the relay tasks; never read. _relay: SwitchRelay, @@ -45,6 +48,10 @@ pub(crate) struct OwnIpGuard { /// The static ingress forwards exposed for this PTask (R2.3), removed on /// teardown. Empty when no ingress was configured. exposed: Vec, + /// The lease ip this guard's attach holds, passed to `detach` so the + /// lease is released with the count (T66) — handed or drawn alike, a + /// lease's life is its attachment's. + lease_ip: Ipv4Addr, } impl NetGuard for OwnIpGuard { @@ -56,7 +63,7 @@ impl NetGuard for OwnIpGuard { if !self.exposed.is_empty() { crate::net::policy::remove_ingress(&self.control, &self.exposed).await; } - if let Err(e) = self.switch.lock().await.detach().await { + if let Err(e) = self.switch.lock().await.detach(self.lease_ip).await { tracing::warn!(error = %e, "detaching OwnIp PTask from switch on session end"); } // `_relay` drops here, aborting the relay tasks. @@ -87,6 +94,18 @@ impl NetGuard for OwnIpGuard { /// here just propagates: the release of that lease stays with the launch /// (`sandbox2::PlannedLaunch`), and detaching as well would double-decrement /// gvproxy's attach count. +/// +/// `handed_from_host` says whether `lease_ip` is the address the VM host +/// daemon allocated for this box's registration and handed back (T66), or +/// one this daemon drew itself — the one debug line per attach names the +/// address and that distinction, so a log tail can tell an attach that +/// honored its handed address from one that did not. +// Left positional: a single call site that has just planned the launch, so a +// struct would be single-use ceremony. +#[expect( + clippy::too_many_arguments, + reason = "left positional: a single call site that has just planned the launch, so a struct would be single-use ceremony" +)] pub(crate) async fn complete_own_ip_attach( switch: &Arc>, tap_fd: OwnedFd, @@ -95,7 +114,14 @@ pub(crate) async fn complete_own_ip_attach( session_name: &str, policy: Option<&sessions::SessionPolicy>, own_address: Option<&crate::net::provider::OwnAddressReporter>, + handed_from_host: bool, ) -> io::Result { + tracing::debug!( + session = session_name, + switch_address = %lease_ip, + handed_from_host, + "own-address box attached to the switch" + ); // The relay's egress carve-out and deprecation notice (NET-004) derive // their addresses from the subnet of the switch this box attaches to, so // a custom-subnet switch is keyed to its own resolver and watched at its @@ -247,20 +273,30 @@ async fn finish_own_ip_attach( switch: Arc::clone(switch), control, exposed, + lease_ip, }) } #[cfg(test)] mod tests { use std::net::Ipv4Addr; + use std::os::fd::FromRawFd as _; use std::path::PathBuf; + use std::sync::Arc; + use switch::MacAddr; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use tokio::net::{UnixListener, UnixStream}; - use tokio::sync::mpsc; + use tokio::sync::{Mutex, mpsc}; + use crate::net::PtaskLease; + use crate::net::SwitchClient; + use crate::net::SwitchTransport; use crate::net::dns; use crate::net::policy::{ControlChannel, register_dns_name}; + use crate::net::provider::network_for; + use crate::test_harness::CaptureWriter; + use sessions::BoxAddresses; /// Reads one control request off `sock` — its head up to the /// end-of-head marker, then exactly its `Content-Length` body — and @@ -442,4 +478,283 @@ mod tests { shared name both daemons publish, got: {bodies:?}" ); } + + /// A switch whose attach/detach are pure bookkeeping: `HostShuttle` + /// leaves the gvproxy process to `minvmd`, so only the count moves — + /// the shape a VM-backed host's daemon carries. + fn vm_host_switch() -> Arc> { + Arc::new(Mutex::new( + SwitchClient::new("/usr/bin/gvproxy", "/run/minimal/gvproxy").with_transport( + SwitchTransport::HostShuttle { + cid: crate::net::VSOCK_HOST_CID, + port: crate::net::VSOCK_GVPROXY_SHUTTLE_PORT, + }, + ), + )) + } + + /// T66, both halves of the daemon-side promise, on a VM host's switch: + /// + /// An own-address attach whose launch carries the addresses the VM host + /// daemon handed its registration attaches **at the handed address** — + /// the address the host-side table's row is keyed by — and **selects + /// none**: the allocator's run shows exactly the handed lease, no + /// self-allocation spent beside it. The lease releases with its attach, + /// so the same box re-attaching — the re-attach an in-process host + /// rebuild produces, where this allocator survives the rebuild — re-hands + /// its own address, and the next unregistered box self-allocates from + /// the daemon's reserve: the plan run's lower half, disjoint from the + /// hand-out run the handed address came from, so the two allocators + /// cannot meet. The attach names the handed address in one debug line, + /// with the handed/self-drawn distinction a log tail reads. A handed + /// address that would put a second tap beside a live one is refused at + /// the attach path rather than shared with it — a reserve address, and + /// an attach's own address alike. + #[tokio::test] + async fn own_ip_attach_uses_handed_address() { + let switch = vm_host_switch(); + // The handed address comes from the hand-out run — the plan run's + // upper half, above the daemon's self-allocation reserve, which is + // where a real host's registrations hand from. + let handed_switch = Ipv4Addr::new(100, 64, 128, 9); + let handed_loopback = Ipv4Addr::new(127, 0, 64, 9); + let net = network_for( + sessions::NetworkMode::OwnIp, + &switch, + "web", + None, + None, + Some(BoxAddresses { + switch_address: handed_switch, + loopback_address: handed_loopback, + }), + ); + + // The plan's lease IS the handed address — the address the tap is + // configured with, whatever mechanism builds it — and the switch's + // table carries exactly that one lease: nothing was drawn beside it. + let plan = net.plan().await.expect("the handed attach plans"); + assert_eq!( + switch.lock().await.attached(), + 1, + "the handed attach counts like any other" + ); + assert_eq!( + switch.lock().await.leases(), + &[PtaskLease { + ip: handed_switch, + mac: MacAddr::for_switch_ip(handed_switch), + }], + "the handed attach records the handed lease and selects nothing else" + ); + let _ = plan; // the plan's resolver/tap carry the same lease; asserted above + + // Releasing it detaches like any other — and the handed lease goes + // with the count: the address is the registered box's, keyed by the + // host-side row, so once its attach ends the same box re-attaching + // re-hands it instead of meeting the collision refusal. That is the + // re-attach an in-process host rebuild produces, where this + // allocator survives the rebuild. + net.abandon().await; + assert_eq!(switch.lock().await.attached(), 0, "the abandon detaches"); + assert!( + switch.lock().await.leases().is_empty(), + "the abandon releases the handed lease with the count" + ); + + // The next box — one the activating client did not register — + // self-allocates from the daemon's reserve: the plan run's lower + // half, a sub-run disjoint from the hand-out run the handed address + // came from, so the two allocators cannot meet and the handed + // address is never drawn. + let unregistered = network_for( + sessions::NetworkMode::OwnIp, + &switch, + "other", + None, + None, + None, + ); + let _ = unregistered + .plan() + .await + .expect("a self-allocated attach plans"); + let leases = switch.lock().await.leases().to_vec(); + assert_eq!( + leases.len(), + 1, + "the released handed lease is out of the table; one draw beside \ + it: {leases:?}" + ); + let drawn = *leases + .first() + .expect("the assert above pinned exactly one lease in the table"); + assert_eq!( + drawn.ip, + Ipv4Addr::new(100, 64, 0, 2), + "the self-allocation draws the reserve's first address — the \ + sub-run is disjoint from the hand-out run, so the handed \ + address is never drawn, got {leases:?}" + ); + + // A handed address inside the reserve is refused at the attach path + // — the shape a host that hands from the run's start would produce — + // and the refusal spends nothing: no lease joins the table, no + // attach is counted. + let colliding_reserve = network_for( + sessions::NetworkMode::OwnIp, + &switch, + "third", + None, + None, + Some(BoxAddresses { + switch_address: drawn.ip, + loopback_address: Ipv4Addr::LOCALHOST, + }), + ); + let err = colliding_reserve + .plan() + .await + .expect_err("a handed address inside the reserve refuses"); + assert!( + err.to_string().contains("self-allocation reserve"), + "the refusal names the reserve: {err}" + ); + assert_eq!( + switch.lock().await.leases().len(), + 1, + "the refusal added no lease to the table" + ); + assert_eq!( + switch.lock().await.attached(), + 1, + "only the self-allocated attach counts; the refused one never did" + ); + + // The same box re-attaching re-hands its own address: the lease its + // previous attach released is back in the table, and the + // select-none half of the promise holds on the re-attach too — the + // table holds exactly the drawn lease and the handed one, nothing + // drawn beside it, and the attach counts like any other. + let reattached = network_for( + sessions::NetworkMode::OwnIp, + &switch, + "web", + None, + None, + Some(BoxAddresses { + switch_address: handed_switch, + loopback_address: handed_loopback, + }), + ); + let _ = reattached + .plan() + .await + .expect("the same box re-attaching re-hands its own address"); + let leases = switch.lock().await.leases().to_vec(); + assert_eq!( + leases.len(), + 2, + "the re-hand records the handed lease beside the drawn one: \ + {leases:?}" + ); + assert!( + leases.contains(&PtaskLease { + ip: handed_switch, + mac: MacAddr::for_switch_ip(handed_switch), + }), + "the re-hand carries the same lease the first attach recorded: \ + {leases:?}" + ); + assert_eq!( + switch.lock().await.attached(), + 2, + "the re-hand counts like any attach" + ); + + // While that attach is live, handing its address again is refused — + // two taps on one address would key one PTask's frames to the + // other's host-side row. The refusal spends nothing. + let colliding_live = network_for( + sessions::NetworkMode::OwnIp, + &switch, + "fourth", + None, + None, + Some(BoxAddresses { + switch_address: handed_switch, + loopback_address: Ipv4Addr::LOCALHOST, + }), + ); + let err = colliding_live + .plan() + .await + .expect_err("a handed address a live attach holds refuses"); + assert!( + err.to_string().contains("already held by a lease"), + "the refusal names the held lease: {err}" + ); + assert_eq!( + switch.lock().await.leases().len(), + 2, + "the refusal added no lease to the table" + ); + assert_eq!( + switch.lock().await.attached(), + 2, + "the refused hand never counted" + ); + + // The attach's one debug line names the address the box carries and + // says it was handed. Driven through the completion the provider's + // attach calls, against a stand-in control channel, with a socketpair + // standing in for the tap. The capture reads at DEBUG — the + // harness's global capture reads at INFO and would never see it. + let capture = CaptureWriter::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(capture.clone()) + .with_max_level(tracing::Level::DEBUG) + .with_ansi(false) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + let dir = tempfile::TempDir::new().unwrap(); + let control_path = dir.path().join("gvproxy.sock"); + // One `/connect` upgrade plus one DNS publish per host id. + let _bodies = spawn_control_channel_capturing(control_path.clone(), 8); + let mut fds = [0 as libc::c_int; 2]; + // SAFETY: `socketpair` with a valid domain/type either returns -1 + // (checked) or fills `fds` with two fresh descriptors. + let rc = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_DGRAM, 0, fds.as_mut_ptr()) }; + assert_eq!(rc, 0, "socketpair: {}", std::io::Error::last_os_error()); + // SAFETY: each fd in `fds` is a fresh, valid, owned descriptor just + // returned by socketpair. + let tap_fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(fds[0]) }; + // SAFETY: each fd in `fds` is a fresh, valid, owned descriptor just + // returned by socketpair. + let _box_end = unsafe { std::fs::File::from_raw_fd(fds[1]) }; + let guard = crate::net::gvproxy_network::complete_own_ip_attach( + &switch, + tap_fd, + ControlChannel::Unix(control_path), + handed_switch, + "web", + None, + None, + true, + ) + .await + .expect("the attach completes against the stand-in"); + drop(guard); + let log = capture.contents(); + assert!( + log.contains(&format!("switch_address={handed_switch}")) + && log.contains("handed_from_host=true"), + "the attach's debug line names the handed address and says it was \ + handed: {log}" + ); + assert!( + !log.contains("handed_from_host=false"), + "this run handed every address; no self-drawn attach was logged: {log}" + ); + } } diff --git a/crates/minimald/src/net/mod.rs b/crates/minimald/src/net/mod.rs index 5f315cdc57..49e0b0231e 100644 --- a/crates/minimald/src/net/mod.rs +++ b/crates/minimald/src/net/mod.rs @@ -94,9 +94,48 @@ const TERM_GRACE: Duration = Duration::from_secs(5); #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum NetError { - /// The configured subnet has no remaining host address to hand out. - #[error("gvproxy subnet {0} is exhausted; no free PTask address remains")] + /// The daemon's self-allocation reserve — the sub-run of the plan's PTask + /// run it draws task sandboxes and unregistered boxes from — has no + /// address left. Explicit, never wrapped: a spent address stays spent. + #[error( + "the self-allocation reserve on subnet {0} is exhausted; no daemon-side address remains" + )] SubnetExhausted(SwitchSubnet), + /// The host handed the box a switch address outside the plan's PTask run + /// (T66): the daemon's subnet cannot honor it, and attaching with it + /// would source frames from an address no row of the host-side table is + /// keyed by. + #[error("handed switch address {address} is outside the plan's PTask run on subnet {subnet}")] + HandedAddressOutsidePlan { + address: Ipv4Addr, + subnet: SwitchSubnet, + }, + /// The host handed the box a switch address an attach still holds (T66): + /// two taps on one address would key one PTask's frames to the other's + /// host-side row, so the collision is refused, never shared. Every lease + /// is withdrawn when its attach ends ([`IpAllocator::release`], via + /// [`SwitchClient::detach`]), so the same box re-attaching — the + /// re-attach an in-process host rebuild produces, where this allocator + /// survives the rebuild — re-hands its own address instead of meeting + /// this refusal. + #[error("handed switch address {address} is already held by a lease (MAC {holder})")] + HandedAddressCollision { address: Ipv4Addr, holder: MacAddr }, + /// The host handed the box a switch address inside this daemon's + /// self-allocation reserve (T66) — the sub-run task sandboxes and + /// unregistered boxes draw from. Host and daemon keep disjoint sub-runs + /// of the plan's PTask run (the daemon's half is [`self_allocation_run`], + /// the host's mirror `minvmd`'s `box_registry::hand_out_run`), so a + /// handed address inside the reserve means the two sides disagree about + /// the split — a skewed pair, a misconfigured host. Attaching with it + /// would put the box's tap beside the daemon's own draws; refused, never + /// shared. + #[error( + "handed switch address {address} is inside this daemon's self-allocation reserve on subnet {subnet}" + )] + HandedAddressInReserve { + address: Ipv4Addr, + subnet: SwitchSubnet, + }, /// Spawning the gvproxy binary failed. #[error("spawning gvproxy at {path:?}: {source}")] Spawn { @@ -138,15 +177,69 @@ pub struct AttachResult { pub exit_signal: watch::Receiver, } -/// Hands out unique switch addresses, never reusing one for the lifetime of -/// the allocator (R1.6). +/// The sub-run of `subnet`'s PTask run this daemon **self-allocates** from: +/// the run's lower half, `[first_ptask, midpoint]`, as an inclusive +/// `(first, last)` pair — the hand-out run above it keeps one address +/// more, a PTask run holding an odd number of addresses. +/// +/// The plan's PTask run is split into two disjoint sub-runs so the two +/// allocators that draw on it cannot meet: this daemon self-allocates task +/// sandboxes and unregistered boxes from the lower half, and the VM host +/// daemon hands registered boxes only from the upper half — the run above +/// this reserve. The host's half is the same midpoint rule mirrored in +/// `minvmd`'s `box_registry::hand_out_run`; one rule, two statements, so +/// change both together — each side's tests pin the default plan's split +/// literally. +/// +/// This is the interim allocation shape (NET-138): task sandboxes +/// self-allocate from the reserve until the task registering every live box +/// host-side retires daemon-side allocation, after which a daemon with a +/// control socket draws nothing — every own-address box arrives handed. +#[must_use] +pub fn self_allocation_run(subnet: SwitchSubnet) -> (u32, u32) { + let first = subnet.first_ptask(); + let last = subnet.last_ptask(); + let reserve_len = (last - first).div_ceil(2); + (first, first + reserve_len - 1) +} + +/// Hands out unique switch addresses, never reusing a drawn one for the +/// lifetime of the allocator (R1.6). +/// +/// Two allocation shapes: [`Self::allocate`] draws a self-allocation from +/// the plan run's lower half — the sub-run [`self_allocation_run`] reserves +/// for this daemon (task sandboxes, unregistered boxes) — and [`Self::hand`] +/// records the address the VM host daemon handed a registered box, from the +/// upper half. The two sub-runs are disjoint, so a self-allocation can never +/// spend a handed address, nor the reverse. A lease's life is its +/// attachment's, handed or drawn alike: it joins the static-lease table the +/// switch is configured from when the tap arrives and leaves with the attach +/// ([`Self::release`], via [`SwitchClient::detach`]), so the table holds only +/// live taps. The never-reuse rule is carried by the draw cursor, not by the +/// table — `next` never regresses, so a withdrawn drawn address is never +/// re-drawn, and it is never handed either (it sits in the reserve, which +/// [`Self::hand`] refuses). A handed address, by contrast, is the registered +/// box's — the host-side row is keyed by it — and the same box re-attaching +/// re-hands it; that re-hand is not a reuse: it is the same row coming back, +/// and the collision refusal guards only an address an attach still holds. +/// +/// The interim this shape ships in: task sandboxes self-allocate from the +/// reserve until the task registering every live box host-side (NET-138) +/// retires self-allocation, after which no daemon-side draw happens once a +/// control socket exists — every own-address box arrives handed. #[derive(Debug)] pub struct IpAllocator { subnet: SwitchSubnet, - /// The next host offset to try; only ever advances. + /// This daemon's self-allocation reserve, inclusive — both bounds from + /// [`self_allocation_run`]. `allocate` draws inside it and nothing else. + reserve: (u32, u32), + /// The next host offset to draw for a self-allocation; only ever + /// advances, within the reserve. next: u32, /// Every address handed out, in allocation order. Doubles as the - /// static-lease table written into gvproxy's config. + /// static-lease table written into gvproxy's config, which therefore + /// holds only live taps: a lease leaves with its attach + /// ([`Self::release`]), handed or drawn alike. leases: Vec, } @@ -157,6 +250,7 @@ impl IpAllocator { pub fn new(subnet: SwitchSubnet) -> Self { Self { next: subnet.first_ptask(), + reserve: self_allocation_run(subnet), subnet, leases: Vec::new(), } @@ -178,7 +272,8 @@ impl IpAllocator { self.subnet } - /// Allocates the next free address and its derived MAC. + /// Allocates the next free address and its derived MAC, drawing from the + /// daemon's self-allocation reserve — the plan run's lower half. /// /// Addresses are sequential and never reused, even after the PTask /// detaches, so a stale frame from a torn-down PTask can never be @@ -186,10 +281,10 @@ impl IpAllocator { /// /// # Errors /// - /// Returns [`NetError::SubnetExhausted`] once the subnet's host range is - /// used up. + /// Returns [`NetError::SubnetExhausted`] once the reserve is used up — + /// explicitly, with no wrap and no reuse; a spent address stays spent. pub fn allocate(&mut self) -> Result { - if self.next > self.subnet.last_ptask() { + if self.next > self.reserve.1 { return Err(NetError::SubnetExhausted(self.subnet)); } let ip = Ipv4Addr::from(self.next); @@ -207,6 +302,88 @@ impl IpAllocator { pub fn leases(&self) -> &[PtaskLease] { &self.leases } + + /// Records `ip` as the switch address a registered box attaches at (T66): + /// the VM host daemon allocated it into its table and handed it back, so + /// the daemon mints nothing. The lease joins [`Self::leases`] — the + /// static-lease table the switch is configured from — and leaves with + /// the box's attach ([`Self::release`]), so the same box re-attaching + /// re-hands it: the host's persisted row still holds the address, and + /// detach never touches that row. A drawn lease ([`Self::allocate`]) + /// leaves the same way, and its address is never seen again — the draw + /// cursor never regresses, and the reserve [`Self::hand`] refuses — so + /// the never-reuse rule (R1.6) is carried by the cursor, not by a lease + /// the table keeps for a tap that is gone. + /// + /// Three refusals, never a share: + /// + /// * an address outside the plan's PTask run + /// ([`NetError::HandedAddressOutsidePlan`]) — the subnet cannot honor + /// it; + /// * an address inside this daemon's self-allocation reserve + /// ([`NetError::HandedAddressInReserve`]) — the host hands only from + /// the run above the reserve, so one there means the two sides + /// disagree about the split; + /// * an address a recorded lease still holds + /// ([`NetError::HandedAddressCollision`]) — two taps on one address + /// would key one PTask's frames to the other's host-side row. What the + /// refusal guards is a live attach: the lease of an attach that ended + /// is withdrawn ([`Self::release`]), so the same box's re-attach + /// re-hands its own address instead of meeting this refusal. + /// + /// # Errors + /// + /// [`NetError::HandedAddressOutsidePlan`] when `ip` is outside the + /// plan's PTask run: the address cannot be honored on this subnet. + /// [`NetError::HandedAddressInReserve`] when `ip` is inside the daemon's + /// self-allocation reserve. [`NetError::HandedAddressCollision`] when a + /// recorded lease still holds `ip`. + pub fn hand(&mut self, ip: Ipv4Addr) -> Result { + let ip_num = u32::from(ip); + if ip_num < self.subnet.first_ptask() || ip_num > self.subnet.last_ptask() { + return Err(NetError::HandedAddressOutsidePlan { + address: ip, + subnet: self.subnet, + }); + } + let (reserve_first, reserve_last) = self.reserve; + if (reserve_first..=reserve_last).contains(&ip_num) { + return Err(NetError::HandedAddressInReserve { + address: ip, + subnet: self.subnet, + }); + } + if let Some(lease) = self.leases.iter().find(|lease| lease.ip == ip) { + return Err(NetError::HandedAddressCollision { + address: ip, + holder: lease.mac, + }); + } + let lease = PtaskLease { + ip, + mac: MacAddr::for_switch_ip(ip), + }; + self.leases.push(lease); + Ok(lease) + } + + /// Withdraws the lease for `ip`, when the allocator holds one: the + /// attachment ended, so the lease ends with it — every attachment's, + /// handed or drawn alike. A handed address is free for the same box to + /// re-hand on the re-attach an in-process host rebuild or a restart + /// produces (T66): the host's persisted row still holds it, and nothing + /// here does. A drawn address is never re-drawn — the draw cursor never + /// regresses (R1.6), so a stale frame from a torn-down PTask can never be + /// misdelivered to a freshly-attached one — and never handed either: it + /// sits in the reserve [`Self::hand`] refuses. An address the allocator + /// never recorded withdraws nothing. + /// + /// Returns whether a lease was withdrawn. + pub(crate) fn release(&mut self, ip: Ipv4Addr) -> bool { + let held = self.leases.iter().any(|lease| lease.ip == ip); + self.leases.retain(|lease| lease.ip != ip); + held + } } impl Default for IpAllocator { @@ -330,6 +507,17 @@ impl SwitchClient { self.allocator.subnet() } + /// Every lease this switch knows — self-allocated and handed (T66) + /// alike, oldest first — the table the generated config is written + /// from. A diagnostic surface for the addresses the switch has handed + /// out and which shape of attach took them, without reaching into the + /// allocator. + #[cfg(test)] + #[must_use] + pub(crate) fn leases(&self) -> &[PtaskLease] { + self.allocator.leases() + } + fn config_path(&self) -> PathBuf { self.state_dir.join("gvproxy.yaml") } @@ -368,14 +556,77 @@ impl SwitchClient { Ok(AttachResult { lease, exit_signal }) } - /// Records that a PTask detached. When the last one leaves, the switch is - /// stopped. + /// Attaches a PTask with the switch address the host handed it (T66) + /// instead of drawing one: the box was registered on the host-side table + /// keyed by this address, so the daemon reuses it and mints nothing. + /// + /// The handed address is recorded like an allocated one — it joins the + /// static-lease table the switch is configured from + /// ([`IpAllocator::hand`]) — and leaves with the attach: [`Self::detach`] + /// withdraws it, so the same box's re-attach re-hands it. That re-handed + /// address comes from the host's persisted row — the registration + /// allocated it into the host-side table and nothing on the host + /// withdraws it — and detach never touches that row: the daemon-side + /// lease ends with the attach, the host-side row does not. What this + /// daemon cannot yet check is that the address it is handed is still the + /// row's for this box: the host-row check by box id — the same address + /// for the same box, a refusal on mismatch — lands with the box id in + /// the registration (T44, #1660, NET-133), which names it as the gap; + /// until then this daemon attaches with whatever the create request + /// carries. A drawn lease, by contrast, is never re-drawn: the draw + /// cursor never regresses (the never-reuse rule). Everything else + /// matches [`Self::attach`], including the DM2 config/spawn steps + /// and the attach count. + /// + /// # Errors + /// + /// [`NetError::HandedAddressOutsidePlan`] when the host handed an + /// address this daemon's subnet cannot honor, + /// [`NetError::HandedAddressInReserve`] when it handed one from the + /// daemon's own self-allocation reserve, and + /// [`NetError::HandedAddressCollision`] when an attach still holds the + /// handed address; propagates config-write, spawn, and socket-readiness + /// failures as [`Self::attach`] does. + pub async fn attach_handed(&mut self, handed: Ipv4Addr) -> Result { + let lease = self.allocator.hand(handed)?; + // DM2 spawns + configures gvproxy locally; DM1/3/4 (HostShuttle) leaves + // gvproxy to `minvmd` on the host, so skip the spawn/config steps and + // only track the attach count — same shape as [`Self::attach`]. + if matches!(self.transport, SwitchTransport::LocalSpawn) { + self.write_config().await?; + self.ensure_running().await?; + } + self.attached += 1; + tracing::info!( + ip = %lease.ip, + mac = %lease.mac, + attached = self.attached, + "attached OwnIp PTask to gvproxy switch at its handed address" + ); + let exit_signal = self.exit_tx.subscribe(); + Ok(AttachResult { lease, exit_signal }) + } + + /// Records that a PTask detached, withdrawing its lease with it (T66): + /// a lease's life is its attachment's, handed or drawn alike — a task + /// sandbox's self-drawn address ends with the sandbox the way a + /// registered box's handed one ends with the box's attach. The handed + /// address is the registered box's, keyed by the host-side row, so once + /// the box's attach ends the same box re-attaching re-hands it — the + /// collision refusal then guards only an address an attach still holds. + /// A drawn address is never re-drawn: the allocator's draw cursor never + /// regresses (R1.6), so the rule holds without the table keeping a lease + /// for a tap that is gone; see [`IpAllocator::release`]. When the last + /// one leaves, the switch is stopped. /// /// # Errors /// /// Propagates teardown failures from [`stop`](Self::stop). - pub async fn detach(&mut self) -> Result<(), NetError> { + pub async fn detach(&mut self, released: Ipv4Addr) -> Result<(), NetError> { self.attached = self.attached.saturating_sub(1); + if self.allocator.release(released) { + tracing::debug!(ip = %released, "released the lease with the attach"); + } tracing::info!(attached = self.attached, "detached OwnIp PTask from switch"); if self.attached == 0 { self.stop().await?; @@ -645,20 +896,185 @@ mod tests { fn allocator_exhausts_a_tiny_subnet() { // /29 => 8 addresses; network, gateway, daemon, host-alias, broadcast // reserved, leaving exactly three allocatable hosts (.2 through .4). + // The run splits at its midpoint, so this daemon's reserve is .2 + // alone and the host's hand-out run is .3 and .4 — exhaustion is + // explicit on each side, with no wrap and no reuse. let subnet = SwitchSubnet::new(Ipv4Addr::new(10, 0, 0, 0), 29).unwrap(); let mut a = IpAllocator::new(subnet); - let got: Vec<_> = std::iter::from_fn(|| a.allocate().ok()) - .map(|l| l.ip) - .collect(); + assert_eq!(a.allocate().unwrap().ip, Ipv4Addr::new(10, 0, 0, 2)); + assert!(matches!(a.allocate(), Err(NetError::SubnetExhausted(_)))); + // The reserve bounds only the daemon's draws: the run's remainder is + // still there for the host's hand-outs, none of them ever drawn. + a.hand(Ipv4Addr::new(10, 0, 0, 3)).unwrap(); + a.hand(Ipv4Addr::new(10, 0, 0, 4)).unwrap(); + assert!(matches!( + a.hand(Ipv4Addr::new(10, 0, 0, 3)), + Err(NetError::HandedAddressCollision { .. }) + )); + } + + #[test] + fn self_allocation_run_splits_the_plan_ptask_run() { + // The default plan's split, pinned literally: the daemon's reserve + // is the PTask run's lower half and the host hands registered boxes + // from the run above it. The mirror is `minvmd`'s + // `box_registry::hand_out_run` — one rule, two statements; change + // both together. + let subnet = SwitchSubnet::default(); + let (reserve_first, reserve_last) = self_allocation_run(subnet); assert_eq!( - got, - vec![ - Ipv4Addr::new(10, 0, 0, 2), - Ipv4Addr::new(10, 0, 0, 3), - Ipv4Addr::new(10, 0, 0, 4), - ] + (reserve_first, reserve_last), + ( + u32::from(Ipv4Addr::new(100, 64, 0, 2)), + u32::from(Ipv4Addr::new(100, 64, 127, 254)), + ), + "the reserve is the run's lower half, starting at the first PTask address" ); - assert!(matches!(a.allocate(), Err(NetError::SubnetExhausted(_)))); + assert_eq!( + reserve_last + 1, + u32::from(Ipv4Addr::new(100, 64, 127, 255)), + "the host's hand-out run starts immediately above the reserve" + ); + assert!( + subnet.first_ptask() <= reserve_first && reserve_last < subnet.last_ptask(), + "both sub-runs are inside the plan's PTask run, and neither is empty" + ); + // A tiny subnet splits in half, so both sides keep at least one + // address: a /29's three PTask addresses give the reserve .2 and the + // hand-out run .3 and .4. + let tiny = SwitchSubnet::new(Ipv4Addr::new(10, 0, 0, 0), 29).unwrap(); + assert_eq!( + self_allocation_run(tiny), + ( + u32::from(Ipv4Addr::new(10, 0, 0, 2)), + u32::from(Ipv4Addr::new(10, 0, 0, 2)), + ) + ); + } + + #[test] + fn hand_refuses_an_address_in_the_self_allocation_reserve() { + // The host hands registered boxes only from the run above this + // daemon's self-allocation reserve, so an address inside the reserve + // means the two sides disagree about the split — a skewed pair, a + // misconfigured host. Refused, never attached beside the daemon's + // own draws: the sequence a shared run once allowed, a task sandbox + // self-allocating an address a box is then handed, ends at the + // refusal. + let mut a = IpAllocator::new(SwitchSubnet::default()); + let task = a.allocate().unwrap(); + assert_eq!(task.ip, Ipv4Addr::new(100, 64, 0, 2)); + let err = a.hand(task.ip).unwrap_err(); + assert!( + matches!( + err, + NetError::HandedAddressInReserve { address, .. } if address == task.ip + ), + "handing a reserve address is refused, held or not" + ); + // The refusal is positional, not lease-dependent: an unheld reserve + // address refuses the same way, and a refusal records nothing — the + // table holds the task's draw alone. + assert!(matches!( + a.hand(Ipv4Addr::new(100, 64, 64, 128)), + Err(NetError::HandedAddressInReserve { .. }) + )); + assert_eq!(a.leases().len(), 1, "a refusal adds no lease"); + // The task's address stays the task's: the next draw takes the + // reserve's next address, never the refused one again. + assert_eq!(a.allocate().unwrap().ip, Ipv4Addr::new(100, 64, 0, 3)); + } + + #[test] + fn hand_refuses_an_address_a_local_lease_holds() { + // An address a recorded lease still holds refuses a second hand: + // two taps on one address would key one PTask's frames to the + // other's host-side row. What the refusal guards is a live attach — + // `handed_lease_releases_on_detach_and_the_box_rehands` covers the + // release that lets the same box re-attach. With the run split, a + // locally-drawn lease always sits in the daemon's reserve + // (`hand_refuses_an_address_in_the_self_allocation_reserve` refuses + // that shape outright), so the lease this one exercises is a handed + // one — the same refusal rule. + let mut a = IpAllocator::new(SwitchSubnet::default()); + let box_a = a.hand(Ipv4Addr::new(100, 64, 127, 255)).unwrap(); + // A task's sandbox self-allocates beside it, from the reserve... + let task = a.allocate().unwrap(); + assert_eq!(task.ip, Ipv4Addr::new(100, 64, 0, 2)); + // ...and handing the box's address again is refused with the holder + // named. The refusal changes nothing: no duplicate joined the table, + // and handing the same address again refuses just the same — the + // address stays with its holder while its attach lives. + let err = a.hand(box_a.ip).unwrap_err(); + assert!( + matches!( + err, + NetError::HandedAddressCollision { address, holder } + if address == box_a.ip && holder == box_a.mac + ), + "handing a lease's address is refused with the holder named" + ); + assert_eq!(a.leases().len(), 2); + assert!(matches!( + a.hand(box_a.ip), + Err(NetError::HandedAddressCollision { .. }) + )); + } + + #[test] + fn handed_lease_releases_on_detach_and_the_box_rehands() { + // T66's re-attach: a lease — handed or drawn alike — leaves the + // table when its attach ends, so the same box re-attaching — the + // re-attach an in-process host rebuild produces, where this + // allocator survives the rebuild — re-hands its own address instead + // of meeting the collision refusal. + let mut a = IpAllocator::new(SwitchSubnet::default()); + let handed_ip = Ipv4Addr::new(100, 64, 128, 9); + let lease = a + .hand(handed_ip) + .expect("the hand-out run's address is handed"); + assert_eq!(a.leases().len(), 1); + + // A drawn lease leaves the same way — a task sandbox's self-drawn + // address ends with its sandbox — and the never-reuse rule (R1.6) is + // carried by the cursor, not by the table: the next draw is a higher + // address, never the withdrawn one. + let task = a.allocate().unwrap(); + assert!(a.release(task.ip), "the drawn lease is withdrawn"); + assert_eq!(a.leases(), &[lease], "only the handed lease remains"); + let redrawn = a.allocate().unwrap(); + assert_eq!( + redrawn.ip, + Ipv4Addr::new(100, 64, 0, 3), + "the withdrawn address is never re-drawn" + ); + // An address the allocator never recorded releases nothing. + assert!(!a.release(Ipv4Addr::new(100, 64, 200, 1))); + assert_eq!(a.leases().len(), 2); + + // A handed address an attach still holds refuses a second hand: the + // refusal guards a live attach, and the lease's withdrawal is what + // lets the re-attach through. + assert!(matches!( + a.hand(handed_ip), + Err(NetError::HandedAddressCollision { .. }) + )); + + // The handed lease releases, and the re-hand records the same lease + // again — the same address, the same derived MAC, the same row the + // host-side table is keyed by — beside the draw that replaced the + // withdrawn one. + assert!(a.release(handed_ip)); + assert_eq!( + a.leases(), + &[redrawn], + "the table holds the live draw alone" + ); + let rehanded = a + .hand(handed_ip) + .expect("the same box re-attaching re-hands its own address"); + assert_eq!(rehanded.ip, lease.ip); + assert_eq!(rehanded.mac, lease.mac); } #[test] @@ -739,7 +1155,9 @@ mod tests { let result = switch.attach().await.expect("host-shuttle attach"); assert_eq!(result.lease.ip, Ipv4Addr::new(100, 64, 0, 2)); - // No gvproxy child was spawned; detach just decrements the count. - switch.detach().await.expect("detach"); + // No gvproxy child was spawned; detach decrements the count and + // withdraws the lease with the attach — the drawn address stays + // spent regardless, the draw cursor never regresses. + switch.detach(result.lease.ip).await.expect("detach"); } } diff --git a/crates/minimald/src/net/provider.rs b/crates/minimald/src/net/provider.rs index 8b79bd743e..d60b1cff9d 100644 --- a/crates/minimald/src/net/provider.rs +++ b/crates/minimald/src/net/provider.rs @@ -76,13 +76,20 @@ impl OwnAddressReporter { /// ingress, the session's own PTask being attached at the same time. /// `own_address` carries the registry handle an own-address launch reports its /// lease through once the box attaches (NET-001); a task launch passes `None` -/// for that too. +/// for that too. `box_addresses` carries the switch and loopback addresses +/// the VM host daemon handed the box's registration (T66): its switch +/// address is what this `OwnIp` PTask attaches with instead of drawing one, +/// because the host-side table's row is keyed by it. A task launch passes +/// `None` deliberately — the task's sandbox is not the box the registration +/// named, and attaching it at the box's address would key its frames to the +/// session's row. pub(crate) fn network_for( mode: NetworkMode, switch: &Arc>, identity: &str, policy: Option, own_address: Option, + box_addresses: Option, ) -> Arc { match mode { NetworkMode::HostNet => Arc::new(HostIpAddressNetwork { @@ -93,6 +100,7 @@ pub(crate) fn network_for( identity: identity.to_string(), policy, own_address, + box_addresses, reserved: std::sync::Mutex::new(None), }), _ => Arc::new(sandbox2::NoNet), @@ -253,6 +261,14 @@ struct OwnIpNetwork { /// box's proxy route exists exactly while the lease does (NET-001). `None` /// for a task launch, which owns no proxy route. own_address: Option, + /// The addresses the VM host daemon handed this box's registration + /// (T66), when it was registered: the switch address this PTask attaches + /// with instead of drawing one — the host-side table's row is keyed by + /// it, so a self-allocated lease would never match — and the published + /// loopback address the host side names the box by. `None` for a launch + /// the activating client did not register, which draws as it always + /// has. + box_addresses: Option, /// Taken by `plan`, taken back out by `attach` or `abandon`. A `std` mutex, /// never held across an await, so a cancelled launch cannot leak it. reserved: std::sync::Mutex>, @@ -263,6 +279,7 @@ impl std::fmt::Debug for OwnIpNetwork { f.debug_struct("OwnIpNetwork") .field("identity", &self.identity) .field("has_policy", &self.policy.is_some()) + .field("handed_addresses", &self.box_addresses.is_some()) .finish_non_exhaustive() } } @@ -271,12 +288,22 @@ impl Network for OwnIpNetwork { /// Lease an address and make sure the switch is up, then describe the tap /// the sandbox layer should build. Before the process starts, because the /// sandbox layer assigns the address as it creates the tap. + /// + /// The lease is the host's handed address when the box was registered + /// (T66) — the address the host-side table's row is keyed by — and a + /// self-allocated one otherwise, exactly as before this existed. fn plan(&self) -> PlanFuture<'_> { Box::pin(async move { let (lease, control, subnet) = { let mut s = self.switch.lock().await; let subnet = s.subnet(); - let attach = s.attach().await.map_err(NetworkError::new)?; + let attach = match self.box_addresses.as_ref() { + Some(handed) => s + .attach_handed(handed.switch_address) + .await + .map_err(NetworkError::new)?, + None => s.attach().await.map_err(NetworkError::new)?, + }; let control = match s.transport() { crate::net::SwitchTransport::LocalSpawn => { ControlChannel::Unix(s.control_socket()) @@ -342,6 +369,7 @@ impl Network for OwnIpNetwork { &self.identity, self.policy.as_ref(), self.own_address.as_ref(), + self.box_addresses.is_some(), ) .await .map_err(NetworkError::new)?; @@ -352,16 +380,23 @@ impl Network for OwnIpNetwork { } /// Give the lease back; the sandbox layer runs this on every path out of a - /// launch that does not reach `attach`. + /// launch that does not reach `attach`. The detach releases the lease + /// with the count (T66), so an abandoned launch leaves no lease in the + /// static-lease table for a tap nothing holds — the handed one included, + /// which is what lets the same box's re-attach re-hand its address. fn abandon(&self) -> AbandonFuture<'_> { Box::pin(async move { - let reserved = self.reserved.lock().unwrap().take(); - if reserved.is_none() { + let Some(reserved) = self + .reserved + .lock() + .expect("reserved mutex poisoned") + .take() + else { // The plan failed or an attach took it; detaching anyway would // decrement the switch's count below the truth. return; - } - if let Err(e) = self.switch.lock().await.detach().await { + }; + if let Err(e) = self.switch.lock().await.detach(reserved.lease.ip).await { tracing::warn!(error = %e, "detaching OwnIp PTask after an abandoned launch"); } }) @@ -391,7 +426,7 @@ mod tests { async fn every_mode_gets_its_provider() { let switch = counting_switch(); - let host = network_for(NetworkMode::HostNet, &switch, "s", None, None) + let host = network_for(NetworkMode::HostNet, &switch, "s", None, None, None) .plan() .await .unwrap(); @@ -405,14 +440,14 @@ mod tests { &Resolver::Nameservers(vec![crate::net::SwitchSubnet::default().dns_server()]) ); - let no_net = network_for(NetworkMode::NoNet, &switch, "s", None, None) + let no_net = network_for(NetworkMode::NoNet, &switch, "s", None, None, None) .plan() .await .unwrap(); assert!(no_net.isolates_netns() && no_net.tap().is_none()); assert_eq!(no_net.resolver(), &Resolver::None); - let own_ip = network_for(NetworkMode::OwnIp, &switch, "s", None, None); + let own_ip = network_for(NetworkMode::OwnIp, &switch, "s", None, None, None); assert!(own_ip.plan().await.unwrap().isolates_netns()); assert_eq!(switch.lock().await.attached(), 1, "own-IP takes a lease"); own_ip.abandon().await; @@ -483,7 +518,7 @@ mod tests { let switch = counting_switch(); let before = switch.lock().await.attached(); - let net = network_for(NetworkMode::OwnIp, &switch, "s", None, None); + let net = network_for(NetworkMode::OwnIp, &switch, "s", None, None, None); net.plan().await.expect("planning leases an address"); assert_eq!(switch.lock().await.attached(), before + 1); @@ -514,7 +549,16 @@ mod tests { async fn concurrent_own_ip_launches_do_not_serialize() { let switch = counting_switch(); let launches: Vec<_> = (0..4) - .map(|i| network_for(NetworkMode::OwnIp, &switch, &format!("p{i}"), None, None)) + .map(|i| { + network_for( + NetworkMode::OwnIp, + &switch, + &format!("p{i}"), + None, + None, + None, + ) + }) .collect(); for net in &launches { net.plan().await.expect("planning leases an address"); @@ -538,7 +582,7 @@ mod tests { async fn host_ip_box_resolves_through_node_dns_layer() { let subnet = crate::net::SwitchSubnet::default(); let switch = counting_switch(); - let plan = network_for(NetworkMode::HostNet, &switch, "s", None, None) + let plan = network_for(NetworkMode::HostNet, &switch, "s", None, None, None) .plan() .await .expect("host-address plans do not fail"); @@ -577,7 +621,7 @@ mod tests { "/usr/bin/gvproxy", "/run/minimal/gvproxy", ))); - let plan = network_for(NetworkMode::HostNet, &native, "s", None, None) + let plan = network_for(NetworkMode::HostNet, &native, "s", None, None, None) .plan() .await .unwrap(); @@ -593,7 +637,7 @@ mod tests { // VM-host host-address: the node's DNS layer answers the same name. let vm = counting_switch(); - let plan = network_for(NetworkMode::HostNet, &vm, "s", None, None) + let plan = network_for(NetworkMode::HostNet, &vm, "s", None, None, None) .plan() .await .unwrap(); diff --git a/crates/minimald/src/rpc.rs b/crates/minimald/src/rpc.rs index 09891c7930..2f5cd6be3a 100644 --- a/crates/minimald/src/rpc.rs +++ b/crates/minimald/src/rpc.rs @@ -2957,6 +2957,7 @@ mod tests { project_path: HostAbsPath::try_new("/uwu").unwrap(), network: NetworkMode::OwnIp, policy: SessionPolicy::new(Some(egress.clone()), None), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }, @@ -2992,6 +2993,7 @@ mod tests { project_path: HostAbsPath::try_new("/uwu").unwrap(), network: NetworkMode::OwnIp, policy, + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }, @@ -3307,6 +3309,7 @@ mod tests { project_path: HostAbsPath::try_new("/uwu").unwrap(), network: NetworkMode::NoNet, policy: SessionPolicy::new(Some(egress), None), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }, diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index 442536aa0a..328aa71b8f 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -2384,6 +2384,12 @@ impl Session { Arc::clone(&self.hostnames), record.id, )), + // The addresses the VM host daemon handed this box's registration + // (T66), persisted on the record: the `OwnIp` attach reuses the + // handed switch address instead of drawing one the host-side row + // would not match — including on a re-attach after a restart, + // where the record is the only thing that still knows it. + box_addresses: record.box_addresses, composition: self.composition(), // A weak handle so in-sandbox `min build` can drive session // side-ops without keeping the actor alive past teardown. diff --git a/crates/minimald/src/session/tests.rs b/crates/minimald/src/session/tests.rs index 3e2cc88108..345aeba3ff 100644 --- a/crates/minimald/src/session/tests.rs +++ b/crates/minimald/src/session/tests.rs @@ -2619,6 +2619,7 @@ async fn own_ip_default_deny_all() { project_path: paths::HostAbsPath::try_new("/uwu").unwrap(), network: sessions::NetworkMode::OwnIp, policy: sessions::SessionPolicy::default(), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }, diff --git a/crates/minimald/src/session_host.rs b/crates/minimald/src/session_host.rs index ecc5ced214..d0d23b74e4 100644 --- a/crates/minimald/src/session_host.rs +++ b/crates/minimald/src/session_host.rs @@ -2147,6 +2147,13 @@ pub(crate) struct SandboxLauncher { /// on attach, so the box's `.min.internal` route exists exactly /// while the box does (NET-001). Ignored by every other network mode. pub(crate) own_address: Option, + /// The addresses the VM host daemon handed this box's registration + /// (T66), when it was registered: the switch address this `OwnIp` PTask + /// attaches with instead of drawing one, and the published loopback + /// address the host side names the box by. Ignored by every other + /// network mode; `None` for a session the activating client did not + /// register, which self-allocates as it always has. + pub(crate) box_addresses: Option, /// Composition to merge into the launcher's baseline packages and /// vars. Patches and lifecycle hooks are ignored today. pub(crate) composition: Option>, @@ -2242,6 +2249,7 @@ impl SessionLauncher for SandboxLauncher { let network_mode = self.network_mode; let net_switch = self.net_switch; let own_address = self.own_address; + let box_addresses = self.box_addresses; // The session name, registered as this PTask's `*.min.internal` hostname on // an own-IP attach (finding #3 / UC6); cloned because `name` is consumed by // the sandbox env below. @@ -2271,6 +2279,7 @@ impl SessionLauncher for SandboxLauncher { &session_name, Some(policy), own_address, + box_addresses, )) .await .map_err(|e| io::Error::other(format!("planning the session network: {e}")))?; diff --git a/crates/minimald/src/sessions.rs b/crates/minimald/src/sessions.rs index 8f334de9a6..7f9ca0b8a5 100644 --- a/crates/minimald/src/sessions.rs +++ b/crates/minimald/src/sessions.rs @@ -78,6 +78,11 @@ fn build_record( project_path: config.project_path, network: config.network, policy: config.policy, + // The handed addresses ride the record: the attach path reads the + // switch address back out of it, so a re-attach — after a restart + // included — reuses the address the host table still holds instead + // of drawing a new one the row would not match. + box_addresses: config.box_addresses, status, hooks_enabled: config.hooks_enabled, attrs: config.attrs, @@ -1248,6 +1253,7 @@ pub(crate) mod tests { project_path: HostAbsPath::try_new("/proj").unwrap(), network: sessions::NetworkMode::default(), policy: Default::default(), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), } diff --git a/crates/minimald/src/store.rs b/crates/minimald/src/store.rs index 52bb51c70b..fb333de54a 100644 --- a/crates/minimald/src/store.rs +++ b/crates/minimald/src/store.rs @@ -391,6 +391,7 @@ mod tests { project_path: HostAbsPath::try_new("/home/alice/proj").unwrap(), network: NetworkMode::default(), policy: Default::default(), + box_addresses: None, hooks_enabled: true, status: SessionStatus::default(), attrs: Default::default(), diff --git a/crates/minimald/src/test_harness.rs b/crates/minimald/src/test_harness.rs index b6d282836d..72d2d4e2f2 100644 --- a/crates/minimald/src/test_harness.rs +++ b/crates/minimald/src/test_harness.rs @@ -564,6 +564,7 @@ pub fn create_session_req(name: &str, project: &str) -> minimald_rpc::CreateSess project_path: paths::HostAbsPath::try_new(project).unwrap(), network: sessions::NetworkMode::default(), policy: Default::default(), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }, diff --git a/crates/minvmd/examples/exec.rs b/crates/minvmd/examples/exec.rs index 9b968dbb87..71e033dd6b 100644 --- a/crates/minvmd/examples/exec.rs +++ b/crates/minvmd/examples/exec.rs @@ -171,6 +171,7 @@ async fn run_session_exec( .map_err(|e| format!("project_path: {e}"))?, network: sessions::NetworkMode::default(), policy: Default::default(), + box_addresses: None, hooks_enabled: true, attrs: Default::default(), }, diff --git a/crates/minvmd/src/box_registry.rs b/crates/minvmd/src/box_registry.rs index 97da910ec5..9e99844e67 100644 --- a/crates/minvmd/src/box_registry.rs +++ b/crates/minvmd/src/box_registry.rs @@ -21,11 +21,13 @@ //! read-only view whose only operations are lookups, so the one component //! that reads guest frames cannot add, replace, or withdraw a row. The //! registration path that carries a client's box declarations into this -//! registry is T66's; what lands here is the table's shape, the gate that -//! reads it, and the rows the host itself can name. +//! registry is [`crate::control`] — the host daemon's control socket, over +//! which the activating client registers a box and reads the addresses the +//! allocation hands back. use std::collections::BTreeMap; use std::net::Ipv4Addr; +use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, RwLock}; use sessions::EgressPolicy; @@ -151,21 +153,142 @@ impl BoxRegistration { } } +/// A box declaration as the activating client carries it over the host's +/// control socket: the facts a row is compiled from, without the addresses — +/// a client-driven registration allocates those on the host, from the +/// address plan this registry's switch serves, and hands them back with the +/// row ([`BoxRegistry::register_client_box`]). +#[derive(Debug, Clone)] +pub struct ClientBoxSpec { + /// The box's name: what the row is named by, and the name the create + /// request will carry — a box's id on the host is its name. + pub name: String, + /// The external ports the box's ingress rules admit, as the client + /// expanded them. + pub ingress_ports: Vec, + /// The box's egress policy, as the client declared it. Absent compiles + /// the allow-all default, the same meaning the create request's absent + /// policy carries. + pub egress: Option, +} + +/// The run of `subnet`'s address plan the host hands registered boxes from: +/// the PTask run's upper half, `[midpoint + 1, last_ptask]`, as an inclusive +/// `(first, last)` pair — one address more than the daemon's reserve below +/// it, a PTask run holding an odd number of addresses. +/// +/// The plan's PTask run is split into two disjoint sub-runs so the two +/// allocators that draw on it cannot meet: this host hands a registered box +/// only from the upper half, and the in-VM daemon self-allocates task +/// sandboxes and unregistered boxes from the lower half — the run below this +/// hand-out run. The daemon's half is the same midpoint rule mirrored in +/// `minimald::net::self_allocation_run`; one rule, two statements, so change +/// both together — each side's tests pin the default plan's split literally. +/// +/// This is the interim allocation shape (NET-138): the daemon keeps +/// self-allocating from the reserve until the task registering every live +/// box host-side retires daemon-side allocation, after which a daemon with a +/// control socket draws nothing — every own-address box arrives handed. +#[must_use] +fn hand_out_run(subnet: SwitchSubnet) -> (u32, u32) { + let first = subnet.first_ptask(); + let last = subnet.last_ptask(); + let reserve_len = (last - first).div_ceil(2); + (first + reserve_len, last) +} + +/// Why a client-driven registration could not be allocated. Both runs a +/// box address comes from — the plan's switch lease run and the published +/// loopback slice — are finite; exhausting one is an answer to hand back +/// over the control socket, not a panic. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum AllocationError { + /// Every switch address in the plan's lease run is published. + #[error("the switch's address plan is exhausted; no box address remains")] + SwitchExhausted, + /// Every address in the loopback slice this subnet's switch publishes + /// at is handed out. + #[error("the host's loopback slice is exhausted; no published address remains")] + LoopbackExhausted, + /// The address plan does not serve this registry's subnet, so no box + /// address can be allocated against it. An explicit registration + /// ([`BoxRegistry::register`]) still works: it brings its own + /// addresses. + #[error("the address plan does not serve subnet {0}; no box address can be allocated")] + UnplannedSubnet(SwitchSubnet), +} + +/// Why a client-driven withdrawal was refused. The pair a withdrawal +/// presents is the proof that its client is the row's creator (T66), so a +/// refusal is the daemon saying the proof does not match the row the +/// switch address publishes — never a failure of the goal state, which +/// [`BoxRegistry::withdraw_client_box`] reports as `Ok(None)`. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum WithdrawError { + /// A row is published at the switch address under another box's name: + /// the requesting client is not its creator. + #[error( + "the row at switch address {switch_addr} is held by box {held_name:?}, \ + not by the withdrawing {asked_name:?}" + )] + NotTheCreatorsRow { + /// The switch address the withdrawal named. + switch_addr: Ipv4Addr, + /// The name the published row carries. + held_name: String, + /// The name the withdrawal presented. + asked_name: String, + }, + /// The row at the switch address carries the requested name but another + /// loopback address than the pair presented: not the pair the + /// registration handed back. + #[error( + "the row at switch address {switch_addr} carries loopback address \ + {held_loopback}, not the {asked_loopback} the withdrawal presented" + )] + NotTheHandedPair { + /// The switch address the withdrawal named. + switch_addr: Ipv4Addr, + /// The loopback address the published row carries. + held_loopback: Ipv4Addr, + /// The loopback address the withdrawal presented. + asked_loopback: Ipv4Addr, + }, +} + /// The writable half of the host-side table, held by the host process: the /// registration surface — the host's own node-namespace row, and T66's /// client-driven path — and the source of the read-only [`BoxTable`] the /// gate reads. /// -/// Cheap to clone, and every clone shares the rows: the table the gate holds -/// is the same one every later registration lands in, which is how a box -/// published after the gate started is decided by its rules from that moment -/// on. The subnet a registry is built with fixes the address plan its rows -/// compile against — the resolver the carve-out is keyed to, and the node -/// address — so it must be the same subnet the gate's switch serves. +/// Cheap to clone, and every clone shares the rows and the allocation +/// cursors: the table the gate holds is the same one every later registration +/// lands in, which is how a box published after the gate started is decided +/// by its rules from that moment on, and an address a registration takes on +/// one clone is never handed twice. The subnet a registry is built with +/// fixes the address plan its rows compile against — the resolver the +/// carve-out is keyed to, the node address, and the runs the client-driven +/// allocation draws from — so it must be the same subnet the gate's switch +/// serves. #[derive(Debug, Clone)] pub struct BoxRegistry { subnet: SwitchSubnet, rows: Arc>, + /// The next switch address the client-driven allocation hands out, + /// shared by every clone of this registry. Draws from the hand-out run + /// — the plan run's upper half, above the daemon's self-allocation + /// reserve (`hand_out_run`) — never from the reserve itself. + next_switch_addr: Arc, + /// The next published loopback address the client-driven allocation + /// hands out, shared the same way. + next_loopback_addr: Arc, + /// The loopback slice this subnet's switch publishes at, when the + /// address plan serves it: the run [`Self::register_client_box`] + /// allocates published addresses from. `None` for a subnet the plan + /// does not serve — such a registry still holds explicit registrations + /// (the node's own row among them), it just cannot allocate for a + /// client box. + loopback_slice: Option, } impl BoxRegistry { @@ -175,9 +298,15 @@ impl BoxRegistry { /// configured with. #[must_use] pub fn new(subnet: SwitchSubnet) -> Self { + let loopback_slice = switch::AddressPlan::default().loopback_slice_for_switch(subnet); Self { subnet, rows: Arc::new(RwLock::new(BTreeMap::new())), + next_switch_addr: Arc::new(AtomicU32::new(hand_out_run(subnet).0)), + next_loopback_addr: Arc::new(AtomicU32::new( + loopback_slice.map_or(0, |slice| u32::from(slice.first())), + )), + loopback_slice, } } @@ -249,6 +378,120 @@ impl BoxRegistry { .remove(&switch_addr.octets()) } + /// Registers a client box: allocates its switch address from the plan's + /// lease run and its published loopback address from the slice this + /// subnet's switch serves at, then fills the row from `spec` — the same + /// compile [`Self::register`] does, addressed at the allocation — and + /// returns the row, whose addresses are the ones to hand back over the + /// control socket. + /// + /// Addresses are handed out in plan order from shared cursors: each + /// registration takes the next address the plan has not spent, and no + /// address is ever handed to two rows (clones of this registry share the + /// cursors, so that holds across every clone). The switch cursor draws + /// only from the hand-out run — the plan run's upper half, above the + /// daemon's self-allocation reserve (`hand_out_run`) — so the two + /// allocators cannot meet. The runs are finite — the hand-out run ends + /// at the plan's last PTask address, the slice ends where the plan's + /// next switch begins — and exhausting one is the [`AllocationError`] + /// the control socket hands back as the registration's failure. + /// + /// An address is spent for good: withdrawing its row does not return it + /// to the cursor, and neither does an allocation whose other run is + /// exhausted. A spent address's host-side state — the gate's + /// rate-limit slots, the switch's static lease table — is keyed by it, + /// and re-issuing it to a new box would inherit all of that; a fresh + /// address starts clean. + /// + /// While a row this registration filled stands, the box's frames are + /// decided by its rules; what a box with **no** row runs as — one whose + /// registration never reached the daemon, or whose row was withdrawn — + /// is the gate's announced unregistered-source interim, and putting the + /// per-box default that eventually refuses it in force is the flip that + /// lands with the last row source (T66's follow-up), not a change this + /// registration makes. + pub fn register_client_box( + &self, + spec: ClientBoxSpec, + ) -> Result, AllocationError> { + let slice = self + .loopback_slice + .ok_or(AllocationError::UnplannedSubnet(self.subnet))?; + let (hand_out_first, hand_out_last) = hand_out_run(self.subnet); + let switch_addr = take_next(&self.next_switch_addr, hand_out_first, hand_out_last) + .ok_or(AllocationError::SwitchExhausted)?; + let loopback_addr = take_next( + &self.next_loopback_addr, + u32::from(slice.first()), + u32::from(slice.last()), + ) + .ok_or(AllocationError::LoopbackExhausted)?; + let mut registration = BoxRegistration::new(spec.name, switch_addr, loopback_addr) + .with_admitted_ports(spec.ingress_ports); + if let Some(policy) = spec.egress { + registration = registration.with_egress_policy(policy); + } + Ok(self.register(registration)) + } + + /// Withdraws the client box's row when the pair `(name, switch_addr, + /// loopback_addr)` proves its client is the row's creator, returning the + /// row removed — `Ok(None)` when no row is published at `switch_addr` at + /// all: the row is already withdrawn, or the daemon restarted since the + /// registration, and either way the goal state — nothing admits the + /// pair's addresses by a row — already holds. Lookup, proof, and removal + /// happen under one write of the row lock, so no registration can land + /// between the proof and the removal. + /// + /// The proof is the same pair the registration handed back + /// ([`ClientBoxSpec`]'s allocation), which only the registering session's + /// record carries; a row published under another name or another + /// loopback is not the requesting client's to remove and is refused with + /// [`WithdrawError`]. The withdrawn addresses are not returned to the + /// allocation cursors — spent for good, as [`Self::register_client_box`] + /// documents — and what their frames do next is the gate phase's to say + /// ([`Self::withdraw`]). + /// + /// This is the host-side half of the withdrawal a destroyed or failed + /// activation sends over the control socket + /// ([`crate::control::serve_request`]) — the guest daemon never asserts + /// or withdraws address→box facts (NET-138). + pub fn withdraw_client_box( + &self, + name: &str, + switch_addr: Ipv4Addr, + loopback_addr: Ipv4Addr, + ) -> Result>, WithdrawError> { + #[expect( + clippy::unwrap_in_result, + reason = "the expect is the lock's poison guard, not this function's error \ + handling: the row lock is never held across a panic, so it cannot be \ + poisoned" + )] + let mut rows = self + .rows + .write() + .expect("the row lock is never held across a panic, so it cannot be poisoned"); + let Some(record) = rows.get(&switch_addr.octets()) else { + return Ok(None); + }; + if record.name() != name { + return Err(WithdrawError::NotTheCreatorsRow { + switch_addr, + held_name: record.name().to_string(), + asked_name: name.to_string(), + }); + } + if record.loopback_addr() != loopback_addr { + return Err(WithdrawError::NotTheHandedPair { + switch_addr, + held_loopback: record.loopback_addr(), + asked_loopback: loopback_addr, + }); + } + Ok(rows.remove(&switch_addr.octets())) + } + /// Publishes the guest **node's** own namespace: the in-VM daemon's /// root-netns tap, the plane design §5.1 names node-plane. The address is /// the host's own derivation from the subnet it configured the switch @@ -281,6 +524,16 @@ impl BoxRegistry { } } +/// Takes the next unspent address from `cursor`, when `first..=last` still +/// holds one. Relaxed ordering: a cursor's only invariant is that no two +/// takes return the same address, which an atomic add gives on every +/// ordering; the run's bounds are checked on the taken value, so even a +/// cursor advanced past its run's end (or wrapped) hands out nothing. +fn take_next(cursor: &AtomicU32, first: u32, last: u32) -> Option { + let next = cursor.fetch_add(1, Ordering::Relaxed); + (first <= next && next <= last).then(|| Ipv4Addr::from(next)) +} + /// The read-only view of the published rows the egress gate decides by: the /// lookup a frame's source address resolves through, and nothing else. The /// registry hands the gate this view precisely because it has no mutation @@ -316,14 +569,18 @@ impl BoxTable { /// ([`SwitchSubnet::first_ptask`] through [`SwitchSubnet::last_ptask`]) — /// the one set of addresses a published row is ever keyed by, and so the /// one set whose rows the host-side creator will supply (T66's registration - /// path). The subnet's own infrastructure sits outside that run: the - /// gateway the resolver carve-out is keyed to, the host alias, and the - /// guest daemon's own tap, which the registry publishes a row for itself. - /// The announced interim the gate ships admits an unregistered source only - /// here, so no amount of it can borrow the plan's infrastructure as a - /// source; the per-box default that replaces the interim admits nothing, - /// and this predicate is what keeps the difference between them one - /// address range wide. + /// path). The run spans both of the plan's sub-runs — the daemon's + /// self-allocation reserve included, because a task sandbox holds a + /// reserve address and stays an unregistered source the interim admits; + /// the host hands registered boxes only from the upper half + /// (`hand_out_run`). The subnet's own infrastructure sits outside that + /// run: the gateway the resolver carve-out is keyed to, the host alias, + /// and the guest daemon's own tap, which the registry publishes a row for + /// itself. The announced interim the gate ships admits an unregistered + /// source only here, so no amount of it can borrow the plan's + /// infrastructure as a source; the per-box default that replaces the + /// interim admits nothing, and this predicate is what keeps the + /// difference between them one address range wide. #[must_use] pub fn is_allocatable(&self, src: [u8; 4]) -> bool { let addr = u32::from(Ipv4Addr::from(src)); @@ -563,6 +820,80 @@ mod tests { ); } + /// The host hands registered boxes only from the hand-out run — the plan + /// run's upper half, above the daemon's self-allocation reserve — and + /// the loopback run's exhaustion stays an explicit refusal, with no + /// wrap. Driven on a planned carved /24, whose runs are small enough to + /// see both edges of. + #[test] + fn client_boxes_hand_out_from_the_run_above_the_reserve() { + // The plan's default subnet splits at 100.64.127.255 — pinned + // literally, mirrored from `minimald::net::self_allocation_run`: + // the first box takes the hand-out run's first address, never the + // PTask run's first (that is the daemon's reserve). + let registry = BoxRegistry::new(SUBNET); + let first = registry + .register_client_box(ClientBoxSpec { + name: "web".to_string(), + ingress_ports: Vec::new(), + egress: None, + }) + .expect("the default plan has hand-out addresses"); + assert_eq!( + first.switch_addr(), + Ipv4Addr::new(100, 64, 127, 255), + "the first box takes the hand-out run's first address, above the \ + daemon's reserve" + ); + + // A planned carved /24: its PTask run is 100.64.1.2 through + // 100.64.1.252, so its hand-out run starts at .127 — and its + // loopback slice holds 32 addresses, which run out first. The + // refusal is explicit and repeats: no wrap, no reuse. + let carved = SwitchSubnet::new(Ipv4Addr::new(100, 64, 1, 0), 24).expect("valid"); + assert!( + switch::AddressPlan::default() + .loopback_slice_for_switch(carved) + .is_some(), + "the carved subnet is planned, so a refusal is a run's exhaustion, not the plan's absence" + ); + let registry = BoxRegistry::new(carved); + let first = registry + .register_client_box(ClientBoxSpec { + name: "web".to_string(), + ingress_ports: Vec::new(), + egress: None, + }) + .expect("the carved subnet has hand-out addresses"); + assert_eq!( + first.switch_addr(), + Ipv4Addr::new(100, 64, 1, 127), + "the carved /24's hand-out run also starts above its reserve" + ); + for index in 1..32 { + registry + .register_client_box(ClientBoxSpec { + name: format!("box{index}"), + ingress_ports: Vec::new(), + egress: None, + }) + .expect("the slice holds 32 published addresses"); + } + for _ in 0..2 { + assert!( + matches!( + registry.register_client_box(ClientBoxSpec { + name: "late".to_string(), + ingress_ports: Vec::new(), + egress: None, + }), + Err(AllocationError::LoopbackExhausted) + ), + "exhaustion is explicit and never wraps" + ); + } + } + /// NET-138's trust boundary: the guest never sources a row. The table the /// gate holds is read-only by construction — `BoxTable`'s only operations /// are lookups — and what that means behaviourally is that no amount of diff --git a/crates/minvmd/src/cmd/run.rs b/crates/minvmd/src/cmd/run.rs index e283090737..acc9a3ce6a 100644 --- a/crates/minvmd/src/cmd/run.rs +++ b/crates/minvmd/src/cmd/run.rs @@ -447,6 +447,29 @@ fn run_foreground() -> Result<()> { let boxes = crate::box_registry::BoxRegistry::new(switch::DEFAULT_SUBNET); boxes.register_node_namespace(); + // The host-side door to the box table (T66): the control socket the + // activating client registers an own-address box on and reads its + // allocated switch and loopback addresses from — the addresses the + // create request then carries, so the in-VM daemon attaches with the + // handed one. Bound before the guest boots, so a session activated + // against this VM can only ever be handed an address this table holds. + // Best-effort at startup, like the switch above: a bind failure is + // warned and the VM still boots — a registration then degrades to the + // gate's announced interim, exactly as against a supervisor predating + // the socket — rather than failing a boot the client could still + // activate against. + let _control = crate::control::resolve_control_sock() + .and_then(|sock_path| crate::control::spawn(sock_path, boxes.clone())) + .inspect_err(|error| { + tracing::warn!( + %error, + "failed to bind the box-registration control socket; own-address \ + activations will not be handed addresses (the egress gate's \ + announced interim applies)" + ); + }) + .ok(); + // Spawn + supervise the host gvproxy switch before the VMM child boots, so // its `-listen` switch socket exists when the gate relays into it for the // guest shuttle. The switch runtime starts the gate on the socket beside diff --git a/crates/minvmd/src/control.rs b/crates/minvmd/src/control.rs new file mode 100644 index 0000000000..097d30043a --- /dev/null +++ b/crates/minvmd/src/control.rs @@ -0,0 +1,838 @@ +//! The VM host daemon's box control socket (T66) — the one host-side door a +//! client has to the box table ([`crate::box_registry`], NET-138). +//! +//! On a minvmd-backed host, `min session activate` registers every +//! own-address box here before it creates the session: it writes one +//! [`minimald_rpc::BoxControlRequest`] line asking `register`, the daemon +//! allocates the box's switch and loopback addresses into the table — the +//! row the host-side egress gate decides every frame by — and answers with +//! the addresses the create request then carries, so the in-VM daemon +//! attaches with the handed address instead of drawing its own. The same +//! client withdraws the row when the session is destroyed or the activation +//! fails after registering: one line asking `withdraw`, answered with the +//! pair echoed back. One connection, one request line in, one reply line +//! out. +//! +//! The socket lives beside the daemon's ssh socket in the provider-instance +//! dir and is created with the same 0700-dir / 0600-socket posture the +//! bridge socket gets ([`crate::sock`]): only the same user may reach the +//! box table. The serving shape mirrors [`crate::net::HostGvproxy::spawn`] +//! — a dedicated OS thread off the supervisor's runtime — because both are +//! long-lived host services whose failures must not take the daemon down +//! with them: a failed registration answers the client with the reason and +//! keeps serving, and a client that never reaches the socket (a supervisor +//! predating this module) still activates, handed no addresses, under the +//! egress gate's announced interim. +//! +//! The v1 posture, stated: the trust is the socket itself — the same-uid +//! 0600 the bind enforces, with no peer-credential check on a connection — +//! and requests are served serially, one connection at a time, each read +//! bounded by a 30-second timeout. That is what v1 ships, not a design +//! endpoint. +//! +//! The interim is also where a box whose row is gone runs — never +//! registered, or withdrawn at destroy — and putting the per-box default +//! that eventually refuses it in force is the flip that lands with the last +//! row source (T66's follow-up), not a change this socket makes. + +use std::io::{Read, Write}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::PathBuf; +use std::thread::JoinHandle; +use std::time::Duration; + +use minimald_rpc::{ + BoxAddresses, BoxControlReply, BoxControlRequest, RegisterBoxRequest, WithdrawBoxRequest, +}; + +use crate::box_registry::{BoxRegistry, ClientBoxSpec}; + +/// The control socket's file name inside the provider-instance dir, beside +/// `paths::SSH_SOCK_FILE`. Deliberately not in the `paths` crate: that crate +/// is shared with consumers that have no box table, and this name only +/// means something where `minvmd` supervises one. +pub const CONTROL_SOCK_FILE: &str = "control.sock"; + +/// How long the server waits for a registration's one request line before +/// dropping the connection. Generous against a slow starter; a hung client +/// must not pin the serving thread — connections are served one at a +/// time — forever. +const REGISTER_READ_TIMEOUT: Duration = Duration::from_secs(30); + +/// The largest request line the server will read. A registration carries a +/// name, a port list, and a policy; anything past this bound is not one. +const MAX_REQUEST_LINE: usize = 64 * 1024; + +/// Resolve the control socket's path: `/control.sock`, the +/// dir the CLI resolves the ssh socket under, so the client finds both by +/// the same rule. +pub fn resolve_control_sock() -> std::io::Result { + Ok(crate::state::provider_dir().join(CONTROL_SOCK_FILE)) +} + +/// Bind the control socket at `sock_path` and serve box control requests +/// — registrations and their withdrawals — against `boxes` on a dedicated +/// thread, whose handle the caller holds for as long as the daemon lives. +/// +/// The bind happens on the calling thread so its failure surfaces to the +/// supervisor's own startup error handling; only the accept loop moves to +/// the thread. The socket gets the bridge socket's posture: path-length +/// check (libkrun aborts on over-long socket paths), a 0700 parent dir, a +/// stale socket removed, and 0600 on the socket itself. +pub fn spawn(sock_path: PathBuf, boxes: BoxRegistry) -> std::io::Result> { + crate::sock::check_uds_path_len(&sock_path)?; + crate::sock::prepare_socket_dir(&sock_path)?; + crate::sock::remove_stale_socket(&sock_path)?; + let listener = UnixListener::bind(&sock_path)?; + crate::sock::enforce_socket_permissions(&sock_path)?; + std::thread::Builder::new() + .name("minvmd-control".to_string()) + .spawn(move || accept_loop(listener, boxes)) +} + +/// Accept and serve box control requests until the daemon exits. One +/// connection at a time: a request is a row's map write or removal, served +/// serially so the table sees its requests in arrival order. +fn accept_loop(listener: UnixListener, boxes: BoxRegistry) { + for stream in listener.incoming() { + match stream { + Ok(stream) => { + if let Err(error) = serve_connection(stream, &boxes) { + tracing::debug!(error = %error, "box control connection failed"); + } + } + Err(error) => tracing::debug!(error = %error, "control socket accept failed"), + } + } +} + +/// Serve one request: read the line, dispatch the verb, answer. +fn serve_connection(stream: UnixStream, boxes: &BoxRegistry) -> std::io::Result<()> { + let mut stream = stream; + stream.set_read_timeout(Some(REGISTER_READ_TIMEOUT))?; + let Some(line) = read_request_line(&mut stream)? else { + // EOF before any byte: a client that connected and left. Nothing to + // answer, nothing to log past the connection level. + return Ok(()); + }; + let request = match parse_request(&line) { + Ok(request) => request, + Err(error) => { + let error = error.to_string(); + tracing::debug!(error = %error, "box control request did not parse"); + return write_reply(&mut stream, &BoxControlReply::Error { error }); + } + }; + serve_request(&mut stream, boxes, request) +} + +/// Dispatch one parsed request to its verb and write its one reply line. +/// +/// The verb dispatch is where the wire's parse refusal pays off: a line +/// that names no verb this build knows never reaches the table at all, so +/// a skewed client cannot make a withdraw look like a register (the +/// [`minimald_rpc::BoxControlRequest`] docs carry that corner). +fn serve_request( + stream: &mut UnixStream, + boxes: &BoxRegistry, + request: BoxControlRequest, +) -> std::io::Result<()> { + match request { + BoxControlRequest::Register(request) => register_and_reply(stream, boxes, request), + BoxControlRequest::Withdraw(request) => withdraw_and_reply(stream, boxes, request), + } +} + +/// Read one line (terminated by `\n`) of the box control request. A +/// connection that closes before sending a line reads as no request; a line +/// past [`MAX_REQUEST_LINE`] is refused. +/// +/// The slices are cut at `read`, the byte count `read()` reported filling +/// `buf` with, or at `newline`, an index `position` found inside +/// `buf[..read]` — so neither range can be out of bounds. +#[expect( + clippy::indexing_slicing, + reason = "cut at `read`, the byte count `read()` reported, or at `newline`, an index `position` found inside `buf[..read]`" +)] +fn read_request_line(stream: &mut UnixStream) -> std::io::Result> { + let mut line = Vec::new(); + let mut buf = [0u8; 1024]; + loop { + let read = stream.read(&mut buf)?; + if read == 0 { + return if line.is_empty() { + Ok(None) + } else { + // A partial line at EOF — a client that died mid-write. + // Still answerable with a parse refusal, so hand what + // arrived back rather than hanging the slot on a timeout. + Ok(Some(String::from_utf8_lossy(&line).into_owned())) + }; + } + if let Some(newline) = buf[..read].iter().position(|byte| *byte == b'\n') { + line.extend_from_slice(&buf[..newline]); + return Ok(Some(String::from_utf8_lossy(&line).into_owned())); + } + line.extend_from_slice(&buf[..read]); + if line.len() > MAX_REQUEST_LINE { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("box control request exceeded {MAX_REQUEST_LINE} bytes without a newline"), + )); + } + } +} + +fn parse_request(line: &str) -> Result { + serde_json_lenient::from_str(line) +} + +/// Allocate the box into the table and write the reply — the addresses on +/// success, the reason on a refusal. One info line per registration names +/// the box and both addresses: the diagnostic a bundle's VM host daemon log +/// is read for. +fn register_and_reply( + stream: &mut UnixStream, + boxes: &BoxRegistry, + request: RegisterBoxRequest, +) -> std::io::Result<()> { + let spec = ClientBoxSpec { + name: request.name.clone(), + ingress_ports: request.ingress_ports, + egress: request.egress, + }; + let reply = match boxes.register_client_box(spec) { + Ok(record) => { + tracing::info!( + box = %record.name(), + switch_address = %record.switch_addr(), + loopback_address = %record.loopback_addr(), + "registered box with the VM host daemon; addresses allocated" + ); + BoxControlReply::Addresses(BoxAddresses { + switch_address: record.switch_addr(), + loopback_address: record.loopback_addr(), + }) + } + Err(error) => { + tracing::debug!( + box = %request.name, + error = %error, + "box registration refused" + ); + BoxControlReply::Error { + error: error.to_string(), + } + } + }; + write_reply(stream, &reply) +} + +/// Remove the row the request's pair proves its client created, and write +/// the reply — the pair echoed back on success, the reason on a refusal. +/// One info line per withdrawal names the box and both addresses, mirroring +/// the registration's; a withdrawal that finds no row is the goal state +/// already holding (already withdrawn, or the daemon restarted since) and +/// is a debug line, not an error. +fn withdraw_and_reply( + stream: &mut UnixStream, + boxes: &BoxRegistry, + request: WithdrawBoxRequest, +) -> std::io::Result<()> { + let reply = match boxes.withdraw_client_box( + &request.name, + request.switch_address, + request.loopback_address, + ) { + Ok(withdrawn) => { + if withdrawn.is_some() { + tracing::info!( + box = %request.name, + switch_address = %request.switch_address, + loopback_address = %request.loopback_address, + "withdrew the box's host row; its addresses admit nothing" + ); + } else { + tracing::debug!( + box = %request.name, + switch_address = %request.switch_address, + "no host row held at the withdrawn switch address; already withdrawn" + ); + } + BoxControlReply::Addresses(BoxAddresses { + switch_address: request.switch_address, + loopback_address: request.loopback_address, + }) + } + Err(error) => { + tracing::debug!( + box = %request.name, + switch_address = %request.switch_address, + error = %error, + "box row withdrawal refused" + ); + BoxControlReply::Error { + error: error.to_string(), + } + } + }; + write_reply(stream, &reply) +} + +fn write_reply(stream: &mut UnixStream, reply: &BoxControlReply) -> std::io::Result<()> { + let mut line = serde_json_lenient::to_string(reply).map_err(|error| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("box control reply did not serialize: {error}"), + ) + })?; + line.push('\n'); + stream.write_all(line.as_bytes()) +} + +#[cfg(test)] +mod tests { + use std::io::{BufRead, BufReader, Write}; + use std::net::Ipv4Addr; + use std::os::unix::net::UnixStream as TestStream; + use std::sync::OnceLock; + use std::time::Duration; + + use minimald_rpc::{ + BoxControlReply, BoxControlRequest, RegisterBoxRequest, WithdrawBoxRequest, + }; + use switch::SwitchSubnet; + + use crate::box_registry::AllocationError; + use crate::net::egress_gate::test_support::CaptureWriter; + + use super::*; + + /// The default switch subnet, the plan the registries below are built + /// for. + const SUBNET: SwitchSubnet = switch::DEFAULT_SUBNET; + + /// A process-global log capture. The server serves on its own OS + /// thread, so the thread-local captures the crate's other tests install + /// do not see its lines; the global default is installed once per + /// process and shadowed by those thread-local ones wherever they exist. + static SERVER_LOG: OnceLock = OnceLock::new(); + + fn server_capture() -> &'static CaptureWriter { + SERVER_LOG.get_or_init(|| { + let capture = CaptureWriter::default(); + tracing::subscriber::set_global_default( + tracing_subscriber::fmt() + .with_writer(capture.clone()) + .with_ansi(false) + .finish(), + ) + .expect("no other test in this binary installs a process-wide subscriber"); + capture + }) + } + + /// Spawns the control server on a temp path over a fresh registry and + /// returns (path, handle keeping the server thread identified, the + /// registry the server serves — the same rows a gate the test brings up + /// later shares). The thread outlives the test the way a daemon's does; + /// the temp dir's drop after the test closes the test's view of the + /// socket. + fn spawn_server( + dir: &std::path::Path, + ) -> std::io::Result<(PathBuf, JoinHandle<()>, BoxRegistry)> { + let sock_path = dir.join(CONTROL_SOCK_FILE); + let boxes = BoxRegistry::new(SUBNET); + let handle = spawn(sock_path.clone(), boxes.clone())?; + // Wait until the socket accepts rather than racing the bind. + for _ in 0..500 { + if TestStream::connect(&sock_path).is_ok() { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + Ok((sock_path, handle, boxes)) + } + + /// A client that writes the request and reads the reply line back, + /// mirroring the CLI's control helper. + fn control( + sock_path: &std::path::Path, + request: &BoxControlRequest, + ) -> std::io::Result { + let mut stream = TestStream::connect(sock_path)?; + let mut line = serde_json_lenient::to_string(request).map_err(|error| { + std::io::Error::other(format!("request did not serialize: {error}")) + })?; + line.push('\n'); + stream.write_all(line.as_bytes())?; + let mut reader = BufReader::new(stream); + let mut reply = String::new(); + reader.read_line(&mut reply)?; + serde_json_lenient::from_str(reply.trim()) + .map_err(|error| std::io::Error::other(format!("reply did not parse: {error}"))) + } + + /// A client that registers a box, wrapping the registration into the + /// wire envelope. + fn register( + sock_path: &std::path::Path, + request: &RegisterBoxRequest, + ) -> std::io::Result { + control(sock_path, &BoxControlRequest::Register(request.clone())) + } + + /// The registered box's two addresses, asserted as the reply the daemon + /// hands back. + fn handed(reply: BoxControlReply) -> BoxAddresses { + match reply { + BoxControlReply::Addresses(addresses) => addresses, + BoxControlReply::Error { error } => { + panic!("a valid request is answered with addresses, refused with {error}") + } + } + } + + /// The end-to-end shape of the layer: the activating client registers + /// over the control socket, the host allocates the box's switch and + /// loopback addresses into its table and hands them back, allocations + /// advance without reuse, a refusal names its reason, and the server + /// keeps serving past one. The registration's one info line names the + /// box and both addresses — the line a bundle's VM host daemon log is + /// read for. + #[test] + fn box_addresses_allocated_on_host_and_handed_to_daemon() { + let capture = server_capture(); + let dir = tempfile::TempDir::new().expect("temp dir"); + let (sock_path, _server, _boxes) = spawn_server(dir.path()).expect("server binds"); + + // The first registration is handed the hand-out run's first switch + // address — the plan's PTask run above the daemon's self-allocation + // reserve, mirrored from `minimald::net::self_allocation_run` — and + // the first published loopback address. + let web = handed( + register( + &sock_path, + &RegisterBoxRequest { + name: "web".to_string(), + ingress_ports: vec![8080, 9090], + egress: Some(sessions::EgressPolicy { + allow_protocols: Some(vec![sessions::IpProto::Tcp]), + allow_subnets: Some(vec!["10.0.0.0/8".to_string()]), + allow_dns_hosts: None, + deny_subnets: None, + }), + }, + ) + .expect("first registration is answered"), + ); + assert_eq!( + web.switch_address, + Ipv4Addr::new(100, 64, 127, 255), + "the first box takes the hand-out run's first address, above the \ + daemon's reserve" + ); + assert_eq!( + web.loopback_address, + switch::AddressPlan::default() + .loopback_slice_for_switch(SUBNET) + .expect("the default subnet is planned") + .first(), + "the first box takes the first address of the slice the host switch publishes at" + ); + + // The registration's one info line names the box and both addresses + // it handed back. + let log = capture.contents(); + assert!( + log.contains("registered box with the VM host daemon; addresses allocated"), + "one info line per registration names the event: {log}" + ); + assert!( + log.contains("box=web") + && log.contains(&format!("switch_address={}", web.switch_address)) + && log.contains(&format!("loopback_address={}", web.loopback_address)), + "the info line names the box and the addresses it handed back: {log}" + ); + + // The second registration takes the next address on both runs — + // sequential allocation, no reuse. + let db = handed( + register( + &sock_path, + &RegisterBoxRequest { + name: "db".to_string(), + ingress_ports: vec![5432], + egress: None, + }, + ) + .expect("second registration is answered"), + ); + assert_eq!( + db.switch_address, + Ipv4Addr::new(100, 64, 128, 0), + "the second box takes the hand-out run's next address, never the first again" + ); + assert_eq!( + db.loopback_address, + Ipv4Addr::from( + u32::from( + switch::AddressPlan::default() + .loopback_slice_for_switch(SUBNET) + .expect("the default subnet is planned") + .first() + ) + 1 + ), + "the second box takes the next published loopback address" + ); + + // A malformed request is answered with the reason, not a hang. + let mut stream = TestStream::connect(&sock_path).expect("socket accepts"); + stream.write_all(b"this is not json\n").expect("write"); + let mut reader = BufReader::new(stream); + let mut reply = String::new(); + reader.read_line(&mut reply).expect("reply read"); + let refused: BoxControlReply = + serde_json_lenient::from_str(reply.trim()).expect("error reply parses"); + assert!( + matches!(refused, BoxControlReply::Error { .. }), + "a malformed request is answered with the reason, got {refused:?}" + ); + + // And the server keeps serving after a refusal. + handed( + register( + &sock_path, + &RegisterBoxRequest { + name: "cache".to_string(), + ingress_ports: Vec::new(), + egress: None, + }, + ) + .expect("server still serves after a refused request"), + ); + } + + /// The registry-level behaviour the socket fronts: a registration fills + /// the row the gate reads from the client's expanded spec, at the + /// addresses the reply would carry — asserted here at the verdict level + /// the gate decides by, not the field level. + #[test] + fn client_registration_fills_the_row_the_gate_reads() { + let registry = BoxRegistry::new(SUBNET); + let table = registry.table(); + let record = registry + .register_client_box(ClientBoxSpec { + name: "web".to_string(), + ingress_ports: vec![8080], + egress: Some(sessions::EgressPolicy { + allow_protocols: Some(vec![sessions::IpProto::Tcp]), + allow_subnets: Some(vec!["10.0.0.0/8".to_string()]), + allow_dns_hosts: None, + deny_subnets: None, + }), + }) + .expect("the default plan has addresses to allocate"); + + let row = table + .by_source(record.switch_addr().octets()) + .expect("the registration filled the row the gate reads"); + assert_eq!(row.name(), "web"); + assert_eq!(row.switch_addr(), record.switch_addr()); + assert_eq!(row.loopback_addr(), record.loopback_addr()); + assert_eq!(row.admitted_ports(), [8080]); + // The compiled rules are the gate's decision surface: a TCP frame to + // an allowed subnet is admitted, and one to a disallowed one is not. + let frame = crate::net::egress_gate::test_support::ipv4_frame( + record.switch_addr().octets(), + 6, + [10, 0, 0, 1], + 80, + ); + let summary = sessions::core::egress::summarize(&frame); + let verdict = sessions::core::egress::verdict(&summary, row.egress()); + assert!( + matches!(verdict, sessions::core::egress::FrameVerdict::Admit), + "the compiled policy admits what the declaration allows, got {verdict:?}" + ); + let outside = crate::net::egress_gate::test_support::ipv4_frame( + record.switch_addr().octets(), + 6, + [203, 0, 113, 7], + 443, + ); + let outside_summary = sessions::core::egress::summarize(&outside); + assert!( + matches!( + sessions::core::egress::verdict(&outside_summary, row.egress()), + sessions::core::egress::FrameVerdict::Drop(_) + ), + "the compiled policy drops what the declaration does not allow" + ); + } + + /// An unplanned subnet serves no loopback slice: an explicit + /// registration still works there (it brings its own addresses), a + /// client-driven one is refused with the reason — never a panic, never + /// a fabricated address. + #[test] + fn client_registration_on_an_unplanned_subnet_is_refused() { + let unplanned = SwitchSubnet::new(Ipv4Addr::new(10, 0, 1, 0), 24).expect("valid"); + let registry = BoxRegistry::new(unplanned); + let error = registry + .register_client_box(ClientBoxSpec { + name: "web".to_string(), + ingress_ports: Vec::new(), + egress: None, + }) + .expect_err("an unplanned subnet has no slice to allocate from"); + assert!( + matches!(error, AllocationError::UnplannedSubnet(_)), + "the refusal names the cause, got {error:?}" + ); + assert!( + registry.table().is_empty(), + "a refused registration publishes no row" + ); + } + + /// The withdrawal round-trip the destroyed session's client drives + /// (T66): it registers the box, presents the pair the registration + /// handed back to prove it is the row's creator, and the daemon removes + /// the row from its table — at both addresses. What the gate then + /// decides at the withdrawn switch address is the phase's to say, and + /// both arms are pinned at relay level through the phase parameter (not + /// the shipped constant), so the flip that ends the interim breaks + /// nothing here: the announced interim admits the withdrawn address's + /// frame under its warn, the in-force per-box default drops it. + #[tokio::test] + async fn host_row_withdrawn_on_destroy() { + use crate::net::egress_gate::{UnregisteredSourcePhase, test_support}; + + let capture = server_capture(); + let dir = tempfile::TempDir::new().expect("temp dir"); + let (sock_path, _server, registry) = spawn_server(dir.path()).expect("server binds"); + + // The registering client holds the pair the registration hands + // back — the destroy-side proof it is the row's creator. + let web = handed( + register( + &sock_path, + &RegisterBoxRequest { + name: "web".to_string(), + ingress_ports: vec![8080], + egress: Some(sessions::EgressPolicy { + allow_protocols: Some(vec![sessions::IpProto::Tcp]), + allow_subnets: Some(vec!["10.0.0.0/8".to_string()]), + allow_dns_hosts: None, + deny_subnets: None, + }), + }, + ) + .expect("the registration is answered"), + ); + let row = registry + .table() + .by_source(web.switch_address.octets()) + .expect("the registration published the row the gate reads"); + assert_eq!(row.loopback_addr(), web.loopback_address); + + // The withdrawal round-trips and is answered with the pair echoed + // back, and the table holds no row at either address after it. + let reply = control( + &sock_path, + &BoxControlRequest::Withdraw(WithdrawBoxRequest { + name: "web".to_string(), + switch_address: web.switch_address, + loopback_address: web.loopback_address, + }), + ) + .expect("the withdrawal is answered"); + match reply { + BoxControlReply::Addresses(echoed) => { + assert_eq!(echoed, web, "the withdrawal echoes the pair it went by"); + } + BoxControlReply::Error { error } => { + panic!("the creator's withdrawal is answered with the pair, refused with {error}") + } + } + assert!( + registry + .table() + .by_source(web.switch_address.octets()) + .is_none(), + "the withdrawn switch address publishes no row" + ); + assert!( + registry + .table() + .rows() + .iter() + .all(|row| row.loopback_addr() != web.loopback_address), + "the withdrawn loopback address publishes no row either" + ); + + // One info line per withdrawal, mirroring the registration's. + let log = capture.contents(); + assert!( + log.contains("withdrew the box's host row; its addresses admit nothing"), + "one info line names the withdrawal: {log}" + ); + assert!( + log.contains("box=web") + && log.contains(&format!("switch_address={}", web.switch_address)) + && log.contains(&format!("loopback_address={}", web.loopback_address)), + "the info line names the box and both addresses: {log}" + ); + + // A row is its creator's to withdraw, so a live row whose proof + // does not match is refused with the reason and stays published: + // another box's name at this row's address, and the right name with + // a loopback the registration did not hand back. + let db = handed( + register( + &sock_path, + &RegisterBoxRequest { + name: "db".to_string(), + ingress_ports: Vec::new(), + egress: None, + }, + ) + .expect("the marker box is registered"), + ); + for (name, loopback, needle) in [ + ("web", db.loopback_address, "held by box"), + ("db", Ipv4Addr::LOCALHOST, "carries loopback address"), + ] { + let refused = control( + &sock_path, + &BoxControlRequest::Withdraw(WithdrawBoxRequest { + name: name.to_string(), + switch_address: db.switch_address, + loopback_address: loopback, + }), + ) + .expect("the withdrawal is answered"); + match refused { + BoxControlReply::Error { error } => { + assert!( + error.contains(needle), + "the refusal names why the pair is not this row's ({needle}): {error}" + ); + } + BoxControlReply::Addresses(..) => { + panic!("a foreign pair's withdrawal must be refused, got addresses") + } + } + } + assert!( + registry + .table() + .by_source(db.switch_address.octets()) + .is_some(), + "a refused withdrawal leaves the row published" + ); + + // No row at the address is the goal state either way, so withdrawing + // the same pair again is answered the same way. + let again = control( + &sock_path, + &BoxControlRequest::Withdraw(WithdrawBoxRequest { + name: "web".to_string(), + switch_address: web.switch_address, + loopback_address: web.loopback_address, + }), + ) + .expect("the repeat withdrawal is answered"); + match again { + BoxControlReply::Addresses(echoed) => { + assert_eq!(echoed, web, "the repeat withdrawal echoes the pair too"); + } + BoxControlReply::Error { error } => { + panic!("no row at the address is success, refused with {error}") + } + } + + // A line that names no verb this build knows is refused at the + // parse, never reaching the table — the tagged wire's corner. + let mut stream = TestStream::connect(&sock_path).expect("socket accepts"); + stream + .write_all(b"{\"verb\":\"retire\",\"name\":\"web\"}\n") + .expect("write the unknown-verb line"); + let mut reply = String::new(); + BufReader::new(stream) + .read_line(&mut reply) + .expect("reply read"); + let refused: BoxControlReply = + serde_json_lenient::from_str(reply.trim()).expect("error reply parses"); + assert!( + matches!(refused, BoxControlReply::Error { .. }), + "an unknown verb is refused, got {refused:?}" + ); + assert!( + registry + .table() + .by_source(db.switch_address.octets()) + .is_some(), + "a refused line publishes nothing and removes nothing" + ); + + // The gate shares the registry's table, as production does: the + // announced interim admits the withdrawn address's frame under its + // warn — the row's rules ended with the row — while a published + // box's frame is the marker that proves nothing else slipped. + let marker = test_support::ipv4_frame(db.switch_address.octets(), 6, [10, 1, 2, 3], 80); + let from_withdrawn = + test_support::ipv4_frame(web.switch_address.octets(), 6, [10, 1, 2, 3], 80); + let mut h = test_support::gate_over_with_phase( + registry.clone(), + UnregisteredSourcePhase::Announced, + ) + .await; + test_support::send_frame(&mut h.guest, &from_withdrawn).await; + test_support::send_frame(&mut h.guest, &marker).await; + let seen = test_support::expect_frame(&mut h.switch).await; + assert_eq!( + seen, from_withdrawn, + "the announced interim admits the withdrawn address's frame" + ); + let seen = test_support::expect_frame(&mut h.switch).await; + assert_eq!(seen, marker, "the published box's frame is the marker"); + test_support::expect_silence(&mut h.switch).await; + test_support::wait_for_log(&h.log, "egress-unregistered-source").await; + let logged = h.log.contents(); + assert!( + logged.contains(&format!("source={}", web.switch_address)), + "the interim's line names the withdrawn source, got: {logged}" + ); + assert!( + logged.contains("T66 (#1711)"), + "the interim's line names the path that ends it, got: {logged}" + ); + + // And the in-force arm drops the same frame — the per-box default + // the flip puts in force — while the marker still passes. + let mut h = + test_support::gate_over_with_phase(registry.clone(), UnregisteredSourcePhase::InForce) + .await; + test_support::send_frame(&mut h.guest, &from_withdrawn).await; + test_support::send_frame(&mut h.guest, &marker).await; + let seen = test_support::expect_frame(&mut h.switch).await; + assert_eq!( + seen, marker, + "the in-force default drops the withdrawn address's frame and \ + passes the published box's" + ); + test_support::expect_silence(&mut h.switch).await; + test_support::wait_for_log(&h.log, "egress-unknown-source").await; + let logged = h.log.contents(); + assert!( + logged.contains(&format!("source={}", web.switch_address)), + "the drop line names the withdrawn source, got: {logged}" + ); + assert!( + !logged.contains("egress-unregistered-source"), + "the in-force arm fires no interim line, got: {logged}" + ); + } +} diff --git a/crates/minvmd/src/lib.rs b/crates/minvmd/src/lib.rs index 4f60a8d2da..eb2d6f7696 100644 --- a/crates/minvmd/src/lib.rs +++ b/crates/minvmd/src/lib.rs @@ -13,6 +13,7 @@ pub mod box_registry; pub mod cmd; pub mod config; +pub mod control; pub mod error; pub mod image; pub mod lifecycle; diff --git a/crates/minvmd/tests/minimald_session_integration.rs b/crates/minvmd/tests/minimald_session_integration.rs index 3daf48cd20..30208cf8b3 100644 --- a/crates/minvmd/tests/minimald_session_integration.rs +++ b/crates/minvmd/tests/minimald_session_integration.rs @@ -276,6 +276,7 @@ async fn run_session_exec( .map_err(|e| format!("project_path: {e}"))?, network: sessions::NetworkMode::default(), policy: Default::default(), + box_addresses: None, // The serde default, and what every non-`--no-hooks` // activation sends. This session only runs an exec, so // it declares no hooks either way. diff --git a/crates/minvmd/tests/vm_escape_integration.rs b/crates/minvmd/tests/vm_escape_integration.rs index d607cc5caa..20dc1e69e9 100644 --- a/crates/minvmd/tests/vm_escape_integration.rs +++ b/crates/minvmd/tests/vm_escape_integration.rs @@ -382,6 +382,9 @@ impl BoxSession { .map_err(|e| format!("project_path: {e}"))?, network, policy, + // No registration happened on this path: the box attaches + // as an unregistered one always has. + box_addresses: None, // The serde default, and what every non-`--no-hooks` // activation sends. This session only runs execs, so it // declares no hooks either way. diff --git a/crates/sessions/src/lib.rs b/crates/sessions/src/lib.rs index c9c8a75da3..d5eb7455c9 100644 --- a/crates/sessions/src/lib.rs +++ b/crates/sessions/src/lib.rs @@ -3,6 +3,7 @@ use std::collections::BTreeMap; use std::fmt; +use std::net::Ipv4Addr; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -554,6 +555,24 @@ where Ok(name.filter(|name| !name.is_empty())) } +/// The pair of addresses a VM host daemon allocates for a box and hands +/// back to its creator (T66): where the box lives on the switch, and where +/// it is published on the guest's loopback. +/// +/// One type because the pair is one fact — handed together over the +/// registration, carried together in the create request, recorded together +/// on the session record — and every consumer of one half needs to be able +/// to say it came from a host allocation, not a local draw. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub struct BoxAddresses { + /// The box's address on the switch: the lease its frames must carry + /// (NET-084) and the key the host-side box table's row is filed under. + pub switch_address: Ipv4Addr, + /// The box's address on the guest's loopback, from the slice the host + /// switch publishes at. + pub loopback_address: Ipv4Addr, +} + /// The on-disk row/record pertaining to a session. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Record { @@ -605,6 +624,19 @@ pub struct Record { #[serde(default = "hooks_enabled_default")] pub hooks_enabled: bool, + /// The addresses the VM host daemon handed this box's registration at + /// activation (T66): the switch address the attach path configures the + /// tap with instead of drawing its own, and the published loopback + /// address the host side names the box by. Persisted so a re-attach — + /// including after a daemon restart — attaches with the same address + /// the host table still holds a row for; drawing a fresh one would + /// silently orphan the row and drop the box to the egress gate's + /// unregistered-source interim. Defaults to `None` for records that + /// predate the field (every pre-T66 session): those boxes self-allocate + /// exactly as they always have. + #[serde(default)] + pub box_addresses: Option, + /// Free-form attributes. pub attrs: BTreeMap, } @@ -757,6 +789,7 @@ mod tests { policy, status: SessionStatus::default(), hooks_enabled: true, + box_addresses: None, attrs: BTreeMap::new(), } } diff --git a/crates/sessions/src/store.rs b/crates/sessions/src/store.rs index 3892b76ad1..d2961d13f2 100644 --- a/crates/sessions/src/store.rs +++ b/crates/sessions/src/store.rs @@ -1115,10 +1115,15 @@ mod tests { None, ), status: SessionStatus::default(), - // Deliberately the non-default (`--no-hooks`): `true` is the - // serde default, so a fixture using it would round-trip + // Deliberately the non-default shapes: `hooks_enabled: false` + // and a present `box_addresses` are both serde defaults to + // `None`/`true`, so a fixture using those would round-trip // green even if the field were dropped on write. hooks_enabled: false, + box_addresses: Some(crate::BoxAddresses { + switch_address: std::net::Ipv4Addr::new(100, 64, 0, 2), + loopback_address: std::net::Ipv4Addr::new(127, 0, 64, 0), + }), attrs: [("color".to_string(), "blue".to_string())] .into_iter() .collect(),