From fbe342a8e4c1c5a085f602081b1254028e1605c1 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:30:11 +0000 Subject: [PATCH 01/34] fix(minimald): proxy runtime-published ports like a direct connection (#2714) T99. The hostname proxy reads the target box's runtime-published ports (an allowed expose, a published listener) from its live relay gate when the name resolves, so its verdict admits exactly the ports a direct connection reaches. Admitted requests log their route source at info, and refusals record route_source=none. Backport of #2714, folded from #2724. Adapted to the line: - BoxGate, BoxPolicy and boxes::DynamicIngress become SessionGate, sessions::SessionPolicy and sessions::DynamicIngress. - crate::traced::spawn becomes tokio::spawn in the new tests. - The line's spawn_backend_on takes a full address, so proxy_forwards_runtime_published_port picks its port with a probe listener. - The parity proptest checks the gate that carries the runtime publishes and asks admits_tcp, since admits_direct_tcp covers only the declared ports here. Also keeps proxy_refuses_a_withdrawn_port_like_direct (NET-069, from #2732): it publishes and withdraws through the gate only, because the line's listen watcher has no record_dynamic_hold. (cherry picked from commit 7097b969bf7ae41ab229df1d623f64f29ce25f67) --- crates/minimald/src/net/dns.rs | 119 ++++++++- crates/minimald/src/net/proxy.rs | 364 ++++++++++++++++++++++++++- crates/minimald/src/net/switch.rs | 34 ++- crates/minimald/src/session/tests.rs | 98 +++++++- 4 files changed, 587 insertions(+), 28 deletions(-) diff --git a/crates/minimald/src/net/dns.rs b/crates/minimald/src/net/dns.rs index 440464307b..54791869ce 100644 --- a/crates/minimald/src/net/dns.rs +++ b/crates/minimald/src/net/dns.rs @@ -191,6 +191,43 @@ pub struct Route { /// proxy's verdict and the relay's port gate stays the one derivation /// ([`super::switch::declared_ingress_ports`]). declared: Option>, + /// The ports the target published at runtime (NET-044, NET-016): the + /// ports a `min net expose` decided `allow` and the ports a listener + /// published, read from the box's own relay gate when the name is + /// resolved ([`HostnameRegistry::resolve`]). The same set the gate admits + /// a direct connection on beside the declared ports, so the proxy's + /// verdict and the direct connection's read one published set + /// (NET-069, NET-071). Empty for a host-address box, whose route gates + /// nothing, and for a box with no live relay. A runtime port is the + /// same number outside and inside the box: an expose and a listen + /// publication both bind `port` to `port`. + runtime: BTreeSet, +} + +/// Why a [`Route`] carries a port: what the proxy's per-request line names +/// as the route source, and what a refusal line names when it carries none +/// (NET-069). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RouteSource { + /// The target's ingress declaration publishes the port. + Declared, + /// The target published the port at runtime: an expose or a listener. + Runtime, + /// The target is a host-address box, whose direct connections no gate + /// filters, so the route carries every port. + Ungated, +} + +impl RouteSource { + /// The value the proxy's log lines carry in their `route_source` field. + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Declared => "declared", + Self::Runtime => "runtime", + Self::Ungated => "ungated", + } + } } /// The upstream a [`Route`] forwards to. @@ -227,6 +264,7 @@ impl Route { target: Target::Loopback { address }, session: session.into(), declared, + runtime: BTreeSet::new(), } } @@ -247,6 +285,40 @@ impl Route { }, session: session.into(), declared: Some(declared_keys(&ports)), + runtime: BTreeSet::new(), + } + } + + /// This route with `runtime` as the ports its target published at + /// runtime (see [`Route::runtime`]). A host-address route gates no port, + /// so it keeps no runtime set: every port is open on it already. + #[must_use] + pub(crate) fn with_runtime(mut self, runtime: BTreeSet) -> Self { + if self.declared.is_some() { + self.runtime = runtime; + } + self + } + + /// The ports the target published at runtime, as this route carries them + /// (see [`Route::runtime`]). + #[must_use] + pub fn runtime_ports(&self) -> &BTreeSet { + &self.runtime + } + + /// Why this route carries `port`, or `None` when it does not: the + /// declaration publishes it, the target published it at runtime, or the + /// target is a host-address box that gates nothing. A declared port wins + /// over a runtime one with the same number, because the declared mapping + /// is the forward bound at that port. + #[must_use] + pub fn source(&self, port: u16) -> Option { + match &self.declared { + None => Some(RouteSource::Ungated), + Some(declared) if declared.contains(&port) => Some(RouteSource::Declared), + Some(_) if self.runtime.contains(&port) => Some(RouteSource::Runtime), + Some(_) => None, } } @@ -268,19 +340,22 @@ impl Route { /// forwards any port: its direct connections are ungated, so its proxied /// requests are too — while an own-address route with an empty set is a /// deny-all gate, the posture of a box that declared no ingress (NET-071). + /// A port the target published at runtime ([`Route::runtime_ports`]) is + /// carried too, at the same number: on a lease route the box's own port, + /// on a loopback route the forward the publish bound at the box's + /// published address (NET-044, NET-016). #[must_use] pub fn upstream(&self, port: u16) -> Option { - if let Some(declared) = self.declared_ports() - && !declared.contains(&port) - { - return None; - } - match &self.target { - Target::Loopback { address } => Some(SocketAddr::new(IpAddr::V4(*address), port)), - Target::Lease { lease, ports } => { + let source = self.source(port)?; + match (&self.target, source) { + (Target::Loopback { address }, _) => Some(SocketAddr::new(IpAddr::V4(*address), port)), + (Target::Lease { lease, ports }, RouteSource::Declared | RouteSource::Ungated) => { let internal = ports.get(&port).copied()?; Some(SocketAddr::new(IpAddr::V4(*lease), internal)) } + (Target::Lease { lease, .. }, RouteSource::Runtime) => { + Some(SocketAddr::new(IpAddr::V4(*lease), port)) + } } } @@ -1506,7 +1581,33 @@ impl HostnameRegistry { // the node's shared address keeps its name held, but nothing may be // forwarded there — the node's own listener at that port would answer // for the dead box. - route.filter(|route| !self.stopped_on_shared_address(route)) + route + .filter(|route| !self.stopped_on_shared_address(route)) + .map(|route| { + let runtime = self.runtime_published(&route); + route.with_runtime(runtime) + }) + } + + /// The ports `route`'s box published at runtime, read from its live relay + /// gate (NET-044, NET-016, NET-069): the ports a `min net expose` decided + /// `allow` and the ports a listener published — the set the gate admits + /// a direct connection on beside the declared ports. Read at resolve + /// time, so a publish is reachable through the proxy from the moment its + /// gate admits it, and a withdrawal is refused from the moment the gate + /// drops it, without a second copy of the set to keep in step. Empty for + /// a host-address route (it gates nothing), for a box that has reported + /// no lease, and for a box whose relay is gone. + fn runtime_published(&self, route: &Route) -> BTreeSet { + if route.declared_ports().is_none() { + return BTreeSet::new(); + } + self.by_session + .get(route.session()) + .and_then(|registration| self.own.get(®istration.id)) + .and_then(|own| super::switch::live_gate(own.lease)) + .map(|gate| gate.runtime_published_tcp()) + .unwrap_or_default() } /// Whether `route`'s box is stopped while publishing on the node's shared diff --git a/crates/minimald/src/net/proxy.rs b/crates/minimald/src/net/proxy.rs index 8369c05778..0d96776629 100644 --- a/crates/minimald/src/net/proxy.rs +++ b/crates/minimald/src/net/proxy.rs @@ -506,6 +506,11 @@ where session = route.session(), caller = caller.as_ref().map_or("none", super::dns::Caller::name), port, + // Whether the target published the port, and how: `none` + // for an ingress refusal, the source for an egress one. + route_source = route + .source(port) + .map_or("none", super::dns::RouteSource::as_str), direction = %refusal.direction, remote_addr = refusal .other @@ -520,6 +525,21 @@ where return write_refusal_status(&mut client, "403 Forbidden", reason).await; } }; + // One line per admitted request, naming why the target's route carries + // the port: its declaration, a runtime publish, or a host-address box + // that gates nothing. At `info`, so the daemon's default filter keeps + // it in the log a `min bug` bundle carries. + tracing::info!( + component = "dns-proxy", + host, + session = route.session(), + port, + upstream = %upstream_addr, + route_source = route + .source(port) + .map_or("none", super::dns::RouteSource::as_str), + "forwarding a proxied request" + ); let mut upstream = match tokio::time::timeout( UPSTREAM_DIAL_TIMEOUT, @@ -3572,6 +3592,277 @@ mod tests { logged.contains(r#"status="403 Forbidden""#), "expected the refusal to name the status it sent, got: {logged}" ); + // The diagnostics bundle reads, from the refusal line alone, that the + // port was neither declared nor published at runtime. + assert!( + logged.contains(r#"route_source="none""#), + "expected the refusal to say the port has no route source, got: {logged}" + ); + } + + /// The target lease [`proxy_forwards_runtime_published_port`] registers + /// its box's live gate under. The live-gate table is process-wide, so the + /// address is this test's alone. + #[cfg(target_os = "linux")] + const RUNTIME_LEASE: Ipv4Addr = Ipv4Addr::new(127, 0, 71, 1); + + /// NET-044, NET-016 and NET-069 through the proxy: a port the box + /// published at runtime — an expose decided `allow`, or a listener — + /// opens through the hostname proxy exactly while the box's own gate + /// admits a direct connection to it, and the proxy refuses it with the + /// direct connection's refusal before the publish and after the + /// withdrawal. The box declares no ingress, so every forward here is a + /// runtime one, and the per-request line names it so. + #[cfg(target_os = "linux")] + #[tokio::test] + async fn proxy_forwards_runtime_published_port() { + let probe = TcpListener::bind((RUNTIME_LEASE, 0)).await.unwrap(); + let backend_port = probe.local_addr().unwrap().port(); + drop(probe); + spawn_backend_on( + SocketAddr::new(IpAddr::V4(RUNTIME_LEASE), backend_port), + "runtime", + ) + .await; + let subnet = SwitchSubnet::default(); + + // The own-address box on a VM host: its lease reported with an empty + // applied map, because it declares no ingress. + let target = SessionId::parse_str("00000000-0000-0000-0000-000000000071").unwrap(); + let mut reg = HostnameRegistry::new(DEFAULT_HOST_ID, true); + reg.report_own_address(target, "rtweb", RUNTIME_LEASE, BTreeMap::new()); + let router = Router::new(Arc::new(RwLock::new(reg)), proxied_request_verdict); + + // The box's relay gate, registered under its lease as the relay's + // spawn registers it: the gate a direct connection meets. + let gate = Arc::new(SessionGate::for_session( + "rtweb".to_string(), + RUNTIME_LEASE, + &SessionPolicy::default(), + subnet, + None, + )); + crate::net::switch::register_live_gate_for_test(RUNTIME_LEASE, &gate); + + let proxy = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let proxy_addr = proxy.local_addr().unwrap(); + tokio::spawn(serve(proxy, router)); + let authority = format!("rtweb.min.internal:{backend_port}"); + + let buf = CaptureWriter::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(buf.clone()) + .with_ansi(false) + .with_max_level(tracing::Level::DEBUG) + .finish(); + let guard = tracing::subscriber::set_default(subscriber); + + // Before any publish: a direct connection is refused, and so is the + // proxied request. + assert!(!gate.admits_tcp(backend_port)); + let before = proxy_get(proxy_addr, &authority).await; + assert!( + before.contains("403 Forbidden"), + "an unpublished port is refused before the publish, got: {before}" + ); + + // An expose decided `allow` admits the port at the gate: the direct + // connection reaches it, and the proxied request does at once. + gate.admit_exposed(backend_port); + assert!(gate.admits_tcp(backend_port)); + let exposed = proxy_get(proxy_addr, &authority).await; + assert!( + exposed.contains("200 OK") && exposed.ends_with("runtime"), + "an exposed port opens through the proxy, got: {exposed}" + ); + + // The expose's revocation drops the admission, and the proxy refuses + // the port again with the direct connection's refusal. + gate.withdraw_exposed(backend_port); + assert!(!gate.admits_tcp(backend_port)); + let revoked = proxy_get(proxy_addr, &authority).await; + assert!( + revoked.contains("403 Forbidden"), + "a withdrawn expose is refused, got: {revoked}" + ); + + // A listener's publication is the other runtime surface: published, + // the port opens; withdrawn, it is refused. + gate.admit_published(backend_port); + let listened = proxy_get(proxy_addr, &authority).await; + assert!( + listened.contains("200 OK") && listened.ends_with("runtime"), + "a listen-published port opens through the proxy, got: {listened}" + ); + gate.withdraw_published(backend_port); + let closed = proxy_get(proxy_addr, &authority).await; + assert!( + closed.contains("403 Forbidden"), + "a closed listener's port is refused, got: {closed}" + ); + + drop(guard); + let logged = buf.contents(); + let forwarded: Vec<&str> = logged + .lines() + .filter(|line| line.contains("forwarding a proxied request")) + .collect(); + assert_eq!( + forwarded.len(), + 2, + "one line per admitted request, got: {logged}" + ); + assert!( + forwarded + .iter() + .all(|line| line.contains(r#"route_source="runtime""#) + && line.contains(r#"session="rtweb""#)), + "each admitted line names the runtime route source, got: {logged}" + ); + assert_eq!( + logged + .lines() + .filter(|line| line.contains("network policy violation") + && line.contains(r#"route_source="none""#)) + .count(), + 3, + "each refusal says the port had no route source, got: {logged}" + ); + } + + /// The target lease [`proxy_refuses_a_withdrawn_port_like_direct`] + /// registers its box's live gate under. The live-gate table is + /// process-wide, so the address is this test's alone. + #[cfg(target_os = "linux")] + const WITHDRAWN_LEASE: Ipv4Addr = Ipv4Addr::new(127, 0, 71, 2); + + /// NET-069 for a port published while the box runs (NET-016): once the + /// listener's publication is withdrawn, a proxied request to the port is + /// refused with the same refusal a direct connection gets — the verdict + /// and the logged rule a port the box never published meets. The publish + /// and the withdrawal are the listen watcher's own steps: the gate's + /// admission, then its withdrawal. + #[cfg(target_os = "linux")] + #[tokio::test] + async fn proxy_refuses_a_withdrawn_port_like_direct() { + let probe = TcpListener::bind((WITHDRAWN_LEASE, 0)).await.unwrap(); + let backend_port = probe.local_addr().unwrap().port(); + drop(probe); + spawn_backend_on( + SocketAddr::new(IpAddr::V4(WITHDRAWN_LEASE), backend_port), + "withdrawn", + ) + .await; + let never_published = 9000; + + // The own-address box on a VM host, declaring no ingress: every port + // it serves is a runtime one. + let target = SessionId::parse_str("00000000-0000-0000-0000-000000000072").unwrap(); + let mut reg = HostnameRegistry::new(DEFAULT_HOST_ID, true); + reg.report_own_address(target, "wdweb", WITHDRAWN_LEASE, BTreeMap::new()); + let registry = Arc::new(RwLock::new(reg)); + let router = Router::new(Arc::clone(®istry), proxied_request_verdict); + + // The box's relay gate, registered under its lease as the relay's + // spawn registers it: the gate a direct connection meets. + let gate = Arc::new(SessionGate::for_session( + "wdweb".to_string(), + WITHDRAWN_LEASE, + &SessionPolicy::default(), + SwitchSubnet::default(), + None, + )); + crate::net::switch::register_live_gate_for_test(WITHDRAWN_LEASE, &gate); + + let proxy = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let proxy_addr = proxy.local_addr().unwrap(); + tokio::spawn(serve(proxy, router)); + let authority = format!("wdweb.min.internal:{backend_port}"); + + // The listener's publication, as the watcher makes it once its + // forward stands: the gate admits the port. + gate.admit_published(backend_port); + assert!(gate.admits_tcp(backend_port)); + let published = proxy_get(proxy_addr, &authority).await; + assert!( + published.contains("200 OK") && published.ends_with("withdrawn"), + "a listen-published port opens through the proxy, got: {published}" + ); + + // The listener closes: the watcher withdraws the admission. + gate.withdraw_published(backend_port); + + // The direct half: the gate refuses a direct connection to the + // withdrawn port, as it refuses one to a port never published. + assert!( + !gate.admits_tcp(backend_port), + "the target's gate refuses a direct connection to the withdrawn port" + ); + assert!(!gate.admits_tcp(never_published)); + + // The proxy's verdict for the withdrawn port is the direct + // connection's refusal: the ingress rule a direct drop carries, and + // exactly the refusal a never-published port meets. + let route = registry + .read() + .unwrap() + .resolve("wdweb.min.internal") + .expect("the target's name still resolves"); + let direct = ProxiedRequest::Refused(Refusal { + rule: NO_INGRESS_MAPPING_RULE, + direction: Direction::Ingress, + other: None, + }); + assert_eq!( + proxied_request_verdict(None, &route, backend_port), + direct, + "the withdrawn port's proxied verdict must equal the direct refusal" + ); + assert_eq!( + proxied_request_verdict(None, &route, never_published), + direct, + "a never-published port meets the same refusal" + ); + + let (buf, guard) = capture_logs(); + let withdrawn = proxy_get(proxy_addr, &authority).await; + let unpublished = + proxy_get(proxy_addr, &format!("wdweb.min.internal:{never_published}")).await; + drop(guard); + assert!( + withdrawn.contains("403 Forbidden"), + "expected the withdrawn port to be refused, got: {withdrawn}" + ); + assert!( + unpublished.contains("403 Forbidden"), + "expected the never-published port to be refused, got: {unpublished}" + ); + + // Both refusals log the same violation, field for field, apart from + // the port they name. + let logged = buf.contents(); + for port in [backend_port, never_published] { + let line = logged + .lines() + .find(|line| { + line.contains("network policy violation") + && line.contains(&format!(" port={port} ")) + }) + .unwrap_or_else(|| panic!("expected a refusal line for port {port}: {logged}")); + for field in [ + r#"session="wdweb""#, + format!("rule_matched=\"{NO_INGRESS_MAPPING_RULE}\"").as_str(), + "direction=ingress", + r#"route_source="none""#, + "the box has not published this port", + r#"status="403 Forbidden""#, + ] { + assert!( + line.contains(field), + "the refusal for port {port} must carry {field}, got: {line}" + ); + } + } } /// NET-070's verify line: a request through the hostname proxy from a @@ -3802,10 +4093,15 @@ mod tests { // request is put to — or from a host-side peer, whose egress is // ungated here exactly as its direct connections are. from_a_box in prop::bool::ANY, + // The ports the target published at runtime: bit 0 exposes port + // 3000 (`min net expose` decided `allow`), bit 1 publishes port + // 3001 for a listener. Only an own-address box has a gate to + // admit them on. + runtime in 0u8..=3, // The requested port: one of the publishable externals, one of the - // internal ports behind them, or an unrelated one. + // internal ports behind them, a runtime one, or an unrelated one. request_port in prop::sample::select(vec![ - 1024u16, 1025, 1026, 1027, 8000, 8001, 8002, 8003, 9000, + 1024u16, 1025, 1026, 1027, 8000, 8001, 8002, 8003, 3000, 3001, 9000, ]), ) { let subnet = SwitchSubnet::default(); @@ -3832,8 +4128,32 @@ mod tests { credentialed_upstream: None, }; // The registry, as the session actor and the attach path fill it - // for each mode. Loopback stands in for the target's lease. + // for each mode. Loopback stands in for the target's lease; no + // other test registers a live gate under it, so the gate below + // is the only one the process-wide table holds for it. let target_lease = Ipv4Addr::LOCALHOST; + // The target's relay gate, registered under its lease as the + // relay's spawn registers it, with the runtime publishes made: + // the gate a direct connection to the target meets. + let gate = Arc::new(SessionGate::for_session( + "web".to_string(), + target_lease, + &target_policy, + subnet, + None, + )); + if runtime & 1 != 0 { + gate.admit_exposed(3000); + } + if runtime & 2 != 0 { + gate.admit_published(3001); + } + crate::net::switch::register_live_gate_for_test(target_lease, &gate); + let runtime_ports: BTreeSet = [(1u8, 3000u16), (2, 3001)] + .into_iter() + .filter(|(bit, _)| runtime & bit != 0) + .map(|(_, port)| port) + .collect(); // The applied external→internal map the attach path reports: the // declaration's own mappings, so a box with no ingress section // reports an empty map, exactly as the attach path derives it. @@ -3937,6 +4257,31 @@ mod tests { "the proxy's verdict must equal the direct connection's" ); + // The runtime half, held to the gate itself rather than to the + // route: an own-address target's runtime port is forwarded, at + // the same number, exactly when its gate admits a direct + // connection to it and the caller's egress lets the frame out. + if mode != TargetMode::HostAddress + && (3000..=3001).contains(&request_port) + && !matches!( + &observed, + ProxiedRequest::Refused(Refusal { direction: Direction::Egress, .. }) + ) + { + let reached = matches!( + &observed, + ProxiedRequest::Forward(SocketAddr::V4(up)) + if *up == SocketAddrV4::new(target_lease, request_port) + ); + prop_assert_eq!( + reached, + gate.admits_tcp(request_port), + "a runtime port is reached through the proxy exactly when \ + the gate admits a direct connection to it" + ); + prop_assert_eq!(reached, runtime_ports.contains(&request_port)); + } + // No reach a direct connection would not have: where the target // has a gate, the port the forwarded connection terminates on is // one its own declaration admits. A host-address target has no @@ -3951,24 +4296,19 @@ mod tests { TargetMode::OwnAddressOnSwitch => Some(up.port()), TargetMode::OwnAddressNative => Some( // The forwarder carries the published external port; - // the connection terminates on the internal one. + // the connection terminates on the internal one. A + // runtime port is the same number on both sides. mappings .iter() .find(|m| m.external_port == up.port()) .map(|m| m.internal_port) + .or_else(|| runtime_ports.contains(&up.port()).then_some(up.port())) .expect("a forwarded request names a published port"), ), }; if let Some(internal) = terminated_on { - let gate = SessionGate::for_session( - "web".to_string(), - target_lease, - &target_policy, - subnet, - None, - ); prop_assert!( - gate.admits_direct_tcp(internal), + gate.admits_tcp(internal), "the proxy forwarded to port {internal}, which the target's gate refuses" ); } diff --git a/crates/minimald/src/net/switch.rs b/crates/minimald/src/net/switch.rs index 162c207971..05171b4ee6 100644 --- a/crates/minimald/src/net/switch.rs +++ b/crates/minimald/src/net/switch.rs @@ -1931,6 +1931,31 @@ impl SessionGate { self.terminate_port(port) } + /// The TCP ports this gate admits because the box published them at + /// runtime: the listen-published ones (NET-016) and the explicitly + /// exposed ones (NET-044), without the declared set. The hostname + /// registry reads this when it resolves the box's name, so the proxy + /// forwards exactly the runtime ports a direct connection reaches + /// (NET-069, NET-071). A runtime port is the same number outside and + /// inside the box, so the set needs no translation. + pub(crate) fn runtime_published_tcp(&self) -> BTreeSet { + let mut ports: BTreeSet = self + .listen_published + .lock() + .expect("gate listen-published lock poisoned") + .iter() + .copied() + .collect(); + ports.extend( + self.exposed + .lock() + .expect("gate exposed lock poisoned") + .iter() + .copied(), + ); + ports + } + /// Whether this gate admits a new inbound TCP connection to `port` — /// the declared ports (NET-121), the listen-published ones (NET-016) /// and the explicitly exposed ones (NET-044) together: the one @@ -2543,9 +2568,12 @@ pub struct Refusal { /// /// The caller's declaration comes first, in the order a direct connection /// meets the two: its frame clears the caller's own egress gate before the -/// target's ingress gate ever sees it (NET-070). The target's declaration is -/// the port gate [`Route::upstream`](crate::net::dns::Route::upstream) already -/// applies — a port its ingress does not publish routes nowhere. +/// target's ingress gate ever sees it (NET-070). The target's half is the +/// port gate [`Route::upstream`](crate::net::dns::Route::upstream) already +/// applies over the target's published set: the declared ports and the ports +/// it published at runtime ([`SessionGate::runtime_published_tcp`], read into +/// the route when its name resolved). A port in neither routes nowhere, and +/// is refused with the rule a direct connection's drop carries. #[must_use] pub fn proxied_request_verdict( caller: Option<&crate::net::dns::Caller>, diff --git a/crates/minimald/src/session/tests.rs b/crates/minimald/src/session/tests.rs index 8a4b1e9446..9b7ebdb460 100644 --- a/crates/minimald/src/session/tests.rs +++ b/crates/minimald/src/session/tests.rs @@ -7926,14 +7926,18 @@ async fn expose_allow_publishes_and_lists() { // binds at the address the box's name answers at. let manager = server.state.sessions_manager().await; manager.land_range_verdict(crate::net::dns::RangeVerdict::Present); + // Addresses no other test's box uses: the live-gate table the proxy and + // the expose both read is process-wide. + let switch = std::net::Ipv4Addr::new(100, 64, 128, 96); + let loopback = std::net::Ipv4Addr::new(127, 0, 64, 96); let web = finalize_dynamic_ingress_session( &mut client, // This test's own box name: under the shared capture every test's // lines ride one buffer, so each assertion finds its line by the // box it names. "listweb", - std::net::Ipv4Addr::new(100, 64, 128, 21), - std::net::Ipv4Addr::new(127, 0, 64, 21), + switch, + loopback, Some(sessions::DynamicIngress::Allow), Some((3000, 3999)), ) @@ -7947,6 +7951,37 @@ async fn expose_allow_publishes_and_lists() { .ensure_host("tester".to_string()) .await .expect("the allowing box launches its host"); + // The box's relay gate, registered under its switch address as the + // relay's spawn registers it, and the lease its attach reports to the + // hostname registry: what a direct connection and the proxy both meet. + let gate = std::sync::Arc::new(crate::net::switch::SessionGate::for_session( + "listweb".to_string(), + switch, + &sessions::SessionPolicy::default(), + crate::net::SwitchSubnet::default(), + None, + )); + crate::net::switch::register_live_gate_for_test(switch, &gate); + let hostnames = manager.hostnames(); + hostnames + .write() + .expect("registry lock") + .report_own_address(web, "listweb", switch, std::collections::BTreeMap::new()); + // Before the expose, the proxy refuses the port with the refusal a + // direct connection gets: the box published nothing at it. + let before = { + let route = hostnames + .read() + .expect("registry lock") + .resolve("listweb.min.internal") + .expect("the box's name resolves"); + crate::net::switch::proxied_request_verdict(None, &route, 3000) + }; + assert!( + matches!(before, crate::net::switch::ProxiedRequest::Refused(_)), + "an unpublished port is refused through the proxy: {before:?}" + ); + assert!(!gate.admits_tcp(3000)); let sock = handle .net_switch() .await @@ -7961,7 +7996,7 @@ async fn expose_allow_publishes_and_lists() { .await .expect("the allowed port publishes"); let expected = minimald_rpc::LiveMapping { - local: "127.0.64.21:3000".to_string(), + local: "127.0.64.96:3000".to_string(), internal_port: 3000, proto: sessions::IpProto::Tcp, pending: Some(false), @@ -8027,10 +8062,65 @@ async fn expose_allow_publishes_and_lists() { assert!( line.contains("port=3000") && line.contains("decision=allow") - && line.contains("local=127.0.64.21:3000"), + && line.contains("local=127.0.64.96:3000"), "the publish line names the box, the port, the decision and the \ outcome: {line}" ); + // The publish is reachable through the hostname proxy at once (NET-044, + // NET-069): the box's gate admits the port, so the proxy forwards a + // request for it to the forward the publish bound — the mapping's own + // `local` — and a request through the proxy reaches what listens there. + assert!( + gate.admits_tcp(3000), + "the publish admits the port at the gate" + ); + let route = hostnames + .read() + .expect("registry lock") + .resolve("listweb.min.internal") + .expect("the box's name resolves"); + assert_eq!( + route.source(3000), + Some(crate::net::dns::RouteSource::Runtime), + "the route carries the exposed port as a runtime publish" + ); + let local: std::net::SocketAddr = mapping.local.parse().expect("the mapping names a socket"); + assert_eq!( + crate::net::switch::proxied_request_verdict(None, &route, 3000), + crate::net::switch::ProxiedRequest::Forward(local), + "the proxy forwards the exposed port to the forward the publish bound" + ); + // The fake forwarder bound nothing, so this test stands in for the box + // behind the forward, at the mapping's own address. + let backend = tokio::net::TcpListener::bind(local) + .await + .expect("the mapping's address is free to stand in for the forward"); + tokio::spawn(async move { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + while let Ok((mut sock, _)) = backend.accept().await { + let mut scratch = [0u8; 1024]; + let _ = sock.read(&mut scratch).await; + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await; + } + }); + let proxy = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .expect("the proxy binds"); + let proxy_addr = proxy.local_addr().expect("the proxy has an address"); + tokio::spawn(crate::net::proxy::serve( + proxy, + crate::net::proxy::Router::new( + std::sync::Arc::clone(&hostnames), + crate::net::switch::proxied_request_verdict, + ), + )); + let reached = crate::net::proxy::proxy_get(proxy_addr, "listweb.min.internal:3000").await; + assert!( + reached.contains("200 OK"), + "the exposed port opens through the hostname proxy: {reached}" + ); } /// NET-044: an expose decided `allow` admits its port at the box's relay From 950f384aabcb9a42a5c09cce510e3b450ee50648 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:54:22 +0000 Subject: [PATCH 02/34] fix(minimald): name the range and refuse --public in min net expose (#2718) T109. A port outside the box's --dynamic-range is refused with the range named, for allow and ask boxes alike, and an ask box refuses it before any prompt. min net expose --public is its own request and is refused because this host has no public surface. The host-refusal and port prompt texts stop naming the VM host daemon, and the session e2e script pins the new out-of-range text. Backport of #2718, folded from #2729. Adapted to the line: - main's box_actor/ports.rs hunks land in session.rs. - The line has no CHANNEL_METHODS list or channel_method helper, so the net-expose-public arm joins impl SessionChannel in env.rs without a list entry, and expose_public_refused_without_public_surface drops its channel_method assertion. The refusal text and log lines it pins are unchanged. - boxes:: becomes sessions:: and BoxKeyPredicate becomes SessionKeyPredicate. (cherry picked from commit 1bbd45c059413bb0faed332365e412e27aa5b5a4) --- crates/minimald/src/env.rs | 73 +++++++++++++++++- crates/minimald/src/env_min_helper.sh | 11 +++ crates/minimald/src/net/listeners.rs | 33 ++++++--- crates/minimald/src/net/policy.rs | 28 ++++++- crates/minimald/src/session.rs | 26 +++++-- crates/minimald/src/session/tests.rs | 90 ++++++++++++++++++++++- crates/minimald/src/session_host/tests.rs | 4 +- scripts/session-e2e.sh | 2 +- 8 files changed, 237 insertions(+), 30 deletions(-) diff --git a/crates/minimald/src/env.rs b/crates/minimald/src/env.rs index 7afe0520b5..6f5ea5f4ae 100644 --- a/crates/minimald/src/env.rs +++ b/crates/minimald/src/env.rs @@ -976,6 +976,13 @@ impl SessionChannel { self.expose_port(stream, port).await; None } + // `` or nothing: the box's own `min net expose --public + // []` (NET-145). No public surface exists on this host, so + // the request is refused and nothing is published. + Some(("net-expose-public", port)) => { + self.refuse_public_expose(stream, port); + None + } _ => { let _ = writeln!(stream, "error: unhandled input '{line}'"); None @@ -1733,6 +1740,30 @@ impl SessionChannel { } } + /// Refuses the box's `min net expose --public []` (NET-145): this + /// host has no public surface, so nothing is published, neither locally + /// nor publicly. The request is never handed to the session, so no + /// mapping can result from it. One info line names the box, the port as + /// given (empty when none was given), the outcome and the reason. + fn refuse_public_expose(&self, stream: &mut UnixStream, port: &str) { + tracing::info!( + name = %self.name, + port = %port, + outcome = "refused: public exposure", + reason = %format_args!("public exposure is not available on this host"), + "dynamic ingress expose" + ); + #[expect( + clippy::let_underscore_must_use, + reason = "the reply channel is best-effort: a peer may be gone before the reply \ + lands, and there is nowhere to report that to" + )] + let _ = writeln!( + stream, + "error: public exposure is not available on this host. Nothing was published." + ); + } + /// Writes an [`mctx::Error`] to the client as `msg:` lines (preserving its /// multi-line, richly-formatted report) followed by an `error:` terminator. fn write_error(e: &Error, stream: &mut UnixStream) { @@ -2556,6 +2587,45 @@ exit $rc ); } + /// NET-145: the box's `min net expose --public []` arrives as its + /// own request, with or without a port, and is refused with the plain + /// line the helper prints before it exits 1. The session is never asked, + /// so nothing is published: the dangling session handle here would + /// answer `session is gone` if it were. The info line names the box, the + /// port as given and the outcome. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn expose_public_refused_without_public_surface() { + let capture = crate::test_harness::captured_log(); + let (_state, _rootfs, _cwd, mut chan) = setup_channel(); + for request in ["net-expose-public%3000", "net-expose-public%"] { + let (mut ours, theirs) = UnixStream::pair().unwrap(); + #[expect( + clippy::large_futures, + reason = "the handle future carries the harness's whole channel; the test \ + awaits it to completion" + )] + chan.handle(request, &mut ours).await; + drop(ours); + assert_eq!( + read_lines(&theirs), + vec![ + "error: public exposure is not available on this host. Nothing was published." + ], + "{request}: the public request is refused, and the session is not asked" + ); + } + let logged = capture.contents(); + let lines: Vec<&str> = logged + .lines() + .filter(|line| { + line.contains("dynamic ingress expose") + && line.contains("outcome=\"refused: public exposure\"") + }) + .collect(); + assert_eq!(lines.len(), 2, "one line per request, got: {logged}"); + assert!(lines[0].contains("port=3000"), "{}", lines[0]); + } + /// NET-044's observability for the paths no session actor ever saw — the /// ones a `min net expose` request can take without one deciding it: a /// port that is not a number, a session that is gone, a reply channel an @@ -3154,7 +3224,8 @@ exit $rc assert_eq!( read_lines(&theirs), vec![ - "error: port 4200 is outside this box's declared dynamic range 3000-3999" + "error: port 4200 is outside this box's --dynamic-range 3000-3999. Nothing \ + was published." .to_string() ], "the out-of-range request is refused with its own typed error" diff --git a/crates/minimald/src/env_min_helper.sh b/crates/minimald/src/env_min_helper.sh index 1c25fdc36c..716a6d208c 100644 --- a/crates/minimald/src/env_min_helper.sh +++ b/crates/minimald/src/env_min_helper.sh @@ -215,6 +215,17 @@ min_net() { case "$subcmd" in expose) + # `--public` is its own request, never a port string. The + # daemon answers it, with or without a port. + if [[ "$1" == "--public" ]]; then + shift + if __min_is_help_flag "$1"; then + __min_usage expose + return 0 + fi + __min_rpc "net-expose-public" "$1" + return + fi local port="$1" if [[ -z "$port" ]]; then echo "Usage: min net expose " >&2 diff --git a/crates/minimald/src/net/listeners.rs b/crates/minimald/src/net/listeners.rs index b222772bb5..3d6c6ef1b4 100644 --- a/crates/minimald/src/net/listeners.rs +++ b/crates/minimald/src/net/listeners.rs @@ -457,7 +457,7 @@ pub(crate) async fn report_ask( let dial_timeout = |_elapsed| { io::Error::new( io::ErrorKind::TimedOut, - "the VM host daemon did not accept the port prompt before its deadline", + "the port prompt did not get an answer in time", ) }; let reply = match &channel { @@ -485,11 +485,15 @@ pub(crate) async fn report_ask( match reply { minimald_rpc::BoxControlReply::AskAdmit(outcome) => Ok(outcome), minimald_rpc::BoxControlReply::Error { error } => Err(io::Error::other(format!( - "the VM host daemon refused the port prompt: {error}" - ))), - other => Err(io::Error::other(format!( - "the VM host daemon sent an unexpected reply to the port prompt: {other:?}" + "the port prompt was refused: {error}" ))), + other => { + // The reply itself goes to the log, not to the person. + tracing::warn!(reply = ?other, "the port prompt got an unexpected reply"); + Err(io::Error::other( + "the port prompt got an answer it could not read", + )) + } } } @@ -543,17 +547,17 @@ pub(crate) async fn report_admitted_port( // the next attempt would draw the same refusal, and the // rate window does not need it asked for again. return Err(io::Error::other(format!( - "the VM host daemon refused to open the port: {error}" + "the request to open the port was refused: {error}" ))); } Ok(other) => { // Not the admit's own answer, so not a decision either: the // host's row may or may not name the port, and the unwind // below withdraws it the way a lost reply's does. - unanswered = Some(io::Error::other(format!( - "the VM host daemon sent an unexpected reply when asked to open the \ - port: {other:?}" - ))); + tracing::warn!(reply = ?other, "the request to open the port got an unexpected reply"); + unanswered = Some(io::Error::other( + "the request to open the port got an answer it could not read", + )); break; } // No reply: a lost answer, retried — the host's admit is @@ -568,6 +572,15 @@ pub(crate) async fn report_admitted_port( } } let unanswered = unanswered.expect("the loop ran at least once without deciding"); + // A deadline reads as the request's own words, not the door's. + let unanswered = if unanswered.kind() == io::ErrorKind::TimedOut { + io::Error::new( + io::ErrorKind::TimedOut, + "the request to open the port did not get an answer in time", + ) + } else { + unanswered + }; // Fail-closed, and clean on the host's side too: the host may have // recorded the port before its reply was lost, so the unwind withdraws // it there — best-effort, because the publish is failing either way and diff --git a/crates/minimald/src/net/policy.rs b/crates/minimald/src/net/policy.rs index 5d910c22f3..ac59059455 100644 --- a/crates/minimald/src/net/policy.rs +++ b/crates/minimald/src/net/policy.rs @@ -733,6 +733,9 @@ impl ExposeRefusal { host_loopback: false, .. } => format!("{self}, so it already answers at {box_name}.min.internal:{port}"), + // The same words for an allow-stance and an ask-stance box: the + // range refused the port before any prompt (NET-047). + Self::OutOfRange { .. } => format!("{self}. Nothing was published."), _ => self.to_string(), } } @@ -758,7 +761,7 @@ impl fmt::Display for ExposeRefusal { ), Self::OutOfRange { requested, range } => write!( f, - "port {requested} is outside this box's declared dynamic range {}-{}", + "port {requested} is outside this box's --dynamic-range {}-{}", range.0, range.1 ), Self::NoPublishedAddress => write!( @@ -853,7 +856,8 @@ impl fmt::Display for ExposeFailure { /// derive — the one derivation this decision and the listen watcher's /// verdict share, so the two runtime ingress surfaces cannot part ways on a /// port — and this is the half that renders each fact as the refusal the -/// box's `min net expose` prints. +/// box's `min net expose` prints. An `ask` box's range is checked here too, +/// so a port outside it is refused without a prompt. pub fn dynamic_ingress_decision( ingress: Option<&IngressPolicy>, port: u16, @@ -861,7 +865,15 @@ pub fn dynamic_ingress_decision( match IngressRules::from_policy(ingress).dynamic_verdict(port) { DynamicPortVerdict::Allow => Ok(sessions::DynamicIngress::Allow), DynamicPortVerdict::Deny => Err(ExposeRefusal::DeniedByPolicy), - DynamicPortVerdict::Ask => Err(ExposeRefusal::AskNeedsAnswer), + // An `ask` box with a range refuses a port outside it here, before + // any prompt is routed: no answer could publish it (NET-047). + DynamicPortVerdict::Ask => match ingress.and_then(|i| i.dynamic_allowed_range) { + Some(range) if port < range.0 || port > range.1 => Err(ExposeRefusal::OutOfRange { + requested: port, + range, + }), + _ => Err(ExposeRefusal::AskNeedsAnswer), + }, DynamicPortVerdict::NoRange => Err(ExposeRefusal::NoDynamicRange), DynamicPortVerdict::OutOfRange { range } => Err(ExposeRefusal::OutOfRange { requested: port, @@ -1833,7 +1845,15 @@ mod tests { // range, in the words `min net expose` prints. let expected_decision = match stance { None | Some(DynamicIngress::Deny) => Err(ExposeRefusal::DeniedByPolicy), - Some(DynamicIngress::Ask) => Err(ExposeRefusal::AskNeedsAnswer), + Some(DynamicIngress::Ask) => match range { + Some((low, high)) if port < low || port > high => { + Err(ExposeRefusal::OutOfRange { + requested: port, + range: (low, high), + }) + } + _ => Err(ExposeRefusal::AskNeedsAnswer), + }, Some(DynamicIngress::Allow) => match range { None => Err(ExposeRefusal::NoDynamicRange), Some((low, high)) if low <= port && port <= high => { diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index ffdacff364..eea70733e0 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -817,14 +817,12 @@ fn host_ask_refusal_text(reason: minimald_rpc::AskRefused) -> &'static str { "too many port prompts from this box are already waiting for an answer" } minimald_rpc::AskRefused::NoRow => { - "this box's network addresses are not registered with the VM host daemon" + "this box's network addresses are not registered on the host" } minimald_rpc::AskRefused::StanceNotAsk => { - "the VM host daemon does not have this box's --dynamic-ingress set to ask" - } - minimald_rpc::AskRefused::OutsideGrant => { - "the port is outside the --dynamic-range the VM host daemon holds for this box" + "the host does not have this box's --dynamic-ingress set to ask" } + minimald_rpc::AskRefused::OutsideGrant => "the port is outside this box's --dynamic-range", minimald_rpc::AskRefused::NoClient => "nobody is attached on the host to answer", minimald_rpc::AskRefused::Denied => "the person attached answered no", minimald_rpc::AskRefused::NoTty => "the attached terminal could not show the prompt", @@ -2973,7 +2971,9 @@ impl Session { .unwrap_or(sessions::DynamicIngress::Deny); // The box's own setting decides first (NET-043), whatever asked: the // deny-all default when nothing was declared, the range gate under an - // `allow`, and an `ask` handed to whoever is attached (NET-045). + // `allow` or an `ask`, and an in-range `ask` handed to whoever is + // attached (NET-045). A port outside an `ask` box's range is refused + // here, so no prompt is routed for it (NET-047). match crate::net::policy::dynamic_ingress_decision(record.policy.ingress.as_ref(), port) { Err(crate::net::policy::ExposeRefusal::AskNeedsAnswer) => { self.route_ask(record, box_name, port, reply).await; @@ -3247,7 +3247,7 @@ impl Session { AskEnd::HostRefused(reason) => Err(crate::net::policy::ExposeFailure::Publish { port: ask.port, source: std::io::Error::other(format!( - "the VM host daemon refused the port prompt: {}", + "the port prompt was refused: {}", host_ask_refusal_text(*reason) )), }), @@ -3417,12 +3417,22 @@ impl Session { ) } Err(crate::net::policy::ExposeFailure::Refused(refusal)) => { + // A port outside the box's range says so in the outcome, so + // the log tail tells it apart from a stance refusal. + let outcome = if matches!( + refusal, + crate::net::policy::ExposeRefusal::OutOfRange { .. } + ) { + "refused: out of range" + } else { + "refused" + }; tracing::info!( name = %box_name, port, decision = %decision, decided_by = %decided_by, - outcome = "refused", + outcome, reason = %refusal, "dynamic ingress expose" ); diff --git a/crates/minimald/src/session/tests.rs b/crates/minimald/src/session/tests.rs index 9b7ebdb460..45ff4b8184 100644 --- a/crates/minimald/src/session/tests.rs +++ b/crates/minimald/src/session/tests.rs @@ -8466,7 +8466,7 @@ async fn refused_admission_report_leaves_no_partial_mapping() { assert!( source .to_string() - .contains("the VM host daemon refused to open the port"), + .contains("the request to open the port was refused"), "the refused publish says who refused it: {source}" ); } @@ -9059,13 +9059,12 @@ async fn expose_rejected_leaves_no_partial_mapping() { .lines() .find(|line| { line.contains("dynamic ingress expose") - && line.contains("outcome=\"refused\"") + && line.contains("outcome=\"refused: out of range\"") && line.contains("port=4500") }) .unwrap_or_else(|| panic!("the out-of-range refusal must be logged, got: {logged}")); assert!( - refused_line - .contains("reason=port 4500 is outside this box's declared dynamic range 3000-3999"), + refused_line.contains("reason=port 4500 is outside this box's --dynamic-range 3000-3999"), "the refusal names the port and the declared range: {refused_line}" ); let failed_line = logged @@ -9082,6 +9081,89 @@ async fn expose_rejected_leaves_no_partial_mapping() { ); } +/// NET-047: a port outside the box's `--dynamic-range` is refused with the +/// range named, for an `allow` box and an `ask` box alike. The `ask` box +/// refuses it before any prompt is routed: the same box's in-range request +/// is routed, and with nobody attached it ends as the nobody-to-answer +/// refusal, which an out-of-range request never reaches. Neither box keeps a +/// mapping, and the request's info line says `refused: out of range`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn expose_out_of_range_refusal_names_range() { + let capture = crate::test_harness::captured_log(); + let server = TestServer::new().await; + let mut client = server.connect().await; + let manager = server.state.sessions_manager().await; + for (name, stance, octet) in [ + ("rangeallow", sessions::DynamicIngress::Allow, 41), + ("rangeask", sessions::DynamicIngress::Ask, 42), + ] { + let id = finalize_dynamic_ingress_session( + &mut client, + name, + std::net::Ipv4Addr::new(100, 64, 128, octet), + std::net::Ipv4Addr::new(127, 0, 64, octet), + Some(stance), + Some((3000, 3999)), + ) + .await; + let handle = manager + .get_session(crate::sessions::SessionKeyPredicate::Id(id)) + .await + .unwrap() + .expect("the box resolves"); + let refusal = match handle.expose_dynamic(4500).await { + Err(crate::net::policy::ExposeFailure::Refused( + refusal @ crate::net::policy::ExposeRefusal::OutOfRange { + requested: 4500, + range: (3000, 3999), + }, + )) => refusal, + other => panic!("{name}: a port outside the range is refused by range: {other:?}"), + }; + assert_eq!( + refusal.for_caller(name, sessions::NetworkMode::OwnIp, 4500), + "port 4500 is outside this box's --dynamic-range 3000-3999. Nothing was published.", + "{name}: the refusal names the port and the range" + ); + if stance == sessions::DynamicIngress::Ask { + // The in-range ask is routed, and nobody is attached to answer: + // the out-of-range one above never got that far. + match handle.expose_dynamic(3000).await { + Err(crate::net::policy::ExposeFailure::Refused( + crate::net::policy::ExposeRefusal::AskNeedsAnswer, + )) => {} + other => panic!("an in-range ask is routed to a prompt: {other:?}"), + } + } + let live: minimald_rpc::Errorable> = client + .call::(&minimald_rpc::GetLiveIngressRequest::Id(id)) + .await; + assert_eq!( + live, + minimald_rpc::Errorable::Ok(vec![]), + "{name}: a refused request leaves no mapping" + ); + } + let logged = capture.contents(); + for (name, decision) in [("rangeallow", "allow"), ("rangeask", "ask")] { + let line = logged + .lines() + .find(|line| { + line.contains("dynamic ingress expose") + && line.contains(&format!("name={name}")) + && line.contains("port=4500") + }) + .unwrap_or_else(|| panic!("{name}: the refusal must be logged, got: {logged}")); + assert!( + line.contains("outcome=\"refused: out of range\"") + && line.contains(&format!("decision={decision}")) + && line + .contains("reason=port 4500 is outside this box's --dynamic-range 3000-3999"), + "{name}: the line names the outcome, the port and the range: {line}" + ); + } +} + /// NET-044 on the shared-address interim: under an absent range verdict every /// box publishes at `127.0.0.1`, so a runtime expose of a port another box /// already holds there cannot be made — the switch refuses the duplicate diff --git a/crates/minimald/src/session_host/tests.rs b/crates/minimald/src/session_host/tests.rs index 120e6a8fb5..7f9a0506dd 100644 --- a/crates/minimald/src/session_host/tests.rs +++ b/crates/minimald/src/session_host/tests.rs @@ -6806,12 +6806,12 @@ async fn vm_ask_host_refusals_are_publish_failures() { ), ( minimald_rpc::AskRefused::OutsideGrant, - "outside the --dynamic-range", + "outside this box's --dynamic-range", ), (minimald_rpc::AskRefused::StanceNotAsk, "set to ask"), ( minimald_rpc::AskRefused::NoRow, - "not registered with the VM host daemon", + "not registered on the host", ), ] { let result = vm_ask_ended_by_host(host_refused(reason, None)).await; diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index 670f2cf88d..c89ccda0d7 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -17801,7 +17801,7 @@ PY fail fi case "$mnx_ref_err" in - *"port $mnx_out_port is outside this box's declared dynamic range $mnx_lo-$mnx_hi"*) ;; + *"port $mnx_out_port is outside this box's --dynamic-range $mnx_lo-$mnx_hi. Nothing was published."*) ;; *) echo "::error::the out-of-range refusal does not name the port and the declared range (got: '$mnx_ref_err')"; fail ;; esac echo "out of range: refused with its own typed error — the range the create declared is the whole gate (NET-043)" From 47df1d65635a58cc29ae81a167c3bfc4e127e929 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:27:51 +0000 Subject: [PATCH 03/34] fix(minimald): record VM refusals on the host and admit RPC only from the host (#2721) T107. The in-VM daemon's vsock listener drops a peer at any CID but 2 at accept, before reading a byte, and writes one rate-limited audited line naming the CID (NET-139). The in-VM daemon reports each expose decision and each refused own-address egress flow over the guest door, and the VM host daemon appends them to its audit log under per-box caps, with a line counting suppressed records (NET-046). Expose decisions keep their own cap, a withdrawn box flushes its window, guest-reported records carry reported_by=guest, and box-admissions.log rotates to one kept generation at 1 MiB. The session e2e script expects the new host audit records. Backport of #2721, folded from #2735. Adapted to the line: - boxes::DynamicIngress becomes sessions::DynamicIngress. - The line has no ProxyListening verb or reply, so ReportExposeDecision, ReportRefusedEgress and Audited stand without it. - The listen watcher keeps tokio::spawn, since crate::traced is absent. The audit route is registered before the spawn and removed after the watcher's withdraw_all. - minvmd has no serde_json here, so its host audit tests parse with serde_json_lenient. - The vsock peer-CID tests join the inline tests in server.rs. Dropped: ask_audit_line_spells_box_id_as_32_hex, which reached the cherry-pick as conflict context from main's proxy work and is not part of #2721. (cherry picked from commit abe99f7c82e647593ef07d46f0894547b6ed0280) --- crates/minimal-client/src/box_registration.rs | 6 + crates/minimald-rpc/src/lib.rs | 95 ++ crates/minimald/src/audit.rs | 40 +- crates/minimald/src/net/listeners.rs | 201 ++++- crates/minimald/src/net/switch.rs | 24 +- crates/minimald/src/rpc.rs | 4 +- crates/minimald/src/server.rs | 217 +++++ crates/minimald/src/session_host/tests.rs | 73 +- crates/minvmd/src/control.rs | 810 +++++++++++++++++- scripts/session-e2e.sh | 110 ++- 10 files changed, 1542 insertions(+), 38 deletions(-) diff --git a/crates/minimal-client/src/box_registration.rs b/crates/minimal-client/src/box_registration.rs index 45337307e5..6035c2a8eb 100644 --- a/crates/minimal-client/src/box_registration.rs +++ b/crates/minimal-client/src/box_registration.rs @@ -283,6 +283,12 @@ pub async fn register_box_with_vm_host( read for {switch_address}; the registration did not happen" ) } + minimald_rpc::BoxControlReply::Audited { .. } => { + anyhow::bail!( + "the VM host daemon answered the box registration with an audit report's \ + answer; the registration did not happen" + ) + } minimald_rpc::BoxControlReply::NameHeld { name, .. } => { anyhow::bail!( "the VM host daemon answered the box registration with the name-hold \ diff --git a/crates/minimald-rpc/src/lib.rs b/crates/minimald-rpc/src/lib.rs index 04a9998084..4eabe26290 100644 --- a/crates/minimald-rpc/src/lib.rs +++ b/crates/minimald-rpc/src/lib.rs @@ -846,6 +846,83 @@ pub struct WithdrawPortRequest { pub source: PortReportSource, } +/// Who decided a dynamic ingress request, as the in-VM daemon's expose +/// decision report names it ([`ExposeDecisionReport`]): the same three +/// deciders, in the same kebab-case spelling, as the in-VM daemon's own +/// audit record. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum ExposeDecidedBy { + /// The box's own `dynamic_ingress` declaration. + BoxPolicy, + /// The attached human, answering an `ask` (NET-045). + AttachedHuman, + /// The in-VM daemon, failing an `ask` closed with nobody attached. + Daemon, +} + +/// What became of a dynamic ingress request, as the expose decision report +/// names it ([`ExposeDecisionReport`]). +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum ExposeDecisionOutcome { + /// The port was published. + Published, + /// The request was refused. + Refused, + /// The publish could not be made. + PublishFailed, +} + +/// The in-VM daemon's report of one dynamic ingress decision (NET-046): the +/// host keeps a copy of each decision in its own audit log, outside the VM, +/// so the record survives a box that erases the in-VM log. +/// +/// Fixed and size-bounded like the port reports: the row key, the port, and +/// three closed enums. The box is named by the host from the row at +/// `switch_address`, never by a string the guest sends. A report keyed at +/// no row is refused and leaves no record. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub struct ExposeDecisionReport { + /// The switch address of the row the decision is about. + pub switch_address: std::net::Ipv4Addr, + /// The port the box asked to publish. + pub port: u16, + /// The decision the box's `dynamic_ingress` setting made. + pub decision: DynamicIngress, + /// Who decided it. + pub decided_by: ExposeDecidedBy, + /// What became of the request. + pub outcome: ExposeDecisionOutcome, +} + +/// The longest `proto` or `rule` text a [`RefusedEgressReport`] carries. +/// The host clamps a longer one, so a report cannot grow the host's log. +pub const REFUSED_EGRESS_TEXT_MAX: usize = 64; + +/// The in-VM daemon's report of one refused own-address egress flow: the +/// relay dropped a box's frame against its egress rules and said so in its +/// rate-limited drop line, and the same event is reported to the host, +/// which appends it to its own audit log outside the VM. +/// +/// Sent at most once per drop line, so under the same rate limit as that +/// line and never once per frame. The host caps the records it writes per +/// box again on its side, whatever the guest sends. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct RefusedEgressReport { + /// The switch address of the row of the box whose frame was dropped. + pub switch_address: std::net::Ipv4Addr, + /// The destination the dropped frame was sent to, when it had an IPv4 + /// destination to read. + pub destination: Option, + /// The frame's protocol as the drop line names it (`tcp`, `udp`, + /// `icmp`, `ipv6`, ...), at most [`REFUSED_EGRESS_TEXT_MAX`] bytes. + pub proto: String, + /// The rule that dropped it, as the drop line names it, at most + /// [`REFUSED_EGRESS_TEXT_MAX`] bytes. + pub rule: String, +} + /// The read-only row verb's key: the box's name, the identity a row is /// registered under ([`RegisterBoxRequest::name`]). Liveness is the table's /// own fact — a name that no live box holds answers @@ -1264,6 +1341,16 @@ pub enum BoxControlRequest { /// ([`RowStandingRequest`]): carried on the daemon's own control /// channel, answered with [`BoxControlReply::RowStanding`]. RowStanding(RowStandingRequest), + /// The in-VM daemon's report of one dynamic ingress decision + /// ([`ExposeDecisionReport`], NET-046): carried on the daemon's own + /// control channel, appended to the host's audit log, and answered with + /// [`BoxControlReply::Audited`]. + ReportExposeDecision(ExposeDecisionReport), + /// The in-VM daemon's report of one refused own-address egress flow + /// ([`RefusedEgressReport`]): carried on the daemon's own control + /// channel, appended to the host's audit log under a per-box cap, and + /// answered with [`BoxControlReply::Audited`]. + ReportRefusedEgress(RefusedEgressReport), } /// The VM host daemon's answerer status: the state of the machine's @@ -1631,6 +1718,14 @@ pub enum BoxControlReply { /// Whether a row stands at it. row_standing: bool, }, + /// An audit report's answer: whether the host wrote a record for it. + /// `false` when the host's per-box cap suppressed the record, which the + /// host counts in its log instead. `audited` is required — the marker + /// that keeps this document from decoding as any other reply. + Audited { + /// Whether a record was written. + audited: bool, + }, } /// The request for a [`CreateSession`] RPC. diff --git a/crates/minimald/src/audit.rs b/crates/minimald/src/audit.rs index c452ece1e4..93fd7a8261 100644 --- a/crates/minimald/src/audit.rs +++ b/crates/minimald/src/audit.rs @@ -149,7 +149,7 @@ pub(crate) fn rotated_path(state_dir: &Path) -> PathBuf { /// diagnostic bundle ships beside this file's tail, carries the failure /// either way. pub(crate) async fn append(state_dir: &Path, record: &DecisionRecord) { - if let Err(e) = try_append(state_dir, record).await { + if let Err(e) = try_append_capped(state_dir, record, MAX_LOG_BYTES).await { tracing::warn!( box = %record.box_name, port = record.port, @@ -157,6 +157,36 @@ pub(crate) async fn append(state_dir: &Path, record: &DecisionRecord) { "the dynamic ingress decision could not be audited" ); } + // A refusal goes to the host whether or not the local line landed: the + // host's copy is the one a box cannot reach, and it names a refusal + // that stands either way. + report_to_host(record); +} + +/// Reports `record` to the VM host daemon, which keeps its own copy of the +/// decision outside the VM (NET-046). Best-effort and in the background: +/// the local record is the decision's own, and a host that cannot be +/// reached never changes it. A daemon with no VM host behind the box sends +/// nothing. +fn report_to_host(record: &DecisionRecord) { + #[cfg(target_os = "linux")] + crate::net::listeners::report_expose_decision( + &record.box_name, + record.port, + record.decision, + match record.decided_by { + DecidedBy::BoxPolicy => minimald_rpc::ExposeDecidedBy::BoxPolicy, + DecidedBy::AttachedHuman => minimald_rpc::ExposeDecidedBy::AttachedHuman, + DecidedBy::Daemon => minimald_rpc::ExposeDecidedBy::Daemon, + }, + match record.outcome { + DecisionOutcome::Published => minimald_rpc::ExposeDecisionOutcome::Published, + DecisionOutcome::Refused => minimald_rpc::ExposeDecisionOutcome::Refused, + DecisionOutcome::PublishFailed => minimald_rpc::ExposeDecisionOutcome::PublishFailed, + }, + ); + #[cfg(not(target_os = "linux"))] + let _ = record; } /// Appends one decision to the log (NET-046), rotating it first when the @@ -164,8 +194,14 @@ pub(crate) async fn append(state_dir: &Path, record: &DecisionRecord) { /// landed: the caller of an allow refuses the exposure on an `Err`, because /// NET-046 requires every decision recorded and an allow is the one decision /// that can still be taken back. +/// +/// A record that landed is also reported to the VM host daemon, in the +/// background. One that did not land is not: the caller withdraws the +/// allow it names, and records that withdrawal through [`append`]. pub(crate) async fn try_append(state_dir: &Path, record: &DecisionRecord) -> std::io::Result<()> { - try_append_capped(state_dir, record, MAX_LOG_BYTES).await + try_append_capped(state_dir, record, MAX_LOG_BYTES).await?; + report_to_host(record); + Ok(()) } /// [`try_append`] with the rotation threshold passed in, so a test can rotate diff --git a/crates/minimald/src/net/listeners.rs b/crates/minimald/src/net/listeners.rs index 3d6c6ef1b4..13b01adf05 100644 --- a/crates/minimald/src/net/listeners.rs +++ b/crates/minimald/src/net/listeners.rs @@ -225,7 +225,7 @@ enum BoxReportChannel { /// polls reach the stand-in exactly the expose path's own reports do. #[cfg(test)] static VM_REPORT_SEAM: LazyLock>> = - LazyLock::new(|| Mutex::new(HashMap::new())); + std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); /// Seed a stand-in report door for the session whose switch control socket /// is `control_socket` (tests only, T94): the door at `door` answers every @@ -641,6 +641,200 @@ pub(crate) async fn unreport_port( } } +/// Where one box's audit reports go (NET-046): the VM host daemon's door, +/// and the row key the host names the box by. Registered while the box's +/// listen watcher runs on a VM host ([`register_host_audit_route`]), so a +/// box with no switch, or a box on a native host, has none and reports +/// nothing. +#[derive(Debug, Clone)] +struct HostAuditRoute { + /// The box's name, as the in-VM audit record names it. + box_name: String, + /// The door the reports go to. + channel: BoxReportChannel, +} + +/// The live audit routes, by the box's switch address: the relay's drop +/// line knows the box by its lease, and the expose decision record by its +/// name, so both look here. +static HOST_AUDIT_ROUTES: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); + +/// The most audit reports in flight at once. A report past it is not sent +/// and the in-VM record stays the only one, so a box that refuses frames +/// in a loop cannot grow the daemon's tasks. The drop line's own rate limit +/// keeps a box far under it. +const HOST_AUDIT_IN_FLIGHT: usize = 64; + +/// The audit reports in flight now ([`HOST_AUDIT_IN_FLIGHT`]). +static HOST_AUDIT_SENDING: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// Registers where `box_name`'s audit reports go, while its watcher runs: +/// the VM host daemon's door over `control`, keyed by the box's switch +/// address. A native host's control channel has no door, so nothing is +/// registered there. +fn register_host_audit_route(box_name: &str, switch_address: Ipv4Addr, control: &ControlChannel) { + let channel = match control { + ControlChannel::Vsock { cid, .. } => BoxReportChannel::Vsock { cid: *cid }, + ControlChannel::Unix(_) => return, + }; + HOST_AUDIT_ROUTES + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + switch_address, + HostAuditRoute { + box_name: box_name.to_string(), + channel, + }, + ); +} + +/// Removes the audit route [`register_host_audit_route`] made for the box +/// at `switch_address`, when it is still that box's. +fn deregister_host_audit_route(box_name: &str, switch_address: Ipv4Addr) { + let mut routes = HOST_AUDIT_ROUTES + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if routes + .get(&switch_address) + .is_some_and(|route| route.box_name == box_name) + { + routes.remove(&switch_address); + } +} + +/// Seeds an audit route for `box_name` at `switch_address` whose reports go +/// to the stand-in door at `door` (tests only): the harness runs boxes on a +/// native-shaped switch, which registers no route of its own. +#[cfg(test)] +pub(crate) fn seed_host_audit_route_for_tests( + box_name: &str, + switch_address: Ipv4Addr, + door: impl Into, +) { + HOST_AUDIT_ROUTES + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + switch_address, + HostAuditRoute { + box_name: box_name.to_string(), + channel: BoxReportChannel::Unix(door.into()), + }, + ); +} + +/// Removes an audit route a test seeded (tests only). +#[cfg(test)] +pub(crate) fn clear_host_audit_route_for_tests(box_name: &str, switch_address: Ipv4Addr) { + deregister_host_audit_route(box_name, switch_address); +} + +/// Sends one audit report to the VM host daemon in the background, once, +/// under [`REPORT_DEADLINE`]. Best-effort: the in-VM record is already +/// written or already failed on its own, so a host that cannot be reached +/// costs only the host's copy. A report the host does not take says so in +/// one debug line. +fn send_host_audit(channel: BoxReportChannel, request: minimald_rpc::BoxControlRequest) { + use std::sync::atomic::Ordering; + if HOST_AUDIT_SENDING.fetch_add(1, Ordering::AcqRel) >= HOST_AUDIT_IN_FLIGHT { + HOST_AUDIT_SENDING.fetch_sub(1, Ordering::AcqRel); + tracing::debug!( + request = ?request, + "too many audit reports are in flight; the host keeps no copy of this one" + ); + return; + } + // A caller off a runtime (a sync test) has nothing to send on. + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + HOST_AUDIT_SENDING.fetch_sub(1, Ordering::AcqRel); + return; + }; + runtime.spawn(async move { + let deadline = tokio::time::Instant::now() + REPORT_DEADLINE; + match report_exchange(&channel, &request, deadline).await { + Ok(minimald_rpc::BoxControlReply::Audited { .. }) => {} + Ok(other) => tracing::debug!( + reply = ?other, + "the VM host daemon did not take an audit report" + ), + Err(error) => tracing::debug!( + %error, + "an audit report did not reach the VM host daemon" + ), + } + HOST_AUDIT_SENDING.fetch_sub(1, Ordering::AcqRel); + }); +} + +/// Reports one dynamic ingress decision to the VM host daemon (NET-046), so +/// the host keeps its own copy outside the VM. Nothing is sent for a box +/// with no audit route: a native host, or a box with no switch. +pub(crate) fn report_expose_decision( + box_name: &str, + port: u16, + decision: sessions::DynamicIngress, + decided_by: minimald_rpc::ExposeDecidedBy, + outcome: minimald_rpc::ExposeDecisionOutcome, +) { + let route = HOST_AUDIT_ROUTES + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .find(|(_, route)| route.box_name == box_name) + .map(|(switch_address, route)| (*switch_address, route.channel.clone())); + let Some((switch_address, channel)) = route else { + return; + }; + send_host_audit( + channel, + minimald_rpc::BoxControlRequest::ReportExposeDecision(minimald_rpc::ExposeDecisionReport { + switch_address, + port, + decision, + decided_by, + outcome, + }), + ); +} + +/// Reports one refused own-address egress flow to the VM host daemon: the +/// relay calls this when its rate-limited drop line fires, so the report +/// shares that line's rate limit and is never sent once per frame. Nothing +/// is sent for a box with no audit route. +pub(crate) fn report_refused_egress( + switch_address: Ipv4Addr, + destination: Option, + proto: &str, + rule: &str, +) { + let channel = HOST_AUDIT_ROUTES + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&switch_address) + .map(|route| route.channel.clone()); + let Some(channel) = channel else { + return; + }; + let clamp = |text: &str| { + let mut end = text.len().min(minimald_rpc::REFUSED_EGRESS_TEXT_MAX); + while !text.is_char_boundary(end) { + end -= 1; + } + text[..end].to_string() + }; + send_host_audit( + channel, + minimald_rpc::BoxControlRequest::ReportRefusedEgress(minimald_rpc::RefusedEgressReport { + switch_address, + destination, + proto: clamp(proto), + rule: clamp(rule), + }), + ); +} + /// The withdrawal's attempts over a resolved report channel, shared by /// [`report_withdrawn_port`] and the unwind of an admit whose reply never /// arrived. @@ -1196,6 +1390,10 @@ impl ListenWatcher { leader = ?leader, "listen watcher started" ); + // The box's audit reports reach the VM host daemon while the + // watcher runs (NET-046), and stop when it ends. + register_host_audit_route(&plan.box_name, plan.lease, &plan.control); + let route = (plan.box_name.clone(), plan.lease); let task = tokio::spawn(async move { let mut state = WatchState::new(plan, leader); loop { @@ -1208,6 +1406,7 @@ impl ListenWatcher { // The stop's own work: everything the box's processes published // comes down here, before the stopper's teardown continues. state.withdraw_all().await; + deregister_host_audit_route(&route.0, route.1); }); Self { stop, task } } diff --git a/crates/minimald/src/net/switch.rs b/crates/minimald/src/net/switch.rs index 05171b4ee6..cac728690d 100644 --- a/crates/minimald/src/net/switch.rs +++ b/crates/minimald/src/net/switch.rs @@ -831,14 +831,30 @@ where gate.dns.admits_flow(dst, pkt.as_ref(), Instant::now()) }); if !pinned { - gate.limiter.warn( + let remote = drop_remote(&summary); + let transport = drop_transport(&reason); + // The VM host daemon keeps its own record of the refused + // flow, outside the VM: reported only when the drop line + // fires, so under the same rate limit, never per frame. + if gate.limiter.warn( &gate.label, Direction::Egress, - drop_remote(&summary), - drop_transport(&reason), + remote, + transport, None, reason.rule(), - ); + ) { + let destination = match remote { + Some(SocketAddr::V4(destination)) => Some(destination), + _ => None, + }; + crate::net::listeners::report_refused_egress( + reject.lease, + destination, + &transport.to_string(), + reason.rule(), + ); + } continue; } } diff --git a/crates/minimald/src/rpc.rs b/crates/minimald/src/rpc.rs index 64693785fb..ed8477abf6 100644 --- a/crates/minimald/src/rpc.rs +++ b/crates/minimald/src/rpc.rs @@ -925,7 +925,9 @@ async fn serve_answerer_control( | BoxControlRequest::RecordAskAnswer(_) | BoxControlRequest::SubscribeAsks(_) | BoxControlRequest::ResumeBox(_) - | BoxControlRequest::RowStanding(_) => BoxControlReply::Error { + | BoxControlRequest::RowStanding(_) + | BoxControlRequest::ReportExposeDecision(_) + | BoxControlRequest::ReportRefusedEgress(_) => BoxControlReply::Error { error: "the native daemon's control socket answers only the answerer \ verbs; boxes register over the daemon's RPC channels" .to_string(), diff --git a/crates/minimald/src/server.rs b/crates/minimald/src/server.rs index 2f0f1d73d7..17bbc14adf 100644 --- a/crates/minimald/src/server.rs +++ b/crates/minimald/src/server.rs @@ -1017,6 +1017,87 @@ pub trait Listener: Send { const IS_LOCAL: bool; fn accept(&self) -> impl Future> + Send; + + /// The vsock CID of a peer this transport refuses at accept, or `None` + /// to serve it. The accept loop asks this before it reads a byte from + /// the connection, and drops a refused one unread. Only the vsock + /// transport refuses a peer ([`vsock_peer_refused`]); a UDS peer is + /// gated by its socket's file mode instead. + fn refused_cid(_peer: &Self::Addr) -> Option { + None + } +} + +/// Whether a vsock peer at `cid` is refused by the in-VM daemon's RPC +/// listener (NET-139): every CID but the host's (`VMADDR_CID_HOST`, 2). The +/// vsock channel into a VM runs host to guest only, so a guest-local peer +/// (loopback CID 1, the VM's own CID, or a sibling) is not the host and is +/// refused, whatever the box seal's state. +#[cfg(target_os = "linux")] +pub(crate) fn vsock_peer_refused(cid: u32) -> bool { + cid != tokio_vsock::VMADDR_CID_HOST +} + +/// How often the refusal line for one peer CID is written (NET-139): the +/// first refusal in a window writes the line, and the line carries the +/// count of refusals since the last one, so a peer that connects in a loop +/// cannot flood the daemon's log. +const PEER_REFUSAL_LOG_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60); + +/// The most peer CIDs the refusal limiter tracks at once. Past it the +/// table is cleared, so a peer that cycles CIDs costs a line per CID, not +/// memory. +const PEER_REFUSAL_TRACKED: usize = 64; + +/// Per peer CID: when its refusal line was last written, and how many +/// refusals came after that line. +static PEER_REFUSALS: std::sync::LazyLock< + std::sync::Mutex>, +> = std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashMap::new())); + +/// Records one refused peer at `cid` and answers how many refusals the +/// line to write now covers, or `None` while the CID's window is still +/// open and the line is suppressed. +fn peer_refusal_due(cid: u32, now: std::time::Instant) -> Option { + let mut refusals = PEER_REFUSALS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if refusals.len() >= PEER_REFUSAL_TRACKED && !refusals.contains_key(&cid) { + refusals.clear(); + } + match refusals.get_mut(&cid) { + Some((last, unlogged)) if now.duration_since(*last) < PEER_REFUSAL_LOG_INTERVAL => { + *unlogged = unlogged.saturating_add(1); + None + } + Some((last, unlogged)) => { + let covered = unlogged.saturating_add(1); + *last = now; + *unlogged = 0; + Some(covered) + } + None => { + refusals.insert(cid, (now, 0)); + Some(1) + } + } +} + +/// Writes the audited line for a refused vsock peer (NET-139), rate-limited +/// per CID by [`peer_refusal_due`]. The line names the CID and counts the +/// refusals it covers. It is a warn line in the daemon's log, which the +/// diagnostics bundle ships. +fn log_refused_peer(transport: &'static str, cid: u32) { + if let Some(refused) = peer_refusal_due(cid, std::time::Instant::now()) { + tracing::warn!( + audit = true, + transport, + peer_cid = cid, + refused, + "refused an RPC connection from a vsock peer that is not the host; \ + only the host (CID 2) may connect" + ); + } } impl Listener for UnixListener { @@ -1053,6 +1134,14 @@ impl Listener for tokio_vsock::VsockListener { async fn accept(&self) -> std::io::Result<(Self::Stream, Self::Addr)> { tokio_vsock::VsockListener::accept(self).await } + + /// The vsock listener serves only the host (NET-139): a vsock listener + /// is what a VM-hosted daemon binds (`--vsock`), and its every peer + /// but CID 2 is refused at accept. + fn refused_cid(peer: &Self::Addr) -> Option { + let cid = peer.cid(); + vsock_peer_refused(cid).then_some(cid) + } } /// How long [`Server::run`] waits for in-flight connections to drain after a @@ -1170,6 +1259,14 @@ impl Server { () = shutdown.cancelled() => break, accepted = listener.accept() => accepted?, }; + // NET-139: a peer the transport refuses is dropped here, at + // accept, before a byte of its stream is read and before it + // takes a connection id or a task. + if let Some(cid) = L::refused_cid(&peer) { + drop(stream); + log_refused_peer(L::TRANSPORT, cid); + continue; + } // One span per accepted connection: every record the connection // and its channels emit carries `conn`, so accept, channel // bindings, and close correlate across the log with a grep @@ -5972,4 +6069,124 @@ mod tests { "the publication line must name the guest port beside it, got: {logged}" ); } + + /// A vsock-shaped listener a test feeds: each accept hands back the next + /// queued stream with the peer address the test chose, so a test can stand + /// a peer at any CID without a VM. It refuses peers exactly as the real + /// vsock listener does, through that listener's own `refused_cid`. + #[cfg(target_os = "linux")] + struct FedVsockListener { + queued: tokio::sync::Mutex< + tokio::sync::mpsc::UnboundedReceiver<(tokio::io::DuplexStream, tokio_vsock::VsockAddr)>, + >, + } + + #[cfg(target_os = "linux")] + impl Listener for FedVsockListener { + type Stream = tokio::io::DuplexStream; + type Addr = tokio_vsock::VsockAddr; + + const TRANSPORT: &'static str = "vsock"; + const IS_LOCAL: bool = true; + + async fn accept(&self) -> std::io::Result<(Self::Stream, Self::Addr)> { + match self.queued.lock().await.recv().await { + Some(accepted) => Ok(accepted), + None => std::future::pending().await, + } + } + + fn refused_cid(peer: &Self::Addr) -> Option { + ::refused_cid(peer) + } + } + + /// NET-139: the in-VM daemon's vsock RPC listener serves only the host. A + /// peer at any other CID (here a sibling at CID 3 and the guest's own + /// loopback at CID 1) is dropped at accept. The daemon never sends it the + /// SSH banner and never reads what it wrote. One audited line names the + /// CID. A peer at CID 2 is served as before. + #[cfg(target_os = "linux")] + #[tokio::test] + async fn vsock_rpc_refuses_non_host_cid() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + assert!(vsock_peer_refused(1), "the guest's loopback CID is refused"); + assert!(vsock_peer_refused(3), "a sibling CID is refused"); + assert!( + !vsock_peer_refused(tokio_vsock::VMADDR_CID_HOST), + "the host's CID is served" + ); + + let capture = crate::test_harness::captured_log(); + let dir = TempDir::new().unwrap(); + let state = ServerStateHandle::new(test_config(&dir), None) + .await + .unwrap(); + let (feed, queued) = tokio::sync::mpsc::unbounded_channel(); + let listener = FedVsockListener { + queued: tokio::sync::Mutex::new(queued), + }; + let run = tokio::spawn(Server::serve(state, listener)); + + for cid in [3, 1] { + let (mut peer, served) = tokio::io::duplex(4096); + // The peer speaks first. A refusal at accept must not read it. + peer.write_all(b"SSH-2.0-not-the-host\r\n").await.unwrap(); + feed.send((served, tokio_vsock::VsockAddr::new(cid, 22))) + .unwrap(); + let mut heard = Vec::new(); + let read = tokio::time::timeout(Duration::from_secs(10), peer.read_to_end(&mut heard)) + .await + .expect("a refused peer's connection is closed at once"); + assert!( + read.is_ok() && heard.is_empty(), + "a peer at CID {cid} hears nothing before its connection closes: {heard:?}" + ); + } + + let (mut host, served) = tokio::io::duplex(4096); + feed.send(( + served, + tokio_vsock::VsockAddr::new(tokio_vsock::VMADDR_CID_HOST, 22), + )) + .unwrap(); + let mut banner = [0u8; 4]; + tokio::time::timeout(Duration::from_secs(10), host.read_exact(&mut banner)) + .await + .expect("the host is answered") + .expect("the host reads the daemon's SSH banner"); + assert_eq!(&banner, b"SSH-", "the host's connection is served"); + + let log = capture.contents(); + for cid in [3, 1] { + assert!( + log.contains(&format!("peer_cid={cid}")) && log.contains("audit=true"), + "the refusal of CID {cid} is one audited line naming it: {log}" + ); + } + assert!( + !log.contains("peer_cid=2"), + "the host's CID is never refused: {log}" + ); + run.abort(); + } + + /// NET-139's rate limit: the first refusal of a CID writes the line, the + /// refusals inside its window are counted and not written, and the first + /// refusal after the window writes a line that covers them all. + #[test] + fn refused_peer_line_is_rate_limited_per_cid() { + // A CID no other test refuses: the limiter is process-wide. + let cid = 0xdead_beef; + let start = std::time::Instant::now(); + assert_eq!(peer_refusal_due(cid, start), Some(1)); + assert_eq!(peer_refusal_due(cid, start + Duration::from_secs(1)), None); + assert_eq!(peer_refusal_due(cid, start + Duration::from_secs(2)), None); + assert_eq!( + peer_refusal_due(cid, start + PEER_REFUSAL_LOG_INTERVAL), + Some(3), + "the next line counts the refusals its window suppressed" + ); + } } diff --git a/crates/minimald/src/session_host/tests.rs b/crates/minimald/src/session_host/tests.rs index 7f9a0506dd..d37e953c23 100644 --- a/crates/minimald/src/session_host/tests.rs +++ b/crates/minimald/src/session_host/tests.rs @@ -5720,7 +5720,16 @@ async fn expose_unenrolled_decision_audited() { .lock() .await .control_socket(); - let (forwarder, _served) = fake_forwarder(sock, 200).await; + let (forwarder, _served) = fake_forwarder(sock.clone(), 200).await; + // On a VM host each decision is also reported to the VM host daemon, + // which keeps its own copy outside the VM. The two boxes' audit routes + // go to one stand-in door, each under a row key of its own. + let web_row = std::net::Ipv4Addr::new(100, 64, 128, 201); + let db_row = std::net::Ipv4Addr::new(100, 64, 128, 202); + let door_sock = sock.with_file_name("audit-door.sock"); + let (door, mut reports, replies) = fake_report_door(&door_sock).await; + crate::net::listeners::seed_host_audit_route_for_tests("web", web_row, &door_sock); + crate::net::listeners::seed_host_audit_route_for_tests("db", db_row, &door_sock); web_handle .expose_dynamic(3000) @@ -5732,10 +5741,70 @@ async fn expose_unenrolled_decision_audited() { )) => {} other => panic!("the deny-all default refuses the request: {other:?}"), } + + // Each decision reaches the host as one report keyed by its box's row. + let mut host_reports = Vec::new(); + for _ in 0..2 { + let report = tokio::time::timeout(Duration::from_secs(10), reports.recv()) + .await + .expect("each decision is reported to the VM host daemon") + .expect("the report door stand-in lives"); + replies + .send(minimald_rpc::BoxControlReply::Audited { audited: true }) + .expect("the report door stand-in lives"); + match report { + minimald_rpc::BoxControlRequest::ReportExposeDecision(report) => { + host_reports.push(report); + } + other => panic!("a decision is reported as an expose decision: {other:?}"), + } + } + host_reports.sort_by_key(|report| report.switch_address); + assert_eq!( + host_reports, + vec![ + minimald_rpc::ExposeDecisionReport { + switch_address: web_row, + port: 3000, + decision: sessions::DynamicIngress::Allow, + decided_by: minimald_rpc::ExposeDecidedBy::BoxPolicy, + outcome: minimald_rpc::ExposeDecisionOutcome::Published, + }, + minimald_rpc::ExposeDecisionReport { + switch_address: db_row, + port: 3000, + decision: sessions::DynamicIngress::Deny, + decided_by: minimald_rpc::ExposeDecidedBy::BoxPolicy, + outcome: minimald_rpc::ExposeDecisionOutcome::Refused, + }, + ], + "the host hears each decision the local log records" + ); + + // With the host unreachable, the local record still lands. + crate::net::listeners::seed_host_audit_route_for_tests( + "db", + db_row, + sock.with_file_name("no-door.sock"), + ); + match db_handle.expose_dynamic(3001).await { + Err(crate::net::policy::ExposeFailure::Refused( + crate::net::policy::ExposeRefusal::DeniedByPolicy, + )) => {} + other => panic!("the deny-all default refuses the request: {other:?}"), + } forwarder.abort(); + door.abort(); + crate::net::listeners::clear_host_audit_route_for_tests("web", web_row); + crate::net::listeners::clear_host_audit_route_for_tests("db", db_row); let records = audit_records(&server.state.minimal_state_dir().await).await; - assert_eq!(records.len(), 2, "each decision is one record: {records:?}"); + assert_eq!(records.len(), 3, "each decision is one record: {records:?}"); + assert_eq!(records[2]["box"], "db"); + assert_eq!( + records[2]["port"], 3001, + "a decision the host cannot hear is still recorded locally: {records:?}" + ); assert_eq!(records[0]["box"], "web"); assert_eq!(records[0]["port"], 3000); assert_eq!(records[0]["decision"], "allow"); diff --git a/crates/minvmd/src/control.rs b/crates/minvmd/src/control.rs index ae49faaa64..b2e3bbd6eb 100644 --- a/crates/minvmd/src/control.rs +++ b/crates/minvmd/src/control.rs @@ -918,7 +918,16 @@ fn serve_request( write_reply(stream, &reply) } (BoxControlRequest::Withdraw(request), ControlDoor::Host) => { - let reply = order.apply(|| withdraw_box(boxes, answerer, request)); + let reply = order.apply(|| { + let (name, switch_address) = (request.name.clone(), request.switch_address); + let reply = withdraw_box(boxes, answerer, request); + // The box is gone: its audit window ends, and the count of + // the records it suppressed lands now. + if matches!(reply, BoxControlReply::Addresses(_)) { + flush_guest_audit(audit_path, &name, switch_address); + } + reply + }); write_reply(stream, &reply) } (BoxControlRequest::ResumeBox(request), ControlDoor::Host) => { @@ -989,6 +998,16 @@ fn serve_request( let reply = order.apply(|| withdraw_report(boxes, &request)); write_reply(stream, &reply) } + // The audit reports touch no row, so they take no ticket: each is + // a line in the host's audit log, under the per-box cap. + (BoxControlRequest::ReportExposeDecision(request), ControlDoor::GuestReports) => { + let reply = audit_expose_decision(boxes, audit_path, &request); + write_reply(stream, &reply) + } + (BoxControlRequest::ReportRefusedEgress(request), ControlDoor::GuestReports) => { + let reply = audit_refused_egress(boxes, audit_path, &request); + write_reply(stream, &reply) + } (BoxControlRequest::ReleaseAnswerer, ControlDoor::Host) => { let reply = answerer.release(); tracing::info!( @@ -1318,6 +1337,8 @@ fn request_verb(request: &BoxControlRequest) -> &'static str { BoxControlRequest::SubscribeAsks(_) => "subscribe_asks", BoxControlRequest::ResumeBox(_) => "resume_box", BoxControlRequest::RowStanding(_) => "row_standing", + BoxControlRequest::ReportExposeDecision(_) => "report_expose_decision", + BoxControlRequest::ReportRefusedEgress(_) => "report_refused_egress", } } @@ -1857,19 +1878,427 @@ fn append_audit_copy( ); } +/// How long one box's audit window lasts: the guest's expose decision +/// reports for a box write at most [`GUEST_AUDIT_RECORDS_PER_WINDOW`] +/// records in it, and its refused egress reports as many again. +const GUEST_AUDIT_WINDOW: std::time::Duration = std::time::Duration::from_secs(60); + +/// The most records each kind of the guest's audit reports (expose +/// decisions, refused egress) writes for one box in one +/// [`GUEST_AUDIT_WINDOW`]. Each kind has its own budget, so a flood of +/// refused flows never hides a later expose decision. Past it a report is +/// counted, not written, and one line with the count precedes the next +/// window's first record, or lands when the box is withdrawn. So a hostile +/// box cannot fill the host's disk through its reports. +const GUEST_AUDIT_RECORDS_PER_WINDOW: u32 = 30; + +/// The most refused flows one box's window remembers to fold repeats. +const GUEST_AUDIT_FLOWS_PER_WINDOW: usize = 256; + +/// The most boxes the audit cap tracks at once, across every door in the +/// process. Past it the windows that ended are pruned. +const GUEST_AUDIT_TRACKED: usize = 1024; + +/// One box's audit window: when it opened, how many records of each kind +/// it wrote, how many it suppressed, and the refused flows it already +/// recorded. +#[derive(Debug)] +struct GuestAuditWindow { + opened: std::time::Instant, + expose_written: u32, + egress_written: u32, + suppressed: u64, + flows: HashSet, +} + +/// The audit windows, by the audit log they write to and the box's row +/// key. The log is part of the key so that two doors in one process (the +/// tests) never share a cap. +static GUEST_AUDIT_WINDOWS: std::sync::LazyLock< + std::sync::Mutex>, +> = std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashMap::new())); + +/// What the per-box cap says about one report. +#[derive(Debug, PartialEq, Eq)] +enum GuestAuditAdmission { + /// Write the record, after a line counting the `suppressed` records of + /// the window that ended, when there were any. + Write { suppressed: u64 }, + /// The same refused flow is already recorded in this window: no record, + /// because a flow is recorded once, never per frame. + Repeat, + /// The window's cap is reached: no record, and the count goes up. + Suppressed, +} + +/// Decides one report against its box's window at `now`. `flow` is the +/// refused flow's identity, or `None` for a report that is not a flow (an +/// expose decision, each of which is its own record). +fn admit_guest_audit( + audit_path: &Path, + switch_address: Ipv4Addr, + flow: Option, + now: std::time::Instant, +) -> GuestAuditAdmission { + let mut windows = GUEST_AUDIT_WINDOWS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let key = (audit_path.to_path_buf(), switch_address); + if windows.len() >= GUEST_AUDIT_TRACKED && !windows.contains_key(&key) { + windows.retain(|_, window| { + now.duration_since(window.opened) < GUEST_AUDIT_WINDOW || window.suppressed > 0 + }); + } + let window = windows.entry(key).or_insert_with(|| GuestAuditWindow { + opened: now, + expose_written: 0, + egress_written: 0, + suppressed: 0, + flows: HashSet::new(), + }); + let mut carried = 0; + if now.duration_since(window.opened) >= GUEST_AUDIT_WINDOW { + carried = window.suppressed; + *window = GuestAuditWindow { + opened: now, + expose_written: 0, + egress_written: 0, + suppressed: 0, + flows: HashSet::new(), + }; + } + if let Some(flow) = &flow + && window.flows.contains(flow) + { + return GuestAuditAdmission::Repeat; + } + // Each kind spends its own budget: refused flows never use up the + // records an expose decision needs. + let written = if flow.is_some() { + &mut window.egress_written + } else { + &mut window.expose_written + }; + if *written >= GUEST_AUDIT_RECORDS_PER_WINDOW { + window.suppressed = window.suppressed.saturating_add(1); + return GuestAuditAdmission::Suppressed; + } + *written += 1; + if let Some(flow) = flow + && window.flows.len() < GUEST_AUDIT_FLOWS_PER_WINDOW + { + window.flows.insert(flow); + } + GuestAuditAdmission::Write { + suppressed: carried, + } +} + +/// The host's copy of one dynamic ingress decision the in-VM daemon +/// reported (NET-046): the box named from the host's own row, never from +/// text the guest sent. +#[derive(serde::Serialize)] +struct ExposeDecisionAudit { + /// When the report was recorded: Unix seconds. + ts: u64, + /// Always `expose_decision`. + event: &'static str, + /// Who asserted this record: always `guest`. The in-VM daemon reported + /// it, and the host did not observe it. + reported_by: &'static str, + /// The box the decision was about, from the row. + #[serde(rename = "box")] + box_name: String, + /// The row key the report named. + switch_address: Ipv4Addr, + /// The port the box asked to publish. + port: u16, + /// The decision the box's `dynamic_ingress` setting made. + decision: sessions::DynamicIngress, + /// Who decided it. + decided_by: minimald_rpc::ExposeDecidedBy, + /// What became of the request. + outcome: minimald_rpc::ExposeDecisionOutcome, +} + +/// The host's copy of one refused own-address egress flow the in-VM +/// daemon reported. +#[derive(serde::Serialize)] +struct RefusedEgressAudit { + /// When the report was recorded: Unix seconds. + ts: u64, + /// Always `egress_refused`. + event: &'static str, + /// Who asserted this record: always `guest`. The in-VM daemon reported + /// it, and the host did not observe it. + reported_by: &'static str, + /// The box whose flow was refused, from the row. + #[serde(rename = "box")] + box_name: String, + /// The row key the report named. + switch_address: Ipv4Addr, + /// The destination the flow was sent to, when it had one. + #[serde(skip_serializing_if = "Option::is_none")] + destination: Option, + /// The flow's protocol, clamped. + proto: String, + /// The rule that refused it, clamped. + rule: String, +} + +/// The line that counts the records a box's audit window suppressed. +#[derive(serde::Serialize)] +struct SuppressedAudit { + /// When the line was written: Unix seconds. + ts: u64, + /// Always `audit_suppressed`. + event: &'static str, + /// The box whose records were suppressed. + #[serde(rename = "box")] + box_name: String, + /// The box's row key. + switch_address: Ipv4Addr, + /// How many records the cap suppressed in the window that ended. + suppressed: u64, +} + +/// Writes the line that counts `suppressed` records for the box, when +/// there are any. +fn append_suppressed_audit( + audit_path: &Path, + box_name: &str, + switch_address: Ipv4Addr, + suppressed: u64, +) { + if suppressed == 0 { + return; + } + tracing::warn!( + box = %box_name, + %switch_address, + suppressed, + "the box's audit reports passed the host's cap; records were suppressed" + ); + append_audit_line( + audit_path, + &SuppressedAudit { + ts: unix_now_secs(), + event: "audit_suppressed", + box_name: box_name.to_string(), + switch_address, + suppressed, + }, + ); +} + +/// Writes the suppressed-count line when the admission carries one, and +/// says whether the record itself is to be written. +fn take_guest_audit( + audit_path: &Path, + box_name: &str, + switch_address: Ipv4Addr, + admission: &GuestAuditAdmission, +) -> bool { + match admission { + GuestAuditAdmission::Write { suppressed } => { + append_suppressed_audit(audit_path, box_name, switch_address, *suppressed); + true + } + GuestAuditAdmission::Repeat | GuestAuditAdmission::Suppressed => false, + } +} + +/// Ends a withdrawn box's audit window. The window is forgotten, so a later +/// box at the same address starts a fresh one. The count of the records it +/// suppressed is written now: a box that stops reporting opens no later +/// window, so without this flush that count is never written. +fn flush_guest_audit(audit_path: &Path, box_name: &str, switch_address: Ipv4Addr) { + let window = GUEST_AUDIT_WINDOWS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&(audit_path.to_path_buf(), switch_address)); + if let Some(window) = window { + append_suppressed_audit(audit_path, box_name, switch_address, window.suppressed); + } +} + +/// Clamps a guest-sent text to [`minimald_rpc::REFUSED_EGRESS_TEXT_MAX`] +/// bytes of printable ASCII: the guest names the protocol and the rule, +/// and the host log takes nothing longer or stranger. +fn clamp_guest_text(text: &str) -> String { + text.chars() + .filter(|c| c.is_ascii_graphic() || *c == ' ') + .take(minimald_rpc::REFUSED_EGRESS_TEXT_MAX) + .collect() +} + +/// Serves the in-VM daemon's expose decision report (NET-046): one record +/// in the host's audit log, the box named from the row at the report's +/// switch address. A report at no row is refused and writes nothing. +fn audit_expose_decision( + boxes: &BoxRegistry, + audit_path: &Path, + request: &minimald_rpc::ExposeDecisionReport, +) -> BoxControlReply { + let Some(record) = boxes.table().by_source(request.switch_address.octets()) else { + tracing::debug!( + switch_address = %request.switch_address, + "refused an expose decision report at an address no row holds" + ); + return BoxControlReply::Error { + error: format!("no box is registered at {}", request.switch_address), + }; + }; + let admission = admit_guest_audit( + audit_path, + request.switch_address, + None, + std::time::Instant::now(), + ); + let audited = take_guest_audit( + audit_path, + record.name(), + request.switch_address, + &admission, + ); + if audited { + tracing::info!( + box = %record.name(), + port = request.port, + decision = ?request.decision, + decided_by = ?request.decided_by, + outcome = ?request.outcome, + reported_by = "guest", + "recorded the box's expose decision in the host audit log" + ); + append_audit_line( + audit_path, + &ExposeDecisionAudit { + ts: unix_now_secs(), + event: "expose_decision", + reported_by: "guest", + box_name: record.name().to_string(), + switch_address: request.switch_address, + port: request.port, + decision: request.decision, + decided_by: request.decided_by, + outcome: request.outcome, + }, + ); + } + BoxControlReply::Audited { audited } +} + +/// Serves the in-VM daemon's refused egress report: one record per refused +/// flow in the host's audit log, under the box's cap, the box named from +/// the row at the report's switch address. A repeat of a flow already +/// recorded in the window writes nothing. A report at no row is refused. +fn audit_refused_egress( + boxes: &BoxRegistry, + audit_path: &Path, + request: &minimald_rpc::RefusedEgressReport, +) -> BoxControlReply { + let Some(record) = boxes.table().by_source(request.switch_address.octets()) else { + tracing::debug!( + switch_address = %request.switch_address, + "refused a refused-egress report at an address no row holds" + ); + return BoxControlReply::Error { + error: format!("no box is registered at {}", request.switch_address), + }; + }; + let proto = clamp_guest_text(&request.proto); + let rule = clamp_guest_text(&request.rule); + let destination = request + .destination + .map_or_else(|| "none".to_string(), |d| d.to_string()); + let flow = format!("{destination}/{proto}/{rule}"); + let admission = admit_guest_audit( + audit_path, + request.switch_address, + Some(flow), + std::time::Instant::now(), + ); + let audited = take_guest_audit( + audit_path, + record.name(), + request.switch_address, + &admission, + ); + if audited { + tracing::info!( + box = %record.name(), + %destination, + %proto, + %rule, + reported_by = "guest", + "recorded the box's refused egress in the host audit log" + ); + append_audit_line( + audit_path, + &RefusedEgressAudit { + ts: unix_now_secs(), + event: "egress_refused", + reported_by: "guest", + box_name: record.name().to_string(), + switch_address: request.switch_address, + destination: request.destination, + proto, + rule, + }, + ); + } + BoxControlReply::Audited { audited } +} + +/// The size the host-side audit log may reach before it rotates (NET-046). +/// The per-box caps bound the rate a guest can report at, not the total, so +/// the file itself is bounded too: the append that would carry it past this +/// size first renames it to `box-admissions.log.1`, replacing the generation +/// rotated before it, and starts a fresh file. One kept generation bounds +/// the log at about twice this size. The same bound and shape as the in-VM +/// daemon's own audit log (`minimald`'s `audit::MAX_LOG_BYTES`). +const MAX_AUDIT_LOG_BYTES: u64 = 1024 * 1024; + +/// Serialises every append to the host-side audit log. The ask threads and +/// the doors append concurrently, and a rotation is a size read, a rename +/// and a reopen: two writers deciding it at once could each rotate, and the +/// second rename would replace the full generation the first one kept. +static AUDIT_APPEND_LOCK: Mutex<()> = Mutex::new(()); + /// Append one JSON line to the daemon's owner-only audit log: the shared /// half of the admission copy and the ask records. fn append_audit_line(path: &Path, line: &impl serde::Serialize) { + append_audit_line_capped(path, line, MAX_AUDIT_LOG_BYTES); +} + +/// [`append_audit_line`] with the rotation size passed in, so a test can +/// rotate the log without writing a mebibyte. +fn append_audit_line_capped(path: &Path, line: &impl serde::Serialize, cap: u64) { let Ok(json) = serde_json_lenient::to_string(line) else { // A plain struct of primitives serializes; this arm is unreachable // in practice and costs nothing to keep honest. tracing::warn!("an audit line did not serialize"); return; }; + let bytes = format!("{json}\n"); + // A poisoned lock guards no data: the append it serialised failed or + // panicked, and the next append is still safe to make. + let _guard = AUDIT_APPEND_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let written = (|| -> std::io::Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } + let len = match std::fs::metadata(path) { + Ok(meta) => meta.len(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => 0, + Err(error) => return Err(error), + }; + if audit_log_rotates(len, bytes.len() as u64, cap) { + std::fs::rename(path, audit_log_rotated_path(path))?; + } use std::io::Write as _; use std::os::unix::fs::OpenOptionsExt as _; let mut file = std::fs::OpenOptions::new() @@ -1877,9 +2306,9 @@ fn append_audit_line(path: &Path, line: &impl serde::Serialize) { .append(true) .mode(0o600) .open(path)?; - // One write per line: the ask threads append concurrently, and a - // line written in two pieces could interleave with another's. - file.write_all(format!("{json}\n").as_bytes()) + // One write per line: a line written in two pieces could interleave + // with another writer's outside this process. + file.write_all(bytes.as_bytes()) })(); if let Err(error) = written { tracing::warn!( @@ -1890,6 +2319,22 @@ fn append_audit_line(path: &Path, line: &impl serde::Serialize) { } } +/// Whether a log already holding `len` bytes rotates before an `incoming` +/// line lands: only when the line would carry it past `cap`, and never when +/// it is empty, so a single line longer than the cap goes into a fresh file +/// rather than rotating on every append. +fn audit_log_rotates(len: u64, incoming: u64, cap: u64) -> bool { + len > 0 && len.saturating_add(incoming) > cap +} + +/// The one rotated generation's path beside the live log: +/// `box-admissions.log.1`. +fn audit_log_rotated_path(path: &Path) -> PathBuf { + let mut name = path.file_name().unwrap_or_default().to_os_string(); + name.push(".1"); + path.with_file_name(name) +} + /// Run one ask verb's connection on a thread of its own (NET-045): the ask /// or the subscription holds the connection open, so it must not hold the /// door's connection slot with it. A thread that cannot be spawned is @@ -2814,6 +3259,77 @@ mod tests { /// for. const SUBNET: SwitchSubnet = switch::DEFAULT_SUBNET; + /// NET-046: the host-side audit log is bounded. Past its cap it rotates + /// to one kept generation, so neither file grows past the cap however + /// many records a guest reports, and the newest records are the ones + /// kept. + #[test] + fn host_audit_log_rotates_within_its_bound_and_keeps_the_newest_records() { + #[derive(serde::Serialize)] + struct Line { + seq: u64, + } + fn seqs(path: &Path) -> Vec { + std::fs::read_to_string(path) + .unwrap_or_default() + .lines() + .map(|line| { + serde_json_lenient::from_str::(line) + .expect("each kept line is whole JSON")["seq"] + .as_u64() + .expect("seq") + }) + .collect() + } + let dir = tempfile::TempDir::new().expect("temp dir"); + let path = dir.path().join(AUDIT_LOG_RELATIVE_PATH); + let rotated = audit_log_rotated_path(&path); + let cap = 512; + let total = 1000; + for seq in 0..total { + append_audit_line_capped(&path, &Line { seq }, cap); + assert!( + std::fs::metadata(&path).unwrap().len() <= cap, + "the live log stays within its cap" + ); + if let Ok(meta) = std::fs::metadata(&rotated) { + assert!( + meta.len() <= cap, + "the kept generation stays within its cap" + ); + } + } + let files: Vec<_> = std::fs::read_dir(path.parent().unwrap()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!( + files.len(), + 2, + "one rotated generation is kept, never more: {files:?}" + ); + let kept: Vec = seqs(&rotated).into_iter().chain(seqs(&path)).collect(); + assert_eq!( + kept.last().copied(), + Some(total - 1), + "the newest record is kept" + ); + let first = kept[0]; + assert_eq!( + kept, + (first..total).collect::>(), + "the kept records are the newest, contiguous and in order" + ); + assert!( + kept.len() as u64 * 12 > cap, + "a rotation keeps at least a full generation: {}", + kept.len() + ); + // The host's own records still land after rotation. + append_audit_line_capped(&path, &Line { seq: total }, cap); + assert_eq!(seqs(&path).last().copied(), Some(total)); + } + /// A process-global log capture. The server serves on its own OS /// thread, so the thread-local captures the crate's other tests install /// do not see its lines; the global default is installed once per @@ -2964,6 +3480,9 @@ mod tests { other @ BoxControlReply::RowStanding { .. } => { panic!("a box verb is never answered with a row-standing read, got {other:?}") } + other @ BoxControlReply::Audited { .. } => { + panic!("a box verb is never answered with an audit report's answer, got {other:?}") + } BoxControlReply::AnswererRelease { detail, .. } => { panic!("a box verb is never answered with a release reply, got {detail}") } @@ -3440,6 +3959,9 @@ mod tests { other @ BoxControlReply::RowStanding { .. } => { panic!("a box verb is never answered with a row-standing read, got {other:?}") } + other @ BoxControlReply::Audited { .. } => { + panic!("a box verb is never answered with an audit report's answer, got {other:?}") + } BoxControlReply::AnswererRelease { detail, .. } => { panic!("a box verb is never answered with a release reply, got {detail}") } @@ -3557,6 +4079,11 @@ mod tests { other @ BoxControlReply::RowStanding { .. } => { panic!("a box verb is never answered with a row-standing read, got {other:?}") } + other @ BoxControlReply::Audited { .. } => { + panic!( + "a box verb is never answered with an audit report's answer, got {other:?}" + ) + } BoxControlReply::AnswererRelease { detail, .. } => { panic!("a box verb is never answered with a release reply, got {detail}") } @@ -3625,6 +4152,9 @@ mod tests { other @ BoxControlReply::RowStanding { .. } => { panic!("a box verb is never answered with a row-standing read, got {other:?}") } + other @ BoxControlReply::Audited { .. } => { + panic!("a box verb is never answered with an audit report's answer, got {other:?}") + } BoxControlReply::AnswererRelease { detail, .. } => { panic!("a box verb is never answered with a release reply, got {detail}") } @@ -6392,4 +6922,276 @@ mod tests { ); host.register_live("api"); } + + /// Registers one box on the host socket for the audit report tests and + /// answers its row: a plain box, so its switch address is a row key the + /// guest door's reports can name. + fn register_audited_box(sock_path: &std::path::Path, name: &str) -> RegisteredBox { + handed( + register( + sock_path, + &RegisterBoxRequest { + name: name.to_string(), + ingress_ports: vec![], + egress: None, + credentialed_upstream: None, + dynamic_ingress: Some(sessions::DynamicIngress::Allow), + dynamic_allowed_range: Some((3000, 3999)), + hold: false, + task_slots: 0, + }, + ) + .expect("the registration is answered"), + ) + } + + /// The host's audit log beside `sock_path`, parsed one record per line. + fn host_audit_records(sock_path: &std::path::Path) -> Vec { + let logged = std::fs::read_to_string(audit_log_path(sock_path)).unwrap_or_default(); + logged + .lines() + .map(|line| serde_json_lenient::from_str(line).expect("one audit line is one record")) + .collect() + } + + /// NET-046 on a VM host: a decision the in-VM daemon reports on the guest + /// door lands in the host's own audit log, outside the VM, the box named + /// from the host's row and not from the report. A denied expose and a + /// published one are each one record. A report at an address no row + /// holds is refused and writes nothing. + #[test] + fn expose_decision_recorded_on_host() { + let dir = tempfile::TempDir::new().expect("temp dir"); + let (sock_path, _server, _registry, _answerer, _proxy_publish) = + spawn_server(dir.path()).expect("server binds"); + let guest = sock_path.with_file_name(GUEST_CONTROL_SOCK_FILE); + let web = register_audited_box(&sock_path, "web"); + + for (port, decision, outcome) in [ + ( + 4000, + sessions::DynamicIngress::Deny, + minimald_rpc::ExposeDecisionOutcome::Refused, + ), + ( + 3000, + sessions::DynamicIngress::Allow, + minimald_rpc::ExposeDecisionOutcome::Published, + ), + ] { + let reply = control( + &guest, + &BoxControlRequest::ReportExposeDecision(minimald_rpc::ExposeDecisionReport { + switch_address: web.switch_address, + port, + decision, + decided_by: minimald_rpc::ExposeDecidedBy::BoxPolicy, + outcome, + }), + ) + .expect("the report is answered"); + assert_eq!(reply, BoxControlReply::Audited { audited: true }); + } + let stray = control( + &guest, + &BoxControlRequest::ReportExposeDecision(minimald_rpc::ExposeDecisionReport { + switch_address: Ipv4Addr::new(192, 0, 2, 1), + port: 3000, + decision: sessions::DynamicIngress::Allow, + decided_by: minimald_rpc::ExposeDecidedBy::BoxPolicy, + outcome: minimald_rpc::ExposeDecisionOutcome::Published, + }), + ) + .expect("the stray report is answered"); + assert!( + matches!(stray, BoxControlReply::Error { .. }), + "a report at no row is refused: {stray:?}" + ); + // The host socket does not take the guest's reports. + let wrong_door = control( + &sock_path, + &BoxControlRequest::ReportExposeDecision(minimald_rpc::ExposeDecisionReport { + switch_address: web.switch_address, + port: 3001, + decision: sessions::DynamicIngress::Allow, + decided_by: minimald_rpc::ExposeDecidedBy::BoxPolicy, + outcome: minimald_rpc::ExposeDecisionOutcome::Published, + }), + ) + .expect("the wrong-door report is answered"); + assert!(matches!(wrong_door, BoxControlReply::Error { .. })); + + let records: Vec<_> = host_audit_records(&guest) + .into_iter() + .filter(|record| record["event"] == "expose_decision") + .collect(); + assert_eq!(records.len(), 2, "one record per decision: {records:?}"); + assert_eq!(records[0]["box"], "web"); + assert_eq!(records[0]["port"], 4000); + assert_eq!(records[0]["decision"], "deny"); + assert_eq!(records[0]["decided_by"], "box-policy"); + assert_eq!(records[0]["outcome"], "refused"); + assert_eq!(records[0]["reported_by"], "guest"); + assert_eq!(records[1]["port"], 3000); + assert_eq!(records[1]["reported_by"], "guest"); + assert_eq!(records[1]["decision"], "allow"); + assert_eq!(records[1]["outcome"], "published"); + } + + /// A refused own-address egress flow the in-VM daemon reports lands in + /// the host's audit log: one record per refused flow, never per frame, so + /// a repeat of the same flow writes nothing more. The records a box + /// writes are capped per window, the rest counted, and the next window + /// opens with one line that says how many were suppressed. The guest's + /// texts are clamped. + #[test] + fn egress_refusal_logged_on_host() { + let dir = tempfile::TempDir::new().expect("temp dir"); + let (sock_path, _server, _registry, _answerer, _proxy_publish) = + spawn_server(dir.path()).expect("server binds"); + let guest = sock_path.with_file_name(GUEST_CONTROL_SOCK_FILE); + let web = register_audited_box(&sock_path, "web"); + let refused = |port: u16, rule: &str| { + control( + &guest, + &BoxControlRequest::ReportRefusedEgress(minimald_rpc::RefusedEgressReport { + switch_address: web.switch_address, + destination: Some(std::net::SocketAddrV4::new( + Ipv4Addr::new(203, 0, 113, 9), + port, + )), + proto: "tcp".to_string(), + rule: rule.to_string(), + }), + ) + .expect("the report is answered") + }; + + assert_eq!( + refused(443, "undeclared subnet"), + BoxControlReply::Audited { audited: true } + ); + assert_eq!( + refused(443, "undeclared subnet"), + BoxControlReply::Audited { audited: false }, + "the same flow again is not a second record" + ); + let records: Vec<_> = host_audit_records(&guest) + .into_iter() + .filter(|record| record["event"] == "egress_refused") + .collect(); + assert_eq!(records.len(), 1, "one record per refused flow: {records:?}"); + assert_eq!(records[0]["box"], "web"); + assert_eq!(records[0]["destination"], "203.0.113.9:443"); + assert_eq!(records[0]["proto"], "tcp"); + assert_eq!(records[0]["rule"], "undeclared subnet"); + assert_eq!(records[0]["reported_by"], "guest"); + + // A flood of distinct flows writes up to the cap, and the rest are + // counted, not written. + let long_rule = "x".repeat(500); + let mut written = 1; + for port in 1..=(GUEST_AUDIT_RECORDS_PER_WINDOW as u16 + 20) { + if refused(port, &long_rule) == (BoxControlReply::Audited { audited: true }) { + written += 1; + } + } + assert_eq!(written, GUEST_AUDIT_RECORDS_PER_WINDOW); + let records: Vec<_> = host_audit_records(&guest) + .into_iter() + .filter(|record| record["event"] == "egress_refused") + .collect(); + assert_eq!(records.len() as u32, GUEST_AUDIT_RECORDS_PER_WINDOW); + assert!( + records.iter().all(|record| record["rule"] + .as_str() + .is_some_and(|rule| rule.len() <= minimald_rpc::REFUSED_EGRESS_TEXT_MAX)), + "a guest's rule text is clamped: {records:?}" + ); + + // The flood used up the egress budget, not the expose one: a denied + // `min net expose` after it is still recorded. + let expose = control( + &guest, + &BoxControlRequest::ReportExposeDecision(minimald_rpc::ExposeDecisionReport { + switch_address: web.switch_address, + port: 4000, + decision: sessions::DynamicIngress::Deny, + decided_by: minimald_rpc::ExposeDecidedBy::BoxPolicy, + outcome: minimald_rpc::ExposeDecisionOutcome::Refused, + }), + ) + .expect("the report is answered"); + assert_eq!( + expose, + BoxControlReply::Audited { audited: true }, + "a flood of refused flows never hides an expose decision" + ); + assert_eq!( + host_audit_records(&guest) + .iter() + .filter(|record| record["event"] == "expose_decision") + .count(), + 1 + ); + + // The window's end: the next record is preceded by the count. + let audit_path = audit_log_path(&guest); + let later = std::time::Instant::now() + GUEST_AUDIT_WINDOW; + let admission = admit_guest_audit( + &audit_path, + web.switch_address, + Some("later".to_string()), + later, + ); + assert_eq!(admission, GuestAuditAdmission::Write { suppressed: 21 }); + assert!(take_guest_audit( + &audit_path, + "web", + web.switch_address, + &admission + )); + let suppressed: Vec<_> = host_audit_records(&guest) + .into_iter() + .filter(|record| record["event"] == "audit_suppressed") + .collect(); + assert_eq!( + suppressed.len(), + 1, + "one line counts the suppressed records" + ); + assert_eq!(suppressed[0]["box"], "web"); + assert_eq!(suppressed[0]["suppressed"], 21); + + // A box that floods and then goes quiet: its withdrawal writes the + // count, since no later report will. + let mut dropped = 0u64; + for port in 1..=(GUEST_AUDIT_RECORDS_PER_WINDOW as u16 + 5) { + if refused(port, "quiet after") == (BoxControlReply::Audited { audited: false }) { + dropped += 1; + } + } + assert!(dropped > 0, "the flood passed the cap"); + let withdrawn = control( + &sock_path, + &BoxControlRequest::Withdraw(WithdrawBoxRequest { + name: "web".to_string(), + switch_address: web.switch_address, + loopback_address: web.loopback_address, + box_id: None, + }), + ) + .expect("the withdrawal is answered"); + assert!( + matches!(withdrawn, BoxControlReply::Addresses(_)), + "the box is withdrawn: {withdrawn:?}" + ); + let suppressed: Vec<_> = host_audit_records(&guest) + .into_iter() + .filter(|record| record["event"] == "audit_suppressed") + .collect(); + assert_eq!(suppressed.len(), 2, "the withdrawal flushes the count"); + assert_eq!(suppressed[1]["box"], "web"); + assert_eq!(suppressed[1]["suppressed"], dropped); + } } diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index c89ccda0d7..5ebc4893d4 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -13793,16 +13793,37 @@ proof_box_without_credentialed_lane_cannot_reach_proxy() { else echo "relay-side warn line skipped (guest-side daemon log on VM lane — its count and fields are pinned by the native switch test)" fi - # What this host CAN observe is the consequence of a first-gate refusal: - # the frame never crossed the switch, so the VM host's gate never saw it - # and minvmd's log says nothing about it. That silence is the pin — a - # proxy-lane line in the host gate's log would mean the frame reached the - # host, the exception the relay no longer makes. The control above was - # reset through the same fabric, so the box and its path are live, and - # the absence below is the drop, not a dead fabric. - if cred_case_log | grep -q -- "egress-uncredentialed-proxy-destination"; then + # What this host CAN observe is the consequence of a first-gate refusal. + # The frame never crossed the switch, so the VM host's gate never saw it + # and minvmd never logs the gate's own drop line (its `rule_matched` field) + # for it. That silence is the pin: a gate drop line would mean the frame + # reached the host, the exception the relay no longer makes. The relay now + # reports its refusal over the guest door, so the host does see the relay's + # audit record, which minvmd::control logs with a `rule` field, never + # `rule_matched`. Its presence proves the relay refused and reported the + # frame. The control above was reset through the same fabric, so the box + # and its path are live, and the gate's silence is the drop, not a dead + # fabric. + if cred_case_log | grep -qF -- '"rule_matched":"egress-uncredentialed-proxy-destination"'; then echo "::error::the minvmd gate logged a proxy-lane drop for the no-lane box — the box's own relay must refuse the frame before the switch, so the host gate never sees it" - cred_case_log | grep -- "egress-uncredentialed-proxy-destination" | tail -n5 | sed 's/^/ /' + cred_case_log | grep -F -- '"rule_matched":"egress-uncredentialed-proxy-destination"' | tail -n5 | sed 's/^/ /' + cred_fail + fi + nolane_audit_found="" + for _ in $(seq 1 20); do + if cred_case_log \ + | grep -F -- "recorded the box's refused egress in the host audit log" \ + | grep -F -- '"box":"e2e-cred-nolane"' \ + | grep -F -- '"reported_by":"guest"' \ + | grep -qF -- '"rule":"egress-uncredentialed-proxy-destination"'; then + nolane_audit_found=1 + break + fi + sleep 0.5 + done + if [ -z "$nolane_audit_found" ]; then + echo "::error::the host audit log has no refused-egress record for e2e-cred-nolane under the rule 'egress-uncredentialed-proxy-destination' — the box's relay must refuse the frame and report it to the host" + cred_case_log | grep -F -- "recorded the box's refused egress in the host audit log" | tail -n5 | sed 's/^/ /' cred_fail fi @@ -14222,12 +14243,33 @@ proof_proxy_sees_boxes_by_address() { cred_case_log | grep -F -- 'connection delivered' | tail -n5 | sed 's/^/ /' cred_fail fi - if cred_case_log | grep -q -- "egress-uncredentialed-proxy-destination"; then + # The gate's own drop line carries `rule_matched`. The relay's refusal + # reaches the host as the audit record minvmd::control logs with a `rule` + # field, so match only the gate's field here, and expect the audit record + # for this box below. + if cred_case_log | grep -qF -- '"rule_matched":"egress-uncredentialed-proxy-destination"'; then echo "::error::the minvmd gate logged a proxy-lane drop for the laneless box — the box's own relay must refuse the frame before the switch, so the host gate never sees it" - cred_case_log | grep -- "egress-uncredentialed-proxy-destination" | tail -n5 | sed 's/^/ /' + cred_case_log | grep -F -- '"rule_matched":"egress-uncredentialed-proxy-destination"' | tail -n5 | sed 's/^/ /' cred_fail fi - echo "the laneless box ($psba_ip_c, open to every address, no lane) was silent at $proxy_ip:$proxy_port for ${probe_ms}ms; the host gate stayed silent" + psba_nolane_audit_found="" + for _ in $(seq 1 20); do + if cred_case_log \ + | grep -F -- "recorded the box's refused egress in the host audit log" \ + | grep -F -- '"box":"e2e-psba-nolane"' \ + | grep -F -- '"reported_by":"guest"' \ + | grep -qF -- '"rule":"egress-uncredentialed-proxy-destination"'; then + psba_nolane_audit_found=1 + break + fi + sleep 0.5 + done + if [ -z "$psba_nolane_audit_found" ]; then + echo "::error::the host audit log has no refused-egress record for e2e-psba-nolane under the rule 'egress-uncredentialed-proxy-destination' — the box's relay must refuse the frame and report it to the host" + cred_case_log | grep -F -- "recorded the box's refused egress in the host audit log" | tail -n5 | sed 's/^/ /' + cred_fail + fi + echo "the laneless box ($psba_ip_c, open to every address, no lane) was silent at $proxy_ip:$proxy_port for ${probe_ms}ms; the host gate stayed silent, and the host audit log holds the relay's refusal" # The host process's leg: NET-132's IF-clause, end to end. A process # outside every box connects straight to the acceptor's own unix socket @@ -18956,9 +18998,9 @@ exit" E2E_PTY_ANSWER=keep \ # registration named (`minvmd status --row`), and the admission's own copy # lands in the VM host daemon's owner-only host audit (T84) — printed, # like every record this case asserts. The deny stance is decided inside -# the box, so nothing crosses and nothing may exist on the host: no live -# row, an empty runtime set, and no audit line at all, asserted all three -# ways. The host's own probe at the published address must reach the +# the box, so nothing is admitted on the host: no live row and an empty +# runtime set. The decision still crosses the report door (NET-046), so the +# host audit holds exactly one deny line for it. The host's own probe at the published address must reach the # in-box server and get its response (NET-044). The ask legs # (`eib_vm_ask_legs` above) are the host-attached human's. And the expose half of T61 lands on this lane: a # box created on the non-default VM (`min --vm alpha session activate`, @@ -19004,8 +19046,8 @@ proof_expose_from_inside_box() { # the report door T94 opened: the guest daemon reports it, the host admits # the runtime port into the row, and the admission's own copy lands in the # host audit. The ask stays with `eib_vm_ask_legs` — the human attached on - # the host answers. The deny is decided inside the box, so nothing crosses - # and nothing may exist on the host. The two-VM leg at the end is the + # the host answers. The deny is decided inside the box, so nothing is + # admitted on the host, and its one audit line is reported there. The two-VM leg at the end is the # expose half of T61: a box on the non-default VM publishes there. if [ -n "${E2E_VM:-}" ]; then local eib_vm_sids="" @@ -19248,9 +19290,10 @@ proof_expose_from_inside_box() { *) echo "::error::the deny box's refusal does not say dynamic ingress is denied for the box (got: '$eib_err')"; fail ;; esac echo "deny box: refused with the deny the stance spells (NET-043)" - # Nothing crosses the report door — the guest decided deny — so nothing - # may exist on the host: no live row in the box's own listing, an empty - # runtime set in the host-held row, and no line at all in the host audit. + # The guest decided deny, so nothing is admitted on the host: no live row + # in the box's own listing and an empty runtime set in the host-held row. + # The decision itself still crosses the report door (NET-046), so the host + # audit holds exactly one expose_decision line for it. eib_policy="$(mnl session policy "$eib_deny_sid" 2>"$WORK/eib-vm-deny-policy.err")" \ || { echo "::error::'min session policy' failed for the deny box on a VM lane"; cat "$WORK/eib-vm-deny-policy.err" 2>/dev/null || true; fail; } case "$eib_policy" in @@ -19276,12 +19319,31 @@ proof_expose_from_inside_box() { fail ;; esac - if grep -qF "\"box\":\"$eib_deny_name\"" "$eib_vm_audit" 2>/dev/null; then - echo "::error::the VM host daemon's host audit carries a line for $eib_deny_name — a deny decided in the box never crosses the report door, so the host must record nothing (T84)" + # The report is sent after the refusal, so poll briefly for its line. + eib_deny_lines="" + eib_deny_count=0 + for _ in $(seq 1 20); do + eib_deny_lines="$(grep -F "\"box\":\"$eib_deny_name\"" "$eib_vm_audit" 2>/dev/null \ + | grep -F '"event":"expose_decision"' || true)" + eib_deny_count="$(printf '%s' "$eib_deny_lines" | grep -c . || true)" + [ "$eib_deny_count" -ge 1 ] && break + sleep 0.25 + done + if [ "$eib_deny_count" -ne 1 ]; then + echo "::error::the VM host daemon's host audit carries $eib_deny_count expose_decision lines for $eib_deny_name — the deny decided in the box must reach the host as exactly one line (NET-046)" grep -F "\"box\":\"$eib_deny_name\"" "$eib_vm_audit" 2>/dev/null | sed 's/^/ /' || true fail fi - echo "the deny recorded nothing: no live row, an empty runtime set in the host row, and no line in the host audit (NET-044, NET-046)" + case "$eib_deny_lines" in + *'"reported_by":"guest"'*'"decision":"deny"'*'"decided_by":"box-policy"'*'"outcome":"refused"'*) ;; + *) + echo "::error::the host audit's expose_decision line for $eib_deny_name is not a deny reported by the guest, decided by box-policy and refused (NET-046)" + echo " $eib_deny_lines" + fail + ;; + esac + echo "host audit: $eib_deny_lines (the deny decided in the box, reported to the host, NET-046)" + echo "the deny admitted nothing: no live row and an empty runtime set in the host row, and the host audit holds the one deny line (NET-044, NET-046)" while IFS= read -r eib_vm_sid; do [ -n "$eib_vm_sid" ] && { mnl session destroy --force "$eib_vm_sid" >/dev/null 2>&1 || true; } @@ -19484,7 +19546,7 @@ exit" E2E_PTY_ANSWER=keep \ echo "two named VMs: the expose inside the box on $eib_two_vm published on $eib_two_vm, and the named VM is stopped again (NET-055, NET-058)" fi - eib_vm_ok="expose from inside the box OK on a VM lane (the allow stance published through the VM host daemon's admission — the row holds the port at the registration's switch address and the owner-only host audit carries the line, and the host reached the in-box server through the published address; the host's attached human's Allow published and their deny recorded nothing, a later expose asked again, nobody attached was refused, the deny box was refused with nothing recorded on the host, and every decision is a line in the VM host daemon's own audit" + eib_vm_ok="expose from inside the box OK on a VM lane (the allow stance published through the VM host daemon's admission — the row holds the port at the registration's switch address and the owner-only host audit carries the line, and the host reached the in-box server through the published address; the host's attached human's Allow published and their deny recorded nothing, a later expose asked again, nobody attached was refused, the deny box was refused with nothing admitted on the host and its deny recorded in the host audit, and every decision is a line in the VM host daemon's own audit" if [ -n "$eib_two_skip" ]; then echo "$eib_vm_ok; the two named VMs leg SKIPPED: $eib_two_skip)" else From 403d8e83dce71c3de3036d3b7a0f021d8b2729ff Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:41:25 +0000 Subject: [PATCH 04/34] fix(minimald): fail activation when a declared host port is already taken (#2352) Prerequisite for T100. A declared TCP host port that another process holds at the box's published address fails the finalize before the Active write (NET-121), with an error that names the port and the next step. The daemon probes on a native host. On a VM-backed host the forwarder binds on the host OS, so the probe is skipped and the spawn's apply reports the port. A forwarder's EADDRINUSE refusal fails the apply the same way. Backport of #2352, folded from #2738. Adapted to the line: main's box_actor/setup.rs hunk lands in session.rs, boxes::IpProto and boxes::NetworkMode become sessions::, and the fake forwarder spawns with tokio::spawn. The split from T100 carries those renames here, so this commit builds alone. (cherry picked from commit 9437ba3c89fea5315d1b16281caa1c342df81520) --- crates/minimald/src/net/policy.rs | 296 +++++++++++++++++++++++++++ crates/minimald/src/session.rs | 44 ++++ crates/minimald/src/session/tests.rs | 31 +++ 3 files changed, 371 insertions(+) diff --git a/crates/minimald/src/net/policy.rs b/crates/minimald/src/net/policy.rs index ac59059455..660a60bc91 100644 --- a/crates/minimald/src/net/policy.rs +++ b/crates/minimald/src/net/policy.rs @@ -334,6 +334,12 @@ impl PortForwarder { /// not a failure. It is skipped, with one warn line naming the port and the /// box that holds it, and the box's other forwards bind as usual. /// +/// A host port another process already holds is a failure like any other +/// refused bind (NET-121): the rollback below runs and the caller withdraws +/// the name. Only the returned error differs. It names the port and says +/// the port is in use on the host, in place of the forwarder's raw +/// refusal, and its kind is [`io::ErrorKind::AddrInUse`]. +/// /// On the first failure the already-bound forwards are rolled back so a /// partial apply does not leak forwards onto the switch, and the original /// error is returned. The failure says its own line first (NET-121's sub 2): @@ -393,6 +399,15 @@ pub async fn apply_ingress( // Roll back what we managed to expose so a half-applied policy // does not leave dangling forwards on the shared switch. release_declared_ingress(control, &bound).await; + // A taken host port fails the apply like any other refusal: + // NET-121 (docs/specs/18-spec-box-networking, design §7.1) + // makes a failed bind a surfaced error, never a silent + // fallback, so the box never publishes a name while one of + // its declared ports is missing. Only the error's words + // differ: they name the port and the way out. + if is_address_in_use(&e) { + return Err(address_in_use_error(mapping.external_port)); + } return Err(e); } } @@ -400,6 +415,74 @@ pub async fn apply_ingress( Ok(bound) } +/// The OS's own text for a bind onto a port another process already holds, +/// as it reaches [`apply_ingress`] inside the forwarder's refusal. +const ADDRESS_IN_USE_BODY: &str = "address already in use"; + +/// Whether `error` is the forwarder refusing an `expose` because the host +/// port it binds is already in use — the OS's `EADDRINUSE` text, matched on +/// the message's own words rather than the forwarder's surrounding wording, +/// so a differently-phrased listener on the same bind is still read as the +/// same fact. +fn is_address_in_use(error: &io::Error) -> bool { + let said = error.to_string(); + said.contains("returned HTTP 500") && said.contains(ADDRESS_IN_USE_BODY) +} + +/// The declared TCP host ports of `ingress` that another process already +/// holds at `published`, found by a short-lived bind of each one in this +/// daemon's network namespace — the same bind the forwarder makes at the +/// box's first spawn when `control` reaches a switch this daemon spawned. +/// When the host unbinds the declared forwards +/// ([`ControlChannel::host_unbinds_declared_forwards`]), the forwarder +/// binds on the host OS, not where this daemon runs, so a bind here would +/// test the wrong ports: the probe reports nothing and the spawn's own +/// apply says a taken port. A port in `yielded` is skipped, because another +/// box at the shared address holds it and the collision report already +/// names it. A bind that fails for any reason other than `EADDRINUSE` +/// reports nothing: the spawn's own apply says that failure. Call this only +/// when the box's forwards are not bound, or it finds the box's own +/// forwards. +#[must_use] +pub fn host_ports_in_use( + control: &ControlChannel, + published: Ipv4Addr, + ingress: &IngressPolicy, + yielded: &[u16], +) -> Vec { + if control.host_unbinds_declared_forwards() { + return Vec::new(); + } + ingress + .port_mappings + .iter() + .filter(|mapping| mapping.proto == sessions::IpProto::Tcp) + .map(|mapping| mapping.external_port) + .filter(|port| !yielded.contains(port)) + .filter(|&port| { + std::net::TcpListener::bind((published, port)) + .is_err_and(|e| e.kind() == io::ErrorKind::AddrInUse) + }) + .collect() +} + +/// The error [`apply_ingress`] returns for a declared port another process +/// already holds on the host, and the error a finalize returns when +/// [`host_ports_in_use`] finds one. A person reads it at the failed +/// activation or command, so +/// it names the port and the next step; the forwarder's own refusal is in +/// the warn line [`apply_ingress`] logs, not here. +pub fn address_in_use_error(port: u16) -> io::Error { + io::Error::new( + io::ErrorKind::AddrInUse, + format!( + "host port {port} is already in use on the host, so the \ + session cannot forward it. Free the port, or activate again \ + with --ingress :." + ), + ) +} + /// The body the forwarder answers an expose with when a forward is already /// bound at the listener, as an HTTP 500. On a VM-backed host the egress /// gate gives the same answer to a publish whose listener another box's row @@ -2653,4 +2736,217 @@ mod tests { ); } } + + // ---- a host port another process already holds (reported as EADDRINUSE) -- + mod host_port_in_use { + use std::net::Ipv4Addr; + + use tokio::io::AsyncWriteExt as _; + use tokio::net::UnixListener; + + use super::super::{ + ControlChannel, IngressPolicy, PortMapping, apply_ingress, host_ports_in_use, + is_address_in_use, + }; + use super::forwarder_probe::read_request; + + const PUBLISHED: Ipv4Addr = Ipv4Addr::new(127, 0, 64, 9); + const LEASE: Ipv4Addr = Ipv4Addr::new(100, 64, 128, 10); + + fn mapping(external_port: u16) -> PortMapping { + PortMapping { + external_port, + internal_port: 80, + proto: sessions::IpProto::Tcp, + } + } + + fn ingress(ports: &[u16]) -> IngressPolicy { + IngressPolicy { + port_mappings: ports.iter().map(|&port| mapping(port)).collect(), + dynamic_allowed_range: None, + dynamic_ingress: None, + } + } + + /// Serves a gvproxy-shaped control channel at `path`: one request per + /// entry of `answers`, each answered with its status and body — the + /// stand-in for the host forwarder whose `expose` of a taken port + /// answers `500` with the OS's bind text in its body. The OS's own + /// words arrive as an HTTP body, so the answers here carry them. + fn forwarder_answering_in_turn( + path: std::path::PathBuf, + answers: Vec<(u16, &'static str)>, + ) -> tokio::task::JoinHandle> { + let listener = UnixListener::bind(&path).unwrap(); + tokio::spawn(async move { + let mut asked = Vec::new(); + for (status, body) in answers { + let Ok((mut sock, _)) = listener.accept().await else { + return asked; + }; + asked + .push(String::from_utf8_lossy(&read_request(&mut sock).await).into_owned()); + sock.write_all( + format!( + "HTTP/1.1 {status} Internal Server Error\r\n\ + Content-Length: {}\r\n\r\n{body}", + body.len() + ) + .as_bytes(), + ) + .await + .expect("the fake forwarder must answer"); + } + asked + }) + } + + /// The OS's bind text as the forwarder returns it to the caller of + /// `expose`: the status line in the wrapper's own words, the OS's + /// message in the body. + const IN_USE: (u16, &str) = ( + 500, + "listen tcp 127.0.0.1:18999: bind: address already in use", + ); + + /// One declared port taken by another process fails the apply + /// (NET-121): the forward already bound is rolled back with an + /// unexpose, and the error names the taken port in place of the + /// forwarder's raw refusal. The caller then withdraws the name, as + /// it does for every failed apply. + #[tokio::test] + async fn a_taken_host_port_fails_the_apply_and_rolls_back() { + let dir = tempfile::TempDir::new().unwrap(); + let sock = dir.path().join("gvproxy.sock"); + // Two declared ports: the first binds, the second is taken, and + // the third request is the rollback of the first. + let forwarder = + forwarder_answering_in_turn(sock.clone(), vec![(200, ""), IN_USE, (200, "")]); + let err = match apply_ingress( + &ControlChannel::Unix(sock), + PUBLISHED, + LEASE, + &ingress(&[18080, 18999]), + None, + &[], + ) + .await + { + Err(e) => e, + Ok(_) => panic!("a taken host port must fail the apply"), + }; + + assert_eq!(err.kind(), std::io::ErrorKind::AddrInUse); + assert_eq!( + err.to_string(), + "host port 18999 is already in use on the host, so the session \ + cannot forward it. Free the port, or activate again with \ + --ingress :.", + "the error names the taken port and the next step, and no \ + forwarder internals" + ); + let asked = forwarder + .await + .expect("the fake forwarder served every round"); + assert_eq!(asked.len(), 3, "expose, expose, rollback: {asked:?}"); + assert!( + asked[2].contains("127.0.64.9:18080"), + "the rollback releases the forward that bound: {asked:?}" + ); + } + + /// The activation probe names exactly the TCP ports another process + /// holds at the published address. A free port, a yielded port and + /// a UDP mapping on the held port all stay out of the list, and a + /// host that binds the forwards itself is not probed at all. + #[test] + fn the_probe_names_only_a_port_another_process_holds() { + let native = ControlChannel::Unix("/nonexistent/gvproxy.sock".into()); + let held = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let held_port = held.local_addr().unwrap().port(); + let free = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let free_port = free.local_addr().unwrap().port(); + drop(free); + + let mut declared = ingress(&[held_port, free_port]); + assert_eq!( + host_ports_in_use(&native, Ipv4Addr::LOCALHOST, &declared, &[]), + [held_port] + ); + assert!( + host_ports_in_use(&native, Ipv4Addr::LOCALHOST, &declared, &[held_port]).is_empty(), + "a yielded port is the collision report's to name" + ); + let vm_backed = ControlChannel::Vsock { cid: 2, port: 1024 }; + assert!( + host_ports_in_use(&vm_backed, Ipv4Addr::LOCALHOST, &declared, &[]).is_empty(), + "on a VM-backed host the forwarder binds on the host OS, so a \ + bind in the guest proves nothing and the probe is skipped" + ); + declared.port_mappings[0].proto = sessions::IpProto::Udp; + assert!( + host_ports_in_use(&native, Ipv4Addr::LOCALHOST, &declared, &[]).is_empty(), + "a UDP mapping is not probed with a TCP bind" + ); + } + + /// Only the OS's own words get the taken-port error: any other + /// refusal of the bind — a permission denied, a missing address, a + /// crashed forwarder — rolls back and fails with the forwarder's + /// refusal as it came. + #[tokio::test] + async fn any_other_refusal_still_rolls_back_and_fails() { + let dir = tempfile::TempDir::new().unwrap(); + let sock = dir.path().join("gvproxy.sock"); + let forwarder = forwarder_answering_in_turn( + sock.clone(), + vec![(200, ""), (500, "listen tcp: permission denied")], + ); + let err = match apply_ingress( + &ControlChannel::Unix(sock), + PUBLISHED, + LEASE, + &ingress(&[18080, 18181]), + None, + &[], + ) + .await + { + Err(e) => e, + Ok(_) => panic!("a refusal that is not the OS's bind text still fails"), + }; + + assert!( + !is_address_in_use(&err), + "a permission-denied refusal is not the bind text: {err}" + ); + forwarder + .await + .expect("the fake forwarder served both rounds"); + } + + /// The classifier's own words: the status line the wrapper writes, + /// the OS's phrase the body carries. A refusal that says either half + /// alone — a 200 that carries the phrase, a 500 that does not — is + /// not the bind text. + #[test] + fn the_classifier_reads_the_wrappers_own_words() { + let said = |body: &str| { + std::io::Error::other(format!( + "gvproxy /services/forwarder/expose returned HTTP 500: {body}" + )) + }; + assert!(is_address_in_use(&said( + "listen tcp 127.0.0.1:18999: bind: address already in use" + ))); + assert!(!is_address_in_use(&said( + "listen tcp 127.0.0.1:18999: bind: permission denied" + ))); + assert!(!is_address_in_use(&said("proxy already running"))); + assert!(!is_address_in_use(&std::io::Error::other( + "connection closed before the answer" + ))); + } + } } diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index eea70733e0..79c450949d 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -2705,6 +2705,50 @@ impl Session { #[cfg(not(target_os = "linux"))] let shared_port_collisions: Vec = Vec::new(); + // A declared TCP host port that another process already + // holds at this box's published address fails the finalize + // before the `Active` write (NET-121, design §7.1: the bind + // succeeds before the box is active, and a failed bind is an + // error). The probe needs the address the registration above + // publishes, so it runs here, and the client's activate + // cleanup withdraws the publish like any other failure after + // that point. On a native host the forwarder binds in the + // daemon's own network namespace, so the daemon probes. On a + // VM-backed host the forwarder binds on the host OS, so the + // probe finds nothing and the spawn's apply reports the port. + // Skipped when activate hooks run: their launch binds the + // forwards, and a taken port fails that apply with the same + // error. + #[cfg(target_os = "linux")] + if let (sessions::NetworkMode::OwnIp, Some(ingress), false) = ( + &record.network, + record.policy.ingress.as_ref(), + self.has_hooks_for(crate::hooks::HookEvent::Activate), + ) { + let control = self.switch_control().await; + let published = self + .hostnames + .read() + .expect("hostname registry lock poisoned") + .published_own_address(record.id); + let yielded: Vec = shared_port_collisions.iter().map(|c| c.port).collect(); + if let Some(published) = published + && let Some(&port) = crate::net::policy::host_ports_in_use( + &control, published, ingress, &yielded, + ) + .first() + { + tracing::warn!( + port, + session = %record.id, + action = "host-port-in-use", + "declared ingress port is already in use on the host; \ + finalize fails" + ); + return Err(crate::net::policy::address_in_use_error(port)); + } + } + let mut ran: Vec = Vec::new(); if self.has_hooks_for(crate::hooks::HookEvent::Activate) { self.launch_host_for_hooks(LaunchPhase::Activating).await?; diff --git a/crates/minimald/src/session/tests.rs b/crates/minimald/src/session/tests.rs index 45ff4b8184..3c489573ed 100644 --- a/crates/minimald/src/session/tests.rs +++ b/crates/minimald/src/session/tests.rs @@ -3492,6 +3492,37 @@ async fn create_handed_own_ip_session( id } +/// NET-121, design §7.1: a declared TCP host port that another process +/// already holds at the box's published address fails the finalize. The +/// error names the port and the next step, and the box never reaches +/// `Active`. +#[cfg(target_os = "linux")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn finalize_fails_on_a_taken_declared_host_port() { + use minimald_rpc::{Errorable, FinalizeSession, FinalizeSessionRequest}; + let handed = std::net::Ipv4Addr::new(127, 0, 64, 77); + let switch = std::net::Ipv4Addr::new(100, 64, 128, 77); + // Another process holds the declared host port at the handed address. + let _holder = std::net::TcpListener::bind((handed, 18080)).expect("hold the declared port"); + let server = TestServer::new().await; + let mut client = server.connect().await; + let id = create_handed_own_ip_session(&mut client, "taken", switch, handed).await; + let reply = client + .call::(&FinalizeSessionRequest { + session_id: id, + report_shared_port_collisions: true, + }) + .await; + match reply { + Errorable::Err { error } => assert!( + error.contains("host port 18080 is already in use") + && error.contains("--ingress :"), + "the error must name the port and the next step, got: {error}" + ), + Errorable::Ok(ok) => panic!("finalize must fail on a taken host port, got {ok:?}"), + } +} + /// Waits until a registration is parked on the range verdict, so the /// landing a test makes next provably answers a waiter rather than a /// registration that has not yet begun to wait (NET-123 §7.1). From e289e5fd2a28aae89ba47231b6824267ca3dd8c8 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:10:47 +0000 Subject: [PATCH 05/34] fix(minimald): refuse an activation whose declared port something else holds (#2725) T100. Finalize refuses a declared ingress host port that a program outside Minimal holds, or that another box holds at the 127.0.0.1 fallback once a landed range verdict keeps the box there. The typed PortInUseRefusal names the port and its holder, the daemon logs one warn line, and min session activate ends with that refusal text alone. An unknown holder is named as another program. On a VM-backed host, finalize binds the declared forwards toward the handed switch address before the Active write, and the first attach takes the host's "already bound" answer as the box's own forward. A box that finalizes at the fallback under a pending verdict gets a watcher that reports an absent landing. Backport of #2725, folded from #2738. Adapted to the line: - The finalize code lives in session.rs, not box_actor/setup.rs, and registry_name is the session module's own helper. In the CLI, activate_refusal applies to activate_session. - boxes:: becomes sessions:: and switch::BoxGate becomes switch::SessionGate. - The fallback watcher spawns with tokio::spawn under the same root span. - OwnAddressReporter::registry comes without NET-129's dynamic-hold tracking. The attach path's holder lookup needs only the accessor. - The line's DestroySessionRequest has no only_if_unfinalized, so the refusal tests assert the refused box is listed and not Active, destroy it plainly, and assert no record stays. (cherry picked from commit 703bb7fc70097130788a38d749007c97ec70b8a5) --- crates/minimal/src/cmd/session.rs | 64 ++++- crates/minimald-rpc/src/lib.rs | 106 +++++++ crates/minimald/src/net/dns.rs | 102 +++++++ crates/minimald/src/net/gvproxy_network.rs | 18 +- crates/minimald/src/net/policy.rs | 303 +++++++++++++++++++-- crates/minimald/src/net/provider.rs | 6 + crates/minimald/src/session.rs | 255 ++++++++++++++--- crates/minimald/src/session/tests.rs | 247 +++++++++++++++-- 8 files changed, 1009 insertions(+), 92 deletions(-) diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index f7920d3779..57e2ea6972 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -24,7 +24,25 @@ pub(crate) fn session_announce_label(id: &sessions::SessionId, name: Option<&str /// Create a new session via the `CreateSession` RPC. pub async fn cmd_activate(global: &GlobalArgs, args: ActivateArgs) -> Result<(), anyhow::Error> { - activate_session(global, args, true).await + activate_session(global, args, true) + .await + .map_err(activate_refusal) +} + +/// The error `min session activate` ends with (NET-144). When the daemon +/// refused the activation because a declared host port is already in use, +/// the refusal's own text replaces the whole chain: it names the port, its +/// holder and the next step, and the stage and RPC context around it add +/// nothing a person can act on. The activate cleanup has already destroyed +/// the box by then. Any other error passes through as it is. +fn activate_refusal(error: anyhow::Error) -> anyhow::Error { + let refusal = error + .chain() + .find_map(|cause| minimald_rpc::PortInUseRefusal::find_in(&cause.to_string())); + match refusal { + Some(refusal) => anyhow::anyhow!("{refusal}"), + None => error, + } } // The box registration exchange with the VM host daemon (T66) — the @@ -7677,4 +7695,48 @@ mod tests { "no session was created for the unregistered box" ); } + + /// NET-144: the daemon's refusal reaches the person as its own text, + /// with the port and the holder named, whatever context the + /// activation wrapped around it on the way up. + #[test] + fn activate_refusal_names_port_and_holder() { + for (holder, first_line) in [ + ( + minimald_rpc::PortHolder::Box("web".to_string()), + "Port 8080 is already in use on this machine by box web.", + ), + ( + minimald_rpc::PortHolder::Host, + "Port 8080 is already in use on this machine by another program on this machine.", + ), + ] { + let refusal = minimald_rpc::PortInUseRefusal { + port: 8080, + box_port: 80, + holder, + }; + let from_daemon = anyhow::anyhow!("could not finish activating the session: {refusal}") + .context("Finalize failed"); + let printed = format!("{:#}", activate_refusal(from_daemon)); + assert_eq!( + printed, + format!( + "{first_line}\nChoose another host port in --ingress (for example \ + --ingress 8081:80) or in [network] ingress in minimal.toml." + ) + ); + assert_eq!( + minimald_rpc::PortInUseRefusal::find_in(&printed), + Some(refusal), + "the text reads back as the same refusal" + ); + } + let other = activate_refusal(anyhow::anyhow!("something else failed")); + assert_eq!( + format!("{other:#}"), + "something else failed", + "any other error passes through" + ); + } } diff --git a/crates/minimald-rpc/src/lib.rs b/crates/minimald-rpc/src/lib.rs index 4eabe26290..4bdf3203e2 100644 --- a/crates/minimald-rpc/src/lib.rs +++ b/crates/minimald-rpc/src/lib.rs @@ -2041,6 +2041,112 @@ pub struct SharedPortCollision { pub held_by: String, } +/// What holds a declared host port that an activation refused (NET-144). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PortHolder { + /// Another box on this machine holds the port. Carries its box name. + Box(String), + /// No box in this daemon's hostname registry holds the port, so + /// another program on this machine does. That program can be this + /// daemon's own proxy or a box of another daemon on the machine. + Host, +} + +impl PortHolder { + /// The holder as the daemon log names it: `box:` or `host`. + #[must_use] + pub fn log_label(&self) -> String { + match self { + Self::Box(name) => format!("box:{name}"), + Self::Host => "host".to_string(), + } + } +} + +/// An activation refused because a declared ingress host port is already in +/// use on this machine (NET-144): the port, the box port the declaration +/// maps it to, and what holds it. +/// +/// The refusal travels as the daemon's error text, which is exactly this +/// type's [`std::fmt::Display`]. The client reads it back with +/// [`Self::find_in`], so the two cannot drift. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PortInUseRefusal { + /// The host port the declaration names. + pub port: u16, + /// The box port the declaration maps the host port to. + pub box_port: u16, + /// What holds the host port. + pub holder: PortHolder, +} + +/// The middle of the refusal's first line, after `Port `. +const PORT_IN_USE_MIDDLE: &str = " is already in use on this machine by "; +/// The holder clause when no box holds the port. +const PORT_IN_USE_HOST: &str = "another program on this machine."; +/// The refusal's second line, up to the suggested host port. +const PORT_IN_USE_REMEDY: &str = "\nChoose another host port in --ingress (for example --ingress "; +/// The refusal's second line, after the suggested mapping. +const PORT_IN_USE_REMEDY_END: &str = ") or in [network] ingress in minimal.toml."; + +impl PortInUseRefusal { + /// The host port the remedy suggests: the next port up, or the one + /// below when the refused port is the highest. + #[must_use] + pub fn suggested_port(&self) -> u16 { + self.port + .checked_add(1) + .unwrap_or_else(|| self.port.saturating_sub(1)) + } + + /// The refusal inside `error`, if it carries one. The daemon's text can + /// arrive inside other context, so the search does not need it to open + /// the string. + #[must_use] + pub fn find_in(error: &str) -> Option { + error + .match_indices("Port ") + .find_map(|(at, _)| Self::parse_at(&error[at + "Port ".len()..])) + } + + fn parse_at(text: &str) -> Option { + let digits = text.find(|c: char| !c.is_ascii_digit())?; + let port: u16 = text[..digits].parse().ok()?; + let rest = text[digits..].strip_prefix(PORT_IN_USE_MIDDLE)?; + let remedy = rest.find(PORT_IN_USE_REMEDY)?; + let holder = match &rest[..remedy] { + PORT_IN_USE_HOST => PortHolder::Host, + clause => PortHolder::Box(clause.strip_prefix("box ")?.strip_suffix('.')?.to_string()), + }; + let mapping = &rest[remedy + PORT_IN_USE_REMEDY.len()..]; + let end = mapping.find(PORT_IN_USE_REMEDY_END)?; + let (_, box_port) = mapping[..end].split_once(':')?; + Some(Self { + port, + box_port: box_port.parse().ok()?, + holder, + }) + } +} + +impl std::fmt::Display for PortInUseRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Port {}{PORT_IN_USE_MIDDLE}", self.port)?; + match &self.holder { + PortHolder::Box(name) => write!(f, "box {name}.")?, + PortHolder::Host => f.write_str(PORT_IN_USE_HOST)?, + } + write!( + f, + "{PORT_IN_USE_REMEDY}{}:{}{PORT_IN_USE_REMEDY_END}", + self.suggested_port(), + self.box_port + ) + } +} + +impl std::error::Error for PortInUseRefusal {} + /// Serde helper: omit [`FinalizeSessionResponse::package_check_skipped`] /// when it is `false`, so the common success payload is unchanged for /// clients that predate the field. diff --git a/crates/minimald/src/net/dns.rs b/crates/minimald/src/net/dns.rs index 54791869ce..a5f1726834 100644 --- a/crates/minimald/src/net/dns.rs +++ b/crates/minimald/src/net/dns.rs @@ -1268,6 +1268,42 @@ impl HostnameRegistry { .unwrap_or_default() } + /// The box that holds `port` at `address`, with its box name (NET-144): + /// the holder the shared-port table records there, declared or dynamic. + /// The name is the box's registered session name, or its stable id when + /// nothing answers for it. `None` when no box holds the port there. + #[must_use] + pub fn own_port_holder(&self, address: Ipv4Addr, port: u16) -> Option<(SessionId, String)> { + let holder = *self.own_holders.get(&(address, port))?; + let name = self + .by_session + .iter() + .find(|(_, registration)| registration.id == holder) + .map_or_else(|| holder.to_string(), |(name, _)| name.clone()); + Some((holder, name)) + } + + /// The first declared port another box already holds where this box + /// publishes at the shared `127.0.0.1` fallback, with the holder's box + /// name (NET-144). Nobody chose to share the fallback, so a port another + /// box holds there is a refusal at activation, never a yielded port. A + /// box at any other address gets `None`: a host-address box that shares + /// its node's address keeps the shared-address rule (NET-129), and + /// yields the port. The registry does not know the range verdict, so + /// the finalize asks [`LoopbackLeaseBook::range_verdict`] whether the + /// box stays at the fallback before it refuses. + #[must_use] + pub fn fallback_port_refusal(&self, session_id: SessionId) -> Option<(u16, String)> { + let own = self.own_published.get(&session_id)?; + if own.address != Ipv4Addr::LOCALHOST { + return None; + } + own.ports.iter().find_map(|&port| { + let (holder, name) = self.own_port_holder(own.address, port)?; + (holder != session_id).then_some((port, name)) + }) + } + /// Elects the next holder of each of `ports` at `address` that `leaving` /// held and no longer publishes there (NET-129): the lowest /// [`SessionId`] among the boxes still publishing the port at the @@ -2168,6 +2204,17 @@ impl LoopbackLeaseBook { RangeVerdict::from_key(self.verdict.load(std::sync::atomic::Ordering::Acquire)) } + /// The verdict over the reserved range as it stands now, for a caller + /// outside the book (NET-144): a finalize refuses a port another box + /// holds at the `127.0.0.1` fallback only once a landed verdict keeps + /// the box there. Under [`RangeVerdict::Pending`] a box at the interim + /// moves to its hand when the verdict lands present, so its collision + /// there is not lasting. + #[must_use] + pub fn range_verdict(&self) -> RangeVerdict { + self.verdict() + } + /// Applies the bind probe's verdict over the reserved local range, when /// the book opened before the probe that produced it could answer. /// @@ -2191,6 +2238,24 @@ impl LoopbackLeaseBook { self.verdict_landing.send_replace(verdict); } + /// Waits, without a bound, for the verdict to land, and returns it. A + /// verdict already landed returns at once. When the book is dropped + /// before a landing, this returns [`RangeVerdict::Pending`]. The watcher + /// of a box finalized at the `127.0.0.1` fallback under a pending verdict + /// waits here (NET-144). + pub async fn await_landing(&self) -> RangeVerdict { + let mut landing = self.verdict_landing.subscribe(); + loop { + let verdict = *landing.borrow_and_update(); + if verdict != RangeVerdict::Pending { + return verdict; + } + if landing.changed().await.is_err() { + return RangeVerdict::Pending; + } + } + } + /// The daemon's one verdict deadline, as an instant: the moment the book /// opened plus [`HAND_VERDICT_WAIT`] (or the instant a test moved it to). fn hand_verdict_deadline(&self) -> std::time::Instant { @@ -3276,6 +3341,43 @@ mod tests { ); } + /// NET-144: at the shared `127.0.0.1` fallback a port another box holds + /// is a refusal that names the holder's box name. At a node's shared + /// address the same collision stays a yielded port (NET-129). + #[test] + fn activation_refused_when_declared_port_is_held_at_the_fallback_only() { + let node = Ipv4Addr::new(127, 0, 64, 200); + let mut reg = HostnameRegistry::new("dev", false).with_node_address(node); + let ports = BTreeSet::from([8080u16]); + for (address, first, second) in [ + (Ipv4Addr::LOCALHOST, id("1"), id("2")), + (node, id("3"), id("4")), + ] { + reg.publish_own_address(first, &format!("first-{first}"), address, ports.clone()); + reg.register_own_ip(first, &format!("first-{first}"), ports.clone()); + reg.publish_own_address(second, "second", address, ports.clone()); + assert_eq!( + reg.fallback_port_refusal(first), + None, + "the holder is never refused" + ); + if address == Ipv4Addr::LOCALHOST { + assert_eq!( + reg.fallback_port_refusal(second), + Some((8080, format!("first-{first}"))), + "the fallback refuses and names the holder by its box name" + ); + } else { + assert_eq!( + reg.fallback_port_refusal(second), + None, + "a node's shared address keeps the yield" + ); + } + reg.unpublish_own_address(second); + } + } + /// NET-129's shared address is first-come: the later box records the /// ports it yields, with the box that holds each, and a re-publish of the /// holder (a rename, a resume) neither reports a collision of its own nor diff --git a/crates/minimald/src/net/gvproxy_network.rs b/crates/minimald/src/net/gvproxy_network.rs index f277fcd002..8bb6983f16 100644 --- a/crates/minimald/src/net/gvproxy_network.rs +++ b/crates/minimald/src/net/gvproxy_network.rs @@ -404,8 +404,17 @@ async fn finish_own_ip_attach( let yielded = own_address .map(crate::net::provider::OwnAddressReporter::shared_port_collisions) .unwrap_or_default(); - match crate::net::policy::apply_ingress( - &control, published, lease_ip, ingress, gate, &yielded, + // A taken port's refusal names its holder (NET-144): the box + // whose publish stands at that address and port, or the host. + let registry = own_address.map(crate::net::provider::OwnAddressReporter::registry); + let holder_of = |port: u16| match ®istry { + Some(registry) => { + crate::net::policy::port_holder(registry, session_name, published, port) + } + None => minimald_rpc::PortHolder::Host, + }; + match crate::net::policy::apply_ingress_naming_holders( + &control, published, lease_ip, ingress, gate, &yielded, &holder_of, ) .await { @@ -416,7 +425,10 @@ async fn finish_own_ip_attach( // route reported, and the name finalize registered is // withdrawn — a bind that failed never leaves a name standing // for a port that refused (NET-121) — and no substitute - // address stands in for the port that could not bind. + // address stands in for the port that could not bind. A taken + // port comes back as the typed refusal, with the port and its + // holder, so the finalize that launched this attach can name + // both (NET-144). Err(e) => { drop(relay); if let Some(own_address) = own_address { diff --git a/crates/minimald/src/net/policy.rs b/crates/minimald/src/net/policy.rs index 660a60bc91..313491bf73 100644 --- a/crates/minimald/src/net/policy.rs +++ b/crates/minimald/src/net/policy.rs @@ -336,9 +336,11 @@ impl PortForwarder { /// /// A host port another process already holds is a failure like any other /// refused bind (NET-121): the rollback below runs and the caller withdraws -/// the name. Only the returned error differs. It names the port and says -/// the port is in use on the host, in place of the forwarder's raw -/// refusal, and its kind is [`io::ErrorKind::AddrInUse`]. +/// the name. Only the returned error differs. It carries a +/// [`minimald_rpc::PortInUseRefusal`] in place of the forwarder's raw +/// refusal, and its kind is [`io::ErrorKind::AddrInUse`]. The refusal names +/// the port and its holder (NET-144). Here the holder is always the host. +/// [`apply_ingress_naming_holders`] reads it from the hostname registry. /// /// On the first failure the already-bound forwards are rolled back so a /// partial apply does not leak forwards onto the switch, and the original @@ -356,6 +358,108 @@ pub async fn apply_ingress( ingress: &IngressPolicy, gate: Option<&Arc>, yielded: &[super::dns::SharedPortCollision], +) -> io::Result> { + apply_ingress_naming_holders( + control, + published, + ptask_ip, + ingress, + gate, + yielded, + &|_| minimald_rpc::PortHolder::Host, + ) + .await +} + +/// [`apply_ingress`], with `holder_of` to name what holds a taken port +/// (NET-144). The attach path passes [`port_holder`] over the hostname +/// registry, so the refusal names the box whose publish stands at that +/// address and port, or the host when none does. +/// +/// # Errors +/// +/// As [`apply_ingress`]. +pub async fn apply_ingress_naming_holders( + control: &ControlChannel, + published: Ipv4Addr, + ptask_ip: Ipv4Addr, + ingress: &IngressPolicy, + gate: Option<&Arc>, + yielded: &[super::dns::SharedPortCollision], + holder_of: &(dyn Fn(u16) -> minimald_rpc::PortHolder + Sync), +) -> io::Result> { + apply_declared( + control, + control.host_unbinds_declared_forwards(), + published, + ptask_ip, + ingress, + gate, + yielded, + holder_of, + ) + .await +} + +/// Binds the declared forwards of a box on a VM-backed host when finalize +/// publishes it, before the record goes `Active` (NET-121, NET-144, design +/// §7.1: the bind succeeds before the name is published). On a VM-backed +/// host the forwarder binds on the host OS, so the daemon cannot probe the +/// ports itself ([`host_ports_in_use`]). It binds them, toward the +/// `switch_address` the VM host handed the box, and a held port fails with +/// the same typed refusal the native probe gives. A failure rolls back as +/// [`apply_ingress`] does. +/// +/// The bound forwards stay on the host. The box's first attach posts the +/// same exposes again, and the host's switch answers each one that the +/// listener is already bound, which the attach takes as bound. The host +/// unbinds the forwards when it withdraws the box's row, as it does for +/// every declared forward ([`release_declared_ingress`]). +/// +/// # Errors +/// +/// As [`apply_ingress`]. +pub async fn bind_declared_at_publish( + control: &ControlChannel, + published: Ipv4Addr, + switch_address: Ipv4Addr, + ingress: &IngressPolicy, + yielded: &[super::dns::SharedPortCollision], + holder_of: &(dyn Fn(u16) -> minimald_rpc::PortHolder + Sync), +) -> io::Result> { + apply_declared( + control, + control.host_unbinds_declared_forwards(), + published, + switch_address, + ingress, + None, + yielded, + holder_of, + ) + .await +} + +/// [`apply_ingress_naming_holders`], with `host_holds_listeners` read off the +/// control channel ([`ControlChannel::host_unbinds_declared_forwards`]) by +/// the caller, so a test can drive the VM-backed answers over a unix socket. +/// Behind a VM host's gate a listener the switch already holds is the +/// box's own forward, bound at publish ([`bind_declared_at_publish`]), or +/// a publish the host keeps for the box until the holder's row is withdrawn +/// ([`HandedBind::HeldByHost`]). Either way the forward is tracked as bound. +#[expect( + clippy::too_many_arguments, + reason = "the public wrappers above fix the flag; a struct would be single-use ceremony" +)] +async fn apply_declared( + control: &ControlChannel, + host_holds_listeners: bool, + published: Ipv4Addr, + ptask_ip: Ipv4Addr, + ingress: &IngressPolicy, + gate: Option<&Arc>, + yielded: &[super::dns::SharedPortCollision], + holder_of: &(dyn Fn(u16) -> minimald_rpc::PortHolder + Sync), ) -> io::Result> { let mut bound: Vec = Vec::with_capacity(ingress.port_mappings.len()); for mapping in &ingress.port_mappings { @@ -374,7 +478,18 @@ pub async fn apply_ingress( continue; } let req = expose_request(mapping, published, ptask_ip); - match post_json(control, "/services/forwarder/expose", &req).await { + let answer = match post_json(control, "/services/forwarder/expose", &req).await { + Err(e) if host_holds_listeners && is_already_running(&e) => { + tracing::debug!( + port = mapping.external_port, + local = %req.local, + "the host already holds this declared forward for the box" + ); + Ok(()) + } + answer => answer, + }; + match answer { Ok(()) => bound.push(PortForwarder { mapping: ExposedMapping { local: req.local, @@ -398,7 +513,7 @@ pub async fn apply_ingress( ); // Roll back what we managed to expose so a half-applied policy // does not leave dangling forwards on the shared switch. - release_declared_ingress(control, &bound).await; + release_declared(control, host_holds_listeners, &bound).await; // A taken host port fails the apply like any other refusal: // NET-121 (docs/specs/18-spec-box-networking, design §7.1) // makes a failed bind a surfaced error, never a silent @@ -406,7 +521,11 @@ pub async fn apply_ingress( // its declared ports is missing. Only the error's words // differ: they name the port and the way out. if is_address_in_use(&e) { - return Err(address_in_use_error(mapping.external_port)); + return Err(address_in_use_error(minimald_rpc::PortInUseRefusal { + port: mapping.external_port, + box_port: mapping.internal_port, + holder: holder_of(mapping.external_port), + })); } return Err(e); } @@ -466,21 +585,51 @@ pub fn host_ports_in_use( .collect() } -/// The error [`apply_ingress`] returns for a declared port another process -/// already holds on the host, and the error a finalize returns when -/// [`host_ports_in_use`] finds one. A person reads it at the failed -/// activation or command, so -/// it names the port and the next step; the forwarder's own refusal is in -/// the warn line [`apply_ingress`] logs, not here. -pub fn address_in_use_error(port: u16) -> io::Error { - io::Error::new( - io::ErrorKind::AddrInUse, - format!( - "host port {port} is already in use on the host, so the \ - session cannot forward it. Free the port, or activate again \ - with --ingress :." - ), - ) +/// The error [`apply_ingress`] returns for a declared port that something +/// else already holds on the host, and the error a finalize returns when it +/// finds one before the box is active. A person reads it at the failed +/// activation or command, so it names the port, its holder and the next step +/// (NET-144). The forwarder's own refusal is in the warn line +/// [`apply_ingress`] logs, not here. The kind is +/// [`io::ErrorKind::AddrInUse`], and the payload is the typed refusal, which +/// [`port_in_use_refusal`] reads back. +pub fn address_in_use_error(refusal: minimald_rpc::PortInUseRefusal) -> io::Error { + io::Error::new(io::ErrorKind::AddrInUse, refusal) +} + +/// The typed refusal inside `error`, when it is one [`address_in_use_error`] +/// built or when its text carries one, as an error wrapped on its way up +/// does. +#[must_use] +pub fn port_in_use_refusal(error: &io::Error) -> Option { + error + .get_ref() + .and_then(|inner| inner.downcast_ref::()) + .cloned() + .or_else(|| minimald_rpc::PortInUseRefusal::find_in(&error.to_string())) +} + +/// What holds `port` at `address` (NET-144): the box whose publish stands +/// there in the hostname registry, or the host when no box does. A holder +/// named `own_name` is the box that asks, so it does not count: then the +/// host holds the port. +#[must_use] +pub fn port_holder( + registry: &std::sync::RwLock, + own_name: &str, + address: Ipv4Addr, + port: u16, +) -> minimald_rpc::PortHolder { + registry + .read() + .expect("hostname registry lock poisoned") + .own_port_holder(address, port) + .map(|(_, name)| name) + .filter(|name| name != own_name) + .map_or( + minimald_rpc::PortHolder::Host, + minimald_rpc::PortHolder::Box, + ) } /// The body the forwarder answers an expose with when a forward is already @@ -626,7 +775,13 @@ pub async fn unexpose_forwarder( /// their connections when it withdraws the box's row. Each forward still /// gets one info line, so the log keeps NET-121's bind/unbind pair. pub async fn release_declared_ingress(control: &ControlChannel, bound: &[PortForwarder]) { - if !control.host_unbinds_declared_forwards() { + release_declared(control, control.host_unbinds_declared_forwards(), bound).await; +} + +/// [`release_declared_ingress`], with `host_unbinds` read off the control +/// channel by the caller. +async fn release_declared(control: &ControlChannel, host_unbinds: bool, bound: &[PortForwarder]) { + if !host_unbinds { remove_ingress(control, bound).await; return; } @@ -2745,8 +2900,8 @@ mod tests { use tokio::net::UnixListener; use super::super::{ - ControlChannel, IngressPolicy, PortMapping, apply_ingress, host_ports_in_use, - is_address_in_use, + ControlChannel, IngressPolicy, PortMapping, apply_declared, apply_ingress, + host_ports_in_use, is_address_in_use, port_in_use_refusal, }; use super::forwarder_probe::read_request; @@ -2838,11 +2993,21 @@ mod tests { }; assert_eq!(err.kind(), std::io::ErrorKind::AddrInUse); + assert_eq!( + port_in_use_refusal(&err), + Some(minimald_rpc::PortInUseRefusal { + port: 18999, + box_port: 80, + holder: minimald_rpc::PortHolder::Host, + }), + "the error carries the typed refusal for the finalize to read" + ); assert_eq!( err.to_string(), - "host port 18999 is already in use on the host, so the session \ - cannot forward it. Free the port, or activate again with \ - --ingress :.", + "Port 18999 is already in use on this machine by another \ + program on this machine.\nChoose another host port in --ingress (for \ + example --ingress 19000:80) or in [network] ingress in \ + minimal.toml.", "the error names the taken port and the next step, and no \ forwarder internals" ); @@ -2856,6 +3021,90 @@ mod tests { ); } + /// NET-144 on a VM-backed host, where the daemon cannot probe the + /// host's ports: finalize binds the declared forwards at publish, and + /// a port a program on the host holds fails that bind with the same + /// refusal text as the native probe. On a VM-backed host the host + /// unbinds the declared forwards, so the rollback asks the switch + /// for nothing: the forwarder sees exactly the one expose. + #[tokio::test] + async fn activation_refused_when_declared_port_is_held_on_a_vm_host_at_publish() { + let dir = tempfile::TempDir::new().unwrap(); + let sock = dir.path().join("gvproxy.sock"); + let forwarder = forwarder_answering_in_turn(sock.clone(), vec![IN_USE]); + let err = match apply_declared( + &ControlChannel::Unix(sock), + true, + PUBLISHED, + LEASE, + &ingress(&[18999]), + None, + &[], + &|_| minimald_rpc::PortHolder::Host, + ) + .await + { + Err(e) => e, + Ok(_) => panic!("a held host port must fail the bind at publish"), + }; + assert_eq!(err.kind(), std::io::ErrorKind::AddrInUse); + assert_eq!( + err.to_string(), + "Port 18999 is already in use on this machine by another \ + program on this machine.\nChoose another host port in --ingress (for \ + example --ingress 19000:80) or in [network] ingress in \ + minimal.toml.", + "the VM-backed refusal is the native refusal's text" + ); + let asked = forwarder.await.expect("the fake forwarder ran"); + assert_eq!(asked.len(), 1, "one expose and no rollback: {asked:?}"); + assert!( + asked[0].contains("\"local\":\"127.0.64.9:18999\"") + && asked[0].contains("\"remote\":\"100.64.128.10:80\""), + "the bind names the published address and the handed switch address: {asked:?}" + ); + } + + /// After the bind at publish, the box's first attach posts the same + /// expose, and the VM host's switch answers that the listener is + /// already bound. Behind a VM host's gate that answer is the box's + /// own forward, so the attach tracks it as bound. On a native host + /// the same answer still fails the apply. + #[tokio::test] + async fn activation_refused_when_declared_port_is_held_not_by_the_forward_bound_at_publish() + { + for host_holds_listeners in [true, false] { + let dir = tempfile::TempDir::new().unwrap(); + let sock = dir.path().join("gvproxy.sock"); + let forwarder = forwarder_answering_in_turn( + sock.clone(), + vec![(500, "proxy already running\n"), (200, "")], + ); + let applied = apply_declared( + &ControlChannel::Unix(sock), + host_holds_listeners, + PUBLISHED, + LEASE, + &ingress(&[18080]), + None, + &[], + &|_| minimald_rpc::PortHolder::Host, + ) + .await; + if host_holds_listeners { + let bound = applied.expect("the forward bound at publish is the box's own"); + assert_eq!(bound.len(), 1); + assert_eq!(bound[0].local(), "127.0.64.9:18080"); + } else { + let Err(err) = applied else { + panic!("on a native host a bound listener fails"); + }; + assert!(port_in_use_refusal(&err).is_none(), "got: {err}"); + } + forwarder.abort(); + } + } + /// The activation probe names exactly the TCP ports another process /// holds at the published address. A free port, a yielded port and /// a UDP mapping on the held port all stay out of the list, and a diff --git a/crates/minimald/src/net/provider.rs b/crates/minimald/src/net/provider.rs index 84c963edc5..4d2bb3c72b 100644 --- a/crates/minimald/src/net/provider.rs +++ b/crates/minimald/src/net/provider.rs @@ -278,6 +278,12 @@ impl OwnAddressReporter { .published_own_address(self.session_id) } + /// The registry this reporter reports to, for the attach path's port + /// refusal to name the box that holds a declared port (NET-144). + pub(crate) fn registry(&self) -> Arc> { + Arc::clone(&self.registry) + } + /// The ports this box yields at a shared address, each with the box that /// holds it (NET-129, first-come): recorded when the box published, read /// here so the attach skips those forwards rather than failing on them. diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index 79c450949d..a51030771e 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -262,6 +262,25 @@ pub(crate) struct SessionConfig { pub creator: Option>, } +/// The daemon log's one line for an activation refused on a declared host +/// port that something else holds (NET-144): the session, the port, the +/// address and the holder (`box:` or `host`). +#[cfg(target_os = "linux")] +fn warn_port_refusal( + record: &Record, + address: Option, + refusal: &minimald_rpc::PortInUseRefusal, +) { + tracing::warn!( + session = %record.id, + port = refusal.port, + address = %address.map_or_else(|| "none".to_string(), |address| address.to_string()), + holder = %refusal.holder.log_label(), + action = "host-port-in-use", + "declared ingress port is already in use; activation refused" + ); +} + /// The egress *section* the gate compiles for a session: the materialized /// form of [`sessions::effective_egress`]'s answer — `None` for the /// allow-all default, the deny-all section for an absent declaration under the @@ -2285,6 +2304,57 @@ impl Session { ControlFlow::Continue(()) } + /// Watches for the range verdict to land, for a box that finalized at + /// the shared `127.0.0.1` fallback under a pending verdict while another + /// box there held one of its declared ports (NET-144). Under a pending + /// verdict the box yields the port, because a present landing moves it + /// to its own address. When the verdict lands absent, the box stays at + /// the fallback with the port unbound. The box is `Active` by then, so + /// the watcher reports the held port in one warn line with the same + /// fields as a refused activation, and does not destroy the box. A box + /// that moved off the fallback, or no longer publishes, is not reported. + #[cfg(target_os = "linux")] + fn watch_fallback_landing(&self, record: &Record, ingress: &sessions::IngressPolicy) { + let book = Arc::clone(&self.loopback); + let hostnames = Arc::clone(&self.hostnames); + let session = record.id; + let mappings = ingress.port_mappings.clone(); + drop(tokio::spawn(tracing::Instrument::instrument( + async move { + if book.await_landing().await != crate::net::dns::RangeVerdict::Absent { + return; + } + let held = hostnames + .read() + .expect("hostname registry lock poisoned") + .fallback_port_refusal(session); + let Some((port, holder)) = held else { + return; + }; + let box_port = mappings + .iter() + .find(|mapping| mapping.external_port == port) + .map_or(port, |mapping| mapping.internal_port); + let refusal = minimald_rpc::PortInUseRefusal { + port, + box_port, + holder: minimald_rpc::PortHolder::Box(holder), + }; + tracing::warn!( + session = %session, + port = refusal.port, + address = %std::net::Ipv4Addr::LOCALHOST, + holder = %refusal.holder.log_label(), + action = "host-port-in-use", + "the range verdict landed absent, so this active box stays at the \ + 127.0.0.1 fallback, where another box holds its declared ingress \ + port; the port is not bound for this box" + ); + }, + tracing::info_span!(parent: None, "session.fallback_port_watch", session = %session), + ))); + } + /// Compose this session's loadout from its project config and the /// client's wire contribution, then either finalize it (`Ok(None)`, the /// session is now `Active`) or park it in `Draft` holding the resume @@ -2693,65 +2763,164 @@ impl Session { // ([`Self::deregister_hostname`]). A retried finalize // registers again, which is idempotent for one session id. #[cfg(target_os = "linux")] - let shared_port_collisions: Vec = self - .register_hostname(&record, true) - .await - .into_iter() + let yielded_ports = self.register_hostname(&record, true).await; + #[cfg(target_os = "linux")] + let shared_port_collisions: Vec = yielded_ports + .iter() .map(|c| minimald_rpc::SharedPortCollision { port: c.port, - held_by: c.other, + held_by: c.other.clone(), }) .collect(); #[cfg(not(target_os = "linux"))] let shared_port_collisions: Vec = Vec::new(); - // A declared TCP host port that another process already - // holds at this box's published address fails the finalize - // before the `Active` write (NET-121, design §7.1: the bind - // succeeds before the box is active, and a failed bind is an - // error). The probe needs the address the registration above - // publishes, so it runs here, and the client's activate - // cleanup withdraws the publish like any other failure after - // that point. On a native host the forwarder binds in the - // daemon's own network namespace, so the daemon probes. On a - // VM-backed host the forwarder binds on the host OS, so the - // probe finds nothing and the spawn's apply reports the port. - // Skipped when activate hooks run: their launch binds the - // forwards, and a taken port fails that apply with the same - // error. + // A declared host port that something else already holds + // fails the finalize before the `Active` write (NET-121, + // NET-144, design §7.1: the bind succeeds before the box is + // active, and a failed bind is an error). The refusal names + // the port and its holder, and the client's activate cleanup + // destroys the box, so no record stays behind. The checks need + // the address the registration above publishes, so they run + // here. + // + // Another box that holds the port at the shared `127.0.0.1` + // fallback refuses the box: nobody chose to share that + // address. Only a landed range verdict keeps a box there. + // Under a pending verdict the interim is a waypoint, the box + // moves to its hand when the verdict lands present, and the + // port is yielded as before. If the verdict then lands absent, + // the box stays at the fallback, and a watcher reports the + // held port in the daemon log ([`Self::watch_fallback_landing`]). + // At a node's shared address the port is yielded too + // (NET-129). Then the daemon checks the ports that something + // outside the registry holds. On a native host the forwarder + // binds in the daemon's own network namespace, so the daemon + // probes. On a VM-backed host the forwarder binds on the host + // OS, so a probe here proves nothing: the daemon binds the + // forwards at publish instead, toward the switch address the + // host handed the box, and a held port fails that bind with + // the same refusal. The check is skipped when activate hooks + // run: their launch binds the forwards, and a taken port + // fails that apply with the same refusal. #[cfg(target_os = "linux")] - if let (sessions::NetworkMode::OwnIp, Some(ingress), false) = ( - &record.network, - record.policy.ingress.as_ref(), - self.has_hooks_for(crate::hooks::HookEvent::Activate), - ) { - let control = self.switch_control().await; - let published = self - .hostnames - .read() - .expect("hostname registry lock poisoned") - .published_own_address(record.id); - let yielded: Vec = shared_port_collisions.iter().map(|c| c.port).collect(); - if let Some(published) = published - && let Some(&port) = crate::net::policy::host_ports_in_use( - &control, published, ingress, &yielded, + if let (sessions::NetworkMode::OwnIp, Some(ingress)) = + (&record.network, record.policy.ingress.as_ref()) + { + let (published, fallback_refusal) = { + let hostnames = self + .hostnames + .read() + .expect("hostname registry lock poisoned"); + ( + hostnames.published_own_address(record.id), + hostnames.fallback_port_refusal(record.id), ) - .first() + }; + let fallback_refusal = if fallback_refusal.is_some() + && self.loopback.range_verdict() == crate::net::dns::RangeVerdict::Pending { - tracing::warn!( + self.watch_fallback_landing(&record, ingress); + None + } else { + fallback_refusal + }; + let box_port = |port: u16| { + ingress + .port_mappings + .iter() + .find(|mapping| mapping.external_port == port) + .map_or(port, |mapping| mapping.internal_port) + }; + let mut refusal = + fallback_refusal.map(|(port, holder)| minimald_rpc::PortInUseRefusal { port, - session = %record.id, - action = "host-port-in-use", - "declared ingress port is already in use on the host; \ - finalize fails" - ); - return Err(crate::net::policy::address_in_use_error(port)); + box_port: box_port(port), + holder: minimald_rpc::PortHolder::Box(holder), + }); + if refusal.is_none() + && !self.has_hooks_for(crate::hooks::HookEvent::Activate) + && let Some(published) = published + { + let control = self.switch_control().await; + let own_name = registry_name(&record); + let holder_of = |port: u16| { + crate::net::policy::port_holder( + &self.hostnames, + &own_name, + published, + port, + ) + }; + let switch_address = record + .box_addresses + .map(|addresses| addresses.switch_address); + match (control.host_unbinds_declared_forwards(), switch_address) { + // A VM-backed host: bind at publish. + (true, Some(switch_address)) => { + if let Err(e) = crate::net::policy::bind_declared_at_publish( + &control, + published, + switch_address, + ingress, + &yielded_ports, + &holder_of, + ) + .await + { + match crate::net::policy::port_in_use_refusal(&e) { + Some(held) => refusal = Some(held), + // Any other failed bind fails the + // finalize as it is (NET-121). + None => return Err(e), + } + } + } + // A VM-backed box the host handed no switch + // address has no forward to bind yet. Its + // first attach binds the forwards. + (true, None) => {} + (false, _) => { + let yielded: Vec = + yielded_ports.iter().map(|c| c.port).collect(); + if let Some(&port) = crate::net::policy::host_ports_in_use( + &control, published, ingress, &yielded, + ) + .first() + { + refusal = Some(minimald_rpc::PortInUseRefusal { + port, + box_port: box_port(port), + holder: holder_of(port), + }); + } + } + } + } + if let Some(refusal) = refusal { + warn_port_refusal(&record, published, &refusal); + return Err(crate::net::policy::address_in_use_error(refusal)); } } let mut ran: Vec = Vec::new(); if self.has_hooks_for(crate::hooks::HookEvent::Activate) { - self.launch_host_for_hooks(LaunchPhase::Activating).await?; + self.launch_host_for_hooks(LaunchPhase::Activating) + .await + .inspect_err(|e| { + // The launch's attach binds the declared forwards. + // A taken port fails it with the typed refusal, + // which this finalize logs like its own (NET-144). + #[cfg(target_os = "linux")] + if let Some(refusal) = crate::net::policy::port_in_use_refusal(e) { + let published = self + .hostnames + .read() + .expect("hostname registry lock poisoned") + .published_own_address(record.id); + warn_port_refusal(&record, published, &refusal); + } + })?; let outcomes = run_session_hooks( &self.inner, &self.record, diff --git a/crates/minimald/src/session/tests.rs b/crates/minimald/src/session/tests.rs index 3c489573ed..ec9edc0bff 100644 --- a/crates/minimald/src/session/tests.rs +++ b/crates/minimald/src/session/tests.rs @@ -3492,14 +3492,73 @@ async fn create_handed_own_ip_session( id } -/// NET-121, design §7.1: a declared TCP host port that another process -/// already holds at the box's published address fails the finalize. The -/// error names the port and the next step, and the box never reaches -/// `Active`. +/// The finalize of a box whose declared port is held fails with the typed +/// refusal (NET-144). The box never reached `Active`, and the destroy the +/// client's activate cleanup sends removes it, so no record is left behind. +#[cfg(target_os = "linux")] +async fn assert_refused_and_cleaned_up( + client: &mut TestClient, + id: SessionId, + expected: minimald_rpc::PortInUseRefusal, +) { + use minimald_rpc::{ + DestroySession, DestroySessionRequest, Errorable, FinalizeSession, FinalizeSessionRequest, + ListSessions, + }; + let reply = client + .call::(&FinalizeSessionRequest { + session_id: id, + report_shared_port_collisions: true, + }) + .await; + match reply { + Errorable::Err { error } => { + assert_eq!( + minimald_rpc::PortInUseRefusal::find_in(&error), + Some(expected.clone()), + "the refusal names the port and its holder, got: {error}" + ); + assert!( + error.contains(&expected.to_string()), + "the daemon's text is the refusal's own text, got: {error}" + ); + } + Errorable::Ok(ok) => panic!("finalize must fail on a held port, got {ok:?}"), + } + // The refused box never reached `Active`. + let listed = client.call::(&()).await; + assert!( + listed + .sessions + .iter() + .any(|entry| entry.id == id && entry.status != sessions::SessionStatus::Active), + "the refused box is listed and not active: {:?}", + listed.sessions + ); + // The client's activate cleanup destroys the refused box. + match client + .call::(&DestroySessionRequest { id }) + .await + { + Errorable::Ok(_) => {} + Errorable::Err { error } => panic!("the activate cleanup's destroy failed: {error}"), + } + let listed = client.call::(&()).await; + assert!( + listed.sessions.iter().all(|entry| entry.id != id), + "a refused activation leaves no record: {:?}", + listed.sessions + ); +} + +/// NET-121, NET-144, design §7.1: a declared TCP host port that a program +/// outside Minimal already holds at the box's published address fails the +/// finalize. The refusal names the port and the host as its holder, and +/// the box leaves no record. #[cfg(target_os = "linux")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn finalize_fails_on_a_taken_declared_host_port() { - use minimald_rpc::{Errorable, FinalizeSession, FinalizeSessionRequest}; +async fn activation_refused_when_declared_port_is_held_by_a_host_program() { + let capture = crate::test_harness::captured_log(); let handed = std::net::Ipv4Addr::new(127, 0, 64, 77); let switch = std::net::Ipv4Addr::new(100, 64, 128, 77); // Another process holds the declared host port at the handed address. @@ -3507,20 +3566,172 @@ async fn finalize_fails_on_a_taken_declared_host_port() { let server = TestServer::new().await; let mut client = server.connect().await; let id = create_handed_own_ip_session(&mut client, "taken", switch, handed).await; - let reply = client - .call::(&FinalizeSessionRequest { - session_id: id, - report_shared_port_collisions: true, + assert_refused_and_cleaned_up( + &mut client, + id, + minimald_rpc::PortInUseRefusal { + port: 18080, + box_port: 80, + holder: minimald_rpc::PortHolder::Host, + }, + ) + .await; + let logged = capture.contents(); + assert!( + logged.lines().any(|line| line.contains("host-port-in-use") + && line.contains(&format!("session={id}")) + && line.contains("port=18080") + && line.contains("127.0.64.77") + && line.contains("holder=host")), + "one warn line names the session, port, address and holder, got: {logged}" + ); +} + +/// NET-144: at the shared `127.0.0.1` fallback a declared port that another +/// box already holds fails the second box's activation, naming the box that +/// holds it, where before the second box yielded the port. The first box +/// keeps the port, and the second leaves no record. +#[cfg(target_os = "linux")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn activation_refused_when_declared_port_is_held_by_another_box_at_the_fallback() { + let capture = crate::test_harness::captured_log(); + let server = TestServer::new().await; + let manager = server.state.sessions_manager().await; + // No reserved range on this host: every box publishes at `127.0.0.1`. + manager.land_range_verdict(crate::net::dns::RangeVerdict::Absent); + let mut client = server.connect().await; + let first = create_handed_own_ip_session( + &mut client, + "first", + std::net::Ipv4Addr::new(100, 64, 128, 78), + std::net::Ipv4Addr::new(127, 0, 64, 78), + ) + .await; + finalize_session(&mut client, first).await; + let second = create_handed_own_ip_session( + &mut client, + "second", + std::net::Ipv4Addr::new(100, 64, 128, 79), + std::net::Ipv4Addr::new(127, 0, 64, 79), + ) + .await; + let registry = manager.hostnames(); + assert_eq!( + registry + .read() + .expect("registry lock") + .published_own_address(first), + Some(std::net::Ipv4Addr::LOCALHOST), + "the first box publishes at the fallback" + ); + assert_refused_and_cleaned_up( + &mut client, + second, + minimald_rpc::PortInUseRefusal { + port: 18080, + box_port: 80, + holder: minimald_rpc::PortHolder::Box("first".to_string()), + }, + ) + .await; + assert_eq!( + registry + .read() + .expect("registry lock") + .own_port_holder(std::net::Ipv4Addr::LOCALHOST, 18080) + .map(|(holder, _)| holder), + Some(first), + "the first box keeps the port" + ); + let logged = capture.contents(); + assert!( + logged.lines().any(|line| line.contains("host-port-in-use") + && line.contains(&format!("session={second}")) + && line.contains("holder=box:first")), + "the warn line names the holding box, got: {logged}" + ); +} + +/// NET-144, under a pending range verdict: a second box at the `127.0.0.1` +/// interim yields the declared port the first box holds there, because a +/// present landing moves both to their own addresses. When the verdict +/// lands absent, the second box stays at the fallback with the port unbound, +/// and the daemon log reports it with the port and the holder. The first +/// box, which holds the port, is not reported. +#[cfg(target_os = "linux")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn activation_refused_when_declared_port_is_held_reported_when_the_verdict_lands_absent() { + let capture = crate::test_harness::captured_log(); + let server = TestServer::new().await; + let manager = server.state.sessions_manager().await; + manager.hold_range_verdict_pending(); + // The bound is shrunk so both finalizes publish at the interim. + manager.reset_hand_verdict_deadline(100); + let mut client = server.connect().await; + let first = finalize_handed_own_ip_session( + &mut client, + "first", + std::net::Ipv4Addr::new(100, 64, 128, 81), + std::net::Ipv4Addr::new(127, 0, 64, 81), + ) + .await; + let second = create_handed_own_ip_session( + &mut client, + "second", + std::net::Ipv4Addr::new(100, 64, 128, 82), + std::net::Ipv4Addr::new(127, 0, 64, 82), + ) + .await; + let finalized = finalize_session(&mut client, second).await; + assert!( + finalized + .shared_port_collisions + .iter() + .any(|collision| collision.port == 18080 && collision.held_by.starts_with("first")), + "under the pending verdict the second box yields the port: {finalized:?}" + ); + let registry = manager.hostnames(); + for id in [first, second] { + assert_eq!( + registry + .read() + .expect("registry lock") + .published_own_address(id), + Some(std::net::Ipv4Addr::LOCALHOST), + "both boxes stand at the interim while the verdict walks" + ); + } + let reported = |logged: &str, id: SessionId| { + logged.lines().any(|line| { + line.contains("host-port-in-use") + && line.contains("127.0.0.1 fallback") + && line.contains(&format!("session={id}")) }) - .await; - match reply { - Errorable::Err { error } => assert!( - error.contains("host port 18080 is already in use") - && error.contains("--ingress :"), - "the error must name the port and the next step, got: {error}" - ), - Errorable::Ok(ok) => panic!("finalize must fail on a taken host port, got {ok:?}"), + }; + assert!( + !reported(&capture.contents(), second), + "nothing is reported while the verdict is pending" + ); + + manager.land_range_verdict(crate::net::dns::RangeVerdict::Absent); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !reported(&capture.contents(), second) && std::time::Instant::now() < deadline { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; } + let logged = capture.contents(); + assert!( + logged.lines().any(|line| line.contains("host-port-in-use") + && line.contains("127.0.0.1 fallback") + && line.contains(&format!("session={second}")) + && line.contains("port=18080") + && line.contains("address=127.0.0.1") + && line.contains("holder=box:first")), + "the absent landing reports the held port, the address and the holder, got: {logged}" + ); + assert!( + !reported(&logged, first), + "the box that holds the port is not reported: {logged}" + ); } /// Waits until a registration is parked on the range verdict, so the From c7087f0002ef4876e4fc0ff83457e02d89061d9c Mon Sep 17 00:00:00 2001 From: Norrie Taylor <91171431+norrietaylor@users.noreply.github.com> Date: Fri, 30 Oct 2026 08:53:05 -0700 Subject: [PATCH 06/34] fix(minimald): retry the finalize port probe so a parallel probe is not a holder (#2754) A finalize at the shared 127.0.0.1 fallback probes each declared port with a real bind. A parallel probe of the same port held it for the moment of its own bind, so the other finalize refused the port as held by another program. The probe now binds up to three times, 20 ms apart, and names a port held only when every bind fails with EADDRINUSE. It runs on the blocking pool, and its rule takes the bind as a closure so the test fails chosen binds in turn. Adapted to the line: the box_actor/setup.rs hunk lands in session.rs, with tokio::task::spawn_blocking for crate::traced::spawn_blocking. (cherry picked from commit 8cac647c78a550f0a40ea7b0f5caaf2c75f6bef3) --- crates/minimald/src/net/policy.rs | 95 +++++++++++++++++++++++++++++-- crates/minimald/src/session.rs | 19 +++++-- 2 files changed, 103 insertions(+), 11 deletions(-) diff --git a/crates/minimald/src/net/policy.rs b/crates/minimald/src/net/policy.rs index 313491bf73..3e6ab87d97 100644 --- a/crates/minimald/src/net/policy.rs +++ b/crates/minimald/src/net/policy.rs @@ -578,13 +578,44 @@ pub fn host_ports_in_use( .filter(|mapping| mapping.proto == sessions::IpProto::Tcp) .map(|mapping| mapping.external_port) .filter(|port| !yielded.contains(port)) - .filter(|&port| { - std::net::TcpListener::bind((published, port)) - .is_err_and(|e| e.kind() == io::ErrorKind::AddrInUse) - }) + .filter(|&port| port_held_at(published, port)) .collect() } +/// How many times [`port_held_at`] binds a port before it reports the port +/// held. +const PORT_HELD_BINDS: u32 = 3; +const _: () = assert!(PORT_HELD_BINDS > 1, "one failed bind is not enough"); + +/// The pause between two binds of [`port_held_at`]. +const PORT_HELD_RETRY: Duration = Duration::from_millis(20); + +/// Whether another process holds `port` at `address`: every one of +/// [`PORT_HELD_BINDS`] binds, [`PORT_HELD_RETRY`] apart, fails with +/// `EADDRINUSE`. A single failed bind is not enough, because a parallel +/// probe of the same port holds it for the moment of its own bind. A real +/// holder fails every bind. The first bind that succeeds, or fails for any +/// other reason, ends the probe at once, so a free port costs no pause. +fn port_held_at(address: Ipv4Addr, port: u16) -> bool { + port_held_by(|| std::net::TcpListener::bind((address, port)).map(drop)) +} + +/// [`port_held_at`]'s rule over any `bind`, so a test can fail chosen +/// binds in turn without racing a listener's release. +fn port_held_by(mut bind: impl FnMut() -> io::Result<()>) -> bool { + for attempt in 1..=PORT_HELD_BINDS { + match bind() { + Err(e) if e.kind() == io::ErrorKind::AddrInUse => { + if attempt < PORT_HELD_BINDS { + std::thread::sleep(PORT_HELD_RETRY); + } + } + _ => return false, + } + } + true +} + /// The error [`apply_ingress`] returns for a declared port that something /// else already holds on the host, and the error a finalize returns when it /// finds one before the box is active. A person reads it at the failed @@ -2894,14 +2925,16 @@ mod tests { // ---- a host port another process already holds (reported as EADDRINUSE) -- mod host_port_in_use { + use std::io; use std::net::Ipv4Addr; use tokio::io::AsyncWriteExt as _; use tokio::net::UnixListener; use super::super::{ - ControlChannel, IngressPolicy, PortMapping, apply_declared, apply_ingress, - host_ports_in_use, is_address_in_use, port_in_use_refusal, + ControlChannel, IngressPolicy, PORT_HELD_BINDS, PORT_HELD_RETRY, PortMapping, + apply_declared, apply_ingress, host_ports_in_use, is_address_in_use, port_held_by, + port_in_use_refusal, }; use super::forwarder_probe::read_request; @@ -3140,6 +3173,56 @@ mod tests { ); } + /// A port held only for a moment, as a parallel probe of the same + /// port holds it, is not reported: the probe binds again before it + /// names the port. A port held across every bind still is. + #[test] + fn a_port_held_only_for_a_moment_is_not_reported() { + let native = ControlChannel::Unix("/nonexistent/gvproxy.sock".into()); + let in_use = || Err(io::Error::from(io::ErrorKind::AddrInUse)); + for free_at in 2..=PORT_HELD_BINDS { + let mut binds = 0; + assert!( + !port_held_by(|| { + binds += 1; + if binds < free_at { in_use() } else { Ok(()) } + }), + "a port freed by bind {free_at} is not held by another program" + ); + assert_eq!( + binds, free_at, + "the probe stops at the first bind that succeeds" + ); + } + let mut binds = 0; + assert!( + port_held_by(|| { + binds += 1; + in_use() + }), + "a port that refuses every bind is held" + ); + assert_eq!( + binds, PORT_HELD_BINDS, + "the probe binds before it names a held port" + ); + + let held = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let port = held.local_addr().unwrap().port(); + let declared = ingress(&[port]); + let started = std::time::Instant::now(); + assert_eq!( + host_ports_in_use(&native, Ipv4Addr::LOCALHOST, &declared, &[]), + [port], + "a port held across every bind is still reported" + ); + assert!( + started.elapsed() >= PORT_HELD_RETRY * (PORT_HELD_BINDS - 1), + "the probe binds again before it names a held port" + ); + drop(held); + } + /// Only the OS's own words get the taken-port error: any other /// refusal of the bind — a permission denied, a missing address, a /// crashed forwarder — rolls back and fails with the forwarder's diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index a51030771e..8a49725d17 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -2883,11 +2883,20 @@ impl Session { (false, _) => { let yielded: Vec = yielded_ports.iter().map(|c| c.port).collect(); - if let Some(&port) = crate::net::policy::host_ports_in_use( - &control, published, ingress, &yielded, - ) - .first() - { + // The probe binds each port and pauses between + // binds of a held one, so on the blocking pool. + let in_use = { + let control = control.clone(); + let ingress = ingress.clone(); + tokio::task::spawn_blocking(move || { + crate::net::policy::host_ports_in_use( + &control, published, &ingress, &yielded, + ) + }) + .await + .map_err(std::io::Error::other)? + }; + if let Some(&port) = in_use.first() { refusal = Some(minimald_rpc::PortInUseRefusal { port, box_port: box_port(port), From 027dfe61c26cfc8da9c5dcfa8575c276152bf838 Mon Sep 17 00:00:00 2001 From: Norrie Taylor <91171431+norrietaylor@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:01:49 +0000 Subject: [PATCH 07/34] test(minimald): give each own-address test box its own host port (#2752) Every own-address test box declared host port 18080. Since T100 a finalize probes a declared port at the box's published address and refuses it when something else holds it. nextest runs tests in parallel processes on one host, so tests that publish at 127.0.0.1 in lockstep can catch each other's probe and refuse each other's box. Each test process now leases one port from 23000-23999, below the kernel's ephemeral range, with a per-port lock file in a per-user directory, and every box in the test declares it, so the collision tests still put two boxes on one port. Assertions read that port back. Adapted to the line: the change lands in session/tests.rs, and boxes:: reads sessions::. Dropped: the hunks for #2749's tests, which the line does not have at this point, and for main-only test text. #2749 carries its own tests with declared_port() when it lands below. (cherry picked from commit 0be7561f698a85bd7c12fb2394e5f013b1492b1f) --- crates/minimald/src/session/tests.rs | 135 +++++++++++++++++++-------- 1 file changed, 95 insertions(+), 40 deletions(-) diff --git a/crates/minimald/src/session/tests.rs b/crates/minimald/src/session/tests.rs index ec9edc0bff..72ffe1c76f 100644 --- a/crates/minimald/src/session/tests.rs +++ b/crates/minimald/src/session/tests.rs @@ -3332,6 +3332,58 @@ async fn own_ip_default_deny_all() { // (NET-010 to NET-013) // --------------------------------------------------------------------------- +/// The running test's [`declared_port`] and the lock file that holds its +/// lease. nextest runs each test in a process of its own, so one lease per +/// process is one port per test, whichever thread asks for it: the test's +/// own thread or a task it spawned onto a runtime worker. The lock is +/// released when the process exits. +static DECLARED_PORT: std::sync::OnceLock<(u16, std::fs::File)> = std::sync::OnceLock::new(); + +/// The ports [`declared_port`] leases from. The band sits below the kernel's +/// default ephemeral range (`net.ipv4.ip_local_port_range` starts at 32768), +/// so no other process's `bind(0)` or outbound `connect` is handed one +/// between the lease and the finalize's probe, and no fixed port in the tree +/// falls inside it. A host that widens the ephemeral range down into this +/// band loses that guarantee. +const DECLARED_PORTS: std::ops::Range = 23_000..24_000; + +/// Leases a port from [`DECLARED_PORTS`]: the first one whose lock file this +/// process can lock, among every test process the same user runs on the +/// host, and that nothing already listens on at `127.0.0.1`. The lock files +/// live in a directory of the user's own, so another user's run cannot leave +/// files this one cannot open; a file that still cannot be opened counts as +/// leased. The probe checks `127.0.0.1` only: a listener on `0.0.0.0` also +/// fails it, but one bound to a specific `127.0.64.x` address does not. The +/// port itself stays unbound, because the finalize under test must find it +/// free. +fn lease_declared_port() -> (u16, std::fs::File) { + let dir = std::env::temp_dir().join(format!("minimald-test-ports-{}", nix::unistd::getuid())); + std::fs::create_dir_all(&dir) + .unwrap_or_else(|e| panic!("create the port lease directory {dir:?}: {e}")); + for port in DECLARED_PORTS { + let Ok(lock) = std::fs::File::create(dir.join(format!("{port}.lock"))) else { + continue; + }; + if lock.try_lock().is_ok() + && std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, port)).is_ok() + { + return (port, lock); + } + } + panic!("every port in {DECLARED_PORTS:?} is leased, in use, or has no lease file in {dir:?}"); +} + +/// The host port every own-address box of the running test declares, in +/// place of one fixed number. A finalize refuses a declared port that +/// something else holds at the box's published address (NET-144), and +/// nextest runs tests side by side on one host: two tests declaring one +/// fixed port at `127.0.0.1` can refuse each other's box. So each test +/// leases its own port, and every box within the test shares it, which the +/// collision tests need. +fn declared_port() -> u16 { + DECLARED_PORT.get_or_init(lease_declared_port).0 +} + /// An own-address session's config: a box that declares one published port — /// the ingress a publish is made of (NET-010) — under the session `name`. fn own_ip_session_req(name: &str) -> minimald_rpc::CreateSessionRequest { @@ -3344,7 +3396,7 @@ fn own_ip_session_req(name: &str) -> minimald_rpc::CreateSessionRequest { egress: None, ingress: Some(sessions::IngressPolicy { port_mappings: vec![sessions::PortMapping { - external_port: 18080, + external_port: declared_port(), internal_port: 80, proto: sessions::IpProto::Tcp, }], @@ -3562,7 +3614,8 @@ async fn activation_refused_when_declared_port_is_held_by_a_host_program() { let handed = std::net::Ipv4Addr::new(127, 0, 64, 77); let switch = std::net::Ipv4Addr::new(100, 64, 128, 77); // Another process holds the declared host port at the handed address. - let _holder = std::net::TcpListener::bind((handed, 18080)).expect("hold the declared port"); + let _holder = + std::net::TcpListener::bind((handed, declared_port())).expect("hold the declared port"); let server = TestServer::new().await; let mut client = server.connect().await; let id = create_handed_own_ip_session(&mut client, "taken", switch, handed).await; @@ -3570,7 +3623,7 @@ async fn activation_refused_when_declared_port_is_held_by_a_host_program() { &mut client, id, minimald_rpc::PortInUseRefusal { - port: 18080, + port: declared_port(), box_port: 80, holder: minimald_rpc::PortHolder::Host, }, @@ -3580,7 +3633,7 @@ async fn activation_refused_when_declared_port_is_held_by_a_host_program() { assert!( logged.lines().any(|line| line.contains("host-port-in-use") && line.contains(&format!("session={id}")) - && line.contains("port=18080") + && line.contains(&format!("port={}", declared_port())) && line.contains("127.0.64.77") && line.contains("holder=host")), "one warn line names the session, port, address and holder, got: {logged}" @@ -3628,7 +3681,7 @@ async fn activation_refused_when_declared_port_is_held_by_another_box_at_the_fal &mut client, second, minimald_rpc::PortInUseRefusal { - port: 18080, + port: declared_port(), box_port: 80, holder: minimald_rpc::PortHolder::Box("first".to_string()), }, @@ -3638,7 +3691,7 @@ async fn activation_refused_when_declared_port_is_held_by_another_box_at_the_fal registry .read() .expect("registry lock") - .own_port_holder(std::net::Ipv4Addr::LOCALHOST, 18080) + .own_port_holder(std::net::Ipv4Addr::LOCALHOST, declared_port()) .map(|(holder, _)| holder), Some(first), "the first box keeps the port" @@ -3687,7 +3740,8 @@ async fn activation_refused_when_declared_port_is_held_reported_when_the_verdict finalized .shared_port_collisions .iter() - .any(|collision| collision.port == 18080 && collision.held_by.starts_with("first")), + .any(|collision| collision.port == declared_port() + && collision.held_by.starts_with("first")), "under the pending verdict the second box yields the port: {finalized:?}" ); let registry = manager.hostnames(); @@ -3723,7 +3777,7 @@ async fn activation_refused_when_declared_port_is_held_reported_when_the_verdict logged.lines().any(|line| line.contains("host-port-in-use") && line.contains("127.0.0.1 fallback") && line.contains(&format!("session={second}")) - && line.contains("port=18080") + && line.contains(&format!("port={}", declared_port())) && line.contains("address=127.0.0.1") && line.contains("holder=box:first")), "the absent landing reports the held port, the address and the holder, got: {logged}" @@ -3960,10 +4014,10 @@ async fn each_box_gets_own_loopback_address() { .resolve(&format!("{name}.min.internal")) .unwrap_or_else(|| panic!("{name}'s name routes")); assert_eq!( - route.upstream(18080), + route.upstream(declared_port()), Some(std::net::SocketAddr::new( std::net::IpAddr::V4(address), - 18080 + declared_port() )), "{name}'s own port number is published at its own address, not translated" ); @@ -4060,13 +4114,13 @@ async fn name_answers_without_attached_client() { let route = registry .read() .expect("registry lock") - .resolve("web.min.internal:18080") + .resolve(&format!("web.min.internal:{}", declared_port())) .expect("the name routes with no client attached"); assert_eq!( - route.upstream(18080), + route.upstream(declared_port()), Some(std::net::SocketAddr::new( std::net::IpAddr::V4(address), - 18080 + declared_port() )), "a request for the box's declared port is forwarded to the box's own address" ); @@ -5645,12 +5699,12 @@ async fn a_resumed_handed_box_publishes_the_interim_without_waiting_and_takes_it registry .read() .expect("registry lock") - .resolve("resumed-hand.min.internal:18080") + .resolve(&format!("resumed-hand.min.internal:{}", declared_port())) .expect("the name routes once the verdict lands") - .upstream(18080), + .upstream(declared_port()), Some(std::net::SocketAddr::new( std::net::IpAddr::V4(handed), - 18080 + declared_port() )), "the box's own port number is published at its own address, not translated" ); @@ -6544,12 +6598,12 @@ async fn an_interim_handed_publish_takes_its_hand_when_the_verdict_lands_late() registry .read() .expect("registry lock") - .resolve("web.min.internal:18080") + .resolve(&format!("web.min.internal:{}", declared_port())) .expect("the name routes at the hand") - .upstream(18080), + .upstream(declared_port()), Some(std::net::SocketAddr::new( std::net::IpAddr::V4(handed), - 18080 + declared_port() )), "the box's declared port is published at its own hand, exactly where \ the attach path binds its forwards" @@ -6787,23 +6841,23 @@ async fn shared_address_port_collision_reported_not_translated() { ); assert_eq!( routes - .resolve("first.min.internal:18080") + .resolve(&format!("first.min.internal:{}", declared_port())) .expect("first's name routes") - .upstream(18080), + .upstream(declared_port()), Some(std::net::SocketAddr::new( std::net::IpAddr::V4(first_address), - 18080, + declared_port(), )), "the first box's port is published at the number it asked for" ); assert_eq!( routes - .resolve("second.min.internal:18080") + .resolve(&format!("second.min.internal:{}", declared_port())) .expect("second's name routes") - .upstream(18080), + .upstream(declared_port()), Some(std::net::SocketAddr::new( std::net::IpAddr::V4(second_address), - 18080, + declared_port(), )), "the second box's port is published at the number it asked for" ); @@ -6816,7 +6870,7 @@ async fn shared_address_port_collision_reported_not_translated() { assert_eq!( routes.shared_port_collisions(second), vec![crate::net::dns::SharedPortCollision { - port: 18080, + port: declared_port(), other: "first.min.internal".to_string(), }], "the second box records the port it yields and the box that holds it" @@ -6835,7 +6889,7 @@ async fn shared_address_port_collision_reported_not_translated() { assert_eq!( second_reply.shared_port_collisions, vec![minimald_rpc::SharedPortCollision { - port: 18080, + port: declared_port(), held_by: "first.min.internal".to_string(), }], "the finalize reply names the yielded port and the box that holds it" @@ -6851,7 +6905,7 @@ async fn shared_address_port_collision_reported_not_translated() { .lines() .filter(|line| { line.contains("action=\"shared-address-port-collision\"") - && line.contains("port=18080") + && line.contains(&format!("port={}", declared_port())) && (line.contains(&format!("session_id={first}")) || line.contains(&format!("session_id={second}"))) }) @@ -6903,7 +6957,7 @@ async fn shared_address_port_collision_survives_a_daemon_restart() { log.lines() .filter(|line| { line.contains("action=\"shared-address-port-collision\"") - && line.contains("port=18080") + && line.contains(&format!("port={}", declared_port())) && line.contains(&format!("session_id={second}")) }) .count() @@ -6938,7 +6992,7 @@ async fn shared_address_port_collision_survives_a_daemon_restart() { assert_eq!( routes.shared_port_collisions(second), vec![crate::net::dns::SharedPortCollision { - port: 18080, + port: declared_port(), other: "holds.min.internal".to_string(), }], "after the restart the second box still yields the port to the first" @@ -7005,7 +7059,7 @@ async fn shared_address_holder_same_in_any_resume_order() { let newer = newer.expect("some newer box's store key sorts before an older box's"); let (older, older_name) = older; let held_by_older = vec![crate::net::dns::SharedPortCollision { - port: 18080, + port: declared_port(), other: format!("{older_name}.min.internal"), }]; { @@ -7069,7 +7123,7 @@ async fn shared_address_port_stays_with_the_box_that_bound_it_across_a_restart() await_echo(&mut channel).await; finalize_session(&mut client, older).await; let held_by_newer = vec![crate::net::dns::SharedPortCollision { - port: 18080, + port: declared_port(), other: "newer.min.internal".to_string(), }]; { @@ -7097,7 +7151,7 @@ async fn shared_address_port_stays_with_the_box_that_bound_it_across_a_restart() .expect("the record reads back"); assert_eq!( record.bound_ports, - std::collections::BTreeSet::from([std::net::SocketAddrV4::new(shared, 18080)]), + std::collections::BTreeSet::from([std::net::SocketAddrV4::new(shared, declared_port())]), "the launch recorded the port it bound on the box's record" ); let registry = manager.hostnames(); @@ -7113,7 +7167,7 @@ async fn shared_address_port_stays_with_the_box_that_bound_it_across_a_restart() ); } -/// Two boxes handed one loopback address, both declaring port 18080: the +/// Two boxes handed one loopback address, both declaring the test's port: the /// first to finalize holds it and the second yields. Returns the manager, /// the two ids, and the shared address the registry publishes them at. #[cfg(target_os = "linux")] @@ -7214,7 +7268,7 @@ async fn a_destroyed_holders_shared_port_is_bound_for_the_running_box_that_takes ); let declared = crate::net::policy::expose_request( &sessions::PortMapping { - external_port: 18080, + external_port: declared_port(), internal_port: 80, proto: sessions::IpProto::Tcp, }, @@ -7234,7 +7288,8 @@ async fn a_destroyed_holders_shared_port_is_bound_for_the_running_box_that_takes "the box holds the port of record" ); } - let bound = std::collections::BTreeSet::from([std::net::SocketAddrV4::new(published, 18080)]); + let bound = + std::collections::BTreeSet::from([std::net::SocketAddrV4::new(published, declared_port())]); let mut recorded = false; for _ in 0..1_000 { if handle @@ -7267,7 +7322,7 @@ async fn a_destroyed_holders_shared_port_is_bound_for_the_running_box_that_takes "the stop unbinds the handed-over forward: {requests:?}" ); assert!( - body.contains(&format!("\"local\":\"{published}:18080\"")), + body.contains(&format!("\"local\":\"{published}:{}\"", declared_port())), "the unexpose names the handed-over listener: {body}" ); assert_eq!(requests.len(), 2, "nothing else was asked: {requests:?}"); @@ -7570,12 +7625,12 @@ async fn register_hostname_promotes_an_interim_to_its_vouched_hand() { registry .read() .expect("registry lock") - .resolve("web.min.internal:18080") + .resolve(&format!("web.min.internal:{}", declared_port())) .expect("the name routes at the hand") - .upstream(18080), + .upstream(declared_port()), Some(std::net::SocketAddr::new( std::net::IpAddr::V4(handed), - 18080 + declared_port() )), "the box's declared port is published at its own hand, exactly where \ the attach path binds its forwards" From 7981f0e2ce2629760b170d14ef2627fe472dd97d Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:56:03 +0000 Subject: [PATCH 08/34] fix(net): hand boxes the loopback interim when the host cannot bind the box range (#2332) Prerequisite for T98. Allocation drew an address out of the reserved 127.0.64.0/24 without measuring whether the host can bind it. Linux routes all of 127.0.0.0/8, but macOS answers only the addresses lo0 carries, so on a host without the range aliases the zone answered an address nothing listens at. The address book now hands a fresh box address only when switch::loopback::probe reads the whole range present, and hands the 127.0.0.1 interim otherwise. A probe refused for any reason but an absent address refuses the allocation. A box keeps its address across the probe: an interim box resumes at the interim, and a resume sends the address it was created at, so a restarted answerer hands it back. Backport of #2332, folded from #2740. Adapted to the line: the answerer imports sessions::core::zone_answer. Dropped: the edit to main's dormant-creation tests, since the line has neither those tests nor dormant rows. (cherry picked from commit 379c2f04ce0def4f6329752f2f68533b9be4884e) --- crates/minvmd/src/control.rs | 224 +++++++++++++- crates/minvmd/src/net/answerer.rs | 471 ++++++++++++++++++++++++++---- 2 files changed, 637 insertions(+), 58 deletions(-) diff --git a/crates/minvmd/src/control.rs b/crates/minvmd/src/control.rs index b2e3bbd6eb..b826df8e36 100644 --- a/crates/minvmd/src/control.rs +++ b/crates/minvmd/src/control.rs @@ -1111,9 +1111,23 @@ fn resume_reply( Err(error) => return refused(error), }; let claim = boxes.begin_registration(&name); - let allocated = match allocate_box_address(answerer, &name) { - Ok(allocated) => allocated, - Err(reply) => return reply, + // A box created at the 127.0.0.1 interim (the host could not bind the + // box range then) keeps it: the answerer records no address for it, so + // asking it now would hand a range address once the host binds the + // range, and the resume would refuse it as moved. `resumable` checked + // the presented address against the creation's. + let interim = request.loopback_address == std::net::Ipv4Addr::LOCALHOST; + // A box created at a range address asks to keep it. A restarted + // answerer holds no record of a dormant box, and where the host cannot + // bind the range now it would hand a fresh ask the 127.0.0.1 interim, + // which the resume refuses as moved. + let allocated = if interim { + std::net::Ipv4Addr::LOCALHOST + } else { + match allocate_box_address_keeping(answerer, &name, Some(request.loopback_address)) { + Ok(allocated) => allocated, + Err(reply) => return reply, + } }; order.apply(move || { match boxes.resume_client_box( @@ -1131,9 +1145,10 @@ fn resume_reply( task_addresses: record.task_addrs().to_vec(), }), Err(error) => { - let released = boxes.release_unless_owned(&claim, || { - answerer.release_address(&name); - }); + let released = !interim + && boxes.release_unless_owned(&claim, || { + answerer.release_address(&name); + }); tracing::info!( address_released = released, box = %name, @@ -1575,7 +1590,19 @@ fn allocate_box_address( answerer: &AnswererStatus, name: &str, ) -> Result { - answerer.allocate(name).map_err(|reason| { + // None: a fresh registration has no earlier address to keep. Only a + // resume knows the address its box was created at. + allocate_box_address_keeping(answerer, name, None) +} + +/// [`allocate_box_address`] for a resumed box created at `keep`, which the +/// answerer hands back while no other box holds it. +fn allocate_box_address_keeping( + answerer: &AnswererStatus, + name: &str, + keep: Option, +) -> Result { + answerer.allocate_keeping(name, keep).map_err(|reason| { tracing::warn!( box = %name, %reason, @@ -5131,6 +5158,189 @@ mod tests { ); } + /// A box created while the host lacked the box range was handed the + /// 127.0.0.1 interim, and keeps it: once the aliases are installed and + /// its row is withdrawn, its creator's resume reinstates it at + /// 127.0.0.1 rather than refusing it as moved, while a new box gets an + /// address out of the range. + #[test] + fn an_interim_box_keeps_the_interim_across_a_resume() { + use std::sync::atomic::{AtomicBool, Ordering}; + let dir = tempfile::TempDir::new().expect("temp dir"); + let sock_path = dir.path().join(CONTROL_SOCK_FILE); + let boxes = BoxRegistry::new(SUBNET); + let aliased = Arc::new(AtomicBool::new(false)); + let probed = Arc::clone(&aliased); + let answerer = AnswererStatus::allocating_for_tests_probed( + "control-test-node", + || None, + |_, _| {}, + move || { + let present = probed.load(Ordering::SeqCst); + switch::loopback::probe_over(switch::loopback::range_hosts(), |_| { + if present { + Ok(()) + } else { + Err(std::io::ErrorKind::AddrNotAvailable.into()) + } + }) + }, + ); + let _server = spawn( + sock_path.clone(), + boxes.clone(), + answerer, + ProxyPublishStatus::new(), + ) + .expect("server binds"); + for _ in 0..500 { + if TestStream::connect(&sock_path).is_ok() { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + let request = |name: &str| RegisterBoxRequest { + name: name.to_string(), + ingress_ports: Vec::new(), + egress: None, + credentialed_upstream: None, + dynamic_ingress: None, + dynamic_allowed_range: None, + hold: false, + task_slots: 0, + }; + let web = handed(register(&sock_path, &request("web")).expect("registered")); + assert_eq!( + web.loopback_address, + Ipv4Addr::LOCALHOST, + "a host without the range hands the interim" + ); + + aliased.store(true, Ordering::SeqCst); + assert!( + boxes.withdraw(web.switch_address).is_some(), + "the row is withdrawn, its creation dormant" + ); + let resumed = handed( + control( + &sock_path, + &BoxControlRequest::ResumeBox(minimald_rpc::ResumeBoxRequest { + name: "web".to_string(), + switch_address: web.switch_address, + loopback_address: web.loopback_address, + box_id: Some(web.box_id), + }), + ) + .expect("the resume is answered"), + ); + assert_eq!( + (resumed.box_id, resumed.loopback_address), + (web.box_id, Ipv4Addr::LOCALHOST), + "the interim box resumes at its own 127.0.0.1" + ); + + let api = handed(register(&sock_path, &request("api")).expect("registered")); + assert!( + api.loopback_address != Ipv4Addr::LOCALHOST, + "a new box gets a range address once the host binds the range: {}", + api.loopback_address + ); + } + + /// A box created at a range address keeps it across a resume after the + /// answerer restarts while the host lacks the range. The restarted + /// answerer holds no record of the dormant box. It hands back the + /// address the box was created at, not the 127.0.0.1 interim, so the + /// resume reinstates the box rather than refusing it as moved. + #[test] + fn a_range_box_keeps_its_address_across_an_answerer_restart() { + let dir = tempfile::TempDir::new().expect("temp dir"); + let sock_path = dir.path().join(CONTROL_SOCK_FILE); + let boxes = BoxRegistry::new(SUBNET); + let wait_for = |path: &std::path::Path| { + for _ in 0..500 { + if TestStream::connect(path).is_ok() { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + }; + let _server = spawn( + sock_path.clone(), + boxes.clone(), + AnswererStatus::allocating_for_tests("control-test-node"), + ProxyPublishStatus::new(), + ) + .expect("server binds"); + wait_for(&sock_path); + let request = |name: &str| RegisterBoxRequest { + name: name.to_string(), + ingress_ports: Vec::new(), + egress: None, + credentialed_upstream: None, + dynamic_ingress: None, + dynamic_allowed_range: None, + hold: false, + task_slots: 0, + }; + let web = handed(register(&sock_path, &request("web")).expect("registered")); + assert!( + web.loopback_address != Ipv4Addr::LOCALHOST, + "a host with the range hands a range address: {}", + web.loopback_address + ); + assert!( + boxes.withdraw(web.switch_address).is_some(), + "the row is withdrawn, its creation dormant" + ); + + // The restarted answerer: an empty book, on a host that lacks the + // range aliases now. + let restarted_dir = tempfile::TempDir::new().expect("temp dir"); + let restarted_path = restarted_dir.path().join(CONTROL_SOCK_FILE); + let _restarted = spawn( + restarted_path.clone(), + boxes.clone(), + AnswererStatus::allocating_for_tests_probed( + "control-test-node", + || None, + |_, _| {}, + || { + switch::loopback::probe_over(switch::loopback::range_hosts(), |_| { + Err(std::io::ErrorKind::AddrNotAvailable.into()) + }) + }, + ), + ProxyPublishStatus::new(), + ) + .expect("server binds"); + wait_for(&restarted_path); + let resumed = handed( + control( + &restarted_path, + &BoxControlRequest::ResumeBox(minimald_rpc::ResumeBoxRequest { + name: "web".to_string(), + switch_address: web.switch_address, + loopback_address: web.loopback_address, + box_id: Some(web.box_id), + }), + ) + .expect("the resume is answered"), + ); + assert_eq!( + (resumed.box_id, resumed.loopback_address), + (web.box_id, web.loopback_address), + "the range box resumes at its own address" + ); + + let api = handed(register(&restarted_path, &request("api")).expect("registered")); + assert_eq!( + api.loopback_address, + Ipv4Addr::LOCALHOST, + "a new box on a host without the range still gets the interim" + ); + } + /// The doors are the verb's access control (NET-138): the read-only row /// verb is refused on the in-VM daemon's channel and an admit report is /// refused on the host's socket — each refusal naming the door that diff --git a/crates/minvmd/src/net/answerer.rs b/crates/minvmd/src/net/answerer.rs index 0c7f30a896..683a139979 100644 --- a/crates/minvmd/src/net/answerer.rs +++ b/crates/minvmd/src/net/answerer.rs @@ -98,6 +98,7 @@ use minimald_rpc::ZoneAnswererStatus; use serde::{Deserialize, Serialize}; use sessions::core::zone_answer::{self, ZoneRow, ZoneView}; +use switch::loopback::RangeProbe; use crate::box_registry::{BoxRegistry, is_host_answerable}; @@ -430,6 +431,12 @@ struct PublishRequest { struct AllocateRequest { /// The box's name, as the node's registry holds it. allocate: String, + /// The address the box was created at, when a resume asks: the book + /// hands it back while no other box holds it, without reading the + /// range, so a resume never moves the box ([`AddressBook::allocate_keeping_at`]). + /// An answerer that predates the field ignores it. + #[serde(default, skip_serializing_if = "Option::is_none")] + keep: Option, } /// A node's release of a box's published address: the box is gone, so the @@ -582,12 +589,29 @@ struct AddressHolder { /// answer TTL the zone serves: until it passes, a host resolver may still /// hold the old name's answer, and a client connecting by it must not reach /// a different box. Only the box that held it may take it back early. -#[derive(Debug, Default)] +#[derive(Debug)] struct AddressBook { held: BTreeMap, /// Released addresses still inside their quarantine: who held each and /// when it was released. released: BTreeMap, + /// The probe [`Self::allocate`] runs over the box range before it hands + /// a fresh address: [`switch::loopback::probe`] in production. + range_probe: fn() -> RangeProbe, + /// Whether the last probe read the box range absent: the interim + /// episode its warning is logged once for. + range_absent: bool, +} + +impl Default for AddressBook { + fn default() -> Self { + Self { + held: BTreeMap::new(), + released: BTreeMap::new(), + range_probe: HOST_RANGE_PROBE, + range_absent: false, + } + } } /// How long a released box address stays out of allocation: the positive @@ -601,19 +625,75 @@ fn in_box_range(address: Ipv4Addr) -> bool { first <= address && address <= last } +/// The range probe a default [`AddressBook`] runs. Unit tests run on hosts +/// that may lack the range aliases (macOS without the lo0 aliases), so their +/// books read the range present; the tests of the interim inject their own. +#[cfg(not(test))] +const HOST_RANGE_PROBE: fn() -> RangeProbe = switch::loopback::probe; +#[cfg(test)] +const HOST_RANGE_PROBE: fn() -> RangeProbe = || RangeProbe { + bound: 1, + probed: 1, + first_failure: None, +}; + impl AddressBook { - /// [`Self::allocate_at`], now. + /// [`Self::allocate_keeping`] with no address to keep. + #[cfg(test)] fn allocate(&mut self, node: &str, name: &str) -> Result { - self.allocate_at(node, name, Instant::now()) + self.allocate_keeping(node, name, None) + } + + /// [`Self::allocate_keeping_at`], now, with the book's own range probe. + fn allocate_keeping( + &mut self, + node: &str, + name: &str, + keep: Option, + ) -> Result { + let probe = self.range_probe; + self.allocate_keeping_at(node, name, keep, Instant::now(), probe) + } + + /// [`Self::allocate_keeping_at`] with no address to keep. + #[cfg(test)] + fn allocate_at( + &mut self, + node: &str, + name: &str, + now: Instant, + range_probe: impl FnOnce() -> RangeProbe, + ) -> Result { + self.allocate_keeping_at(node, name, None, now, range_probe) } /// The address `name` of `node` holds, or the lowest free one whose - /// quarantine has passed by `now`, recorded against them. Idempotent - /// per box; a box re-registered inside its old address's quarantine - /// gets that address back. When every free address is still - /// quarantined, the allocation fails naming when the first one frees — - /// it never hands an address out early. - fn allocate_at(&mut self, node: &str, name: &str, now: Instant) -> Result { + /// quarantine has passed by `now`, recorded against them. Idempotent per box; a box re-registered + /// inside its old address's quarantine gets that address back. When + /// every free address is still quarantined, the allocation fails + /// naming when the first one frees — it never hands an address out + /// early. A fresh address is handed out only when `range_probe` reads + /// the whole box range present (NET-123): a range the host carries + /// none or only part of hands the caller the 127.0.0.1 interim + /// instead, recorded nowhere, so a name it publishes always answers + /// where something listens. A box's own address, held or quarantined, + /// comes back without the probe, so its address never moves under it. + /// A probe refused for any reason other than an absent address refuses + /// the allocation. + /// + /// `keep` is the address a resumed box was created at. The book can + /// hold no record of it, as after an answerer restart. It comes back + /// without the probe too, when it is in the box range and no other box + /// holds it or released it inside its quarantine. A taken one is not + /// kept, and the allocation goes on as for a fresh box. + fn allocate_keeping_at( + &mut self, + node: &str, + name: &str, + keep: Option, + now: Instant, + range_probe: impl FnOnce() -> RangeProbe, + ) -> Result { if let Some((address, _)) = self .held .iter() @@ -631,7 +711,14 @@ impl AddressBook { .released .iter() .find(|(_, (was, _))| *was == holder) - .map(|(address, _)| *address); + .map(|(address, _)| *address) + .or_else(|| { + keep.filter(|address| { + in_box_range(*address) + && !self.held.contains_key(address) + && !self.released.contains_key(address) + }) + }); let (first, last) = switch::box_loopback_interior(); let free = own.or_else(|| { (u32::from(first)..=u32::from(last)) @@ -668,6 +755,38 @@ impl AddressBook { ), }); }; + if own.is_none() { + let probe = range_probe(); + if !probe.present() { + if let Some((address, kind)) = probe.first_failure + && kind != io::ErrorKind::AddrNotAvailable + { + return Err(format!( + "this host could not check that it can listen on the box range: \ + {address} refused the bind ({kind:?})" + )); + } + if !self.range_absent { + tracing::warn!( + component = COMPONENT, + probe = %probe.summary(), + "this host cannot bind the whole box address range; boxes get the \ + 127.0.0.1 interim until the range aliases are installed" + ); + self.range_absent = true; + } + return Ok(Ipv4Addr::LOCALHOST); + } + if self.range_absent { + tracing::info!( + component = COMPONENT, + probe = %probe.summary(), + "this host binds the whole box address range again; new boxes get \ + addresses out of it" + ); + self.range_absent = false; + } + } self.released.remove(&free); self.held.insert(free, holder); Ok(free) @@ -970,8 +1089,8 @@ impl RegisteredTables { } /// Hands `name` of `node` a box address (see [`AddressBook::allocate`]). - fn allocate(&self, node: &str, name: &str) -> Result { - self.book().allocate(node, name) + fn allocate(&self, node: &str, name: &str, keep: Option) -> Result { + self.book().allocate_keeping(node, name, keep) } /// Releases the box address `name` of `node` holds. @@ -2035,28 +2154,29 @@ fn accept_publish( let publish = match parse_node_line(line) { Ok(NodeLine::Publish(publish)) => Ok(publish), Ok(NodeLine::Allocate(request)) => { - let reply = match registered.allocate(node, &box_zone_name(&request.allocate)) { - Ok(address) => { - tracing::info!( - component = COMPONENT, - node = %node, - name = %request.allocate, - %address, - "handed a box its published address" - ); - RegistrationReply::allocated(address) - } - Err(reason) => { - tracing::warn!( - component = COMPONENT, - node = %node, - name = %request.allocate, - %reason, - "could not hand a box a published address" - ); - RegistrationReply::refused(reason) - } - }; + let reply = + match registered.allocate(node, &box_zone_name(&request.allocate), request.keep) { + Ok(address) => { + tracing::info!( + component = COMPONENT, + node = %node, + name = %request.allocate, + %address, + "handed a box its published address" + ); + RegistrationReply::allocated(address) + } + Err(reason) => { + tracing::warn!( + component = COMPONENT, + node = %node, + name = %request.allocate, + %reason, + "could not hand a box a published address" + ); + RegistrationReply::refused(reason) + } + }; return reply_or_end(stream, connection, registered, &reply); } Ok(NodeLine::Release(request)) => { @@ -2201,9 +2321,14 @@ impl Registration { /// Asks the answerer for box `name`'s published address over the held /// connection. The outer error is the connection's (the caller /// reconnects); the inner one is the answerer's refusal. - fn allocate(&mut self, name: &str) -> io::Result> { + fn allocate( + &mut self, + name: &str, + keep: Option, + ) -> io::Result> { let reply = self.request(&AllocateRequest { allocate: name.to_string(), + keep, })?; Ok(if reply.ok { reply @@ -2438,6 +2563,8 @@ enum HandoverCommand { Allocate { /// The box's registry name. name: String, + /// The address a resumed box was created at, kept while it is free. + keep: Option, /// Where the address (or the refusal) goes. reply: std::sync::mpsc::Sender>, }, @@ -2698,6 +2825,17 @@ impl AnswererStatus { /// The reason no address was handed out: no answerer reachable, the /// range exhausted, or no answer in time. pub fn allocate(&self, name: &str) -> Result { + self.allocate_keeping(name, None) + } + + /// [`Self::allocate`] for a resumed box created at `keep`: the answerer + /// hands `keep` back while no other box holds it, even where the host + /// cannot bind the box range now, so a resume never moves the box. + /// + /// # Errors + /// + /// As [`Self::allocate`]. + pub fn allocate_keeping(&self, name: &str, keep: Option) -> Result { let Some(sender) = self.sender() else { return Err(NO_ANSWERER.to_string()); }; @@ -2705,6 +2843,7 @@ impl AnswererStatus { if sender .send(HandoverCommand::Allocate { name: name.to_string(), + keep, reply: reply_to, }) .is_err() @@ -2758,9 +2897,27 @@ impl AnswererStatus { #[cfg(any(test, feature = "test-support"))] #[must_use] pub fn allocating_for_tests_with( + node: &str, + hold_reply: impl FnMut() -> Option> + Send + 'static, + on_release: impl FnMut(&str, Option) + Send + 'static, + ) -> Self { + Self::allocating_for_tests_probed(node, hold_reply, on_release, || RangeProbe { + bound: 1, + probed: 1, + first_failure: None, + }) + } + + /// [`Self::allocating_for_tests_with`] whose book reads the box range + /// with `range_probe` before each fresh address, so a test can stand in + /// for a host that lacks the range aliases, or gains them mid-test. + #[cfg(any(test, feature = "test-support"))] + #[must_use] + pub fn allocating_for_tests_probed( node: &str, mut hold_reply: impl FnMut() -> Option> + Send + 'static, mut on_release: impl FnMut(&str, Option) + Send + 'static, + mut range_probe: impl FnMut() -> RangeProbe + Send + 'static, ) -> Self { let status = Self::starting(); let commands = status.attach_commands(); @@ -2769,8 +2926,14 @@ impl AnswererStatus { let mut book = AddressBook::default(); while let Ok(command) = commands.recv() { match command { - HandoverCommand::Allocate { name, reply } => { - let answer = book.allocate(&node, &box_zone_name(&name)); + HandoverCommand::Allocate { name, keep, reply } => { + let answer = book.allocate_keeping_at( + &node, + &box_zone_name(&name), + keep, + Instant::now(), + &mut range_probe, + ); match hold_reply() { Some(gate) => { std::thread::spawn(move || { @@ -2993,7 +3156,8 @@ fn wait_out_wake( /// connection survived. fn serve_over_channel(registration: &mut Registration, command: HandoverCommand) -> bool { match command { - HandoverCommand::Allocate { name, reply } => match registration.allocate(&name) { + HandoverCommand::Allocate { name, keep, reply } => match registration.allocate(&name, keep) + { Ok(result) => { let _ = reply.send(result); true @@ -3417,8 +3581,8 @@ fn host_interim( } // The interim is the answerer: its own boxes are handed // addresses from the same book its channel's nodes are. - Ok(HandoverCommand::Allocate { name, reply }) => { - let _ = reply.send(registered.allocate(node, &box_zone_name(&name))); + Ok(HandoverCommand::Allocate { name, keep, reply }) => { + let _ = reply.send(registered.allocate(node, &box_zone_name(&name), keep)); } Ok(HandoverCommand::ReleaseAddress { name }) => { registered.release_address(node, &box_zone_name(&name)); @@ -5781,8 +5945,12 @@ mod tests { ); let t0 = Instant::now(); let mut book = AddressBook::default(); - let a = book.allocate_at("node-a", "a", t0).expect(".2"); - let b = book.allocate_at("node-a", "b", t0).expect(".3"); + let a = book + .allocate_at("node-a", "a", t0, range_present) + .expect(".2"); + let b = book + .allocate_at("node-a", "b", t0, range_present) + .expect(".3"); assert_eq!( (a, b), (Ipv4Addr::new(127, 0, 64, 2), Ipv4Addr::new(127, 0, 64, 3)) @@ -5792,7 +5960,7 @@ mod tests { assert_eq!(book.release_at("node-a", "a", t0), Some(a)); let inside = (t0 + ttl).checked_sub(Duration::from_millis(1)).unwrap(); let c = book - .allocate_at("node-b", "c", inside) + .allocate_at("node-b", "c", inside, range_present) .expect("a fresh address"); assert_eq!( c, @@ -5812,12 +5980,12 @@ mod tests { ); let after_a = t0 + ttl; assert_eq!( - book.allocate_at("node-b", "d", after_a), + book.allocate_at("node-b", "d", after_a, range_present), Ok(a), "the lowest address whose quarantine has passed" ); let e = book - .allocate_at("node-b", "e", after_a) + .allocate_at("node-b", "e", after_a, range_present) .expect("the next fresh one"); assert_eq!( e, @@ -5828,7 +5996,7 @@ mod tests { // The box that held a quarantined address may take it back. assert_eq!(book.release_at("node-a", "b", after_a), Some(b)); assert_eq!( - book.allocate_at("node-a", "b", after_a), + book.allocate_at("node-a", "b", after_a, range_present), Ok(b), "a box re-registered gets its own address back" ); @@ -5838,24 +6006,225 @@ mod tests { let mut full = AddressBook::default(); let (first, last) = switch::box_loopback_interior(); for (index, _) in (u32::from(first)..=u32::from(last)).enumerate() { - full.allocate_at("node-a", &format!("box{index}"), t0) + full.allocate_at("node-a", &format!("box{index}"), t0, range_present) .expect("the range holds it"); } let freed = full.release_at("node-a", "box0", t0).expect("box0 held .2"); let refused = full - .allocate_at("node-b", "late", t0 + Duration::from_secs(10)) + .allocate_at( + "node-b", + "late", + t0 + Duration::from_secs(10), + range_present, + ) .expect_err("nothing is free outside the quarantine"); assert!( refused.contains("released in the last 15 s") && refused.contains("frees in 5 s"), "the refusal names the quarantine and the wait: {refused}" ); assert_eq!( - full.allocate_at("node-b", "late", t0 + ttl), + full.allocate_at("node-b", "late", t0 + ttl, range_present), Ok(freed), "once the TTL has passed the address is handed out" ); } + /// A host that cannot bind the box range is never handed a dead + /// address: the allocation answers the 127.0.0.1 interim instead and + /// records nothing, so the hand never collides with a later one and a + /// re-ask redraws. Driven on an injected bindability probe, no sleeps. + #[test] + fn an_unbindable_host_is_handed_the_interim_and_the_book_stays_empty() { + let now = Instant::now(); + let mut book = AddressBook::default(); + assert_eq!( + book.allocate_at("node-a", "web", now, range_absent), + Ok(Ipv4Addr::LOCALHOST), + "a host that cannot bind the range gets the interim" + ); + assert!( + book.held.is_empty() && book.released.is_empty(), + "the interim hand records nothing" + ); + for name in ["api", "db", "worker"] { + assert_eq!( + book.allocate_at("node-a", name, now, range_absent), + Ok(Ipv4Addr::LOCALHOST), + "every box of an unbindable host shares the interim" + ); + } + assert_eq!( + book.allocate_at("node-a", "web", now, range_absent), + Ok(Ipv4Addr::LOCALHOST), + "the interim hand is idempotent per box" + ); + assert!( + book.held.is_empty() && book.released.is_empty(), + "repeated interim hands never exhaust or quarantine" + ); + } + + /// The box range as a host whose `lo0` carries every alias reads it. + fn range_present() -> RangeProbe { + switch::loopback::probe_over(switch::loopback::range_hosts(), |_| Ok(())) + } + + /// The box range as a host whose `lo0` carries none of the aliases (a + /// stock macOS before `min finalize-install`) reads it. + fn range_absent() -> RangeProbe { + switch::loopback::probe_over(switch::loopback::range_hosts(), |_| { + Err(io::ErrorKind::AddrNotAvailable.into()) + }) + } + + /// The interim hand is only a stand-in for a box created while the + /// host lacks the range: once the host binds the range again (the + /// aliases have arrived) a NEW box gets a range address. The box created + /// at the interim keeps 127.0.0.1: its resume never asks the book + /// (`control::resume_reply`, proved by + /// `control::tests::an_interim_box_keeps_the_interim_across_a_resume`). + #[test] + fn a_host_that_becomes_bindable_hands_out_range_addresses_again() { + let now = Instant::now(); + let mut book = AddressBook::default(); + assert_eq!( + book.allocate_at("node-a", "web", now, range_absent), + Ok(Ipv4Addr::LOCALHOST), + "before the aliases arrive the interim stands in" + ); + let api = book + .allocate_at("node-a", "api", now, range_present) + .expect("the range answers once the host binds it"); + assert!( + in_box_range(api), + "a new box gets an address out of the range: {api}" + ); + assert_eq!( + book.allocate_at("node-a", "api", now, range_absent), + Ok(api), + "a re-ask hands back the recorded address" + ); + } + + /// The probe's verdict is whole-range (`switch::loopback`): a host + /// that carries only some of the aliases reads absent and hands the + /// interim, even when the address the book would pick is one it carries, + /// consistent with `min list` advising `min finalize-install`. + #[test] + fn a_partial_alias_set_is_handed_the_interim() { + let mut book = AddressBook::default(); + let partial = || { + switch::loopback::probe_over(switch::loopback::range_hosts(), |address| { + if address.octets()[3] < 128 { + Ok(()) + } else { + Err(io::ErrorKind::AddrNotAvailable.into()) + } + }) + }; + assert_eq!( + book.allocate_at("node-a", "web", Instant::now(), partial), + Ok(Ipv4Addr::LOCALHOST), + "a partial alias set reads as absent" + ); + assert!(book.held.is_empty() && book.released.is_empty()); + } + + /// A box's own address comes back without the probe: one it holds, and + /// one it released inside the quarantine, even when the host has since + /// lost the range. Its address never moves under it. + #[test] + fn a_box_gets_its_own_address_back_without_the_probe() { + let t0 = Instant::now(); + let mut book = AddressBook::default(); + let web = book + .allocate_at("node-a", "web", t0, range_present) + .expect("the range binds"); + assert_eq!( + book.allocate_at("node-a", "web", t0, range_absent), + Ok(web), + "a held address is handed back as it stands" + ); + assert_eq!(book.release_at("node-a", "web", t0), Some(web)); + assert_eq!( + book.allocate_at("node-a", "web", t0, range_absent), + Ok(web), + "a quarantined own address is handed back too" + ); + } + + /// A resumed box asks to keep the address it was created at. A book + /// with no record of it, as after a restart, hands it back without the + /// probe and records it, even where the host lacks the range. A kept + /// address another box holds is not handed out: that ask is a fresh + /// one, and it gets the interim. + #[test] + fn a_resume_keeps_its_address_in_a_book_with_no_record_of_it() { + let mut book = AddressBook::default(); + let (first, _) = switch::box_loopback_interior(); + let kept = Ipv4Addr::from(u32::from(first) + 3); + assert_eq!( + book.allocate_keeping_at("node-a", "web", Some(kept), Instant::now(), range_absent), + Ok(kept), + "the box's own address comes back without the probe" + ); + assert_eq!( + book.allocate_at("node-a", "web", Instant::now(), range_absent), + Ok(kept), + "and it is recorded against the box" + ); + assert_eq!( + book.allocate_keeping_at("node-b", "api", Some(kept), Instant::now(), range_absent), + Ok(Ipv4Addr::LOCALHOST), + "an address another box holds is not kept" + ); + assert_eq!( + book.allocate_keeping_at( + "node-a", + "db", + Some(Ipv4Addr::new(127, 0, 0, 9)), + Instant::now(), + range_absent + ), + Ok(Ipv4Addr::LOCALHOST), + "an address outside the box range is not kept" + ); + } + + /// A probe refused for a reason other than an absent address refuses + /// the allocation and names the refusal, rather than moving the box to + /// the shared 127.0.0.1. Nothing is recorded. + #[test] + fn a_probe_error_refuses_the_allocation() { + let mut book = AddressBook::default(); + let refused = book + .allocate_at("node-a", "web", Instant::now(), || { + switch::loopback::probe_over(switch::loopback::range_hosts(), |_| { + Err(io::ErrorKind::PermissionDenied.into()) + }) + }) + .expect_err("a probe error is not an absent range"); + assert!( + refused.contains("could not check that it can listen on the box range") + && refused.contains("127.0.64.1 refused the bind (PermissionDenied)"), + "the refusal names the address the probe was refused at: {refused}" + ); + assert!(book.held.is_empty() && book.released.is_empty()); + } + + /// A default book reads the range with the probe it was built with. + /// Under test that probe reads the range present, so the service-path + /// tests hand distinct range addresses on hosts without the aliases. + #[test] + fn a_default_book_allocates_through_its_own_probe() { + let mut book = AddressBook::default(); + assert!(in_box_range( + book.allocate("node-a", "web").expect("present") + )); + book.range_probe = range_absent; + assert_eq!(book.allocate("node-a", "api"), Ok(Ipv4Addr::LOCALHOST)); + } + /// Box addresses are host-global (design §7.1): the answerer hands /// each node's boxes the lowest free address of `.2`-`.254`, so two /// nodes under separate state dirs never share one, even for boxes of @@ -6017,7 +6386,7 @@ mod tests { .expect("node d connects") .registration; let d_box = leaving - .allocate("box") + .allocate("box", None) .expect("the channel serves node d") .expect("node d's box is handed an address"); drop(leaving); @@ -6027,7 +6396,7 @@ mod tests { .expect("node e connects") .registration; let e_box = arriving - .allocate("box") + .allocate("box", None) .expect("the channel serves node e") .expect("node e's box is handed an address"); assert_ne!( From 947b955fd9d0faa3ee27e423cfd34d28d7f2c7e9 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:16:00 +0000 Subject: [PATCH 09/34] fix(minimald): answer a macOS box's ports at the address its name resolves to (#2713) T98. A microVM daemon walked the host's loopback once, at its start, so after min finalize-install new boxes stood at the 127.0.0.1 interim until the VM restarted. The create handler on a VM-backed daemon now asks the forwarder's walk, not the guest's own lo, and every session start walks again, so a present landing moves boxes at the interim onto their range addresses and a present-to-absent walk moves them back. Walks run one at a time on the shared probe port. A held probe port counts as bound, a leftover probe forward is released before it is read, and a walk that gets no answer keeps a present verdict. The absent landing also moves the node's own address to the interim. The NET-040 spec line and the macOS e2e case come with the change. Backport of #2713, folded from #2740. Adapted to the line: - box_manager.rs and box_manager/handle.rs hunks land in sessions.rs: RangeWalk, RangeWalkSource and RangeWalkAnswer, the range_walk cell on ManagerHandle and WeakManagerHandle, session_start_range_walk and install_range_walk_standin. - rpc/tests.rs hunks land in the inline tests in rpc.rs. - crate::traced::spawn becomes tokio::spawn in the stand-in forwarders. - ADDRESS_IN_USE_BODY is shared with #2352: one copy stays in net/policy.rs. (cherry picked from commit 64d33586da1e9809526d0663b155ab501edfff6b) --- crates/minimald/src/net/policy.rs | 395 +++++++++++++-- crates/minimald/src/rpc.rs | 450 +++++++++++++++++- crates/minimald/src/session/tests.rs | 32 +- crates/minimald/src/sessions.rs | 433 ++++++++++++++--- .../18-spec-box-networking.md | 2 +- scripts/session-e2e.sh | 189 +++++++- 6 files changed, 1386 insertions(+), 115 deletions(-) diff --git a/crates/minimald/src/net/policy.rs b/crates/minimald/src/net/policy.rs index 3e6ab87d97..6413439cd1 100644 --- a/crates/minimald/src/net/policy.rs +++ b/crates/minimald/src/net/policy.rs @@ -20,7 +20,7 @@ //! plumbing: [`PolicyWarnLimiter`], whose rate limit is keyed by box and //! rule, and [`Proto`], the transport a dropped frame is logged under. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fmt; use std::io; use std::net::{Ipv4Addr, SocketAddr}; @@ -1210,7 +1210,10 @@ pub async fn expose_dynamic( /// local probe classifies its binds — every address bound reads the range /// present; the first refusal names the first missing alias; anything that /// is not a bind — a refused request, a stall, an unreachable control -/// channel — reads absent, never present. +/// channel — reads absent, never present. The first failure's kind tells +/// the two apart: `AddrNotAvailable` is a refusal the forwarder answered, +/// and any other kind is a fault of the channel. The session-start walk +/// keeps a present verdict over a fault (`RangeWalk` in `sessions`). /// /// Why a second probe beside the local one: an own-address box's declared /// ports are bound by the **forwarder**, not by the daemon. On a native host @@ -1243,6 +1246,19 @@ async fn probe_publish_surface_within( budget: Duration, request: Duration, ) -> ::switch::loopback::RangeProbe { + // One walk at a time in this process. The daemon-start walk and each + // session-start walk ask the same forwarder to bind the same probe port, + // so a second walk that overlapped the first would read the first's + // bind as a refusal. The wait is outside the budget: a session-start + // walk queued behind the daemon-start one still gets its whole walk. + let mut turn = RANGE_WALK_TURN.lock().await; + // The addresses at which an earlier walk on this channel may have left + // its own probe forward bound: an unexpose that failed, or a budget that + // expired between a round's expose and its unexpose. The forwarder + // answers `already running` from its own forward table, so a leftover + // probe forward would read as a held address, and so as bound, on every + // later walk, even after the host lost the address. + let held = turn.entry(format!("{control:?}")).or_default(); let mut probe = ::switch::loopback::RangeProbe::failed_to_run(); let hosts: Vec = ::switch::loopback::range_hosts().collect(); let walk = tokio::time::timeout(budget, async { @@ -1253,34 +1269,101 @@ async fn probe_publish_surface_within( remote: format!("{}:1", Ipv4Addr::LOCALHOST), protocol: "tcp".to_string(), }; - match post_json_within(control, "/services/forwarder/expose", &expose, request).await { - Ok(()) => { - probe.bound += 1; - let unexpose = UnexposeRequest { - local: expose.local, - protocol: expose.protocol, - }; - if let Err(error) = post_json_within( - control, - "/services/forwarder/unexpose", - &unexpose, - request, - ) - .await - { - // Best-effort, like teardown's: a probe round left - // bound holds one obscure port at one address, while - // a walk that stopped here would leave the range - // unread — the worse leak by far. + // Marked before the expose, so a budget cut inside this round + // leaves the address marked for the next walk to release. + let leftover = !held.insert(*address); + let mut answer = + post_json_within(control, "/services/forwarder/expose", &expose, request).await; + if leftover + && answer + .as_ref() + .is_err_and(|error| error.kind() == io::ErrorKind::AddrInUse) + { + // The holder can be this walk's own earlier probe forward. + // Release it and ask again, so the round reads the host and + // not the leftover. Only a probe round binds this port at a + // marked address, so the release takes no box's forward. + match release_range_probe(control, &expose.local, request).await { + Ok(()) => { + tracing::info!( + local = %expose.local, + "released a loopback range probe forward that an earlier walk left bound", + ); + answer = post_json_within( + control, + "/services/forwarder/expose", + &expose, + request, + ) + .await; + } + Err(error) => { + // The leftover stays, so this round cannot vouch for + // the address. Read it as the release's own failure. + probe.first_failure.get_or_insert((*address, error.kind())); tracing::warn!( - local = %unexpose.local, + local = %expose.local, error = %error, - "releasing the loopback range probe's forwarder bind failed", + "releasing a leftover loopback range probe forward failed, so the address does not count as bound", ); + continue; + } + } + } + match answer { + Ok(()) => { + probe.bound += 1; + match release_range_probe(control, &expose.local, request).await { + Ok(()) => { + held.remove(address); + } + Err(error) => { + // Best-effort, like teardown's: a probe round left + // bound holds one obscure port at one address, while + // a walk that stopped here would leave the range + // unread — the worse leak by far. The address stays + // marked, so the next walk releases it first. + tracing::warn!( + local = %expose.local, + error = %error, + "releasing the loopback range probe's forwarder bind failed", + ); + } } } + Err(error) if error.kind() == io::ErrorKind::AddrInUse => { + // Something else already holds the probe port at this + // address: a box's forward or a host_ip mirror on the same + // port. The bind reached the address, so the address + // exists and counts as bound (NET-123). Nothing of the + // probe's is bound, so no unexpose runs: one would tear + // down the holder's forward. + held.remove(address); + probe.bound += 1; + tracing::debug!( + local = %expose.local, + error = %error, + "the loopback range probe port is held at this address, so it counts as bound", + ); + } Err(error) => { - probe.first_failure.get_or_insert((*address, error.kind())); + // A refusal the forwarder answered is recorded as the + // bind refusal the local probe records, `AddrNotAvailable`: + // the host said this address does not bind. Every other + // error keeps its own kind (a timeout, a missing socket, + // a malformed reply), so a session-start walk can tell a + // fault of the channel from an answer about the range. + let refused = is_forwarder_refusal(&error); + if refused { + // A refused bind holds nothing at the address. + held.remove(address); + } + let kind = if refused { + io::ErrorKind::AddrNotAvailable + } else { + error.kind() + }; + probe.first_failure.get_or_insert((*address, kind)); if error.kind() == io::ErrorKind::TimedOut { // A control channel that does not answer is not a // fact about the host's loopback at all, so the walk @@ -1315,6 +1398,20 @@ async fn probe_publish_surface_within( probe } +/// Releases the range probe's forward at `local` (`POST +/// /services/forwarder/unexpose`), within the walk's round bound. +async fn release_range_probe( + control: &ControlChannel, + local: &str, + request: Duration, +) -> io::Result<()> { + let unexpose = UnexposeRequest { + local: local.to_string(), + protocol: "tcp".to_string(), + }; + post_json_within(control, "/services/forwarder/unexpose", &unexpose, request).await +} + /// A gvproxy DNS zone-add request body for `POST /services/dns/add` /// (gvproxy's `types.Zone`): registers `records` under the DNS zone `name`. /// @@ -1456,13 +1553,49 @@ pub(crate) async fn post_json_within( if (200..300).contains(&status) { Ok(()) } else { - Err(io::Error::other(format!( - "gvproxy {path} returned HTTP {status}: {}", - String::from_utf8_lossy(body_after_headers(&response)) - ))) + let said = String::from_utf8_lossy(body_after_headers(&response)); + // A refusal that says the listener is already held, by another host + // process (the OS's bind text) or by a forward this forwarder already + // runs, carries `AddrInUse`: the address exists, and a caller that + // reads the kind (the range walk) can tell it from a missing one. + let kind = if status == 500 + && (said.contains(ADDRESS_IN_USE_BODY) || said.contains(ALREADY_RUNNING_BODY)) + { + io::ErrorKind::AddrInUse + } else { + io::ErrorKind::Other + }; + Err(io::Error::new( + kind, + ForwarderRefusal(format!("gvproxy {path} returned HTTP {status}: {said}")), + )) } } +/// The error a control request carries when the forwarder answered with a +/// non-2xx status: the forwarder was reached and said no. The range walk +/// reads it apart from a fault of the channel itself (a timeout, a missing +/// socket, a malformed reply), because only an answer is a fact about the +/// host's loopback (NET-123). +#[derive(Debug)] +struct ForwarderRefusal(String); + +impl std::fmt::Display for ForwarderRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for ForwarderRefusal {} + +/// Whether `error` is a [`ForwarderRefusal`]: the forwarder answered the +/// request and refused it. +fn is_forwarder_refusal(error: &io::Error) -> bool { + error + .get_ref() + .is_some_and(|inner| inner.is::()) +} + /// Writes `request` to a freshly-connected control stream and reads the response /// framed by its `Content-Length`. Transport-agnostic so the unix (DM2) and vsock /// (DM1/3/4) control channels share one exchange. @@ -1588,9 +1721,21 @@ const MAX_CONTROL_RESPONSE: u64 = 64 * 1024; /// probes a given forwarder — the one that publishes through it, and a host /// gvproxy has one of those (the second-daemon-on-a-host shape is native, /// where the local bind probe runs instead) — so no concurrent walk can -/// collide on it. +/// collide on it. That daemon walks at its start and again at each session +/// start, and [`RANGE_WALK_TURN`] runs those walks one at a time. A round +/// that finds the port already held at an address (a box's forward or a +/// host_ip mirror) counts that address as bound. const RANGE_PROBE_PORT: u16 = 21064; +/// The turn every forwarder-conducted range walk in this process takes +/// before its first round, so the daemon-start walk and the session-start +/// walks never bind [`RANGE_PROBE_PORT`] at the same time. It also keeps, +/// per control channel, the addresses at which a walk may have left its own +/// probe forward bound, so the next walk releases that forward before it +/// reads the address. +static RANGE_WALK_TURN: tokio::sync::Mutex>> = + tokio::sync::Mutex::const_new(BTreeMap::new()); + /// Upper bound on the whole forwarder-conducted range walk — the backstop for /// the one pathology the per-request [`GVPROXY_CONTROL_TIMEOUT`] cannot /// price: a forwarder that keeps answering, but slowly, once per address @@ -2496,6 +2641,187 @@ mod tests { ); } + /// A range address whose probe port something already holds: a box's + /// forward through this forwarder (`proxy already running`) at one + /// address, a host_ip mirror held by another host process (the OS's + /// bind text) at another. Both refusals say the address exists, so + /// the walk counts each one bound and reads the range present. The + /// walk sends no unexpose for a held address, so the holder's forward + /// stays up. + #[tokio::test] + async fn counts_an_address_whose_probe_port_is_held_as_bound() { + let dir = tempfile::TempDir::new().unwrap(); + let sock = dir.path().join("gvproxy.sock"); + let listener = UnixListener::bind(&sock).unwrap(); + let box_held = format!("{}:{RANGE_PROBE_PORT}", Ipv4Addr::new(127, 0, 64, 3)); + let mirror_held = format!("{}:{RANGE_PROBE_PORT}", Ipv4Addr::new(127, 0, 64, 9)); + let (tx, mut asked) = mpsc::channel::<(String, String)>(1024); + let (box_held_s, mirror_held_s) = (box_held.clone(), mirror_held.clone()); + let forwarder = tokio::spawn(async move { + loop { + let Ok((mut conn, _)) = listener.accept().await else { + return; + }; + // An expose body carries `remote`, an unexpose body + // does not: that tells the two verbs apart. + let body = read_request(&mut conn).await; + let local = local_of(&body); + let verb = if String::from_utf8_lossy(&body).contains("\"remote\"") { + "expose" + } else { + "unexpose" + }; + let said = if verb == "expose" && local == box_held_s { + Some("proxy already running".to_string()) + } else if verb == "expose" && local == mirror_held_s { + Some(format!("listen tcp {local}: bind: address already in use")) + } else { + None + }; + let answer = match said { + Some(said) => format!( + "HTTP/1.1 500 Internal Server Error\r\nContent-Length: {}\r\n\r\n{said}", + said.len() + ), + None => "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n".to_string(), + }; + conn.write_all(answer.as_bytes()) + .await + .expect("the fake forwarder must answer"); + if tx.send((verb.to_string(), local)).await.is_err() { + return; + } + } + }); + let probe = probe_publish_surface(&ControlChannel::Unix(sock)).await; + forwarder.abort(); + + assert!( + probe.present(), + "a held probe port says the address exists: {}", + probe.summary() + ); + assert_eq!(probe.first_failure, None); + assert_eq!(probe.bound, 254, "{}", probe.summary()); + + let mut unexposed = Vec::new(); + while let Some((verb, local)) = asked.recv().await { + if verb == "unexpose" { + unexposed.push(local); + } + } + assert_eq!( + unexposed.len(), + 252, + "one release per address the probe bound" + ); + assert!( + !unexposed.contains(&box_held) && !unexposed.contains(&mirror_held), + "the walk never releases a forward it did not bind: {unexposed:?}" + ); + } + + /// A probe forward that an earlier walk left bound is released, not + /// counted. The first walk's unexpose fails at one address, so its + /// probe forward stays in the forwarder's table. The host then loses + /// that address. The forwarder still answers `already running` there + /// from its table, but the next walk releases its own leftover, asks + /// again, and reads the refusal: the range is absent. + #[tokio::test] + async fn a_probe_forward_left_by_an_earlier_walk_is_released_not_counted() { + use std::collections::BTreeSet; + use std::sync::{Arc, Mutex}; + + let dir = tempfile::TempDir::new().unwrap(); + let sock = dir.path().join("gvproxy.sock"); + let listener = UnixListener::bind(&sock).unwrap(); + let stuck = format!("{}:{RANGE_PROBE_PORT}", Ipv4Addr::new(127, 0, 64, 5)); + // The forwarder's forward table, the addresses the host binds, + // and how many unexposes at the stuck address still fail. + let table: Arc>> = Arc::default(); + let host_has_stuck = Arc::new(Mutex::new(true)); + let failing_unexposes = Arc::new(Mutex::new(1u32)); + let (table_s, host_s, failing_s, stuck_s) = ( + table.clone(), + host_has_stuck.clone(), + failing_unexposes.clone(), + stuck.clone(), + ); + let forwarder = tokio::spawn(async move { + loop { + let Ok((mut conn, _)) = listener.accept().await else { + return; + }; + let body = read_request(&mut conn).await; + let local = local_of(&body); + let expose = String::from_utf8_lossy(&body).contains("\"remote\""); + let said: Option = { + let mut table = table_s.lock().unwrap(); + if expose { + if table.contains(&local) { + Some("proxy already running".to_string()) + } else if local == stuck_s && !*host_s.lock().unwrap() { + Some(format!( + "listen tcp {local}: bind: can't assign requested address" + )) + } else { + table.insert(local.clone()); + None + } + } else { + let mut failing = failing_s.lock().unwrap(); + if local == stuck_s && *failing > 0 { + *failing -= 1; + Some("unexpose failed".to_string()) + } else { + table.remove(&local); + None + } + } + }; + let answer = match said { + Some(said) => format!( + "HTTP/1.1 500 Internal Server Error\r\nContent-Length: {}\r\n\r\n{said}", + said.len() + ), + None => "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n".to_string(), + }; + conn.write_all(answer.as_bytes()) + .await + .expect("the fake forwarder must answer"); + } + }); + let control = ControlChannel::Unix(sock); + + let first = probe_publish_surface(&control).await; + assert!(first.present(), "{}", first.summary()); + assert!( + table.lock().unwrap().contains(&stuck), + "the failed unexpose left the probe forward bound" + ); + + *host_has_stuck.lock().unwrap() = false; + let second = probe_publish_surface(&control).await; + forwarder.abort(); + + assert!( + !second.present(), + "a leftover probe forward never vouches for an address the host lost: {}", + second.summary() + ); + assert_eq!( + second.first_failure, + Some(( + Ipv4Addr::new(127, 0, 64, 5), + io::ErrorKind::AddrNotAvailable + )), + ); + assert!( + table.lock().unwrap().is_empty(), + "the second walk released its leftover" + ); + } + /// The stock macOS host's shape (`docs/spikes/2026-09-22-macos-loopback-alias.md`): /// every bind on the range refused, so every expose is answered `500` /// — the range reads absent and the daemon behind that forwarder @@ -2516,11 +2842,16 @@ mod tests { assert!(!probe.present()); assert!(probe.interim()); assert_eq!(probe.surface(), "127.0.0.1-interim"); - // The refused request comes back as gvproxy's folded error, whose - // kind is `Other` — the address is the record's, never guessed. + // The forwarder answered and refused, so the walk records the + // bind refusal's kind, `AddrNotAvailable` — the answer a + // session-start walk lands absent on. The address is the + // record's, never guessed. assert_eq!( probe.first_failure, - Some((Ipv4Addr::new(127, 0, 64, 1), io::ErrorKind::Other)) + Some(( + Ipv4Addr::new(127, 0, 64, 1), + io::ErrorKind::AddrNotAvailable + )) ); } diff --git a/crates/minimald/src/rpc.rs b/crates/minimald/src/rpc.rs index ed8477abf6..8990997a36 100644 --- a/crates/minimald/src/rpc.rs +++ b/crates/minimald/src/rpc.rs @@ -469,11 +469,23 @@ async fn serve_create_session( // NET-123: the session-start loopback probe, before any // of this session's names publish. Its interim flag on // the reply is what tells the client to surface the - // naming advisory again (NET-122). NET-018 reads the + // naming advisory again (NET-122). On a VM-backed daemon + // the guest's own `lo` always carries the range, so the + // forwarder walks the host's loopback instead, and the + // flag is the host's verdict. NET-018 reads the // answerer's half straight off the state, not off this - // probe: the range the probe covers is the guest's on a - // VM-backed host, so it is not the reply's fact to read. - let interim_loopback = session_start_loopback_probe(&id).await.interim(); + // probe. + #[cfg(target_os = "linux")] + let walked = mngr + .session_start_range_walk() + .await + .map(|answer| session_start_range_walk_line(&id, &answer)); + #[cfg(not(target_os = "linux"))] + let walked: Option = None; + let interim_loopback = match walked { + Some(interim) => interim, + None => session_start_loopback_probe(&id).await.interim(), + }; let hostname_proxy_port = s.hostname_proxy_port().await; let zone_answerer_port = s.zone_answerer_port().await; Errorable::Ok(minimald_rpc::CreateSessionResponse { @@ -653,6 +665,40 @@ async fn session_start_loopback_probe(session_id: &SessionId) -> crate::net::loo probe } +/// The session-start line of a VM-backed daemon (NET-123): the same record +/// [`session_start_loopback_probe`] writes, over the forwarder's walk of the +/// host's loopback, with whether this start walked and how many boxes the +/// landing moved off the `127.0.0.1` interim. Returns the reply's interim +/// flag. +#[cfg(target_os = "linux")] +fn session_start_range_walk_line( + session_id: &SessionId, + answer: &crate::sessions::RangeWalkAnswer, +) -> bool { + let probe = &answer.probe; + // A walk that got no answer over a present verdict keeps that verdict, + // so the box publishes in the range and the surface is the range's. + let interim = probe.interim() && !answer.kept_present; + let surface = if answer.kept_present { + "reserved-range" + } else { + probe.surface() + }; + tracing::info!( + session_id = %session_id, + via = "forwarder", + walked = answer.walked, + moved_off_interim = answer.moved, + verdict_kept = answer.kept_present, + probe = %probe.summary(), + surface = %surface, + first_failure = ?probe.first_failure, + interim_loopback = interim, + "session-start loopback probe picked the publish surface" + ); + interim +} + /// How long [`log_live_name_surface`] lets the proxy's driver finish /// recording its port before the line names the proxy's state: the two /// listeners start together and normally land within milliseconds of one @@ -5932,6 +5978,402 @@ mod tests { ); } + /// NET-123 on a VM-backed daemon, after `min finalize-install`: the + /// create handler asks the forwarder's walk of the host's loopback, not the + /// guest's own `lo`. Each session start walks again, whatever verdict + /// landed last. The walk after the host gains the range lands present: the + /// box standing at the `127.0.0.1` interim moves onto a range address, the + /// new box's grant comes from the range, and its reply clears the interim + /// flag. A present verdict is walked again too. When the range goes away + /// under the running VM, the next walk lands absent: the reply carries the + /// interim flag and the earlier box moves back to `127.0.0.1`. The stand-in + /// walk stands in for the host forwarder, which a harness daemon does not + /// have. + #[allow(clippy::await_holding_lock)] + #[cfg(target_os = "linux")] + // Multi-thread: the held-forward step runs the real forwarder walk from + // inside the sync stand-in, through `block_in_place`. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn session_start_rewalks_range_after_finalize_install() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + // No local-probe stand-in may leak into these creates: the walk, not + // the local probe, must decide every reply here. + let _standin_window = PROBE_TEST_MUTEX.lock().unwrap_or_else(|p| p.into_inner()); + let log = crate::test_harness::captured_log(); + let server = TestServer::new().await; + let mngr = server.state.sessions_manager().await; + let installed = Arc::new(AtomicBool::new(false)); + // When set, the stand-in walk times out at its first round: the + // forwarder never answered. + let stalled = Arc::new(AtomicBool::new(false)); + let walks = Arc::new(AtomicUsize::new(0)); + // When set, the stand-in runs the real forwarder walk against this + // control socket, in place of a fixed verdict. + let forwarder: Arc>> = + Arc::new(std::sync::Mutex::new(None)); + { + let installed = Arc::clone(&installed); + let stalled = Arc::clone(&stalled); + let walks = Arc::clone(&walks); + let forwarder = Arc::clone(&forwarder); + mngr.install_range_walk_standin(move || { + walks.fetch_add(1, Ordering::SeqCst); + let sock = forwarder.lock().expect("forwarder lock").clone(); + if stalled.load(Ordering::SeqCst) { + crate::net::loopback::RangeProbe { + bound: 0, + probed: ::switch::loopback::range_hosts().count(), + first_failure: Some(( + std::net::Ipv4Addr::new(127, 0, 64, 1), + std::io::ErrorKind::TimedOut, + )), + } + } else if let Some(sock) = sock { + let control = crate::net::policy::ControlChannel::Unix(sock); + tokio::task::block_in_place(|| { + tokio::runtime::Handle::current() + .block_on(crate::net::policy::probe_publish_surface(&control)) + }) + } else if installed.load(Ordering::SeqCst) { + let hosts = ::switch::loopback::range_hosts().count(); + crate::net::loopback::RangeProbe { + bound: hosts, + probed: hosts, + first_failure: None, + } + } else { + absent_range_probe() + } + }); + } + // The daemon-start walk's verdict on a host without the range. + mngr.land_range_verdict(crate::net::dns::RangeVerdict::Absent); + let mut client = server.connect().await; + + let before = client + .call::(&req("before-install", "/uwu")) + .await + .ok() + .expect("an absent range must not refuse the session"); + assert!( + before.interim_loopback, + "the host's walk found the range absent, so the reply carries the interim" + ); + let line = probe_line(&log.contents(), &before.id); + assert!( + line.contains("via=\"forwarder\"") && line.contains("walked=true"), + "a session start on an absent verdict walks the forwarder: {line}" + ); + assert!( + line.contains("surface=127.0.0.1-interim"), + "the line names the interim as the surface picked: {line}" + ); + assert_eq!(walks.load(Ordering::SeqCst), 1); + + // The box the absent verdict published at the interim. + let ports = std::collections::BTreeSet::from([8080]); + { + let registry = mngr.hostnames(); + let mut routes = registry.write().expect("registry lock"); + routes.publish_own_address( + before.id, + "before-install", + std::net::Ipv4Addr::LOCALHOST, + ports.clone(), + ); + routes.register_own_ip(before.id, "before-install", ports); + } + + // `min finalize-install` applies the range on the host. + installed.store(true, Ordering::SeqCst); + let after = client + .call::(&req("after-install", "/uwu")) + .await + .ok() + .expect("a create with the range present"); + assert!( + !after.interim_loopback, + "the walk found the range present, so the reply clears the interim" + ); + assert_eq!( + walks.load(Ordering::SeqCst), + 2, + "the session start walked again" + ); + let line = probe_line(&log.contents(), &after.id); + assert!( + line.contains("surface=reserved-range") && line.contains("moved_off_interim=1"), + "the line names the range and the box it moved off the interim: {line}" + ); + + let moved = mngr + .hostnames() + .read() + .expect("registry lock") + .published_own_address(before.id) + .expect("the earlier box is still published"); + assert!( + ::switch::loopback::range_hosts().any(|host| host == moved), + "the box at the interim moves onto a range address, got {moved}" + ); + let after_address = match mngr + .loopback_book() + .grant(crate::net::dns::LeaseNamespace::Box { session: after.id }) + { + crate::net::dns::LoopbackGrant::Granted(address) => { + assert!( + ::switch::loopback::range_hosts().any(|host| host == address), + "the new box is granted a range address, got {address}" + ); + assert_ne!(address, moved, "the two boxes hold distinct addresses"); + address + } + other => panic!("the new box's grant must come from the range, got {other:?}"), + }; + assert!( + log.contents().lines().any(|line| { + line.contains("loopback-range-present-box") + && line.contains(&format!("session_id={}", before.id)) + && line.contains(&format!("to={moved}")) + }), + "the move is logged with the session id and both addresses" + ); + + // A present verdict is walked again: the range can go away under a + // running VM, so every session start bind-probes it. + let later = client + .call::(&req("later", "/uwu")) + .await + .ok() + .expect("a create with the range present"); + assert!(!later.interim_loopback); + assert_eq!( + walks.load(Ordering::SeqCst), + 3, + "a present verdict is walked again at the next session start" + ); + let line = probe_line(&log.contents(), &later.id); + assert!( + line.contains("walked=true") && line.contains("surface=reserved-range"), + "the session start walked the range and found it present: {line}" + ); + + // A box holds a forward at its range address on the probe's own port: + // the real walk asks the stand-in forwarder, which refuses that + // address's round as in use. The address exists, so the walk counts it + // bound, the range stays present, and every box keeps its address. + let dir = tempfile::TempDir::new().unwrap(); + let sock = dir.path().join("gvproxy.sock"); + let held = format!("{moved}:21064"); + let (unexposed_tx, mut unexposed_rx) = tokio::sync::mpsc::unbounded_channel::(); + let stand_in = spawn_holding_forwarder(sock.clone(), held.clone(), unexposed_tx); + *forwarder.lock().expect("forwarder lock") = Some(sock); + let held_start = client + .call::(&req("forward-held", "/uwu")) + .await + .ok() + .expect("a create with the range present"); + *forwarder.lock().expect("forwarder lock") = None; + stand_in.abort(); + assert!( + !held_start.interim_loopback, + "a held probe port at one range address does not read the range absent" + ); + assert_eq!(walks.load(Ordering::SeqCst), 4); + let line = probe_line(&log.contents(), &held_start.id); + assert!( + line.contains("walked=true") && line.contains("surface=reserved-range"), + "the walk found the range present beside the held forward: {line}" + ); + { + let registry = mngr.hostnames(); + let routes = registry.read().expect("registry lock"); + assert_eq!( + routes.published_own_address(before.id), + Some(moved), + "the box holding the forward keeps its range address" + ); + } + assert_eq!( + mngr.loopback_book() + .grant(crate::net::dns::LeaseNamespace::Box { session: after.id }), + crate::net::dns::LoopbackGrant::Granted(after_address), + "the other box keeps its range address" + ); + let mut unexposed = Vec::new(); + while let Ok(local) = unexposed_rx.try_recv() { + unexposed.push(local); + } + assert!( + !unexposed.contains(&held), + "the walk never releases the forward the box holds: {unexposed:?}" + ); + + // The forwarder stalls: the walk times out and gets no answer. That is + // a fault of the channel, not an absent range, so the present verdict + // stays and every box keeps its range address. + stalled.store(true, Ordering::SeqCst); + let stalled_start = client + .call::(&req("forwarder-stalled", "/uwu")) + .await + .ok() + .expect("a stalled walk must not refuse the session"); + stalled.store(false, Ordering::SeqCst); + assert!( + !stalled_start.interim_loopback, + "a walk with no answer keeps the present verdict, so the reply stays off the interim" + ); + assert_eq!(walks.load(Ordering::SeqCst), 5); + let line = probe_line(&log.contents(), &stalled_start.id); + assert!( + line.contains("walked=true") + && line.contains("verdict_kept=true") + && line.contains("surface=reserved-range"), + "the line says the walk kept the present verdict: {line}" + ); + assert!( + mngr.loopback_book() + .vouches_for(::switch::loopback::range_hosts().next().unwrap()), + "the book still holds the present verdict" + ); + assert_eq!( + mngr.hostnames() + .read() + .expect("registry lock") + .published_own_address(before.id), + Some(moved), + "a stalled walk moves no box off its range address" + ); + let node = mngr + .hostnames() + .read() + .expect("registry lock") + .node_address(); + assert!( + ::switch::loopback::range_hosts().any(|host| host == node), + "the present landing granted the node a range address, got {node}" + ); + + // The range goes away under the running VM. + installed.store(false, Ordering::SeqCst); + let gone = client + .call::(&req("range-gone", "/uwu")) + .await + .ok() + .expect("an absent range must not refuse the session"); + assert!( + gone.interim_loopback, + "the walk found the range absent after a present landing, so the \ + reply carries the interim" + ); + assert_eq!(walks.load(Ordering::SeqCst), 6); + let line = probe_line(&log.contents(), &gone.id); + assert!( + line.contains("walked=true") && line.contains("surface=127.0.0.1-interim"), + "the line names the interim as the surface picked: {line}" + ); + assert_eq!( + mngr.hostnames() + .read() + .expect("registry lock") + .published_own_address(before.id), + Some(std::net::Ipv4Addr::LOCALHOST), + "the absent landing moves the earlier box off its range address onto the interim" + ); + assert!( + log.contents().lines().any(|line| { + line.contains("loopback-range-absent-box") + && line.contains(&format!("session_id={}", before.id)) + && line.contains(&format!("from={moved}")) + }), + "the move back to the interim is logged with the session id and both addresses" + ); + assert_eq!( + mngr.hostnames() + .read() + .expect("registry lock") + .node_address(), + std::net::Ipv4Addr::LOCALHOST, + "the absent landing moves the node's address off the range onto the interim" + ); + assert!( + log.contents().lines().any(|line| { + line.contains("loopback-range-absent-node") + && line.contains(&format!("from={node}")) + && line.contains("to=127.0.0.1") + }), + "the node's move to the interim is logged with both addresses" + ); + } + + /// A stand-in host forwarder on `path` for the range walk: it answers every + /// expose `200`, except an expose of `held`, which it refuses as a forward + /// it already runs (`proxy already running`, HTTP 500). It reports each + /// unexpose's `local` on `unexposed`. + #[cfg(target_os = "linux")] + fn spawn_holding_forwarder( + path: std::path::PathBuf, + held: String, + unexposed: tokio::sync::mpsc::UnboundedSender, + ) -> tokio::task::JoinHandle<()> { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + let listener = tokio::net::UnixListener::bind(&path).unwrap(); + tokio::spawn(async move { + loop { + let Ok((mut conn, _)) = listener.accept().await else { + return; + }; + let mut buf = Vec::new(); + let mut scratch = [0u8; 512]; + let body = loop { + let n = conn.read(&mut scratch).await.unwrap_or(0); + if n == 0 { + break None; + } + buf.extend_from_slice(&scratch[..n]); + let Some(end) = buf.windows(4).position(|w| w == b"\r\n\r\n") else { + continue; + }; + let head = String::from_utf8_lossy(&buf[..end]).to_ascii_lowercase(); + let len: usize = head + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if buf.len() >= end + 4 + len { + break Some( + String::from_utf8_lossy(&buf[end + 4..end + 4 + len]).into_owned(), + ); + } + }; + let Some(body) = body else { continue }; + let local = body + .split("\"local\":\"") + .nth(1) + .and_then(|rest| rest.split('"').next()) + .unwrap_or_default() + .to_string(); + let is_expose = body.contains("\"remote\""); + let answer = if is_expose && local == held { + let said = "proxy already running"; + format!( + "HTTP/1.1 500 Internal Server Error\r\nContent-Length: {}\r\n\r\n{said}", + said.len() + ) + } else { + "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n".to_string() + }; + let _ = conn.write_all(answer.as_bytes()).await; + if !is_expose { + let _ = unexposed.send(local); + } + } + }) + } + /// NET-018: once the answerer is serving beside the proxy, the replies /// report the one fact that is the daemon's to know — its answerer /// bound — and the one log line, emitted the moment the answerer binds, diff --git a/crates/minimald/src/session/tests.rs b/crates/minimald/src/session/tests.rs index 72ffe1c76f..67a872b822 100644 --- a/crates/minimald/src/session/tests.rs +++ b/crates/minimald/src/session/tests.rs @@ -3967,20 +3967,35 @@ async fn a_box_nobody_handed_an_address_publishes_at_the_answerers_grant() { /// NET-129's sub-requirement (report, don't translate) covers the boxes that /// *do* share an address; this is the case the shared-address mode exists to /// avoid. +/// +/// The boxes are created after the range lands the way a microVM daemon +/// lands it once `min finalize-install` ran: the daemon-start walk found the +/// range absent, and the first create's session-start walk finds it present. +/// A stand-in walk stands in for the host forwarder's binds. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn each_box_gets_own_loopback_address() { let alpha_handed = std::net::Ipv4Addr::new(127, 0, 64, 9); let beta_handed = std::net::Ipv4Addr::new(127, 0, 64, 10); let server = TestServer::new().await; + let manager = server.state.sessions_manager().await; + manager.install_range_walk_standin(|| { + let hosts = ::switch::loopback::range_hosts().count(); + crate::net::loopback::RangeProbe { + bound: hosts, + probed: hosts, + first_failure: None, + } + }); + manager.land_range_verdict(crate::net::dns::RangeVerdict::Absent); let mut client = server.connect().await; - let _alpha = finalize_handed_own_ip_session( + let alpha = finalize_handed_own_ip_session( &mut client, "alpha", std::net::Ipv4Addr::new(100, 64, 128, 9), alpha_handed, ) .await; - let _beta = finalize_handed_own_ip_session( + let beta = finalize_handed_own_ip_session( &mut client, "beta", std::net::Ipv4Addr::new(100, 64, 128, 10), @@ -4009,6 +4024,19 @@ async fn each_box_gets_own_loopback_address() { // number each box asked for — no translation, no remap around the pair. let registry = server.state.sessions_manager().await.hostnames(); let routes = registry.read().expect("registry lock"); + // Each box stands at its own hand, not the interim: the publish is the + // address the attach path binds the box's forwards at. + for (name, id, handed, address) in [ + ("alpha", alpha, alpha_handed, alpha_address), + ("beta", beta, beta_handed, beta_address), + ] { + assert_eq!( + routes.published_own_address(id), + Some(handed), + "{name}'s forwards bind at the address it was handed, after the range landed" + ); + assert_eq!(address, handed, "{name}'s name answers at its own address"); + } for (name, address) in [("alpha", alpha_address), ("beta", beta_address)] { let route = routes .resolve(&format!("{name}.min.internal")) diff --git a/crates/minimald/src/sessions.rs b/crates/minimald/src/sessions.rs index edddd94378..6c760198de 100644 --- a/crates/minimald/src/sessions.rs +++ b/crates/minimald/src/sessions.rs @@ -327,6 +327,8 @@ impl Manager { // host-address boxes answer at. A native node's address is the host // loopback itself, which it already owns. #[cfg(target_os = "linux")] + let range_walk: Arc>> = Arc::default(); + #[cfg(target_os = "linux")] let (hostnames, loopback) = { // This daemon's start, on the record's own clock: the stamp the // sweep's race guard reads. A grant another daemon on this host @@ -491,70 +493,26 @@ impl Manager { // each box under the registry's write lock before it // publishes, and releases a grant drawn for a box whose // publish is gone. - let book = Arc::clone(&loopback); - let registry = Arc::clone(&hostnames); + // + // The same walk runs again at each session start + // ([`RangeWalk::session_start`]), so a host that gains the + // range after this daemon started publishes its next box in + // the range, and a host that loses it moves its boxes to the + // interim. The walk lets one walk + // run at a time, so the session-start walk never overlaps + // this one on the forwarder. + let walk = Arc::new(RangeWalk::new( + RangeWalkSource::Forwarder(crate::net::policy::ControlChannel::Vsock { + cid, + port, + }), + Arc::clone(&loopback), + Arc::clone(&hostnames), + )); + // The cell is new and empty here, so the set cannot fail. + let _ = range_walk.set(Arc::clone(&walk)); tokio::spawn(async move { - let probe = crate::net::policy::probe_publish_surface( - &crate::net::policy::ControlChannel::Vsock { cid, port }, - ) - .await; - tracing::info!( - via = "forwarder", - surface = probe.surface(), - interim = probe.interim(), - first_failure = ?probe.first_failure, - "the daemon-start range probe walked the forwarder's loopback \ - and picked the publish surface", - ); - land_range_verdict( - &book, - ®istry, - if probe.present() { - crate::net::dns::RangeVerdict::Present - } else { - crate::net::dns::RangeVerdict::Absent - }, - ); - if !probe.present() { - tracing::warn!( - first_failure = ?probe.first_failure, - "the reserved local range is absent on the publish surface; \ - publishing boxes on the 127.0.0.1 interim", - ); - return; - } - let node = match book.grant(crate::net::dns::LeaseNamespace::Node) { - crate::net::dns::LoopbackGrant::Granted(address) => address, - crate::net::dns::LoopbackGrant::RecordUnavailable => { - // The shutdown shape: a daemon that went down - // mid-walk has closed its book, and the grant that - // comes back is the closed book's, not a fault on - // the host — the manager stops every session - // before it closes, so there is nothing left to - // publish an interim for either. An unreadable - // record lands here too, said at the same level: - // neither is a verdict about this host's range. - tracing::debug!( - grant = ?crate::net::dns::LoopbackGrant::RecordUnavailable, - "the node's own loopback address was not granted — \ - the book was closed for shutdown, or its record could \ - not be read or written", - ); - return; - } - other => { - tracing::warn!( - grant = ?other, - "the node's own loopback address was not granted; \ - publishing host-address boxes on the 127.0.0.1 interim", - ); - return; - } - }; - registry - .write() - .expect("hostname registry lock poisoned") - .set_node_address(node); + walk.daemon_start().await; }); } (hostnames, loopback) @@ -565,6 +523,8 @@ impl Manager { in_shutdown: in_shutdown.clone(), #[cfg(target_os = "linux")] hostnames: Arc::clone(&hostnames), + #[cfg(target_os = "linux")] + range_walk, #[cfg(all(test, target_os = "linux"))] loopback: Arc::clone(&loopback), }; @@ -622,6 +582,260 @@ impl Manager { } } +/// What asks whether the host binds the reserved local range, for a daemon +/// whose publish surface is the host's loopback, not its own. +#[cfg(target_os = "linux")] +pub(crate) enum RangeWalkSource { + /// The host forwarder, asked over the control channel the publishes + /// ride ([`crate::net::policy::probe_publish_surface`]). + Forwarder(crate::net::policy::ControlChannel), + /// A test's stand-in for the forwarder's binds: no Linux host can make + /// the range absent, and no harness daemon has a host forwarder. + #[cfg(test)] + StandIn(Arc crate::net::loopback::RangeProbe + Send + Sync>), +} + +/// The answer a session-start range walk gives the create handler. +#[cfg(target_os = "linux")] +#[derive(Debug, Clone)] +pub(crate) struct RangeWalkAnswer { + /// The probe the verdict rests on: this walk's, or the concurrent + /// walk's when this call took that walk's answer. + pub(crate) probe: crate::net::loopback::RangeProbe, + /// Whether this session start walked the range. `false` only when a + /// walk that ended while this one waited for its turn gave the answer. + pub(crate) walked: bool, + /// How many boxes the landing moved off the `127.0.0.1` interim onto + /// their range addresses. + pub(crate) moved: usize, + /// Whether this walk was a fault of the channel over a landed present + /// verdict, so the verdict stayed present and `probe` is the failed + /// walk's, not the verdict's. + pub(crate) kept_present: bool, +} + +/// Whether a probe that did not bind the whole range rests on a refusal +/// the forwarder answered, recorded as `AddrNotAvailable`, rather than on +/// a fault of the channel. +#[cfg(target_os = "linux")] +fn range_probe_refused(probe: &crate::net::loopback::RangeProbe) -> bool { + matches!( + probe.first_failure, + Some((_, std::io::ErrorKind::AddrNotAvailable)) + ) +} + +/// The forwarder-conducted walk of the reserved local range (NET-123) and +/// the landing of its verdict, shared by the daemon-start walk and every +/// session-start walk of a microVM daemon. +/// +/// One walk runs at a time: the walks share one probe port on one +/// forwarder, and a session start that waits behind a walk takes that +/// walk's answer, not a second walk of its own. +#[cfg(target_os = "linux")] +pub(crate) struct RangeWalk { + source: RangeWalkSource, + book: Arc, + registry: Arc>, + /// How many walks ended, read before a caller waits for its turn. + walks: std::sync::atomic::AtomicU64, + /// The turn, and the last walk's probe. + last: Mutex>, +} + +#[cfg(target_os = "linux")] +impl std::fmt::Debug for RangeWalk { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let source = match &self.source { + RangeWalkSource::Forwarder(_) => "forwarder", + #[cfg(test)] + RangeWalkSource::StandIn(_) => "stand-in", + }; + f.debug_struct("RangeWalk") + .field("source", &source) + .field( + "walks", + &self.walks.load(std::sync::atomic::Ordering::Acquire), + ) + .finish_non_exhaustive() + } +} + +#[cfg(target_os = "linux")] +impl RangeWalk { + pub(crate) fn new( + source: RangeWalkSource, + book: Arc, + registry: Arc>, + ) -> Self { + Self { + source, + book, + registry, + walks: std::sync::atomic::AtomicU64::new(0), + last: Mutex::new(None), + } + } + + /// Asks the source to bind the whole range once. + async fn probe(&self) -> crate::net::loopback::RangeProbe { + match &self.source { + RangeWalkSource::Forwarder(control) => { + crate::net::policy::probe_publish_surface(control).await + } + #[cfg(test)] + RangeWalkSource::StandIn(walk) => walk(), + } + } + + /// Walks the range and lands the verdict. The caller holds the turn. + /// + /// A walk that neither bound the whole range nor got a refusal from the + /// forwarder (a timeout, a missing socket, a malformed reply) is a fault + /// of the channel, not a fact about the host's loopback. Over a landed + /// present verdict such a walk lands nothing: the verdict stays present + /// and every box keeps its range address. The returned flag says so. + /// Over a pending or absent verdict the walk lands absent as before, + /// because the interim is what the surface can promise without an answer. + async fn walk_and_land(&self) -> (crate::net::loopback::RangeProbe, usize, bool) { + let probe = self.probe().await; + if !probe.present() && !range_probe_refused(&probe) && self.present_landed() { + tracing::warn!( + probe = %probe.summary(), + first_failure = ?probe.first_failure, + action = "loopback-range-walk-inconclusive", + "the range walk got no answer from the forwarder; \ + the present verdict stays and every box keeps its address", + ); + self.walks.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + return (probe, 0, true); + } + let moved = land_range_verdict( + &self.book, + &self.registry, + if probe.present() { + crate::net::dns::RangeVerdict::Present + } else { + crate::net::dns::RangeVerdict::Absent + }, + ); + if probe.present() { + self.grant_node_address(); + } + self.walks.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + (probe, moved, false) + } + + /// Whether the book holds a landed present verdict: it vouches for an + /// address inside the reserved range only then. + fn present_landed(&self) -> bool { + ::switch::loopback::range_hosts() + .next() + .is_some_and(|host| self.book.vouches_for(host)) + } + + /// Hands the node its own loopback address once the range is present + /// (NET-129). The grant is idempotent by namespace, so a later present + /// landing asks again at no cost. + fn grant_node_address(&self) { + let node = match self.book.grant(crate::net::dns::LeaseNamespace::Node) { + crate::net::dns::LoopbackGrant::Granted(address) => address, + crate::net::dns::LoopbackGrant::RecordUnavailable => { + // The shutdown shape: a daemon that went down mid-walk has + // closed its book, and the grant that comes back is the + // closed book's, not a fault on the host — the manager stops + // every session before it closes, so there is nothing left + // to publish an interim for either. An unreadable record + // lands here too, said at the same level: neither is a + // verdict about this host's range. + tracing::debug!( + grant = ?crate::net::dns::LoopbackGrant::RecordUnavailable, + "the node's own loopback address was not granted — \ + the book was closed for shutdown, or its record could \ + not be read or written", + ); + return; + } + other => { + tracing::warn!( + grant = ?other, + "the node's own loopback address was not granted; \ + publishing host-address boxes on the 127.0.0.1 interim", + ); + return; + } + }; + self.registry + .write() + .expect("hostname registry lock poisoned") + .set_node_address(node); + } + + /// The daemon-start walk: spawned off `Manager::init`'s critical + /// path, it lands the first verdict the book opened pending on. + pub(crate) async fn daemon_start(&self) { + let mut last = self.last.lock().await; + let (probe, _, _) = self.walk_and_land().await; + tracing::info!( + via = "forwarder", + surface = probe.surface(), + interim = probe.interim(), + first_failure = ?probe.first_failure, + "the daemon-start range probe walked the forwarder's loopback \ + and picked the publish surface", + ); + if !probe.present() { + tracing::warn!( + first_failure = ?probe.first_failure, + "the reserved local range is absent on the publish surface; \ + publishing boxes on the 127.0.0.1 interim", + ); + } + *last = Some(probe); + } + + /// The session-start walk: walks the range again at every session + /// start, whatever verdict landed last, and lands what it finds. The + /// range can go away under a running VM, so a present verdict is + /// bind-probed again too: a present-to-absent walk lands the absent + /// arm, which moves every box in the range onto the `127.0.0.1` + /// interim. Only a refusal the forwarder answered lands that arm over a + /// present verdict. A walk that got no answer keeps the verdict + /// ([`Self::walk_and_land`]). A walk that ended while this call waited for its turn gives + /// this call its answer. + pub(crate) async fn session_start(&self) -> RangeWalkAnswer { + let seen = self.walks.load(std::sync::atomic::Ordering::Acquire); + let mut last = self.last.lock().await; + let ended_while_waiting = self.walks.load(std::sync::atomic::Ordering::Acquire) != seen; + if let Some(probe) = last.as_ref().filter(|_| ended_while_waiting) { + return RangeWalkAnswer { + probe: probe.clone(), + walked: false, + moved: 0, + kept_present: false, + }; + } + let (probe, moved, kept_present) = self.walk_and_land().await; + if kept_present { + // The verdict did not move, so the last walk's probe stays the + // one a waiting session start takes as its answer. + return RangeWalkAnswer { + probe, + walked: true, + moved, + kept_present, + }; + } + *last = Some(probe.clone()); + RangeWalkAnswer { + probe, + walked: true, + moved, + kept_present, + } + } +} + /// One box's move off the `127.0.0.1` interim, as the present landing's /// sweep draws it: the box (`session`, `name`, `ports`) and the address it /// is to move to — its own hand for a box a creator handed one (`hand`, @@ -646,7 +860,7 @@ pub(crate) struct InterimUpgrade { } /// Lands a range verdict the way the deferred walk lands it -/// ([`crate::sessions::Manager::init`]'s spawned probe): the state moves +/// ([`RangeWalk`]'s daemon-start and session-start walks): the state moves /// first, then the arm that follows it runs over the publishes the old /// verdict left standing — the one moment a verdict's landing has effects /// beside the atomic it sets, and the reason the walk cannot simply store @@ -658,7 +872,9 @@ pub(crate) struct InterimUpgrade { /// destroy. The landing is the one moment the daemon holds both facts — the /// verdict and every publish it contradicts — so it re-publishes each such /// box at the interim, name and route with it, and the box's attach binds -/// where the surface listens instead of failing until destroy. The arm is +/// where the surface listens instead of failing until destroy. The node's +/// own address moves to the interim too when it stands in the range, so +/// the names of host-address boxes stop answering at it. The arm is /// a belt behind the pending window's gate: no production path publishes a /// reserved-range address under anything but a landed present verdict — a /// hand waits for one or stands at the interim, and a resumed box the @@ -692,19 +908,21 @@ pub(crate) struct InterimUpgrade { /// the verdict under the registry's write lock before it publishes — its /// ask by the verdict's own arm, and each sweep only re-publishes what /// still stands. +/// +/// Returns how many boxes the present arm moved off the interim, for the +/// session-start walk's log line. The other arms return `0`. #[cfg(target_os = "linux")] fn land_range_verdict( book: &crate::net::dns::LoopbackLeaseBook, registry: &RwLock, verdict: crate::net::dns::RangeVerdict, -) { +) -> usize { book.set_range_verdict(verdict); match verdict { - crate::net::dns::RangeVerdict::Present => { - for upgrade in draw_interim_upgrades(book, registry) { - apply_interim_upgrade(book, registry, &upgrade); - } - } + crate::net::dns::RangeVerdict::Present => draw_interim_upgrades(book, registry) + .iter() + .filter(|upgrade| apply_interim_upgrade(book, registry, upgrade)) + .count(), crate::net::dns::RangeVerdict::Absent => { // The boxes the window published at reserved-range addresses: // the surface cannot bind what they hold, and their attach @@ -726,10 +944,27 @@ fn land_range_verdict( reg.publish_own_address(session, &name, Ipv4Addr::LOCALHOST, ports.clone()); reg.register_own_ip(session, &name, ports); } + // The node's own address a present landing granted: host-address + // boxes route at it, and the surface cannot bind it now. The + // node's lease stays in the book, so the next present landing + // grants the same address back. + let node = reg.node_address(); + if ::switch::loopback::range_hosts().any(|host| host == node) { + tracing::warn!( + from = %node, + to = %Ipv4Addr::LOCALHOST, + action = "loopback-range-absent-node", + "the node's own loopback address stood inside the reserved \ + local range; the range is absent on the publish surface, \ + so the node's names move to the 127.0.0.1 interim", + ); + reg.set_node_address(Ipv4Addr::LOCALHOST); + } + 0 } // The state a book opens on and the state a test re-opens it in: // nothing has been published under it to sweep. - crate::net::dns::RangeVerdict::Pending => {} + crate::net::dns::RangeVerdict::Pending => 0, } } @@ -1508,6 +1743,11 @@ pub struct ManagerHandle { /// through the actor mainloop. #[cfg(target_os = "linux")] hostnames: Arc>, + /// The forwarder-conducted range walk of a microVM daemon, set at init + /// when the daemon publishes through the host's forwarder. Empty on a + /// native daemon, whose own loopback is its publish surface. + #[cfg(target_os = "linux")] + range_walk: Arc>>, /// The actor's lease book, held only for the one test seam below: a test /// that needs this daemon to be in the state a daemon whose deferred /// probe is still walking holds its book in. Production callers reach @@ -1532,6 +1772,9 @@ pub struct WeakManagerHandle { /// keep the actor alive (only live senders do). #[cfg(target_os = "linux")] hostnames: Arc>, + /// Mirrors [`ManagerHandle::range_walk`]. + #[cfg(target_os = "linux")] + range_walk: Arc>>, /// Mirrors [`ManagerHandle::loopback`], for the same reason: the test seam /// must survive an upgrade. #[cfg(all(test, target_os = "linux"))] @@ -1548,6 +1791,8 @@ impl WeakManagerHandle { in_shutdown: self.in_shutdown.clone(), #[cfg(target_os = "linux")] hostnames: Arc::clone(&self.hostnames), + #[cfg(target_os = "linux")] + range_walk: Arc::clone(&self.range_walk), #[cfg(all(test, target_os = "linux"))] loopback: Arc::clone(&self.loopback), }) @@ -1624,6 +1869,8 @@ impl ManagerHandle { in_shutdown: self.in_shutdown.clone(), #[cfg(target_os = "linux")] hostnames: Arc::clone(&self.hostnames), + #[cfg(target_os = "linux")] + range_walk: Arc::clone(&self.range_walk), #[cfg(all(test, target_os = "linux"))] loopback: Arc::clone(&self.loopback), } @@ -1696,6 +1943,48 @@ impl ManagerHandle { self.loopback.reset_hand_verdict_deadline(millis); } + /// The session-start range walk (NET-123) of a daemon that publishes + /// through the host's forwarder: the forwarder walks the reserved local + /// range again at every session start, whatever verdict landed last, + /// and lands what it finds. A present landing moves every box that + /// stands at the `127.0.0.1` interim onto its range address, the same + /// present arm [`Self::land_range_verdict`] runs. So a host that gained + /// the range after the daemon started (`min finalize-install`) + /// publishes the next box in the range, without a VM restart. An absent + /// landing after a present one moves every box in the range onto the + /// interim, because the range can go away under a running VM. + /// + /// `None` on a native daemon: its own loopback is the publish surface, + /// and the create handler's local bind probe is the honest one there. + #[cfg(target_os = "linux")] + pub(crate) async fn session_start_range_walk(&self) -> Option { + let walk = Arc::clone(self.range_walk.get()?); + Some(walk.session_start().await) + } + + /// Makes this daemon a forwarder-walking one whose walk is `walk`, a + /// stand-in for the host forwarder's bind of the reserved local range. + /// The create handler then asks [`Self::session_start_range_walk`], as + /// it does on a microVM daemon, and the stand-in decides the verdict. + /// + /// Test-only: a production daemon gets its walk from its switch + /// transport at init. Panics if a walk is already installed. + #[cfg(all(test, target_os = "linux"))] + pub(crate) fn install_range_walk_standin( + &self, + walk: impl Fn() -> crate::net::loopback::RangeProbe + Send + Sync + 'static, + ) { + let walk = RangeWalk::new( + RangeWalkSource::StandIn(Arc::new(walk)), + Arc::clone(&self.loopback), + Arc::clone(&self.hostnames), + ); + assert!( + self.range_walk.set(Arc::new(walk)).is_ok(), + "a range walk is already installed on this daemon" + ); + } + /// How many registrations are waiting on the range verdict right now /// ([`crate::net::dns::LoopbackLeaseBook::verdict_waiters`]), so a test /// lands the verdict only once a registration provably waits for it. diff --git a/docs/specs/18-spec-box-networking/18-spec-box-networking.md b/docs/specs/18-spec-box-networking/18-spec-box-networking.md index 60a8c9afc0..3a4aad6e57 100644 --- a/docs/specs/18-spec-box-networking/18-spec-box-networking.md +++ b/docs/specs/18-spec-box-networking/18-spec-box-networking.md @@ -334,7 +334,7 @@ included, with every refusal logged (NET-001 to NET-004). verify: cargo nextest run -p minimald network_none_attach_works -- **NET-040** WHEN `min session activate --network own_ip --ingress 8080:8080` is run on a fresh install THE SYSTEM SHALL publish `127.0.0.1:8080` on the host so that a request to it returns the in-box server's response. +- **NET-040** WHEN `min session activate --network own_ip --ingress 8080:8080` is run on a fresh install THE SYSTEM SHALL publish the box's port 8080 on the host at the loopback address its name resolves to, its own reserved-range address (NET-010) or `127.0.0.1` under NET-123's interim, so that a request to it returns the in-box server's response. tier: T0 verify: ./scripts/session-e2e.sh fresh_install_own_ip_ingress_publishes_loopback diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index 5ebc4893d4..8be35b1729 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -4492,10 +4492,11 @@ proof_vm_host_row_read_tracks_create_and_destroy() { # record in the daemon's log per exposed mapping (host, port, session). # # Gating, decided by where the proof's pieces actually run: -# * Native Linux only. A VM lane's daemon runs in the guest while the pair -# this proof installs lives host-side (macOS is always VM-backed), so -# there is no fresh-install story to drive there — those lanes' -# own-address boxes are the proxy proof's own-address half. +# * The install half is native Linux only. A VM lane's daemon runs in the +# guest while the pair this proof installs lives host-side. On the macOS +# lane the case runs fresh_install_range_lands_on_macos instead: the +# range that `min finalize-install` applies lands on the running VM. A +# Linux VM lane skips. # * The box needs a TUN device: its session program opens /dev/net/tun for # its in-namespace tap, and a host that is itself a sandbox (the # agent-runtime boxes this harness runs on) has none and cannot mknod one, @@ -4535,6 +4536,11 @@ local fi_h_minimald fi_h_minimal fi_h_gvmin fi_sid fi_sid2 fi_log fi_rec local fi_ready fi_answered fi_status fi_bucket_host fi_hport fi_portpat local fi_host fi_host2 fi_pkg_log local fi_lane_minimald fi_restore_profile +if [ -n "$E2E_VM" ] && [ "$(uname -s)" = Darwin ]; then + # The macOS lane's half: the range lands on a running VM (NET-010/NET-123). + fresh_install_range_lands_on_macos + return $? +fi if [ -n "$E2E_VM" ] || [ "$(uname -s)" != Linux ]; then echo "fresh-install loopback publish SKIPPED (VM-backed lane: the pair this proof installs lives host-side)" return 0 @@ -4949,6 +4955,181 @@ echo "fresh-install loopback publish OK (installed pair, shipped switch, host-lo echo "::endgroup::" } +# The fresh-install case on the macOS lane (NET-010, NET-040, NET-123): the +# VM is already up, and its daemon walked the host's loopback at its start. +# The case runs the script `min finalize-install` runs, which applies the +# reserved local range to lo0. Then, WITHOUT a VM restart, a new own-IP box +# with `--ingress 8080:8080` must publish at its own 127.0.64.N address: the +# session start walks the range again, finds it present, and the box's name +# answers there. Three facts are asserted: +# * `http://.min.internal:8080` returns the in-box server's response, +# through the resolver file the same script wrote; +# * the box's own range address (the zone answerer's A record for its name) +# returns the same response at port 8080; +# * `min ls` names that address. +# The host is restored on every exit: the range unit, its aliases, the +# resolver file and the answerer service go (range_teardown_unit, +# answerer_service_teardown — the teardown repeats both if the case dies). +fresh_install_range_lands_on_macos() { + local frm_name="e2e-fi-range" frm_port=8080 frm_marker="FRESH_INSTALL_RANGE_OK" + local frm_seed="" frm_sid="" frm_setup="" frm_cmd="" frm_ready="" frm_aliases=0 + local frm_ans_port="" frm_addr="" frm_status="" frm_body="" frm_rc=0 frm_url="" + echo "::group::fresh install publishes an own-IP ingress at the box's range address on macOS" + if ! command -v launchctl >/dev/null 2>&1 || ! sudo -n true >/dev/null 2>&1; then + echo "fresh-install range publish on macOS SKIPPED (needs launchctl and passwordless sudo to run the script min finalize-install runs)" + echo "::endgroup::" + return 0 + fi + if [ -z "${MINVMD_GVPROXY_BIN:-}" ]; then + echo "fresh-install range publish on macOS SKIPPED (no MINVMD_GVPROXY_BIN: this target has no switch, so no box publishes a port)" + echo "::endgroup::" + return 0 + fi + if [ "$(ifconfig lo0 2>/dev/null | grep -c -- '127\.0\.64\.')" != 0 ]; then + echo "::error::lo0 already carries addresses of 127.0.64.0/24 before the case ran min finalize-install; the case proves the range landing on a running VM, so it needs a host without the range" + fail + fi + + # A box before the range: the VM's daemon read the range absent, so the + # walk this case proves is the one a later session start makes. + frm_setup="$WORK/fi-range-finalize-install.out" + mnl finalize-install --show --script >"$frm_setup" 2>"$frm_setup.note" || true + frm_cmd="$(setup_script_from "$frm_setup")" + case "$frm_cmd" in + '#!/bin/sh'*) ;; + *) + echo "::error::no setup script from 'min finalize-install --show --script' (got: '$frm_cmd')" + cat "$frm_setup" "$frm_setup.note" 2>/dev/null || true + fail + ;; + esac + if [ -f /etc/resolver/min.internal ]; then + RANGE_RESOLVER_BEFORE="$WORK/fi-range-resolver.before" + cp /etc/resolver/min.internal "$RANGE_RESOLVER_BEFORE" + fi + # Record every undo before the run, so a half-failed script still leaves + # the teardown its targets. + RANGE_INSTALLED=yes + ANSWERER_SERVICE_CHANNEL="$(answerer_channel_of "$frm_cmd")" + # shellcheck disable=SC2024 # the output files are this user's, not root's + if ! sudo -n sh "$frm_setup" >"$WORK/fi-range-cmd.out" 2>"$WORK/fi-range-cmd.err"; then + echo "::error::the script min finalize-install runs did not run" + cat "$WORK/fi-range-cmd.out" "$WORK/fi-range-cmd.err" 2>/dev/null || true + fail + fi + for _ in $(seq 1 40); do + frm_aliases="$(ifconfig lo0 2>/dev/null | grep -c -- '127\.0\.64\.')" + [ "$frm_aliases" = 254 ] && break + sleep 0.25 + done + if [ "$frm_aliases" != 254 ]; then + echo "::error::the script applied $frm_aliases of 254 range addresses to lo0" + fail + fi + echo "min finalize-install applied the range: 254 of 254 addresses on lo0; the VM keeps running" + + # The new box, on the VM that read the range absent at its start. + frm_seed="$(hook_mktemp /tmp/mnlfr.XXXXXX)" + hook_seed_preamble > "$frm_seed/minimal.toml" + mkdir "$frm_seed/.git" + frm_sid="$(cd "$frm_seed" && mnl session activate . --no-prompt \ + --name "$frm_name" --network own_ip --ingress "$frm_port:$frm_port" \ + 2>"$WORK/fi-range-activate.err")" || { + echo "::error::'min session activate --network own_ip --ingress $frm_port:$frm_port' failed after min finalize-install" + cat "$WORK/fi-range-activate.err" 2>/dev/null || true + fail + } + frm_sid="$(printf '%s\n' "$frm_sid" | tail -n1 | tr -d '\r')" + mnl session exec "$frm_sid" \ + "body=$frm_marker; printf \"HTTP/1.1 200 OK\r\nContent-Length: \${#body}\r\nConnection: close\r\n\r\n%s\" \"\$body\" > /home/fr200" \ + >/dev/null 2>"$WORK/fi-range-responder.err" || { + echo "::error::could not write the box's responder response" + cat "$WORK/fi-range-responder.err" 2>/dev/null || true + fail + } + mnl session exec "$frm_sid" \ + "nohup /usr/bin/socat TCP-LISTEN:$frm_port,reuseaddr,fork SYSTEM:\"cat /home/fr200\" >/dev/null 2>&1 &" \ + >/dev/null 2>"$WORK/fi-range-responder.err" || { + echo "::error::could not start the box's responder" + cat "$WORK/fi-range-responder.err" 2>/dev/null || true + fail + } + for _ in $(seq 1 40); do + if [ "$(mnl session exec "$frm_sid" \ + "curl -sS --max-time 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:$frm_port/" \ + 2>/dev/null || true)" = "200" ]; then + frm_ready=1; break + fi + sleep 0.25 + done + [ -n "$frm_ready" ] || { + echo "::error::the box's responder never answered inside its own box" + fail + } + + # The box's own address: the zone answerer's A record for its name. + for _ in $(seq 1 40); do + frm_ans_port="$(mnl ls 2>/dev/null \ + | sed -n 's/^DNS SERVICE:.*listening on 127\.0\.0\.1:\([0-9][0-9]*\) (UDP).*/\1/p' \ + | head -n1)" + [ -n "$frm_ans_port" ] && break + sleep 0.25 + done + if [ -z "$frm_ans_port" ]; then + echo "::error::'min ls' carries no DNS SERVICE line after min finalize-install, so the box's address cannot be asked for" + mnl ls 2>&1 | head -40 || true + fail + fi + frm_addr="$(zone_answerer_a "$frm_ans_port" "$frm_name.min.internal" 2>"$WORK/fi-range-ans.err")" || true + case "$frm_addr" in + 127.0.64.*) ;; + *) + echo "::error::$frm_name.min.internal answers ${frm_addr:-nothing}, not an address of 127.0.64.0/24: the session start did not land the range on the running VM (NET-123's re-walk), so the box stands at the 127.0.0.1 interim" + cat "$WORK/fi-range-ans.err" 2>/dev/null || true + fail + ;; + esac + echo "the zone answers $frm_name.min.internal with $frm_addr, an address of the range" + + # Both URLs, with every proxy variable stripped: no client needs setup. + for frm_url in "http://$frm_name.min.internal:$frm_port/" "http://$frm_addr:$frm_port/"; do + frm_status=""; frm_body="" + for _ in $(seq 1 20); do + env -u HTTP_PROXY -u HTTPS_PROXY -u http_proxy -u https_proxy \ + -u ALL_PROXY -u all_proxy -u NO_PROXY -u no_proxy \ + curl -sS --max-time 8 -o "$WORK/fi-range.body" -w 'HTTP:%{http_code}' \ + "$frm_url" >"$WORK/fi-range.out" 2>"$WORK/fi-range.err" + frm_rc=$? + frm_status="$(cat "$WORK/fi-range.out" 2>/dev/null)" + frm_body="$(cat "$WORK/fi-range.body" 2>/dev/null || true)" + [ "$frm_rc" -eq 0 ] && [ "$frm_status" = "HTTP:200" ] \ + && [[ "$frm_body" == *"$frm_marker"* ]] && break + sleep 0.5 + done + echo "GET $frm_url -> ${frm_status:-} ${frm_body:-} (curl exit $frm_rc)" + if [ "$frm_rc" -ne 0 ] || [ "$frm_status" != "HTTP:200" ] \ + || [[ "$frm_body" != *"$frm_marker"* ]]; then + echo "::error::GET $frm_url did not return the in-box server's response" + cat "$WORK/fi-range.err" 2>/dev/null || true + fail + fi + done + + # `min ls` names the address the box answers at. + if ! mnl ls 2>/dev/null | grep -qF -- "$frm_addr"; then + echo "::error::'min ls' does not name $frm_addr, the address $frm_name.min.internal answers at" + mnl ls 2>&1 | head -40 || true + fail + fi + echo "'min ls' names $frm_addr" + + mnl session destroy --force "$frm_sid" >/dev/null 2>&1 || true + range_teardown_unit + answerer_service_teardown + echo "fresh-install range publish on macOS OK (min finalize-install on a running VM, $frm_name.min.internal and $frm_addr answer at :$frm_port, min ls names the address, host restored)" + echo "::endgroup::" +} + # --------------------------------------------------------------------------- # The network postures of a stock install, end to end (the S2 integration # case). From a REAL scripts/install.sh run into a fresh HOME — the same From c36cd6e2ad68399cf57265f0d3b025617adb8bd9 Mon Sep 17 00:00:00 2001 From: Norrie Taylor <91171431+norrietaylor@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:22:38 -0700 Subject: [PATCH 10/34] fix(minimald): move a box's forward to its range address when the range lands present (#2749) Fixes a regression from T98. When a box's range probe lands present, its declared forwards, its min net expose forwards and its listen-published forwards move from the 127.0.0.1 interim to the range address, so the name and the forward answer at one address. The host egress gate admits the bounded range probe, the walk is contention-safe, and VM host-net boxes dial their listeners at the guest's 127.0.0.1. Backport of #2749. Adapted to the line: - box_actor.rs and box_actor/{handle,ports}.rs hunks land in session.rs, box_manager.rs and box_manager/handle.rs hunks in sessions.rs, and box_actor/tests.rs in session/tests.rs. The line has no box_actor/kind.rs, so its message-name hunk has no target. - crate::traced becomes tokio::spawn, and box names become session names throughout. egress_gate.rs gains the tracing::Instrument import that main already had. - ANONYMOUS_SESSION becomes pub(crate) for the actor's move log lines. - The tests' port 18080 reads declared_port(), matching #2752, which landed earlier here. - ListenPlan gains the hostnames and session_id fields the move reads. On main they came with NET-129's dynamic holds, which the line lacks; only the two fields come here. Dropped because the line has no NET-129 dynamic holds: - ListenPlan's record, release and move hold helpers, their calls in the watcher, and the bound_address helpers that fed them. A listen forward's move needs none of them: a port another box holds by declaration still stops the move, read through shared_port_holder. - The test a_listen_port_whose_move_failed_releases_its_hold_where_it_was_bound. - The dynamic-hold assertions in the runtime-publications landing test and in a_listen_port_does_not_move_onto_a_port_another_box_holds. - The host_net.insert in register_host_net: that field arrives with T106, next. (cherry picked from commit aa6964d55fac46ba980d70f0de797f4b853dbf6d) --- crates/minimald/src/net/dns.rs | 161 +++- crates/minimald/src/net/gvproxy_network.rs | 10 +- crates/minimald/src/net/listeners.rs | 223 ++++- crates/minimald/src/net/policy.rs | 126 ++- crates/minimald/src/net/provider.rs | 119 ++- crates/minimald/src/rpc.rs | 2 +- crates/minimald/src/session.rs | 226 ++++- crates/minimald/src/session/tests.rs | 687 +++++++++++++++ crates/minimald/src/session_host.rs | 4 + crates/minimald/src/sessions.rs | 73 ++ crates/minvmd/src/net/egress_gate.rs | 925 ++++++++++++++++++++- crates/switch/src/loopback.rs | 11 + scripts/session-e2e.sh | 43 +- 13 files changed, 2535 insertions(+), 75 deletions(-) diff --git a/crates/minimald/src/net/dns.rs b/crates/minimald/src/net/dns.rs index a5f1726834..a231f3debe 100644 --- a/crates/minimald/src/net/dns.rs +++ b/crates/minimald/src/net/dns.rs @@ -238,7 +238,16 @@ enum Target { /// an `OwnIp` box's published-loopback forwarder, where the requested /// port is the published external one and `address` is the host loopback /// address a creator handed the box's own ports to publish at (NET-010). - Loopback { address: Ipv4Addr }, + /// `dial` is where the proxy connects for it when that is not `address`: + /// on a VM node `address` is the node's address on the *host* loopback, + /// where the host's forwarder mirrors the guest's listeners (NET-129). + /// The proxy runs in the guest beside those listeners, so it dials them + /// at the guest's own `127.0.0.1` wherever the node's address moves + /// (#2748). `None` dials `address`. + Loopback { + address: Ipv4Addr, + dial: Option, + }, /// An `OwnIp` box at its lease on the switch (a VM host): the daemon is on /// the switch, so a request reaches the box itself instead of the guest /// loopback. `ports` is the box's ingress declaration as an @@ -261,7 +270,10 @@ impl Route { declared: Option>, ) -> Self { Self { - target: Target::Loopback { address }, + target: Target::Loopback { + address, + dial: None, + }, session: session.into(), declared, runtime: BTreeSet::new(), @@ -348,7 +360,9 @@ impl Route { pub fn upstream(&self, port: u16) -> Option { let source = self.source(port)?; match (&self.target, source) { - (Target::Loopback { address }, _) => Some(SocketAddr::new(IpAddr::V4(*address), port)), + (Target::Loopback { address, dial }, _) => { + Some(SocketAddr::new(IpAddr::V4(dial.unwrap_or(*address)), port)) + } (Target::Lease { lease, ports }, RouteSource::Declared | RouteSource::Ungated) => { let internal = ports.get(&port).copied()?; Some(SocketAddr::new(IpAddr::V4(*lease), internal)) @@ -375,7 +389,7 @@ impl Route { #[must_use] pub fn address(&self) -> Ipv4Addr { match &self.target { - Target::Loopback { address } => *address, + Target::Loopback { address, .. } => *address, Target::Lease { lease, .. } => *lease, } } @@ -387,7 +401,7 @@ impl Route { /// host-loopback fact and never carries the node's address, and a box's /// own granted address is not the node's either. pub(crate) fn repoint(&mut self, from: Ipv4Addr, to: Ipv4Addr) { - if let Target::Loopback { address } = &mut self.target + if let Target::Loopback { address, .. } = &mut self.target && *address == from { *address = to; @@ -450,11 +464,31 @@ struct OwnPublished { ports: BTreeSet, } +/// A box's publish moved to a new address (#2748): the present landing moved +/// it off the `127.0.0.1` interim ([`HostnameRegistry::hand_over_moved_publish`]). +/// Sent as data so the receiver can bind the box's declared ports at the new +/// address and move the forwards its runtime `min net expose` bound at the +/// old one, while the box runs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublishMove { + /// The session whose box moved. + pub holder: SessionId, + /// The address the box published at before the move. + pub from: Ipv4Addr, + /// The address the box publishes at now. + pub to: Ipv4Addr, + /// The declared ports the box holds at `to`, one hand-over each: a port + /// another box holds there is left out, since the box yields it. + pub hand_overs: Vec, +} + /// A shared-address port passing to a new holder (NET-129): the holder's /// publish was withdrawn, left the address, or stopped naming the port, and /// the box created earliest among the running boxes still publishing it -/// there took it — or among the stopped ones, when none is running. -/// Returned as data so the caller can bind the new holder's forward. +/// there took it — or among the stopped ones, when none is running. The +/// present landing makes one too, for a box it moved off the `127.0.0.1` +/// interim, and sends it inside the move ([`PublishMove`]). Returned as data +/// so the caller can bind the new holder's forward. #[derive(Debug, Clone, PartialEq, Eq)] pub struct PortHandOver { /// The session whose box holds the port now. @@ -653,6 +687,12 @@ pub struct HostnameRegistry { /// box runs ([`Self::notify_hand_overs`]). `None` leaves a hand-over a /// change of record alone, bound at the new holder's next attach. hand_overs: Option>, + /// Where each move of a box's publish is sent + /// ([`Self::hand_over_moved_publish`]), for the sessions manager to move + /// the forwards the box's `min net expose` bound while its box runs + /// ([`Self::notify_publish_moves`]). `None` leaves those forwards where + /// they are until the box's spawn ends. + publish_moves: Option>, /// The loopback address a creator handed each own-address box, by stable /// session id (T66, NET-123 §7.1): recorded at the box's registration /// whether the publish lands at it or at the `127.0.0.1` interim — a @@ -724,6 +764,7 @@ impl HostnameRegistry { own_holders: HashMap::new(), own_bound: HashMap::new(), hand_overs: None, + publish_moves: None, hands: HashMap::new(), stopped: HashSet::new(), by_host: HashMap::new(), @@ -753,6 +794,25 @@ impl HostnameRegistry { self.hand_overs = Some(sender); } + /// Sends every move of a box's publish from here on to `sender` + /// (#2748): the receiver moves the forwards the box's runtime + /// `min net expose` bound at the old address, if its box is running. + /// The send never waits, so the registry's lock is held no longer for + /// it. + pub fn notify_publish_moves( + &mut self, + sender: tokio::sync::mpsc::UnboundedSender, + ) { + self.publish_moves = Some(sender); + } + + /// The box holding `port` at `address` of record (NET-129): what the + /// listen watcher reads before it moves a port to a new address (#2748). + #[must_use] + pub(crate) fn shared_port_holder(&self, address: Ipv4Addr, port: u16) -> Option { + self.own_holders.get(&(address, port)).copied() + } + /// Whether `session_id` holds `port` at `address` now (NET-129): it /// still publishes the port there, and no other box holds it. What a /// hand-over's receiver reads before it binds, since the hand-over may @@ -766,6 +826,55 @@ impl HostnameRegistry { .is_some_and(|own| own.address == address && own.ports.contains(&port)) } + /// Sends the move of the box `session_id`'s publish from `from` to the + /// address it publishes at now (NET-123 §7.1, #2748), with one hand-over + /// per declared port it holds there, and returns the hand-overs. The + /// present landing calls it for each box it moves off the `127.0.0.1` + /// interim: a box that runs has its forwards bound at the interim, so + /// the name would answer at an address no forward listens on. A port + /// another box holds at the address is left out: the box yields it. + /// + /// The move goes to [`Self::notify_publish_moves`]'s receiver, never to + /// [`Self::notify_hand_overs`]'s. The receiver passes it on only to a + /// box whose actor runs, which binds each declared port at the new + /// address and moves the forwards its `min net expose` bound at `from`. + /// A box with no actor gets nothing and binds at the new address at its + /// next attach. The listen watcher reads the box's published address + /// here on every poll, and moves the ports it published itself. + pub fn hand_over_moved_publish( + &self, + session_id: SessionId, + from: Ipv4Addr, + ) -> Vec { + let Some(own) = self.own_published.get(&session_id) else { + return Vec::new(); + }; + let hand_overs: Vec = own + .ports + .iter() + .filter(|&&port| self.holds_shared_port(session_id, own.address, port)) + .map(|&port| PortHandOver { + holder: session_id, + address: own.address, + port, + }) + .collect(); + if own.address != from + && let Some(sender) = &self.publish_moves + { + // A receiver that is gone moves nothing: the box binds its + // declared ports at the new address at its next attach, and its + // runtime forwards come down with its spawn. + drop(sender.send(PublishMove { + holder: session_id, + from, + to: own.address, + hand_overs: hand_overs.clone(), + })); + } + hand_overs + } + /// Sets the node's host loopback address once the answerer granted it — /// the half of [`Self::with_node_address`] a daemon whose range verdict /// landed *after* its registry was built needs (a microVM daemon: its @@ -850,12 +959,19 @@ impl HostnameRegistry { /// rejects one on every network mode but `own_ip` — and a direct /// connection to it is ungated, so the proxy gates nothing either /// (NET-071). + /// + /// On a VM node (`on_switch`) the name answers at the node's host-side + /// address, where the host mirrors the guest's listeners, and the proxy, + /// which runs in the guest, dials those listeners at the guest's own + /// `127.0.0.1` (#2748). pub fn register_host_net(&mut self, session_id: SessionId, session_name: &str) -> Hostname { - self.register( - session_id, - session_name, - Route::loopback(session_name, self.node, None), - ) + let mut route = Route::loopback(session_name, self.node, None); + if self.on_switch + && let Target::Loopback { dial, .. } = &mut route.target + { + *dial = Some(Ipv4Addr::LOCALHOST); + } + self.register(session_id, session_name, route) } /// Registers an `OwnIp` PTask's box name at the address its declaration @@ -3078,7 +3194,9 @@ mod tests { ); // A VM node: the node holds one allocated address of its own, and its - // host-address boxes answer at it. + // host-address boxes answer at it. The host's forwarder mirrors the + // guest's listeners there, and the proxy, in the guest beside them, + // dials them at the guest's own loopback (#2748). let node = Ipv4Addr::new(127, 0, 64, 200); let mut vm = HostnameRegistry::new("dev", true).with_node_address(node); vm.register_host_net(id("2"), "web"); @@ -3093,14 +3211,19 @@ mod tests { let route = vm .resolve("web.min.internal:8080") .expect("a host-address box's name routes"); + assert_eq!( + route.address(), + node, + "the route answers at the node's address" + ); assert_eq!( route.upstream(8080), - Some(SocketAddr::new(IpAddr::V4(node), 8080)), - "the box's own port number at the node's allocated address" + Some(SocketAddr::new(loopback_addr(), 8080)), + "the proxy dials the box's own port number on the guest's loopback" ); assert_eq!( route.upstream(9090), - Some(SocketAddr::new(IpAddr::V4(node), 9090)), + Some(SocketAddr::new(loopback_addr(), 9090)), "a host-address route gates no port, so no port is translated" ); } @@ -3141,8 +3264,10 @@ mod tests { reg.resolve("shared.min.internal:8080") .expect("a host-address box's name routes") .upstream(8080), - Some(SocketAddr::new(IpAddr::V4(node), 8080)), - "the proxy forwards a host-address box at the granted address too" + Some(SocketAddr::new(loopback_addr(), 8080)), + "the proxy, in the guest, still dials the host-address box on the guest's \ + own loopback: the granted address is the host's, where its forwarder \ + mirrors the guest's listeners (#2748)" ); assert_eq!( reg.resolve("handed.min.internal:18080") diff --git a/crates/minimald/src/net/gvproxy_network.rs b/crates/minimald/src/net/gvproxy_network.rs index 8bb6983f16..8bd3faf2b0 100644 --- a/crates/minimald/src/net/gvproxy_network.rs +++ b/crates/minimald/src/net/gvproxy_network.rs @@ -541,6 +541,13 @@ async fn finish_own_ip_attach( own_address.report(session_name, lease_ip, ports); } + let runtime_ingress = + own_address.map(crate::net::provider::OwnAddressReporter::runtime_ingress); + // The box's actor unbinds a declared forward a move of its publish left + // at the old address (#2748), so it holds clones of the attach's. + if let Some(runtime_ingress) = &runtime_ingress { + runtime_ingress.record_attached(&exposed); + } Ok(OwnIpGuard { attach: Some(OwnIpAttach { _relay: relay, @@ -548,8 +555,7 @@ async fn finish_own_ip_attach( control, exposed, lease, - runtime_ingress: own_address - .map(crate::net::provider::OwnAddressReporter::runtime_ingress), + runtime_ingress, }), }) } diff --git a/crates/minimald/src/net/listeners.rs b/crates/minimald/src/net/listeners.rs index 13b01adf05..ef5304c19d 100644 --- a/crates/minimald/src/net/listeners.rs +++ b/crates/minimald/src/net/listeners.rs @@ -126,7 +126,7 @@ use std::net::Ipv4Addr; use std::path::{Path, PathBuf}; #[cfg(test)] use std::sync::LazyLock; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; use tokio::sync::watch; @@ -904,6 +904,13 @@ pub struct ListenPlan { /// The daemon's state directory: the audit log the publication's /// decision record is appended to lives under it ([`crate::audit`]). state_dir: PathBuf, + /// The hostname registry, the one source of the address the box's name + /// answers at, which the watcher follows when the box moves (#2748). + /// `None` when the launch carried no reporter: a box with no registry + /// behind it stays where it published. + hostnames: Option>>, + /// The box's own stable id, keying the registry's tables. + session_id: sessions::SessionId, } impl ListenPlan { @@ -924,6 +931,8 @@ impl ListenPlan { gate: Arc, publications: BoxPublications, state_dir: PathBuf, + hostnames: Option>>, + session_id: sessions::SessionId, ) -> Self { Self { box_name, @@ -933,8 +942,35 @@ impl ListenPlan { gate, publications, state_dir, + hostnames, + session_id, } } + + /// The address the hostname registry publishes the box at now + /// (NET-010), the source the plan's own `published` was read from at + /// the launch. `None` when the launch carried no registry, or the box + /// publishes nowhere any more. + fn registry_published(&self) -> Option { + self.hostnames.as_ref().and_then(|registry| { + registry + .read() + .expect("hostname registry lock poisoned") + .published_own_address(self.session_id) + }) + } + + /// The box holding `port` at the plan's published address of record + /// (NET-129). `None` when nobody holds it, or the launch carried no + /// registry. + fn holder_at_published(&self, port: u16) -> Option { + self.hostnames.as_ref().and_then(|registry| { + registry + .read() + .expect("hostname registry lock poisoned") + .shared_port_holder(self.published, port) + }) + } } /// Which of the box's two runtime ingress surfaces owns a publication: the @@ -1163,6 +1199,22 @@ impl BoxPublications { } } + /// Re-points the row `owner`'s listed publication of `port` carries at + /// `local`, the forward's new `host:port` after the box's published + /// address moved (#2748). A publication with no row stays unlisted: + /// the move never lists a port its gate refuses. + pub fn relist_local(&self, port: u16, owner: PublicationOwner, local: &str) { + let mut set = self.set.lock().expect("box publications lock poisoned"); + if let Some(row) = set + .ports + .get_mut(&port) + .filter(|entry| entry.owner == owner && entry.published) + .and_then(|entry| entry.listed.as_mut()) + { + row.local = local.to_string(); + } + } + /// Marks `port` as a permitted listen the watcher could not publish /// because the audit log refused its record (`refused`), or clears the /// mark once the record is written or the listener is gone. @@ -1527,6 +1579,14 @@ struct WatchState { /// is written — an allow nobody can read is never admitted, not even /// twice. The entry goes when the forward comes down, recorded or not. unrecorded: HashSet, + /// Forwards left at the box's old published address after a move + /// (#2748) because their unexpose failed. Each poll asks again, and the + /// stop's withdrawal passes do too. + retired: Vec, + /// The ports whose failed move has been said once already: a bind at + /// the new address that keeps failing is retried on every poll, and is + /// the same failure each time. + reported_move_failures: HashSet, } /// One permitted port whose publish the switch refused: the book @@ -1595,9 +1655,147 @@ impl WatchState { reported_contention: HashSet::new(), reported_withdrawal_failures: HashSet::new(), unrecorded: HashSet::new(), + retired: Vec::new(), + reported_move_failures: HashSet::new(), + } + } + + /// Follows the box's published address when it moved (#2748): the + /// present landing moves a box off the `127.0.0.1` interim while it + /// runs, and the registry is the one source of the address the box's + /// name answers at. The plan takes the new address, so every publish + /// from here binds there. Each forward this watcher bound at another + /// address is bound again at the new one, toward the same lease, and + /// the old one is unbound after it, so the port is served the whole way + /// through. The box's gate and the VM host daemon's row name the port, + /// not the address, so neither changes. + /// + /// A bind at the new address that fails leaves the forward where it is, + /// and the next poll asks again. An unexpose at the old address that + /// fails is asked again on every poll and at the stop. + async fn follow_published_address(&mut self) { + if let Some(moved) = self.plan.registry_published() + && moved != self.plan.published + { + tracing::info!( + session = %self.plan.box_name, + from = %self.plan.published, + to = %moved, + "the box's published address moved; its listening ports follow" + ); + self.plan.published = moved; + } + let host = self.plan.published.to_string(); + let mut stray: Vec = self + .forwards + .iter() + .filter(|(_, mapping)| mapping.host_port().is_none_or(|(bound, _)| bound != host)) + .map(|(port, _)| *port) + .collect(); + stray.sort_unstable(); + for port in stray { + self.move_forward(port).await; + } + self.unbind_retired().await; + } + + /// Binds the forward of `port` again at the plan's published address + /// and unbinds the one at the address it was bound at before + /// ([`Self::follow_published_address`]). + async fn move_forward(&mut self, port: u16) { + let Some(old) = self.forwards.get(&port).cloned() else { + return; + }; + // Another box holding the port at the new address keeps it, as a + // declared publish there would (NET-129, first-come): the forward + // stays where it is, and a later poll moves it once that box gives + // the port up. The `127.0.0.1` interim an absent landing moves the + // box to is shared, and a stopped box's hold stands there with no + // forward behind it. + if let Some(holder) = self.plan.holder_at_published(port) + && holder != self.plan.session_id + { + if self.reported_move_failures.insert(port) { + tracing::info!( + session = %self.plan.box_name, + host = %self.plan.published, + port, + from = old.local(), + other = %holder, + owner = %PublicationOwner::Listen.as_str(), + "another box holds the listening port at the box's new address; \ + it stays at the old address until that box gives it up" + ); + } + return; + } + let moved = match expose_mapping( + &self.plan.control, + self.plan.published, + self.plan.lease, + port, + ) + .await + { + Ok(moved) => moved, + Err(error) => { + if self.reported_move_failures.insert(port) { + tracing::warn!( + session = %self.plan.box_name, + host = %self.plan.published, + port, + from = old.local(), + owner = %PublicationOwner::Listen.as_str(), + %error, + "binding a listening port at the box's new address failed; \ + retrying on every poll" + ); + } + return; + } + }; + self.reported_move_failures.remove(&port); + self.plan + .publications + .relist_local(port, PublicationOwner::Listen, moved.local()); + self.forwards.insert(port, moved); + tracing::info!( + session = %self.plan.box_name, + host = %self.plan.published, + port, + from = old.local(), + owner = %PublicationOwner::Listen.as_str(), + "moved a listening port to the box's new address" + ); + if let Err(error) = unexpose_mapping(&self.plan.control, &old).await { + tracing::warn!( + session = %self.plan.box_name, + port, + from = old.local(), + owner = %PublicationOwner::Listen.as_str(), + %error, + "unbinding a listening port at the box's old address failed; \ + retrying on every poll" + ); + self.retired.push(old); } } + /// Asks again for the unexpose of each forward a move left at the old + /// address ([`Self::move_forward`]). Said once, when the move failed it. + async fn unbind_retired(&mut self) { + let mut still = Vec::new(); + for mapping in std::mem::take(&mut self.retired) { + if unexpose_mapping(&self.plan.control, &mapping) + .await + .is_err() + { + still.push(mapping); + } + } + self.retired = still; + } + /// The PID this poll reads the box's table through, resolving the leader /// first when it is still owed: the resolution is retried on every poll /// that fails, so a refusal that passes costs the moment between two @@ -1657,6 +1855,9 @@ impl WatchState { /// into a permitted port that stays unpublished until its server /// restarts. async fn poll(&mut self) { + // The publications follow the box's address before anything else: + // a move needs no table read, only the forwards already standing. + self.follow_published_address().await; // The leader first, when the box's host could not resolve it. let Some(leader) = self.resolve_leader() else { return; @@ -2573,7 +2774,8 @@ impl WatchState { /// about. async fn withdraw_all(&mut self) { for attempt in 0..WITHDRAW_PASSES { - if self.forwards.is_empty() { + self.unbind_retired().await; + if self.forwards.is_empty() && self.retired.is_empty() { return; } if attempt > 0 { @@ -2586,14 +2788,19 @@ impl WatchState { self.close(port, "box stopped").await; } } - if self.forwards.is_empty() { + if self.forwards.is_empty() && self.retired.is_empty() { // The last pass brought the last forward down: every port the // box's processes published is unpublished, and a warning that // named none of them would only say the stop happened — which // the stop itself already says. return; } - let still: Vec<&str> = self.forwards.values().map(ExposedMapping::local).collect(); + let still: Vec<&str> = self + .forwards + .values() + .chain(&self.retired) + .map(ExposedMapping::local) + .collect(); tracing::warn!( session = %self.plan.box_name, still_published = ?still, @@ -2945,6 +3152,8 @@ mod tests { Arc::clone(&gate), BoxPublications::default(), state_dir, + None, + sessions::SessionId::nil(), ); (ListenWatcher::start(plan, leader), gate) } @@ -3019,6 +3228,8 @@ mod tests { Arc::clone(&gate), publications, dir.path().to_path_buf(), + None, + sessions::SessionId::nil(), ); let watcher = ListenWatcher::start(plan, Leader::Resolved(std::process::id())); (watcher, gate, server, served) @@ -3435,6 +3646,8 @@ mod tests { Arc::clone(&gate), publications.clone(), dir.path().to_path_buf(), + None, + sessions::SessionId::nil(), ), Leader::Resolved(std::process::id()), ); @@ -4999,6 +5212,8 @@ mod tests { Arc::clone(&gate), BoxPublications::default(), dir.path().to_path_buf(), + None, + sessions::SessionId::nil(), ), Leader::Resolved(std::process::id()), ); diff --git a/crates/minimald/src/net/policy.rs b/crates/minimald/src/net/policy.rs index 6413439cd1..9615cb4083 100644 --- a/crates/minimald/src/net/policy.rs +++ b/crates/minimald/src/net/policy.rs @@ -316,6 +316,24 @@ impl PortForwarder { } } } + + /// Unbinds the forward a move of the box's publish left at the old + /// address (#2748). The gate keeps admitting the port, since the forward + /// bound at the new address serves it now. The forwarder is marked + /// revoked once the unbind stands, so the attach's teardown, which holds + /// a clone, asks nothing more of it. + /// + /// # Errors + /// + /// Returns the I/O error from the failing `unexpose` call. + pub async fn unbind_moved(&self, control: &ControlChannel) -> io::Result<()> { + if self.is_revoked() { + return Ok(()); + } + unexpose_mapping(control, &self.mapping).await?; + self.revoked.store(true, Ordering::Release); + Ok(()) + } } /// Exposes every static port mapping in `ingress` on the switch's `control_sock` @@ -1266,7 +1284,7 @@ async fn probe_publish_surface_within( probe.probed += 1; let expose = ExposeRequest { local: format!("{address}:{RANGE_PROBE_PORT}"), - remote: format!("{}:1", Ipv4Addr::LOCALHOST), + remote: ::switch::loopback::RANGE_PROBE_REMOTE.to_string(), protocol: "tcp".to_string(), }; // Marked before the expose, so a budget cut inside this round @@ -1274,6 +1292,21 @@ async fn probe_publish_surface_within( let leftover = !held.insert(*address); let mut answer = post_json_within(control, "/services/forwarder/expose", &expose, request).await; + // The VM host's egress gate admits one probe round at a time, and + // another one is still bound there (#2748): a contention, not an + // answer about this address. The round waits it out and asks + // again. Contention that outlasts the walk's budget ends the walk + // inconclusive, never absent, so it cannot replace a present + // verdict. + while answer + .as_ref() + .is_err_and(|error| error.kind() == io::ErrorKind::ResourceBusy) + { + tokio::time::sleep(RANGE_PROBE_CONTENTION_WAIT).await; + answer = + post_json_within(control, "/services/forwarder/expose", &expose, request) + .await; + } if leftover && answer .as_ref() @@ -1394,6 +1427,13 @@ async fn probe_publish_surface_within( probe .first_failure .get_or_insert((stuck, io::ErrorKind::TimedOut)); + } else if walk.is_err() + && let Some(last) = hosts.last() + { + // The budget expired inside the last round: the same overrun. + probe + .first_failure + .get_or_insert((*last, io::ErrorKind::TimedOut)); } probe } @@ -1562,6 +1602,12 @@ pub(crate) async fn post_json_within( && (said.contains(ADDRESS_IN_USE_BODY) || said.contains(ALREADY_RUNNING_BODY)) { io::ErrorKind::AddrInUse + } else if status == RANGE_PROBE_CONTENTION_STATUS { + // The VM host's egress gate holds another range probe round and + // admits one at a time (#2748). gvproxy itself never answers 409, + // so this says nothing about the host's loopback: the walk waits + // it out. + io::ErrorKind::ResourceBusy } else { io::ErrorKind::Other }; @@ -1724,8 +1770,10 @@ const MAX_CONTROL_RESPONSE: u64 = 64 * 1024; /// collide on it. That daemon walks at its start and again at each session /// start, and [`RANGE_WALK_TURN`] runs those walks one at a time. A round /// that finds the port already held at an address (a box's forward or a -/// host_ip mirror) counts that address as bound. -const RANGE_PROBE_PORT: u16 = 21064; +/// host_ip mirror) counts that address as bound. The host-side egress gate +/// of a VM-backed host admits each round by this port and +/// [`::switch::loopback::RANGE_PROBE_REMOTE`]. +const RANGE_PROBE_PORT: u16 = ::switch::loopback::RANGE_PROBE_PORT; /// The turn every forwarder-conducted range walk in this process takes /// before its first round, so the daemon-start walk and the session-start @@ -1744,6 +1792,16 @@ static RANGE_WALK_TURN: tokio::sync::Mutex>> /// range reads absent, the interim. const RANGE_PROBE_BUDGET: Duration = Duration::from_secs(20); +/// The status the VM host's egress gate answers a range probe round with +/// while another round is still bound there (#2748). gvproxy itself never +/// answers it. +const RANGE_PROBE_CONTENTION_STATUS: u16 = 409; + +/// How long a contended range probe round waits before it asks again. The +/// gate releases a round left bound within seconds, and the walk's whole +/// budget bounds the waiting. +const RANGE_PROBE_CONTENTION_WAIT: Duration = Duration::from_millis(250); + /// Upper bound on **one** request/response round of the forwarder-conducted /// range walk — tighter than the [`GVPROXY_CONTROL_TIMEOUT`] the publish verbs /// run under, because the probe is a walk, two rounds per address of the range @@ -2641,6 +2699,68 @@ mod tests { ); } + /// #2748: the VM host's egress gate admits one probe round at a time + /// and answers 409 while another is still bound there. That is + /// contention, not an answer about the address: the walk waits it + /// out and asks again, and reads the range present once the round + /// binds. + #[tokio::test] + async fn waits_out_the_gates_outstanding_round_and_reads_the_answer_after_it() { + let dir = tempfile::TempDir::new().unwrap(); + let sock = dir.path().join("gvproxy.sock"); + let contended = format!("{}:{RANGE_PROBE_PORT}", Ipv4Addr::new(127, 0, 64, 5)); + let asked = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let counted = std::sync::Arc::clone(&asked); + let (forwarder, _asked) = + spawn_forwarder_answering(sock.clone(), Duration::ZERO, move |local| { + if local == contended + && counted.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 3 + { + 409 + } else { + 200 + } + }); + let probe = probe_publish_surface(&ControlChannel::Unix(sock)).await; + forwarder.abort(); + assert!( + probe.present(), + "contention is waited out, not read as a missing address: {}", + probe.summary() + ); + assert_eq!(probe.first_failure, None); + assert!(asked.load(std::sync::atomic::Ordering::SeqCst) >= 4); + } + + /// #2748: contention that outlasts the walk's budget ends the walk + /// inconclusive, a fault of the channel, never a refusal. The landing + /// keeps a present verdict over such a walk, so a guest that holds a + /// probe round cannot flip the range to absent. + #[tokio::test] + async fn contention_past_the_budget_reads_inconclusive_never_refused() { + let dir = tempfile::TempDir::new().unwrap(); + let sock = dir.path().join("gvproxy.sock"); + let contended = format!("{}:{RANGE_PROBE_PORT}", Ipv4Addr::new(127, 0, 64, 5)); + let (forwarder, _asked) = + spawn_forwarder_answering(sock.clone(), Duration::ZERO, move |local| { + if local == contended { 409 } else { 200 } + }); + let probe = probe_publish_surface_within( + &ControlChannel::Unix(sock), + Duration::from_secs(2), + RANGE_PROBE_REQUEST_BUDGET, + ) + .await; + forwarder.abort(); + assert!(!probe.present(), "{}", probe.summary()); + assert_eq!( + probe.first_failure, + Some((Ipv4Addr::new(127, 0, 64, 5), io::ErrorKind::TimedOut)), + "contention reads as a fault of the channel, not a refusal: {}", + probe.summary() + ); + } + /// A range address whose probe port something already holds: a box's /// forward through this forwarder (`proxy already running`) at one /// address, a host_ip mirror held by another host process (the OS's diff --git a/crates/minimald/src/net/provider.rs b/crates/minimald/src/net/provider.rs index 4d2bb3c72b..4b546a89c4 100644 --- a/crates/minimald/src/net/provider.rs +++ b/crates/minimald/src/net/provider.rs @@ -68,6 +68,15 @@ struct RuntimeIngressState { /// They deliver to the spawn's lease as its attach's own forwards do, /// and come down with it the same way. handed: Vec, + /// Runtime forwards a move of the box's publish left behind at the old + /// address because their unbind failed (#2748). Nothing lists them, and + /// they come down with the spawn or the box's stop as the live ones do. + retired: Vec, + /// Clones of the declared forwards this spawn's attach bound (NET-121), + /// so a move of the box's publish can unbind the one at the old address + /// (#2748). A clone shares its forwarder's revoked mark, so the attach's + /// teardown skips one unbound here. The attach's guard still owns them. + attached: Vec, } /// The ports a box published at runtime with its own `min net expose` @@ -154,7 +163,45 @@ impl RuntimeIngress { /// The box stopped: takes the forwards bound at a hand-over, for the /// session actor to release as declared forwards are. pub(crate) fn take_handed(&self) -> Vec { - std::mem::take(&mut self.state().handed) + let mut state = self.state(); + state.attached.clear(); + std::mem::take(&mut state.handed) + } + + /// Keeps clones of the declared forwards a spawn's attach bound, for a + /// move of the box's publish to unbind the old one (#2748). + pub(crate) fn record_attached(&self, forwarders: &[crate::net::policy::PortForwarder]) { + self.state().attached = forwarders.to_vec(); + } + + /// The declared forwards still bound at `address:port`, the attach's and + /// the hand-overs' alike: what a move of the box's publish off `address` + /// unbinds once the port's forward stands at the new address (#2748). + pub(crate) fn declared_at( + &self, + address: std::net::Ipv4Addr, + port: u16, + ) -> Vec { + let host = address.to_string(); + let state = self.state(); + state + .attached + .iter() + .chain(&state.handed) + .filter(|forwarder| !forwarder.is_revoked()) + .filter(|forwarder| forwarder.host_port() == Some((host.as_str(), port))) + .cloned() + .collect() + } + + /// Whether a declared forward for `port` bound at a hand-over stands at + /// `address`. + pub(crate) fn holds_handed_at(&self, address: std::net::Ipv4Addr, port: u16) -> bool { + let host = address.to_string(); + self.state() + .handed + .iter() + .any(|forwarder| forwarder.host_port() == Some((host.as_str(), port))) } /// The spawn ended: takes the forwards bound at a hand-over, for the @@ -163,6 +210,7 @@ impl RuntimeIngress { pub(crate) fn detach_handed(&self) -> Vec { let mut state = self.state(); state.detached = true; + state.attached.clear(); std::mem::take(&mut state.handed) } @@ -176,12 +224,76 @@ impl RuntimeIngress { pub(crate) fn detach(&self) -> Vec { let mut state = self.state(); state.detached = true; + let retired = std::mem::take(&mut state.retired); std::mem::take(&mut state.forwards) .into_iter() .map(|live| live.forwarder) + .chain(retired) + .collect() + } + + /// The internal port and protocol of each runtime forward bound at + /// `address`: the forwards a move of the box's publish off `address` + /// must bind again at the new one (#2748). + pub(crate) fn bound_at(&self, address: std::net::Ipv4Addr) -> Vec<(u16, sessions::IpProto)> { + let host = address.to_string(); + self.state() + .forwards + .iter() + .filter(|live| { + live.forwarder + .host_port() + .is_some_and(|(bound, _)| bound == host) + }) + .map(|live| (live.mapping.internal_port, live.mapping.proto)) .collect() } + /// Puts `moved`, bound at the box's new address, in the place of the + /// forward for the same port bound at `from` (#2748), and returns the + /// old forward for the caller to unbind. The row keeps what it said + /// about the port's admission: the gate names the port, not the + /// address, so a move changes neither. Refused, handing `moved` back + /// for the caller to unbind, when the spawn has ended or no forward for + /// the port stands at `from` any more. + pub(crate) fn rebind( + &self, + from: std::net::Ipv4Addr, + mut moved: LiveIngressForward, + ) -> Result { + let host = from.to_string(); + let mut state = self.state(); + if state.detached { + return Err(moved); + } + let Some(standing) = state.forwards.iter_mut().find(|live| { + live.mapping.internal_port == moved.mapping.internal_port + && live + .forwarder + .host_port() + .is_some_and(|(bound, _)| bound == host) + }) else { + return Err(moved); + }; + moved.mapping.pending = standing.mapping.pending; + Ok(std::mem::replace(standing, moved)) + } + + /// Keeps a forward a move left behind because its unbind failed + /// (#2748), so the spawn's end or the box's stop unbinds it. Refused, + /// handing it back, when the spawn has ended. + pub(crate) fn retire( + &self, + forwarder: crate::net::policy::PortForwarder, + ) -> Result<(), crate::net::policy::PortForwarder> { + let mut state = self.state(); + if state.detached { + return Err(forwarder); + } + state.retired.push(forwarder); + Ok(()) + } + /// Takes the one runtime forward published for `port`, for the session /// actor to unbind: the publish it recorded is being withdrawn because /// its allow could not be audited (NET-046). `None` when no forward for @@ -201,9 +313,12 @@ impl RuntimeIngress { /// The box stopped: takes every runtime forward for the session actor to /// unbind. pub(crate) fn take_all(&self) -> Vec { - std::mem::take(&mut self.state().forwards) + let mut state = self.state(); + let retired = std::mem::take(&mut state.retired); + std::mem::take(&mut state.forwards) .into_iter() .map(|live| live.forwarder) + .chain(retired) .collect() } } diff --git a/crates/minimald/src/rpc.rs b/crates/minimald/src/rpc.rs index 8990997a36..8d469ae309 100644 --- a/crates/minimald/src/rpc.rs +++ b/crates/minimald/src/rpc.rs @@ -274,7 +274,7 @@ async fn serve_get_session_record( /// Stand-in for `session_name` on the lifecycle records when the session /// carries no user-assigned name. `SessionConfig::name` is `None` for an /// anonymous session, and an absent field reads like a logging bug. -const ANONYMOUS_SESSION: &str = ""; +pub(crate) const ANONYMOUS_SESSION: &str = ""; /// The egress rule counts a session config parses to, one per egress field. /// Logged beside the record the daemon stores at session start, they are the diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index 8a49725d17..c6bcbb6001 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -696,6 +696,19 @@ enum SessionMessage { address: std::net::Ipv4Addr, port: u16, }, + /// The box's publish moved from `from` to `to` (#2748): bind each of + /// `ports`, the declared ports it holds at `to`, and unbind their + /// forwards at `from`, then move the forwards its `min net expose` + /// bound at `from`, if the box is running, so they listen where its + /// name answers. Sent by the sessions manager for each move the name + /// registry makes, to a box whose actor runs. Linux only, as the + /// registry's hand-overs are: on macOS minimald runs inside the Linux guest. + #[cfg(target_os = "linux")] + PublishMoved { + from: std::net::Ipv4Addr, + to: std::net::Ipv4Addr, + ports: Vec, + }, /// The live dynamic-ingress mappings this box published at runtime /// (NET-044), by the surface that published them — what the /// `GetLiveIngress` RPC serves. Empty for a box that published none. @@ -2239,6 +2252,14 @@ impl Session { SessionMessage::BindHandedPort { address, port } => { self.bind_handed_port(address, port).await; } + #[cfg(target_os = "linux")] + SessionMessage::PublishMoved { from, to, ports } => { + for port in ports { + self.bind_handed_port(to, port).await; + self.release_moved_declared(from, to, port).await; + } + self.move_exposed_ports(from, to).await; + } SessionMessage::LiveIngress(r) => { #[expect( clippy::let_underscore_must_use, @@ -4146,7 +4167,11 @@ impl Session { /// it runs (NET-129): the port's holder was destroyed, moved off the /// address or stopped naming the port, and the registry elected this /// box. Its attach skipped the port while it yielded, so nothing is - /// bound for it until this, or until its next attach. + /// bound for it until this, or until its next attach. The present + /// landing sends the same hand-over to a box it moved off the + /// `127.0.0.1` interim (NET-123 §7.1): its attach bound the forward at + /// the interim, and this binds it at the address its name now answers + /// at. /// /// Nothing is asked for a box with no host running or whose spawn has /// ended: that box holds the port of record and its next attach binds @@ -4267,6 +4292,181 @@ impl Session { } } + /// Unbinds the declared forward of `port` still bound at `from` once the + /// move's hand-over bound it at `to` (#2748). Without this the box keeps + /// a listener at the old address: the `127.0.0.1` interim is shared, and + /// the registry may give the port there to another box, whose bind would + /// meet it. The old forward is unbound only after the new one stands, so + /// the port is served the whole way through. A bind at `to` that failed + /// leaves the old forward as it is. + /// + /// On a VM-backed host the host refuses a declared port's withdrawal and + /// unbinds the box's declared forwards itself + /// ([`crate::net::policy::ControlChannel::host_unbinds_declared_forwards`]), + /// so nothing is asked there. An unbind that fails is logged, and the + /// spawn's end unbinds the forward as it would have. + #[cfg(target_os = "linux")] + async fn release_moved_declared( + &mut self, + from: std::net::Ipv4Addr, + to: std::net::Ipv4Addr, + port: u16, + ) { + if !self.has_live_host() || !self.live_ingress.holds_handed_at(to, port) { + return; + } + let stale = self.live_ingress.declared_at(from, port); + if stale.is_empty() { + return; + } + let control = self.switch_control().await; + if control.host_unbinds_declared_forwards() { + return; + } + for forwarder in stale { + match forwarder.unbind_moved(&control).await { + Ok(()) => tracing::info!( + port, + %from, + %to, + "unbound a declared port at the box's old address after its move" + ), + Err(error) => tracing::warn!( + port, + %from, + %error, + "unbinding a declared port at the box's old address failed; \ + it is unbound when the box's spawn ends" + ), + } + } + } + + /// Moves the forwards this box's `min net expose` bound at `from` to + /// `to`, the address its publish moved to (#2748). The present landing + /// moves a box off the `127.0.0.1` interim while it runs, and a runtime + /// publish binds at the address the box published at when it was asked, + /// so without this the name would answer at `to` while the exposed port + /// listens at `from`. + /// + /// Each forward is bound at `to` first, toward the same lease, and only + /// then is the one at `from` unbound, so the port is served the whole + /// way through. The box's gate and the VM host daemon's row name the port, + /// not the address, so neither changes. A bind that fails is logged and + /// leaves the forward at `from`. An unbind that fails keeps the old + /// forward for the box's stop or its spawn's end to unbind. + /// + /// Nothing is moved for a box with no host running or whose spawn has + /// ended, or when the registry no longer publishes the box at `to`, + /// since the move is read after other changes may have landed. + #[cfg(target_os = "linux")] + async fn move_exposed_ports( + &mut self, + from: std::net::Ipv4Addr, + to: std::net::Ipv4Addr, + ) { + if !self.has_live_host() || self.live_ingress.is_detached() { + return; + } + let moving = self.live_ingress.bound_at(from); + if moving.is_empty() { + return; + } + let Ok(record) = self.record.record().await else { + return; + }; + let switch_address = { + let registry = self + .hostnames + .read() + .expect("hostname registry lock poisoned"); + if registry.published_own_address(record.id) != Some(to) { + return; + } + match record.box_addresses { + Some(addresses) => Some(addresses.switch_address), + None => registry.own_lease(record.id), + } + }; + let Some(switch_address) = switch_address else { + return; + }; + let session_name = record + .name + .as_deref() + .unwrap_or(crate::rpc::ANONYMOUS_SESSION); + let control = self.switch_control().await; + for (port, proto) in moving { + let forwarder = match crate::net::policy::expose_dynamic( + &control, + to, + switch_address, + port, + proto, + None, + ) + .await + { + Ok(forwarder) => forwarder, + Err(error) => { + tracing::warn!( + session_id = %record.id, + session_name, + port, + %from, + %to, + %error, + owner = %crate::net::listeners::PublicationOwner::Expose.as_str(), + "binding an exposed port at the box's new address failed; \ + it stays at the old address until the box stops" + ); + continue; + } + }; + let mapping = minimald_rpc::LiveMapping { + local: forwarder.local().to_string(), + internal_port: forwarder.internal_port(), + proto, + pending: None, + }; + let moved = crate::net::provider::LiveIngressForward { forwarder, mapping }; + let old = match self.live_ingress.rebind(from, moved) { + Ok(old) => old, + Err(moved) => { + // The spawn ended, or the port was withdrawn, while the + // bind was in flight: the new forward comes down here. + crate::net::policy::remove_ingress(&control, &[moved.forwarder]).await; + continue; + } + }; + if let Err(error) = + crate::net::policy::unexpose_forwarder(&control, &old.forwarder).await + { + tracing::warn!( + session_id = %record.id, + session_name, + port, + %from, + %error, + "unbinding an exposed port at the box's old address failed; \ + it is unbound again when the box stops" + ); + if let Err(stale) = self.live_ingress.retire(old.forwarder) { + crate::net::policy::remove_ingress(&control, &[stale]).await; + } + } + tracing::info!( + session_id = %record.id, + session_name, + port, + %from, + %to, + owner = %crate::net::listeners::PublicationOwner::Expose.as_str(), + "moved an exposed port to the box's new address" + ); + } + } + /// The switch's control channel, read off the daemon's shared switch under /// a lock held only for the two sync reads — never across the publish it /// feeds. @@ -5670,6 +5870,8 @@ impl Session { .as_utf8_path() .as_std_path() .to_path_buf(), + Some(Arc::clone(&self.hostnames)), + record.id, )); // The set this launch is running on, handed back so a test can // contend with its two real surfaces over the real port — a @@ -6273,6 +6475,28 @@ impl SessionHandle { .await; } + /// Tells the session's actor its publish moved from `from` to `to` + /// (#2748), so it binds `ports`, the declared ports it holds at `to`, + /// unbinds their forwards at `from`, and moves the forwards its + /// `min net expose` bound at `from`, if its box is running. Not + /// answered: a box that is gone or stopped has no forward to move. + #[cfg(target_os = "linux")] + pub(crate) async fn publish_moved( + &self, + from: std::net::Ipv4Addr, + to: std::net::Ipv4Addr, + ports: Vec, + ) { + #[expect( + clippy::let_underscore_must_use, + reason = "an actor that is gone has no box to move the forwards for" + )] + let _ = self + .0 + .send(SessionMessage::PublishMoved { from, to, ports }) + .await; + } + /// Continues a parked runtime port-publish ask (NET-045) with the attached /// human's answer, or its absence — sent by the task /// [`Session::route_ask`] spawned, never by the actor itself. Keyed by the diff --git a/crates/minimald/src/session/tests.rs b/crates/minimald/src/session/tests.rs index 67a872b822..f496e50ce4 100644 --- a/crates/minimald/src/session/tests.rs +++ b/crates/minimald/src/session/tests.rs @@ -6638,6 +6638,693 @@ async fn an_interim_handed_publish_takes_its_hand_when_the_verdict_lands_late() ); } +/// #2748, the fresh install on a VM-backed host: the daemon-start walk +/// lands absent, so a handed box finalizes at the `127.0.0.1` interim and +/// its running attach binds its forwards there. The host then gains the +/// range, and the next session start's walk lands present. The landing +/// moves the box's name to its own hand, and the box's forward follows it: +/// the running box binds its declared port at the hand, toward its switch +/// lease, so the name and the forward answer at one address. Once that bind +/// stands, the forward its attach bound at the interim is unbound, so no +/// listener is left on the shared interim for another box's bind to meet. +#[cfg(target_os = "linux")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_present_landing_moves_a_running_interim_boxs_forward_to_its_hand() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let server = TestServer::new().await; + let manager = server.state.sessions_manager().await; + let installed = Arc::new(AtomicBool::new(false)); + { + let installed = Arc::clone(&installed); + manager.install_range_walk_standin(move || { + let hosts = ::switch::loopback::range_hosts().count(); + if installed.load(Ordering::SeqCst) { + crate::net::loopback::RangeProbe { + bound: hosts, + probed: hosts, + first_failure: None, + } + } else { + crate::net::loopback::RangeProbe { + bound: 0, + probed: hosts, + first_failure: Some(( + std::net::Ipv4Addr::new(127, 0, 64, 1), + std::io::ErrorKind::AddrNotAvailable, + )), + } + } + }); + } + // The daemon-start walk's verdict on a host without the range. + manager.land_range_verdict(crate::net::dns::RangeVerdict::Absent); + let mut client = server.connect().await; + let handed = std::net::Ipv4Addr::new(127, 0, 64, 2); + let lease = std::net::Ipv4Addr::new(100, 64, 128, 2); + let web = finalize_handed_own_ip_session(&mut client, "web", lease, handed).await; + let registry = manager.hostnames(); + assert_eq!( + registry + .read() + .expect("registry lock") + .published_own_address(web), + Some(std::net::Ipv4Addr::LOCALHOST), + "the absent verdict stands the handed box on the interim" + ); + let handle = manager + .get_session(crate::sessions::SessionKeyPredicate::Id(web)) + .await + .unwrap() + .expect("the box resolves"); + handle + .ensure_host("tester".to_string()) + .await + .expect("the box launches its host"); + let sock = handle + .net_switch() + .await + .unwrap() + .lock() + .await + .control_socket(); + let (forwarder, served) = fake_forwarder(sock.clone(), 200).await; + let mapping = sessions::PortMapping { + external_port: declared_port(), + internal_port: 80, + proto: sessions::IpProto::Tcp, + }; + // The declared forward the box's attach bound at the interim, as the + // attach shares it with the box's actor. The mock launch binds none. + let interim = std::net::Ipv4Addr::LOCALHOST; + let control = crate::net::policy::ControlChannel::Unix(sock); + let Ok(crate::net::policy::HandedBind::Bound(attached)) = + crate::net::policy::bind_handed_port(&control, interim, lease, &mapping, None).await + else { + panic!("the stand-in binds the attach's forward"); + }; + let (ingress, _) = handle.ingress_cells().await.expect("the actor answers"); + ingress.record_attached(std::slice::from_ref(&attached)); + + // `min finalize-install` applies the range, and the next session start + // walks it again. + installed.store(true, Ordering::SeqCst); + let answer = manager + .session_start_range_walk() + .await + .expect("the daemon walks the range through the forwarder"); + assert_eq!(answer.moved, 1, "the landing moves the box off the interim"); + + assert_eq!( + registry + .read() + .expect("registry lock") + .published_own_address(web), + Some(handed), + "the present landing moves the box's publish to its own hand" + ); + let (_, address) = zone_answer_for(&server, "web.min.internal") + .await + .expect("the name survives the landing"); + assert_eq!(address, handed, "the name answers at the hand"); + + // The attach's bind, the hand's bind, then the interim's unbind. + let requests = await_served(&served, 3).await; + forwarder.abort(); + let (line, body) = requests[1] + .split_once('\n') + .expect("the served request carries its request line and body"); + assert!( + line.starts_with("POST /services/forwarder/expose "), + "the landing binds the running box's forward at its hand: {requests:?}" + ); + let declared = crate::net::policy::expose_request(&mapping, handed, lease); + assert_eq!( + body, + String::from_utf8(serde_json_lenient::to_vec(&declared).unwrap()).unwrap(), + "the forward listens where the name answers, toward the box's lease" + ); + assert!( + requests[2].starts_with("POST /services/forwarder/unexpose ") + && requests[2].contains(&format!("\"local\":\"{interim}:{}\"", declared_port())), + "the forward at the interim is unbound after the hand's bind stands: {requests:?}" + ); + assert!( + attached.is_revoked(), + "the attach's teardown asks nothing more of the forward unbound here" + ); +} + +/// #2748, a declared port whose bind at the hand fails: the landing moves the +/// box's name to its hand, but the switch refuses the declared port's bind +/// there. The forward the attach bound at the interim is the only one that +/// serves the port, so it stays bound: it is unbound only once the bind at +/// the hand stands. +#[cfg(target_os = "linux")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_declared_port_whose_bind_at_the_hand_fails_keeps_its_interim_forward() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let server = TestServer::new().await; + let manager = server.state.sessions_manager().await; + let installed = Arc::new(AtomicBool::new(false)); + { + let installed = Arc::clone(&installed); + manager.install_range_walk_standin(move || { + let hosts = ::switch::loopback::range_hosts().count(); + if installed.load(Ordering::SeqCst) { + crate::net::loopback::RangeProbe { + bound: hosts, + probed: hosts, + first_failure: None, + } + } else { + crate::net::loopback::RangeProbe { + bound: 0, + probed: hosts, + first_failure: Some(( + std::net::Ipv4Addr::new(127, 0, 64, 1), + std::io::ErrorKind::AddrNotAvailable, + )), + } + } + }); + } + manager.land_range_verdict(crate::net::dns::RangeVerdict::Absent); + let mut client = server.connect().await; + let handed = std::net::Ipv4Addr::new(127, 0, 64, 8); + let lease = std::net::Ipv4Addr::new(100, 64, 128, 8); + let interim = std::net::Ipv4Addr::LOCALHOST; + let web = finalize_handed_own_ip_session(&mut client, "web", lease, handed).await; + let handle = manager + .get_session(crate::sessions::SessionKeyPredicate::Id(web)) + .await + .unwrap() + .expect("the box resolves"); + handle + .ensure_host("tester".to_string()) + .await + .expect("the box launches its host"); + let sock = handle + .net_switch() + .await + .unwrap() + .lock() + .await + .control_socket(); + // The switch binds everything at the interim and nothing at the hand. + let refused_at = format!("\"local\":\"{handed}:"); + let (forwarder, served) = + deciding_forwarder(sock.clone(), move |request| !request.contains(&refused_at)).await; + let mapping = sessions::PortMapping { + external_port: declared_port(), + internal_port: 80, + proto: sessions::IpProto::Tcp, + }; + let control = crate::net::policy::ControlChannel::Unix(sock); + let Ok(crate::net::policy::HandedBind::Bound(attached)) = + crate::net::policy::bind_handed_port(&control, interim, lease, &mapping, None).await + else { + panic!("the stand-in binds the attach's forward"); + }; + let (ingress, _) = handle.ingress_cells().await.expect("the actor answers"); + ingress.record_attached(std::slice::from_ref(&attached)); + + installed.store(true, Ordering::SeqCst); + let answer = manager + .session_start_range_walk() + .await + .expect("the daemon walks the range through the forwarder"); + assert_eq!(answer.moved, 1, "the landing moves the box off the interim"); + // The actor asked for the port at the hand, and the switch refused it. + soon(|| { + served_naming(&served, handed, declared_port()) + .iter() + .any(|request| request.starts_with("POST /services/forwarder/expose ")) + }) + .await; + // The move's handling ends in the actor's own turn: a round trip to it + // after the refused bind is after the release would have been asked. + handle.live_ingress().await.expect("the actor answers"); + forwarder.abort(); + assert!( + served_naming(&served, interim, declared_port()) + .iter() + .all(|request| !request.starts_with("POST /services/forwarder/unexpose ")), + "the forward at the interim is not unbound while no bind at the hand stands: {:?}", + served.lock().expect("served lock") + ); + assert!( + !attached.is_revoked(), + "the attach's forward at the interim still serves the port" + ); +} + +/// #2748, the runtime publications: a handed box standing on the +/// `127.0.0.1` interim publishes one port with `min net expose` and one by +/// listening on it, and both forwards bind at the interim. When the range +/// lands present and the box's name moves to its hand, both forwards move +/// with it: each is bound at the hand toward the box's lease and unbound at +/// the interim, and the listed rows name the hand. The box declares no port, +/// so no declared hand-over is what moves them. +#[cfg(target_os = "linux")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_present_landing_moves_a_running_interim_boxs_runtime_publications_to_its_hand() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let server = TestServer::new().await; + let manager = server.state.sessions_manager().await; + let installed = Arc::new(AtomicBool::new(false)); + { + let installed = Arc::clone(&installed); + manager.install_range_walk_standin(move || { + let hosts = ::switch::loopback::range_hosts().count(); + if installed.load(Ordering::SeqCst) { + crate::net::loopback::RangeProbe { + bound: hosts, + probed: hosts, + first_failure: None, + } + } else { + crate::net::loopback::RangeProbe { + bound: 0, + probed: hosts, + first_failure: Some(( + std::net::Ipv4Addr::new(127, 0, 64, 1), + std::io::ErrorKind::AddrNotAvailable, + )), + } + } + }); + } + manager.land_range_verdict(crate::net::dns::RangeVerdict::Absent); + let mut client = server.connect().await; + // One port the box listens on, and a free one it exposes. + let listener = listening_socket(); + let listened = port_of(&listener); + let exposed = port_of(&listening_socket()); + let range = (listened.min(exposed), listened.max(exposed)); + let interim = std::net::Ipv4Addr::LOCALHOST; + let handed = std::net::Ipv4Addr::new(127, 0, 64, 3); + let lease = std::net::Ipv4Addr::new(100, 64, 128, 3); + let web = finalize_dynamic_ingress_session( + &mut client, + "web", + lease, + handed, + Some(sessions::DynamicIngress::Allow), + Some(range), + ) + .await; + let registry = manager.hostnames(); + assert_eq!( + registry + .read() + .expect("registry lock") + .published_own_address(web), + Some(interim), + "the absent verdict stands the handed box on the interim" + ); + let handle = manager + .get_session(crate::sessions::SessionKeyPredicate::Id(web)) + .await + .unwrap() + .expect("the box resolves"); + let sock = handle + .net_switch() + .await + .unwrap() + .lock() + .await + .control_socket(); + let (forwarder, served) = fake_forwarder(sock.clone(), 200).await; + let gate = Arc::new(crate::net::switch::SessionGate::for_session( + "web".to_string(), + lease, + &listen_policy_over(range), + crate::net::SwitchSubnet::default(), + None, + )); + crate::session::listen_plan_seam::seed( + web, + crate::session::listen_plan_seam::Seeded { + lease, + published: interim, + control: crate::net::policy::ControlChannel::Unix(sock), + gate, + }, + ); + handle + .ensure_host("tester".to_string()) + .await + .expect("the box launches its host"); + handle + .expose_dynamic(exposed) + .await + .expect("a port in the box's range is exposed"); + let at = |address: std::net::Ipv4Addr, port: u16| format!("{address}:{port}"); + listed_once(&handle, |rows| { + rows.exposed + .iter() + .any(|row| row.local == at(interim, exposed)) + && rows + .listened + .iter() + .any(|row| row.local == at(interim, listened)) + }) + .await; + + // `min finalize-install` applies the range, and the next session start + // walks it again. + installed.store(true, Ordering::SeqCst); + let answer = manager + .session_start_range_walk() + .await + .expect("the daemon walks the range through the forwarder"); + assert_eq!(answer.moved, 1, "the landing moves the box off the interim"); + + let rows = listed_once(&handle, |rows| { + rows.exposed + .iter() + .any(|row| row.local == at(handed, exposed)) + && rows + .listened + .iter() + .any(|row| row.local == at(handed, listened)) + }) + .await; + assert!( + rows.exposed + .iter() + .chain(&rows.listened) + .all(|row| !row.local.starts_with("127.0.0.1:")), + "no row names the interim after the move: {rows:?}" + ); + for port in [exposed, listened] { + soon(|| { + served_naming(&served, interim, port) + .iter() + .any(|request| request.starts_with("POST /services/forwarder/unexpose ")) + }) + .await; + let moved = served_naming(&served, handed, port); + assert!( + moved.iter().any( + |request| request.starts_with("POST /services/forwarder/expose ") + && request.contains(&format!("\"remote\":\"{lease}:{port}\"")) + ), + "the port is bound at the hand toward the box's lease: {moved:?}" + ); + } + forwarder.abort(); + drop(listener); +} + +/// #2748, a listen port whose new address another box holds: an absent +/// landing moves a box at its range address onto the shared `127.0.0.1` +/// interim, where a box with no forward behind it, such as a stopped one, +/// declared the port first. The listening port does not move there: the +/// other box keeps the port of record, and the forward stays at the old +/// address. Once that box gives the port up, the next poll moves it. +#[cfg(target_os = "linux")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_listen_port_does_not_move_onto_a_port_another_box_holds() { + let server = TestServer::new().await; + let mut client = server.connect().await; + let manager = server.state.sessions_manager().await; + let listener = listening_socket(); + let listened = port_of(&listener); + let interim = std::net::Ipv4Addr::LOCALHOST; + let handed = std::net::Ipv4Addr::new(127, 0, 64, 6); + let (handle, _gate, served, _gate_sender, web) = box_with_scripted_plan( + &mut client, + &manager, + "web", + true, + std::net::Ipv4Addr::new(100, 64, 128, 6), + handed, + (listened, listened), + Vec::new(), + ) + .await; + let at_hand = format!("{handed}:{listened}"); + listed_once(&handle, |rows| { + rows.listened.iter().any(|row| row.local == at_hand) + }) + .await; + // Another box declared the port at the interim and binds nothing there. + let registry = manager.hostnames(); + let other = SessionId::parse_str("00000000-0000-0000-0000-0000000000db").unwrap(); + registry + .write() + .expect("registry lock") + .publish_own_address( + other, + "db", + interim, + std::collections::BTreeSet::from([listened]), + ); + + manager.land_range_verdict(crate::net::dns::RangeVerdict::Absent); + assert_eq!( + registry + .read() + .expect("registry lock") + .published_own_address(web), + Some(interim), + "the absent landing moves the box onto the interim" + ); + // A few of the watcher's polls. + tokio::time::sleep(Duration::from_secs(1)).await; + assert!( + served_naming(&served, interim, listened).is_empty(), + "the listening port is not bound where another box holds it" + ); + assert_eq!( + registry + .read() + .expect("registry lock") + .shared_port_holder(interim, listened), + Some(other), + "the other box keeps the port of record" + ); + let rows = handle.live_ingress().await.expect("the actor answers"); + assert!( + rows.listened.iter().any(|row| row.local == at_hand), + "the row names where the forward is still bound: {rows:?}" + ); + + // The other box gives the port up, and the port follows the box. + let (_, hand_overs) = registry + .write() + .expect("registry lock") + .unpublish_own_address(other); + assert!(hand_overs.is_empty(), "no box waits on the port"); + soon(|| { + served_naming(&served, interim, listened) + .iter() + .any(|request| request.starts_with("POST /services/forwarder/expose ")) + }) + .await; + drop(listener); +} + +/// #2748: a range walk that ends inconclusive, as one whose probe round the +/// VM host's gate kept contended past the walk's budget does, never replaces +/// a present verdict. Every box keeps its range address, so a guest that +/// holds a probe round cannot move the host's boxes back to the interim. +#[cfg(target_os = "linux")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_contended_walk_never_replaces_a_present_verdict() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let server = TestServer::new().await; + let manager = server.state.sessions_manager().await; + let contended = Arc::new(AtomicBool::new(false)); + { + let contended = Arc::clone(&contended); + manager.install_range_walk_standin(move || { + let hosts = ::switch::loopback::range_hosts().count(); + if contended.load(Ordering::SeqCst) { + crate::net::loopback::RangeProbe { + bound: hosts - 1, + probed: hosts, + first_failure: Some(( + std::net::Ipv4Addr::new(127, 0, 64, 5), + std::io::ErrorKind::TimedOut, + )), + } + } else { + crate::net::loopback::RangeProbe { + bound: hosts, + probed: hosts, + first_failure: None, + } + } + }); + } + manager.land_range_verdict(crate::net::dns::RangeVerdict::Present); + let mut client = server.connect().await; + let handed = std::net::Ipv4Addr::new(127, 0, 64, 7); + let web = finalize_handed_own_ip_session( + &mut client, + "web", + std::net::Ipv4Addr::new(100, 64, 128, 7), + handed, + ) + .await; + let registry = manager.hostnames(); + assert_eq!( + registry + .read() + .expect("registry lock") + .published_own_address(web), + Some(handed), + "the present verdict publishes the box at its hand" + ); + + contended.store(true, Ordering::SeqCst); + let answer = manager + .session_start_range_walk() + .await + .expect("the daemon walks the range through the forwarder"); + assert_eq!(answer.moved, 0, "an inconclusive walk moves nothing"); + assert_eq!( + registry + .read() + .expect("registry lock") + .published_own_address(web), + Some(handed), + "the box keeps its range address over an inconclusive walk" + ); +} + +/// #2748, a box with no actor: the present landing moves every box standing +/// on the `127.0.0.1` interim, stopped ones included, and the move of a +/// box's publish is passed only to a live actor, with the hand-over for +/// each port the box declares. A box whose actor is not running has no +/// forward to move, so the landing starts none for it, not even for the +/// port it declares, and its next attach binds at the new address. +#[cfg(target_os = "linux")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_present_landing_starts_no_actor_for_a_box_without_one() { + let server = TestServer::new().await; + let mut client = server.connect().await; + let handed = std::net::Ipv4Addr::new(127, 0, 64, 5); + // The box declares its test port. + let web = finalize_handed_own_ip_session( + &mut client, + "web", + std::net::Ipv4Addr::new(100, 64, 128, 5), + handed, + ) + .await; + let declared = std::collections::BTreeSet::from([declared_port()]); + // A restarted daemon that has not resumed the box: its record stands, + // and no actor runs for it. + let server = restart(server, client).await; + let manager = server.state.sessions_manager().await; + manager.install_range_walk_standin(|| { + let hosts = ::switch::loopback::range_hosts().count(); + crate::net::loopback::RangeProbe { + bound: hosts, + probed: hosts, + first_failure: None, + } + }); + manager.land_range_verdict(crate::net::dns::RangeVerdict::Absent); + // The box stands on the interim with its hand, as a box that published + // under the absent verdict does. + { + let registry = manager.hostnames(); + let mut registry = registry.write().expect("registry lock"); + registry.record_own_hand(web, handed); + registry.publish_own_address(web, "web", std::net::Ipv4Addr::LOCALHOST, declared.clone()); + registry.register_own_ip(web, "web", declared.clone()); + } + assert_eq!( + manager.running_count().await, + 0, + "no actor runs for the box" + ); + + let answer = manager + .session_start_range_walk() + .await + .expect("the daemon walks the range through the forwarder"); + assert_eq!(answer.moved, 1, "the landing moves the box off the interim"); + { + let registry = manager.hostnames(); + let registry = registry.read().expect("registry lock"); + assert_eq!( + registry.published_own_address(web), + Some(handed), + "the box's publish moves to its hand" + ); + assert!( + registry.holds_shared_port(web, handed, declared_port()), + "the box holds its declared port at the hand, so the move made its hand-over" + ); + } + // The move is passed on off the landing's task: give it the time an + // actor's start would take, then read that none started. + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!( + manager.running_count().await, + 0, + "the landing starts no actor for a box without one" + ); +} + +/// A stand-in for the host gvproxy that answers each request 200 when +/// `accept` takes it and 500 when it does not, recording every request as +/// [`fake_forwarder`] does — so a test can have the switch refuse binds at +/// one address for as long as it runs, however often they are asked. +#[cfg(target_os = "linux")] +async fn deciding_forwarder( + sock: std::path::PathBuf, + accept: impl Fn(&str) -> bool + Send + 'static, +) -> ( + tokio::task::JoinHandle<()>, + std::sync::Arc>>, +) { + use tokio::io::AsyncWriteExt; + if let Some(parent) = sock.parent() { + std::fs::create_dir_all(parent).expect("create the switch state dir"); + } + let listener = tokio::net::UnixListener::bind(&sock).expect("bind the control socket"); + let served = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let recorded = served.clone(); + let server = tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let Some(request) = read_control_request(&mut stream).await else { + continue; + }; + let answer = if accept(&request) { + "200 OK" + } else { + "500 Internal Server Error" + }; + recorded.lock().expect("served lock").push(request); + #[expect( + clippy::let_underscore_must_use, + reason = "the answer's fate is not what the stand-in records; the request is" + )] + let _ = stream + .write_all(format!("HTTP/1.1 {answer}\r\nContent-Length: 0\r\n\r\n").as_bytes()) + .await; + } + }); + (server, served) +} + /// The wiring test for the hand's verdict gate (the `.filter(vouches_for)` /// and the bounded wait on the hand's read, §7.1): a handed reserved address /// finalizes to `published_own_address == 127.0.0.1` under an absent book — diff --git a/crates/minimald/src/session_host.rs b/crates/minimald/src/session_host.rs index 5bdbfd5cae..37f4d9c54f 100644 --- a/crates/minimald/src/session_host.rs +++ b/crates/minimald/src/session_host.rs @@ -4800,6 +4800,10 @@ impl SessionLauncher for SandboxLauncher { gate, self.publications, self.state_dir, + own_address + .as_ref() + .map(crate::net::provider::OwnAddressReporter::registry), + session_id, )) } _ => None, diff --git a/crates/minimald/src/sessions.rs b/crates/minimald/src/sessions.rs index 6c760198de..2f3ff1e2b4 100644 --- a/crates/minimald/src/sessions.rs +++ b/crates/minimald/src/sessions.rs @@ -167,6 +167,13 @@ enum ManagerMessage { List(Responder>), GetRecord(SessionKeyPredicate, Responder>), GetSession(SessionKeyPredicate, Responder>), + /// The handle of the session's live actor, if it has one. Read-only + /// against the `running` map: unlike [`GetSession`](Self::GetSession) + /// it never starts an actor, so a stopped box stays without one. Linux + /// only, like the publish-move receiver that asks it: on macOS minimald + /// runs inside the Linux guest. + #[cfg(target_os = "linux")] + GetRunning(SessionId, Responder>), /// Snapshot a running session's terminal screen (`min dash`'s /// `GetSessionScreen`). Read-only against the `running` map — unlike /// [`GetSession`](Self::GetSession) it never starts an actor, and a @@ -560,6 +567,35 @@ impl Manager { } }); } + // #2748: each move of a box's publish is passed to that box's + // session, which binds its declared ports at the new address and + // moves the forwards its `min net expose` bound at the old one, if + // the box is running. Off the registry's lock, like the hand-overs + // above. Only a live actor is asked: the landing moves stopped boxes + // too, and a box with no actor has no forward to move, so none is + // started for it. Its next attach binds at the new address. + #[cfg(target_os = "linux")] + { + let (sender, mut moves) = tokio::sync::mpsc::unbounded_channel(); + hostnames + .write() + .expect("hostname registry lock poisoned") + .notify_publish_moves(sender); + let manager = weak_self.clone(); + tokio::spawn(async move { + while let Some(moved) = moves.recv().await { + let Some(manager) = manager.upgrade() else { + return; + }; + if let Ok(Some(holder)) = manager.running_session(moved.holder).await { + let ports = moved.hand_overs.iter().map(|hand_over| hand_over.port); + holder + .publish_moved(moved.from, moved.to, ports.collect()) + .await; + } + } + }); + } let mngr = Self { in_shutdown, receiver, @@ -1055,6 +1091,16 @@ pub(crate) fn draw_interim_upgrades( /// stopped box keeps its name, so the move applies to it with its stopped /// marker intact. /// +/// A move also sends the move itself, with a hand-over for each port the box +/// declares ([`crate::net::dns::HostnameRegistry::hand_over_moved_publish`]). +/// A box that runs has its forwards at the interim, where its attach, +/// its `min net expose` and its listen watcher bound them, so without these +/// its name would answer at an address no forward listens on. Its actor +/// binds the declared forwards and moves the exposed ones to the new +/// address. Its listen watcher reads the new address on its next poll and +/// moves the ports it published. A box with no actor running gets nothing: +/// none is started for it, and its next attach binds at the new address. +/// /// The re-check is a runtime one, not an ordering argument: the landing /// runs on the deferred walk's own spawned task /// ([`crate::sessions::Manager::init`]), never through the manager's @@ -1097,6 +1143,11 @@ pub(crate) fn apply_interim_upgrade( upgrade.ports.clone(), ); reg.register_own_ip(upgrade.session, &upgrade.name, upgrade.ports.clone()); + // The forwards follow the name (#2748): a box that runs bound + // them at the interim, so its actor binds each port again at the + // address the name now answers at, and moves the ports its + // `min net expose` published there. + reg.hand_over_moved_publish(upgrade.session, Ipv4Addr::LOCALHOST); true }; (applied, standing.is_none()) @@ -1540,6 +1591,12 @@ impl Manager { ManagerMessage::GetSession(pred, r) => { r.handle(self.resolve_session(pred)).await; } + #[cfg(target_os = "linux")] + ManagerMessage::GetRunning(id, r) => { + let running = self.running.get(&id).cloned(); + r.handle(async move { Ok::<_, SessionsError>(running) }) + .await; + } // Snapshot a running session's screen without starting its // actor. A name resolves through the store to its id, then the // `running` map answers only for live actors. @@ -2163,6 +2220,22 @@ impl ManagerHandle { recv.await.expect("corresponding sessions manager is dead") } + /// The handle of the session's live actor, if it has one. Never starts + /// an actor. See [`ManagerMessage::GetRunning`]. + #[cfg(target_os = "linux")] + pub(crate) async fn running_session( + &self, + id: SessionId, + ) -> Result, SessionsError> { + let (send, recv) = Responder::channel(); + #[expect( + clippy::let_underscore_must_use, + reason = "a send that fails drops the responder, and the recv below reports it" + )] + let _ = self.sender.send(ManagerMessage::GetRunning(id, send)).await; + recv.await.expect("corresponding sessions manager is dead") + } + /// Snapshots the terminal screen of a running session. `Ok(None)` when /// the session is unknown or has no live host — never starts an actor. pub async fn get_screen( diff --git a/crates/minvmd/src/net/egress_gate.rs b/crates/minvmd/src/net/egress_gate.rs index 7a819a7057..675d63d111 100644 --- a/crates/minvmd/src/net/egress_gate.rs +++ b/crates/minvmd/src/net/egress_gate.rs @@ -157,6 +157,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, LazyLock, Mutex, MutexGuard, OnceLock}; use std::task::{Context, Poll}; use std::time::{Duration, Instant}; +use tracing::Instrument as _; use sessions::EgressDefaultPhase; use sessions::core::egress::{self, DropReason, FrameFamily, FrameSummary, FrameVerdict, Ipv4Cidr}; @@ -1424,6 +1425,7 @@ async fn serve_connection( limiter, replies, &forwards, + &switch_sock, handshake_timeout, phase, ) @@ -1846,7 +1848,8 @@ async fn relay_control( table: BoxTable, limiter: Arc, replies: ReplyTables, - forwards: &PublishedForwards, + forwards: &Arc, + switch_sock: &std::path::Path, drain_timeout: Duration, phase: UnregisteredSourcePhase, ) { @@ -1892,18 +1895,141 @@ async fn relay_control( return; } }; - let decision = match decide_control_request(verb, &request, &table, phase, forwards) { - Ok(decision) => decision, - Err(refused) => { - // Refused, and said so at the frame drops' cadence: a guest can - // attempt a publish on a fresh connection as cheaply as it can - // send a frame, and the refusal must not become the flood. The - // connection's halves come down with the return — nothing was - // written on, so gvproxy never saw the request, and the guest's - // read of its own side ends it. - refuse_request(&limiter, &refused); + // One round of the guest daemon's loopback range probe is decided by the + // probe's own bounds, never by a row ([`range_probe_listener`]). The + // guest is untrusted here, so a round binds only at a range address no + // box holds, one round at a time, and no longer than [`RANGE_PROBE_HOLD`]: + // an expose at a held address is answered as held and never relayed, a + // second outstanding round is refused, and a release names only the + // round outstanding. A round's relay takes no ledger note. + // + // A request naming a listener a round can bind takes the probe turn for + // the rest of this exchange, so its classification, its ledger note and + // its write are one step against every other such request and against + // the gate's own release of a round. + let _probe_turn = if names_probe_listener(verb, &request) { + Some(forwards.probe_turn.lock().await) + } else { + None + }; + let mut probe_round = None; + // What an admitted round writes to the switch: the request rebuilt from + // the fields the gate checked, never the guest's bytes + // ([`canonical_probe_request`]). + let mut probe_spoken = None; + let probe = range_probe_listener(verb, &request); + if let Some(listener) = probe + && !probe_body_is_strict(verb, &request) + { + // A probe-shaped body with a key the gate's shape check does not + // read: unknown, repeated, or a case variant gvproxy's decoder would + // take for one of the probe's fields. Refused with an answer, never + // relayed. + refuse_request( + &limiter, + &RefusedRequest { + rule: RANGE_PROBE_RULE, + addr: Some(listener.0), + what: Some(format!("port {}", listener.1)), + reason: "a loopback range probe round carried a key beyond its own fields" + .to_string(), + }, + ); + answer_unrelayed( + &mut guest, + &mut guest_tx, + &probe_malformed_answer(), + drain_timeout, + ) + .await; + return; + } + let decision = match probe { + Some(listener) if !range_address_held(&table, forwards, listener.0) => { + probe_spoken = Some(canonical_probe_request(verb, listener)); + if verb == ControlVerb::Expose { + match forwards.begin_probe(listener) { + Ok(generation) => { + tokio::spawn( + release_probe_after_hold( + switch_sock.to_path_buf(), + Arc::downgrade(forwards), + generation, + ) + .in_current_span(), + ); + probe_round = Some(generation); + } + Err(held) => { + refuse_request( + &limiter, + &RefusedRequest { + rule: RANGE_PROBE_RULE, + addr: Some(listener.0), + what: Some(format!("port {}", listener.1)), + reason: format!( + "another loopback range probe round is still bound at {}", + forward_revoke::listener_local(held.0, held.1) + ), + }, + ); + answer_unrelayed( + &mut guest, + &mut guest_tx, + &probe_refused_answer(), + drain_timeout, + ) + .await; + return; + } + } + } else { + match forwards.outstanding_probe() { + Some((held, generation)) if held == listener => probe_round = Some(generation), + // No round of the probe is bound at this listener, so the + // release has nothing to take: answered as the switch + // answers one that stood, and never relayed. + _ => { + answer_unrelayed( + &mut guest, + &mut guest_tx, + forward_revoke::bound_answer(), + drain_timeout, + ) + .await; + return; + } + } + } + tracing::debug!(?verb, "relaying a loopback range probe round"); + None + } + Some(_) if verb == ControlVerb::Expose => { + // A box holds the address. The round is answered as the switch + // answers a listener already held, which the walk counts as + // present, and never relayed. + answer_unrelayed( + &mut guest, + &mut guest_tx, + &forward_revoke::held_answer(), + drain_timeout, + ) + .await; return; } + _ => match decide_control_request(verb, &request, &table, phase, forwards) { + Ok(decision) => Some(decision), + Err(refused) => { + // Refused, and said so at the frame drops' cadence: a guest + // can attempt a publish on a fresh connection as cheaply as + // it can send a frame, and the refusal must not become the + // flood. The connection's halves come down with the return — + // nothing was written on, so gvproxy never saw the request, + // and the guest's read of its own side ends it. + refuse_request(&limiter, &refused); + return; + } + }, }; // A publish's two ends go to the ledger before the request is written // on: the listener it is applied at, for a later retraction's @@ -1940,8 +2066,8 @@ async fn relay_control( // not written on either: the switch refuses a second expose of a // listener it holds, and the daemon fails the attach on any refused // bind. The gate answers it as bound, which it is. - let noted = match (verb, forward_listener(verb, &request)) { - (ControlVerb::Expose, Some(listener)) => { + let noted = match (verb, forward_listener(verb, &request), &decision) { + (ControlVerb::Expose, Some(listener), Some(decision)) => { match (published_inside(&request), published_protocol(&request)) { (Some(inside), Some(proto)) => { let addr = decision.request.switch_addr(); @@ -2033,9 +2159,13 @@ async fn relay_control( }; // Admitted: the head and body are forwarded together, in one write, so // the request gvproxy sees is exactly the request the guest sent and - // exactly the request the table admitted. - let mut spoken = head; - spoken.extend_from_slice(&request); + // exactly the request the table admitted. A probe round is the + // exception: it is written as the gate rebuilt it from what it checked. + let spoken = probe_spoken.unwrap_or_else(|| { + let mut spoken = head; + spoken.extend_from_slice(&request); + spoken + }); if let Err(error) = switch.write_all(&spoken).await { tracing::warn!(%error, "egress gate could not forward an admitted control request"); if let Some(listener) = noted { @@ -2043,10 +2173,12 @@ async fn relay_control( } return; } - if decision.applied == Applied::Interim { + if let Some(decision) = &decision + && decision.applied == Applied::Interim + { // The interim's admission is the one publish the gate applies whose // reach no row bounds, and the host must be able to see it pass. - warn_interim_publish(&limiter, &decision); + warn_interim_publish(&limiter, decision); // And the lease it named is one the guest daemon vouched for while // no row held it, whatever the request published: the fact the // live-lease drop line reads ([`PublishedForwards::lease_live_at`]). @@ -2169,6 +2301,14 @@ async fn relay_control( replies.end_port_at(addr, proto, inside); } } + // A probe round's expose the switch refused bound nothing, and a release + // the switch answered stood: either way the round is over. A round left + // unanswered stays accounted, and its hold releases it. + if let (Some(generation), Some(&status)) = (probe_round, answered.get()) + && (200..300).contains(&status) == (verb == ControlVerb::Unexpose) + { + forwards.end_probe(generation); + } } /// The box whose row declared `port` at `addr`, when a row applied the @@ -2445,6 +2585,28 @@ struct PublishedForwards { /// the state a reset for it is built from: what the box's end terminates /// once its forwards are unbound ([`revoke_box_forwards`], design §7.1). flows: ForwardedFlows, + /// The guest daemon's outstanding loopback range probe round, if any + /// (#2748): at most one at a time per gate, from its admitted expose + /// until its unexpose stands ([`Self::begin_probe`]). + probe: Mutex, + /// The turn every control request naming a listener a probe round can + /// bind takes, from its classification through the switch's answer, and + /// the turn the gate's own release of a round takes + /// ([`names_probe_listener`]). One at a time, a probe's release can never + /// be decided against a ledger an ordinary publish of the same listener + /// is still changing, nor unbind a forward that publish bound. + probe_turn: tokio::sync::Mutex<()>, +} + +/// The gate's account of the range probe ([`PublishedForwards::probe`]). +#[derive(Debug, Default)] +struct ProbeSlot { + /// The round bound or binding now, with the generation it was admitted + /// under. + outstanding: Option<(Listener, u64)>, + /// The next round's generation, so a release meant for an ended round + /// never ends a later one. + next: u64, } /// A forwarder listener: its loopback address and port. @@ -2532,9 +2694,69 @@ impl PublishedForwards { handed: Mutex::new(Vec::new()), vouched: Mutex::new(Vec::new()), flows: ForwardedFlows::default(), + probe: Mutex::new(ProbeSlot::default()), + probe_turn: tokio::sync::Mutex::new(()), + } + } + + /// Admits one range probe round at `listener`, returning its generation, + /// or the listener of the round still outstanding: at most one round is + /// bound at a time, as the guest daemon's walk binds and releases one + /// address after another. + fn begin_probe(&self, listener: Listener) -> Result { + let mut slot = self.lock_probe(); + if let Some((held, _)) = slot.outstanding { + return Err(held); + } + slot.next += 1; + let generation = slot.next; + slot.outstanding = Some((listener, generation)); + Ok(generation) + } + + /// The outstanding round, if any, and its generation. + fn outstanding_probe(&self) -> Option<(Listener, u64)> { + self.lock_probe().outstanding + } + + /// The listener of round `generation` while it is still outstanding. + fn probe_listener(&self, generation: u64) -> Option { + self.lock_probe() + .outstanding + .filter(|(_, held)| *held == generation) + .map(|(listener, _)| listener) + } + + /// Ends round `generation`: its expose bound nothing, or its unexpose + /// stood. A later round is left alone. + fn end_probe(&self, generation: u64) { + let mut slot = self.lock_probe(); + if slot.outstanding.is_some_and(|(_, held)| held == generation) { + slot.outstanding = None; } } + /// Whether an applied or waiting publish's listener is bound at the + /// loopback address `addr`. + fn holds_loopback_address(&self, addr: [u8; 4]) -> bool { + let applied = self.lock(); + applied + .iter() + .any(|(listener, _, _, _, _)| listener.0 == addr) + || self + .lock_waiting() + .iter() + .any(|waiting| waiting.listener.0 == addr) + } + + /// The range probe's account. + fn lock_probe(&self) -> MutexGuard<'_, ProbeSlot> { + self.probe.lock().expect( + "the probe slot's lock is held only across a lookup or an update, never \ + across a panic", + ) + } + /// Every applied publish at the box address `addr`: what the box's end /// unbinds ([`revoke_box_forwards`]). Each stays in the ledger until its /// unbind succeeds, so a forward whose unexpose failed is still the @@ -3681,6 +3903,241 @@ struct DnsRecordBody { /// truncated, because a truncated request is a different request. const MAX_ZONE_RECORDS: usize = MAX_REQUEST_RECORDS; +/// The listener a control request names when it has the exact shape of one +/// round of the guest daemon's loopback range probe (NET-123): the expose +/// that binds [`switch::loopback::RANGE_PROBE_PORT`] at one address of the +/// reserved local range, tcp, toward [`switch::loopback::RANGE_PROBE_REMOTE`], +/// which is never dialed, or the unexpose that releases it. `None` for any +/// other request, which the row decision decides. +/// +/// The guest cannot see the host's loopback, so it asks the forwarder to bind +/// there, and the answer is the range verdict its publishes follow. No row +/// holds the probe's placeholder remote, so the row decision would refuse +/// every round, and the guest would read the range as absent while this host +/// answers its boxes' names inside it (#2748). The shape alone admits +/// nothing: the guest is untrusted here, so [`relay_control`] bounds what a +/// probe may bind ([`PublishedForwards::begin_probe`], +/// [`range_address_held`], [`RANGE_PROBE_HOLD`]). +fn range_probe_listener(verb: ControlVerb, body: &[u8]) -> Option { + // Read as a plain object, keyed as written, a repeated key's last value + // winning as it does in gvproxy's decoder, so a body that would bind as a + // probe round there is recognised as one here, and the strict read that + // follows ([`probe_body_is_strict`]) refuses it if it carries anything + // beyond the round's own fields. + let value: serde_json_lenient::Value = serde_json_lenient::from_slice(body).ok()?; + let field = |name: &str| value.get(name).and_then(serde_json_lenient::Value::as_str); + match verb { + ControlVerb::Expose if field("remote")? != switch::loopback::RANGE_PROBE_REMOTE => { + return None; + } + ControlVerb::Expose | ControlVerb::Unexpose => {} + ControlVerb::DnsAdd => return None, + } + let listener = loopback_listener(field("local")?).ok()?; + (field("protocol")? == "tcp" + && listener.1 == switch::loopback::RANGE_PROBE_PORT + && in_reserved_local_range(Ipv4Addr::from(listener.0))) + .then_some(listener) +} + +/// A range probe round's expose body, read strictly: exactly these fields, +/// once each, spelled exactly. gvproxy's Go decoder matches keys without +/// regard to case and lets a repeated key win, so a body carrying anything +/// else could bind there what the gate never checked. +#[derive(serde::Deserialize, serde::Serialize)] +#[serde(deny_unknown_fields)] +struct ProbeExposeBody { + local: String, + remote: String, + protocol: String, +} + +/// A range probe round's unexpose body, read as strictly as +/// [`ProbeExposeBody`]. +#[derive(serde::Deserialize, serde::Serialize)] +#[serde(deny_unknown_fields)] +struct ProbeUnexposeBody { + local: String, + protocol: String, +} + +/// Whether a probe-shaped body carries its own fields and nothing else: +/// no unknown key, no key twice, no case variant of one. +fn probe_body_is_strict(verb: ControlVerb, body: &[u8]) -> bool { + match verb { + ControlVerb::Expose => serde_json_lenient::from_slice::(body).is_ok(), + ControlVerb::Unexpose => serde_json_lenient::from_slice::(body).is_ok(), + ControlVerb::DnsAdd => false, + } +} + +/// The request an admitted range probe round writes to the switch, built +/// from the listener the gate checked and the probe's own constants, in the +/// guest daemon's own spelling: gvproxy sees exactly what the gate decided, +/// whatever bytes the guest sent. +fn canonical_probe_request(verb: ControlVerb, listener: Listener) -> Vec { + let local = forward_revoke::listener_local(listener.0, listener.1); + let protocol = "tcp".to_string(); + let (path, body) = match verb { + ControlVerb::Unexpose => ( + "/services/forwarder/unexpose", + serde_json_lenient::to_vec(&ProbeUnexposeBody { local, protocol }), + ), + _ => ( + "/services/forwarder/expose", + serde_json_lenient::to_vec(&ProbeExposeBody { + local, + remote: switch::loopback::RANGE_PROBE_REMOTE.to_string(), + protocol, + }), + ), + }; + let body = body.expect("three strings always serialize"); + let mut request = format!( + "POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\n\ + Content-Length: {}\r\n\r\n", + body.len() + ) + .into_bytes(); + request.extend_from_slice(&body); + request +} + +/// The answer to a probe-shaped round the gate refuses for a key beyond its +/// own fields: an HTTP refusal, so the guest reads a refusal, never a fault +/// of the channel. +fn probe_malformed_answer() -> Vec { + const BODY: &str = "a loopback range probe round carried a key beyond its own fields"; + format!( + "HTTP/1.1 400 Bad Request\r\nContent-Type: text/plain; charset=utf-8\r\n\ + Content-Length: {}\r\n\r\n{BODY}\n", + BODY.len() + 1 + ) + .into_bytes() +} + +/// Whether a control request names a listener a range probe round can bind: +/// the probe port at an address of the reserved range, whatever else the +/// request is. Every such request takes [`PublishedForwards::probe_turn`]. +fn names_probe_listener(verb: ControlVerb, body: &[u8]) -> bool { + forward_listener(verb, body).is_some_and(|listener| { + listener.1 == switch::loopback::RANGE_PROBE_PORT + && in_reserved_local_range(Ipv4Addr::from(listener.0)) + }) +} + +/// Whether a box holds the loopback address `addr`: a row registered at it, +/// or an applied or waiting publish whose listener is bound there. A range +/// probe never binds at such an address, so it can never squat a port a box +/// declares there or skew what the box's own forwards hold. The host bound +/// or granted the address for that box, so the probe's round is answered as +/// held, which the walk counts as present (NET-123). +fn range_address_held(table: &BoxTable, forwards: &PublishedForwards, addr: [u8; 4]) -> bool { + forwards.holds_loopback_address(addr) + || table + .rows() + .iter() + .any(|row| row.loopback_addr().octets() == addr) +} + +/// How long one range probe round may stay bound before the gate releases it +/// itself ([`release_probe_after_hold`]). The guest daemon releases each +/// round the moment it answers, within its own one-second round budget, so +/// only a guest that never releases its round meets this bound. +#[cfg(not(test))] +const RANGE_PROBE_HOLD: Duration = Duration::from_secs(5); +/// [`RANGE_PROBE_HOLD`] for the gate's own tests: long enough for a test's +/// release to arrive first, short enough to wait out. +#[cfg(test)] +const RANGE_PROBE_HOLD: Duration = Duration::from_secs(2); + +/// The rule a refused range probe round is logged under. +const RANGE_PROBE_RULE: &str = "egress-range-probe"; + +/// The answer to a range probe round the gate refuses because another round +/// is still outstanding: an HTTP refusal, so the guest reads the forwarder's +/// refusal it reads for an address that does not bind, never a fault of the +/// channel. +fn probe_refused_answer() -> Vec { + const BODY: &str = "another loopback range probe round is still bound"; + format!( + "HTTP/1.1 409 Conflict\r\nContent-Type: text/plain; charset=utf-8\r\n\ + Content-Length: {}\r\n\r\n{BODY}\n", + BODY.len() + 1 + ) + .into_bytes() +} + +/// How many times the gate asks the switch to release a round the guest +/// left bound before it gives up ([`release_probe_after_hold`]). +const RANGE_PROBE_RELEASE_ATTEMPTS: usize = 3; + +/// Releases the range probe round `generation` once [`RANGE_PROBE_HOLD`] has +/// passed, if the guest has not released it: a guest that binds a round and +/// never unexposes it holds the probe port at that address no longer than +/// the bound. The round stays accounted until the switch answers its +/// release, unbound or never bound, so a release the switch does not answer +/// is asked again a hold later, [`RANGE_PROBE_RELEASE_ATTEMPTS`] times in all. +/// After the last failure the round stays outstanding, and with it the gate +/// admits no other round, failing closed. The task holds the gate's ledger +/// only weakly, and ends with the gate. +async fn release_probe_after_hold( + switch_sock: PathBuf, + forwards: std::sync::Weak, + generation: u64, +) { + for attempt in 1..=RANGE_PROBE_RELEASE_ATTEMPTS { + tokio::time::sleep(RANGE_PROBE_HOLD).await; + let Some(forwards) = forwards.upgrade() else { + return; + }; + let _turn = forwards.probe_turn.lock().await; + let Some(listener) = forwards.probe_listener(generation) else { + return; + }; + // A box's publish holds the listener now. Its forward could bind only + // once the round's was gone, so it is the box's, never the round's, + // and it stays. + if forwards.protocol_of(listener).is_some() { + forwards.end_probe(generation); + return; + } + let local = forward_revoke::listener_local(listener.0, listener.1); + match forward_revoke::unexpose(&switch_sock, &local, "tcp", forward_revoke::REVOKE_TIMEOUT) + .await + { + Ok(_) => { + forwards.end_probe(generation); + tracing::warn!( + rule_matched = RANGE_PROBE_RULE, + %local, + "released a loopback range probe round the guest left bound past its hold" + ); + return; + } + Err(error) if attempt == RANGE_PROBE_RELEASE_ATTEMPTS => { + tracing::error!( + rule_matched = RANGE_PROBE_RULE, + %local, + %error, + attempts = attempt, + "gave up releasing a loopback range probe round the guest left bound; \ + it stays accounted, and the gate admits no other round" + ); + } + Err(error) => { + tracing::warn!( + rule_matched = RANGE_PROBE_RULE, + %local, + %error, + attempt, + "releasing a loopback range probe round failed; asking again" + ); + } + } + } +} + /// Summarizes an expose request into a [`SwitchRequest`]: the remote is the /// switch address the forwarder dials and the address the publish is at; the /// record is the host-side listener's port — the end the registration wire @@ -15639,6 +16096,438 @@ mod tests { expect_teardown(&mut guest).await; } + /// #2748: one round of the guest daemon's loopback range probe reaches + /// the switch, its bind and its release both, though no row holds the + /// probe's placeholder remote. Refused, every round would read as a fault + /// of the channel, and the guest would land the range absent while this + /// host answers its boxes' names inside it. + #[tokio::test] + async fn a_range_probe_round_reaches_the_switch_with_no_row_behind_it() { + let h = gate_connected(BoxRegistry::new(SUBNET)).await; + relayed_and_answered(&h, &probe_round("127.0.64.1", true)).await; + relayed_and_answered(&h, &probe_round("127.0.64.1", false)).await; + } + + /// One range probe round's control request, as the guest daemon's walk + /// spells it: the expose that binds the probe port at `address`, or the + /// unexpose that releases it. + fn probe_round(address: &str, expose: bool) -> Vec { + let local = format!("{address}:{}", switch::loopback::RANGE_PROBE_PORT); + let (path, body) = if expose { + ( + "/services/forwarder/expose", + format!( + r#"{{"local":"{local}","remote":"{}","protocol":"tcp"}}"#, + switch::loopback::RANGE_PROBE_REMOTE + ), + ) + } else { + ( + "/services/forwarder/unexpose", + format!(r#"{{"local":"{local}","protocol":"tcp"}}"#), + ) + }; + control_request(path, &body) + } + + /// A control request in the guest daemon's own spelling + /// (`post_json_within`, `crates/minimald/src/net/policy.rs`). + fn control_request(path: &str, body: &str) -> Vec { + let mut request = format!( + "POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\n" + ) + .into_bytes(); + request.extend_from_slice(format!("Content-Length: {}\r\n\r\n", body.len()).as_bytes()); + request.extend_from_slice(body.as_bytes()); + request + } + + /// #2748: gvproxy's Go decoder matches a key without regard to case and + /// lets a repeated key win, so a probe-shaped body with any key beyond the + /// round's own fields could bind there what the gate never checked. Each + /// is refused with an HTTP refusal and never relayed. + #[tokio::test] + async fn a_probe_round_with_a_key_beyond_its_own_fields_is_refused() { + let h = gate_connected(BoxRegistry::new(SUBNET)).await; + let port = switch::loopback::RANGE_PROBE_PORT; + let remote = switch::loopback::RANGE_PROBE_REMOTE; + for body in [ + format!( + r#"{{"local":"127.0.64.5:{port}","remote":"{remote}","protocol":"tcp","LOCAL":"0.0.0.0:2222","REMOTE":"10.0.0.1:22"}}"# + ), + format!( + r#"{{"local":"0.0.0.0:2222","local":"127.0.64.5:{port}","remote":"{remote}","protocol":"tcp"}}"# + ), + format!( + r#"{{"local":"127.0.64.5:{port}","remote":"{remote}","protocol":"tcp","extra":1}}"# + ), + ] { + let refused = + answered_unrelayed(&h, &control_request("/services/forwarder/expose", &body)).await; + assert!( + refused.starts_with("HTTP/1.1 400 "), + "{body} is refused with an HTTP refusal, got: {refused}" + ); + } + let release = format!(r#"{{"local":"127.0.64.5:{port}","protocol":"tcp","Local":"x"}}"#); + let refused = answered_unrelayed( + &h, + &control_request("/services/forwarder/unexpose", &release), + ) + .await; + assert!( + refused.starts_with("HTTP/1.1 400 "), + "a release with a case variant is refused too, got: {refused}" + ); + // No round was admitted by any of them. + relayed_and_answered(&h, &probe_round("127.0.64.5", true)).await; + } + + /// #2748: an admitted round reaches the switch as the gate rebuilt it from + /// the fields it checked, never as the guest's own bytes. + #[tokio::test] + async fn an_admitted_probe_round_is_relayed_as_its_canonical_request() { + let h = gate_connected(BoxRegistry::new(SUBNET)).await; + let port = switch::loopback::RANGE_PROBE_PORT; + // The round's own fields, in another order and spacing, under a + // different head. + let body = format!( + r#"{{ "protocol": "tcp", "remote": "{}", "local": "127.0.64.5:{port}" }}"#, + switch::loopback::RANGE_PROBE_REMOTE + ); + let mut sent = b"POST /services/forwarder/expose HTTP/1.1\r\nHost: x\r\n".to_vec(); + sent.extend_from_slice(format!("Content-Length: {}\r\n\r\n", body.len()).as_bytes()); + sent.extend_from_slice(body.as_bytes()); + let canonical = probe_round("127.0.64.5", true); + assert_ne!(sent, canonical); + assert_eq!( + canonical, + super::canonical_probe_request(super::ControlVerb::Expose, ([127, 0, 64, 5], port)), + "the gate's canonical request is the guest daemon's own spelling" + ); + + let (mut guest, mut switch) = connect_control(&h).await; + guest.write_all(&sent).await.expect("writing the round"); + let mut spoken = vec![0u8; canonical.len()]; + read_within(&mut switch, &mut spoken).await; + assert_eq!( + String::from_utf8_lossy(&spoken), + String::from_utf8_lossy(&canonical), + "the switch reads the canonical request, not the guest's bytes" + ); + } + + /// Speaks `request` on a fresh control connection and answers it from + /// the switch with 200 once it arrives whole, as gvproxy answers a bind + /// or a release that stood. + async fn relayed_and_answered(h: &super::test_support::GateHarness, request: &[u8]) { + let (mut guest, mut switch) = connect_control(h).await; + guest.write_all(request).await.expect("writing the round"); + let mut spoken = vec![0u8; request.len()]; + read_within(&mut switch, &mut spoken).await; + assert_eq!(spoken, request, "the round reached the switch whole"); + let answer = super::forward_revoke::bound_answer(); + switch + .write_all(answer) + .await + .expect("writing gvproxy's answer"); + let mut seen = vec![0u8; answer.len()]; + read_within(&mut guest, &mut seen).await; + assert_eq!(seen, answer, "the answer reaches the guest verbatim"); + } + + /// Speaks `request` on a fresh control connection, reads the answer the + /// gate gives it itself, and checks nothing of it reached the switch. + async fn answered_unrelayed(h: &super::test_support::GateHarness, request: &[u8]) -> String { + let (mut guest, mut switch) = connect_control(h).await; + guest.write_all(request).await.expect("writing the round"); + let mut answer = Vec::new(); + let mut chunk = [0u8; 256]; + while !String::from_utf8_lossy(&answer).contains("\r\n\r\n") { + let n = tokio::time::timeout(DEADLINE, guest.read(&mut chunk)) + .await + .expect("the gate answers within the deadline") + .expect("reading the gate's answer"); + assert!(n > 0, "the gate closed without answering"); + answer.extend_from_slice(&chunk[..n]); + } + drop(guest); + let mut probe = [0u8; 1]; + match tokio::time::timeout(DEADLINE, switch.read(&mut probe)).await { + Ok(Ok(0)) => {} + Ok(Ok(n)) => panic!("{n} byte(s) of an unrelayed round reached the switch"), + Ok(Err(e)) => panic!("reading the switch end failed: {e}"), + Err(_) => panic!("the gate left the switch side hanging"), + } + String::from_utf8_lossy(&answer).into_owned() + } + + /// #2748, the probe's bound: one round at a time. While a round is bound, + /// a second round's expose is refused with an HTTP refusal the guest + /// reads as an address that does not bind, never relayed. A release + /// names only the round outstanding: one of any other listener is + /// answered as done and never relayed. Once the round is released, the + /// next one is relayed. + #[tokio::test] + async fn a_second_outstanding_range_probe_round_is_refused() { + let h = gate_connected(BoxRegistry::new(SUBNET)).await; + relayed_and_answered(&h, &probe_round("127.0.64.1", true)).await; + + let refused = answered_unrelayed(&h, &probe_round("127.0.64.2", true)).await; + assert!( + refused.starts_with("HTTP/1.1 409 "), + "a second round is refused with an HTTP refusal, got: {refused}" + ); + wait_for_log(&h.log, super::RANGE_PROBE_RULE).await; + let other = answered_unrelayed(&h, &probe_round("127.0.64.2", false)).await; + assert!( + other.starts_with("HTTP/1.1 200 "), + "a release of a listener no round holds is answered as done, got: {other}" + ); + + relayed_and_answered(&h, &probe_round("127.0.64.1", false)).await; + relayed_and_answered(&h, &probe_round("127.0.64.2", true)).await; + } + + /// #2748, the probe's bound: a round at an address a box holds is never + /// relayed, so the probe never binds the port at another box's address + /// or skews what its forwards hold. It is answered as a held listener, + /// which the walk counts as present: the host bound or granted that + /// address for the box. + #[tokio::test] + async fn a_range_probe_round_at_a_boxs_address_is_answered_held_and_not_relayed() { + let registry = BoxRegistry::new(SUBNET); + registry.register(BoxRegistration::new( + "web", + Ipv4Addr::from(LEASE), + Ipv4Addr::new(127, 0, 64, 9), + )); + let h = gate_connected(registry).await; + let held = answered_unrelayed(&h, &probe_round("127.0.64.9", true)).await; + assert!( + held.starts_with("HTTP/1.1 500 ") && held.contains("proxy already running"), + "the round is answered as a held listener, got: {held}" + ); + // A round elsewhere is still relayed. + relayed_and_answered(&h, &probe_round("127.0.64.10", true)).await; + } + + /// #2748, the probe's bound: a round the guest binds and never releases + /// is released by the gate itself once its hold has passed, and the next + /// round is admitted. + #[tokio::test] + async fn a_range_probe_round_left_bound_is_released_after_its_hold() { + let h = gate_connected(BoxRegistry::new(SUBNET)).await; + relayed_and_answered(&h, &probe_round("127.0.64.1", true)).await; + + let (mut release, _) = tokio::time::timeout( + super::RANGE_PROBE_HOLD + DEADLINE, + h.switch_listener.accept(), + ) + .await + .expect("the gate dials the switch to release the round") + .expect("accepting the gate's release"); + let mut request = Vec::new(); + let mut chunk = [0u8; 512]; + while !String::from_utf8_lossy(&request).contains(r#""protocol":"tcp"}"#) { + let n = tokio::time::timeout(DEADLINE, release.read(&mut chunk)) + .await + .expect("the unexpose arrives within the deadline") + .expect("reading the unexpose"); + assert!(n > 0, "the gate closed before its unexpose was whole"); + request.extend_from_slice(&chunk[..n]); + } + let request = String::from_utf8_lossy(&request); + assert!( + request.starts_with("POST /services/forwarder/unexpose HTTP/1.1\r\n") + && request.ends_with(&format!( + r#"{{"local":"127.0.64.1:{}","protocol":"tcp"}}"#, + switch::loopback::RANGE_PROBE_PORT + )), + "the gate releases the round it admitted, got: {request}" + ); + release + .write_all(super::forward_revoke::bound_answer()) + .await + .expect("answering the release"); + drop(release); + wait_for_log(&h.log, "left bound past its hold").await; + relayed_and_answered(&h, &probe_round("127.0.64.2", true)).await; + } + + /// The gate's release of a round leaves a listener alone once a box's + /// publish holds it: that forward could bind only after the round's was + /// gone, so it is the box's. The round ends without the switch being + /// asked anything. Every request naming the probe port at a range + /// address takes the probe turn, whatever else it is. + #[tokio::test] + async fn the_hold_never_releases_a_listener_a_box_now_holds() { + use super::{ControlVerb, PublishedForwards, names_probe_listener}; + let port = switch::loopback::RANGE_PROBE_PORT; + let forwards = std::sync::Arc::new(PublishedForwards::new()); + let listener = ([127, 0, 64, 7], port); + let generation = forwards + .begin_probe(listener) + .expect("no round is outstanding"); + forwards + .note_published( + listener, + LEASE, + port, + super::egress::IPPROTO_TCP, + sessions::core::switch_request::Applied::Row, + ) + .expect("the ledger has room"); + // No switch listens at this path: a release that dialed it would fail + // and ask again until the bound below. + let dir = TempDir::new().unwrap(); + tokio::time::timeout( + super::RANGE_PROBE_HOLD * 3, + super::release_probe_after_hold( + dir.path().join("no-switch.sock"), + std::sync::Arc::downgrade(&forwards), + generation, + ), + ) + .await + .expect("the release ends without asking the switch"); + assert_eq!(forwards.outstanding_probe(), None, "the round is over"); + + let box_publish = + format!(r#"{{"local":"127.0.64.7:{port}","remote":"100.64.0.9:80","protocol":"tcp"}}"#); + assert!(names_probe_listener( + ControlVerb::Expose, + box_publish.as_bytes() + )); + let elsewhere = r#"{"local":"127.0.64.7:8080","remote":"100.64.0.9:80","protocol":"tcp"}"#; + assert!(!names_probe_listener( + ControlVerb::Expose, + elsewhere.as_bytes() + )); + } + + /// The gate's release of a round is bounded: it ends with the gate, and a + /// switch that never answers is asked [`super::RANGE_PROBE_RELEASE_ATTEMPTS`] + /// times, after which the round stays accounted and no other is admitted. + #[tokio::test] + async fn the_holds_release_ends_with_the_gate_and_gives_up_bounded() { + use super::PublishedForwards; + let dir = TempDir::new().unwrap(); + let listener = ([127, 0, 64, 7], switch::loopback::RANGE_PROBE_PORT); + + // The gate is gone: the release ends without asking anything. + let forwards = std::sync::Arc::new(PublishedForwards::new()); + let generation = forwards + .begin_probe(listener) + .expect("no round is outstanding"); + let gone = std::sync::Arc::downgrade(&forwards); + drop(forwards); + tokio::time::timeout( + super::RANGE_PROBE_HOLD * 2, + super::release_probe_after_hold(dir.path().join("no-switch.sock"), gone, generation), + ) + .await + .expect("the release ends with the gate"); + + // No switch answers: the release gives up after its attempts, and the + // round stays accounted. + let forwards = std::sync::Arc::new(PublishedForwards::new()); + let generation = forwards + .begin_probe(listener) + .expect("no round is outstanding"); + let attempts = u32::try_from(super::RANGE_PROBE_RELEASE_ATTEMPTS).unwrap(); + tokio::time::timeout( + super::RANGE_PROBE_HOLD * (attempts + 2), + super::release_probe_after_hold( + dir.path().join("no-switch.sock"), + std::sync::Arc::downgrade(&forwards), + generation, + ), + ) + .await + .expect("the release gives up after a bounded number of attempts"); + assert_eq!( + forwards.outstanding_probe(), + Some((listener, generation)), + "a round the switch never confirmed released stays accounted" + ); + assert!( + forwards.begin_probe(([127, 0, 64, 8], listener.1)).is_err(), + "and no other round is admitted" + ); + } + + /// The probe's shape is exact: an address of the reserved range, the probe + /// port, tcp, and for the expose the placeholder remote. Anything else is + /// not a round, and the row decision decides it. An address is held when a + /// row is registered at it or a publish's listener is bound there. + #[test] + fn the_range_probe_rule_reads_only_the_probes_own_shape() { + use super::{ControlVerb, PublishedForwards, range_address_held, range_probe_listener}; + let port = switch::loopback::RANGE_PROBE_PORT; + let remote = switch::loopback::RANGE_PROBE_REMOTE; + let expose = |local: &str, remote: &str, protocol: &str| { + format!(r#"{{"local":"{local}","remote":"{remote}","protocol":"{protocol}"}}"#) + }; + let round = format!("127.0.64.7:{port}"); + assert_eq!( + range_probe_listener( + ControlVerb::Expose, + expose(&round, remote, "tcp").as_bytes() + ), + Some(([127, 0, 64, 7], port)) + ); + assert_eq!( + range_probe_listener( + ControlVerb::Unexpose, + format!(r#"{{"local":"{round}","protocol":"tcp"}}"#).as_bytes() + ), + Some(([127, 0, 64, 7], port)) + ); + for (local, remote, protocol) in [ + (format!("127.0.0.1:{port}"), remote, "tcp"), + (format!("127.0.64.7:{}", port + 1), remote, "tcp"), + (round.clone(), "100.64.0.9:1", "tcp"), + (round.clone(), remote, "udp"), + ] { + assert_eq!( + range_probe_listener( + ControlVerb::Expose, + expose(&local, remote, protocol).as_bytes() + ), + None, + "{local} {remote} {protocol} is not a probe round" + ); + } + + let registry = BoxRegistry::new(SUBNET); + registry.register(BoxRegistration::new( + "web", + Ipv4Addr::from(LEASE), + Ipv4Addr::new(127, 0, 64, 9), + )); + let table = registry.table(); + let forwards = PublishedForwards::new(); + assert!(!range_address_held(&table, &forwards, [127, 0, 64, 7])); + assert!( + range_address_held(&table, &forwards, [127, 0, 64, 9]), + "a row's own loopback address is held" + ); + forwards + .note_published( + ([127, 0, 64, 7], 8080), + LEASE, + 8080, + super::egress::IPPROTO_TCP, + sessions::core::switch_request::Applied::Row, + ) + .expect("the ledger has room"); + assert!( + range_address_held(&table, &forwards, [127, 0, 64, 7]), + "an address a publish's listener is bound at is held" + ); + } + /// The two accepted spellings are the daemon's own, and a `local` /// outside both is refused as a body the gate does not summarize — /// before any decision the row could have made about its port, which is diff --git a/crates/switch/src/loopback.rs b/crates/switch/src/loopback.rs index 2217546676..2427c7786e 100644 --- a/crates/switch/src/loopback.rs +++ b/crates/switch/src/loopback.rs @@ -98,6 +98,17 @@ impl RangeProbe { } } +/// The host port the forwarder-conducted range probe binds at each address +/// of the reserved local range (NET-123): fixed and deliberately obscure, so +/// no box's declaration is likely to name it. Shared by the guest daemon that +/// walks the range and the host-side egress gate that admits each round. +pub const RANGE_PROBE_PORT: u16 = 21064; + +/// The `remote` every round of the forwarder-conducted range probe names: a +/// placeholder, never dialed, since the probe accepts no connection. The +/// host-side egress gate admits a round by this exact spelling. +pub const RANGE_PROBE_REMOTE: &str = "127.0.0.1:1"; + /// Every usable address in the reserved local range: host part 1..=254. The /// `.0` and `.255` spellings of a /24 are the network and broadcast forms, /// and the step that installs the range installs exactly these 254 aliases — diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index 8be35b1729..1d035425b9 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -4961,12 +4961,12 @@ echo "::endgroup::" # reserved local range to lo0. Then, WITHOUT a VM restart, a new own-IP box # with `--ingress 8080:8080` must publish at its own 127.0.64.N address: the # session start walks the range again, finds it present, and the box's name -# answers there. Three facts are asserted: +# answers there. Two facts are asserted: # * `http://.min.internal:8080` returns the in-box server's response, # through the resolver file the same script wrote; # * the box's own range address (the zone answerer's A record for its name) -# returns the same response at port 8080; -# * `min ls` names that address. +# returns the same response at port 8080. +# That `min ls` names the address belongs to T110 (#2696), which adds it. # The host is restored on every exit: the range unit, its aliases, the # resolver file and the answerer service go (range_teardown_unit, # answerer_service_teardown — the teardown repeats both if the case dies). @@ -5115,18 +5115,10 @@ fresh_install_range_lands_on_macos() { fi done - # `min ls` names the address the box answers at. - if ! mnl ls 2>/dev/null | grep -qF -- "$frm_addr"; then - echo "::error::'min ls' does not name $frm_addr, the address $frm_name.min.internal answers at" - mnl ls 2>&1 | head -40 || true - fail - fi - echo "'min ls' names $frm_addr" - mnl session destroy --force "$frm_sid" >/dev/null 2>&1 || true range_teardown_unit answerer_service_teardown - echo "fresh-install range publish on macOS OK (min finalize-install on a running VM, $frm_name.min.internal and $frm_addr answer at :$frm_port, min ls names the address, host restored)" + echo "fresh-install range publish on macOS OK (min finalize-install on a running VM, $frm_name.min.internal and $frm_addr answer at :$frm_port, host restored)" echo "::endgroup::" } @@ -16485,18 +16477,17 @@ proof_own_ip_deny_all_box_answers_published_port() { # On a Linux VM lane the record is the guest's, so the listener is read # from the host's own socket table: /proc/net/tcp spells an IPv4 socket's # local address as the address' four octets in little-endian order with the - # port in big-endian hex — 127.64.0.14:18086 reads `0E00407F:46A6` — so - # the match pins the port, the LISTEN state, and the loopback half of - # either spelling the publish surface can bind a box's port at: `127.64.0.x`, - # the reserved local range a granted publish answers on (NET-010), or - # `127.0.0.1`, the interim a publish stands at where the surface's range - # verdict read the range absent (NET-123). The KVM lane lands absent: the - # range walk rides the same shuttle the publishes do, and the host gate - # refuses every one of its probe rounds — the probe's `remote` is a - # placeholder, so no registered row holds the address its publish is keyed - # at — so the interim is that lane's own answer, not a broken publish. - # The reserved-range arm is tried first and stays for a lane whose verdict - # lands present, and the marker the dial below demands keeps either + # port in big-endian hex — 127.0.64.14:18086 reads `0E40007F:46A6`, the + # third octet in characters 3-4 and the second in 5-6 — so the match pins + # the port, the LISTEN state, and the loopback half of either spelling the + # publish surface can bind a box's port at: `127.0.64.x`, the reserved + # local range a granted publish answers on (NET-010), or `127.0.0.1`, the + # interim a publish stands at where the surface's range verdict read the + # range absent (NET-123). The KVM lane lands present: the range walk rides + # the same shuttle the publishes do, and the host gate admits its probe + # rounds (#2748), so the reserved-range arm is that lane's answer. The + # interim arm stays for a lane whose verdict lands absent, and the marker + # the dial below demands keeps either # spelling honest: a stray host listener at the same port does not carry # this box's answer. A macOS VM host has neither a readable record nor # /proc, so that leg names itself skipped there. @@ -16525,7 +16516,7 @@ proof_own_ip_deny_all_box_answers_published_port() { da_row="" for _ in $(seq 1 40); do da_row="$(awk -v want="$(printf '%04X' "$DA_EXT")" \ - 'substr($2, 7, 2) == "7F" && substr($2, 5, 2) == "40" && substr($2, 3, 2) == "00" \ + 'substr($2, 7, 2) == "7F" && substr($2, 5, 2) == "00" && substr($2, 3, 2) == "40" \ && $4 == "0A" && index($2, ":" want) == 9 { print $2 }' /proc/net/tcp 2>/dev/null | tail -n1)" if [ -z "$da_row" ]; then da_row="$(awk -v want="$(printf '%04X' "$DA_EXT")" \ @@ -16990,7 +16981,7 @@ proof_port_publishes_on_listen_and_box_outlives_client() { po_row="" for _ in $(seq 1 40); do po_row="$(awk -v want="$(printf '%04X' "$PO_EXT")" \ - 'substr($2, 7, 2) == "7F" && substr($2, 5, 2) == "40" && substr($2, 3, 2) == "00" \ + 'substr($2, 7, 2) == "7F" && substr($2, 5, 2) == "00" && substr($2, 3, 2) == "40" \ && $4 == "0A" && index($2, ":" want) == 9 { print $2 }' /proc/net/tcp 2>/dev/null | tail -n1)" if [ -z "$po_row" ]; then po_row="$(awk -v want="$(printf '%04X' "$PO_EXT")" \ From 8acbf4d067db44a6a0f3b605be827549341c7162 Mon Sep 17 00:00:00 2001 From: Norrie Taylor Date: Wed, 4 Nov 2026 15:16:14 -0700 Subject: [PATCH 11/34] fix(minimald): satisfy fmt and clippy on the publish-move backport (#2749) Follow-up to the #2749 backport. That commit reached the branch before the group's fmt and clippy gates ran, and pushed history is not rewritten, so the correction is its own commit. ListenPlan::new now takes nine arguments, so it carries main's too_many_arguments expect, which came with NET-129 there. move_exposed_ports fits one line at the line's shorter indent. No behaviour changes. --- crates/minimald/src/net/listeners.rs | 5 +++++ crates/minimald/src/session.rs | 6 +----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/minimald/src/net/listeners.rs b/crates/minimald/src/net/listeners.rs index ef5304c19d..35b0d8e6ea 100644 --- a/crates/minimald/src/net/listeners.rs +++ b/crates/minimald/src/net/listeners.rs @@ -922,6 +922,11 @@ impl ListenPlan { /// same port. `state_dir` is the daemon's state directory, the one the /// runtime expose path audits its decisions under, so both surfaces' /// records land in the one log. + #[expect( + clippy::too_many_arguments, + reason = "the plan is built once per launch from facts only the launch holds; \ + a builder would add a type for one call site" + )] #[must_use] pub fn new( box_name: String, diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index c6bcbb6001..9a0eac4586 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -4360,11 +4360,7 @@ impl Session { /// ended, or when the registry no longer publishes the box at `to`, /// since the move is read after other changes may have landed. #[cfg(target_os = "linux")] - async fn move_exposed_ports( - &mut self, - from: std::net::Ipv4Addr, - to: std::net::Ipv4Addr, - ) { + async fn move_exposed_ports(&mut self, from: std::net::Ipv4Addr, to: std::net::Ipv4Addr) { if !self.has_live_host() || self.live_ingress.is_detached() { return; } From 8746a3c7f26684650f55f699484f1ccc1e0f511e Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:42:37 +0000 Subject: [PATCH 12/34] fix(minimald): refuse unmatched names for partial allow lists and gate proxy callers (#2716) T106. NET-141: the relay's DNS gate answers REFUSED, before any onward query, for every name outside the box zone that a declared allow_dns_hosts list does not match. It did this only for a deny-all box, so a box with a partial list still sent the name off the machine. An empty list that admits subnets keeps forwarding. Each refusal writes one rate-limited warning, keyed by a bounded hash bucket of the name. NET-070: the hostname proxy names the box behind a connection from the node's address through the socket's inode, its holder and that process's classifier leaf, and applies that box's egress rules. While a host-address box is live, a connection whose owner cannot be named is refused. Owner lookups are capped at four at once, and a wait over five seconds is refused as unattributed. session-e2e.sh asserts the partial allow list refuses fast. Backport of #2716, folded from #2755. Adapted to the line: - Box actor and box runtime hunks land in session.rs and session_host.rs. - boxes, sandbox and box_runtime read sessions, sandbox2 and session_host, with SessionPolicy and SessionKeyPredicate. crate::traced becomes tokio::spawn and tokio::task::spawn_blocking, and the line's policy has no quic443 field. - register_host_net keeps #2749's dial at the guest's 127.0.0.1 beside the host_net record. Dropped: the edits to the BEP steering tests in dns_gate.rs, since the line has no BEP steering, and the NET spec hunk, since the line's NET-141 text differs from main's. (cherry picked from commit 302edaf097df13d6ada532f8f305d64bcb729a4a) --- crates/minimald/src/net/dns.rs | 503 +++++++++++++++++- crates/minimald/src/net/dns_gate.rs | 440 +++++++++++++--- crates/minimald/src/net/provider.rs | 224 +++++--- crates/minimald/src/net/proxy.rs | 744 ++++++++++++++++++++++++++- crates/minimald/src/session.rs | 29 ++ crates/minimald/src/session/tests.rs | 32 ++ crates/minimald/src/session_host.rs | 12 +- scripts/session-e2e.sh | 20 +- 8 files changed, 1810 insertions(+), 194 deletions(-) diff --git a/crates/minimald/src/net/dns.rs b/crates/minimald/src/net/dns.rs index a231f3debe..f7a5bc16f7 100644 --- a/crates/minimald/src/net/dns.rs +++ b/crates/minimald/src/net/dns.rs @@ -536,13 +536,14 @@ pub struct InterimPublish { /// egress rules its own outbound frames are decided by — the same rules a /// request from it is put to, exactly as a direct connection from it would be /// ([`super::switch::proxied_request_verdict`]). Named by the -/// lease-to-session map ([`HostnameRegistry::by_lease`]), so only a box on the -/// switch is ever a caller: a host-side client has no declaration to honour -/// and no gate of its own on a direct connection either. +/// lease-to-session map ([`HostnameRegistry::by_lease`]) for a box on the +/// switch, and by the box that owns the connecting socket for a host-address +/// box ([`HostnameRegistry::caller_owning`]). A host-side client is no +/// caller: it has no declaration to honour. #[derive(Debug, Clone)] pub struct Caller { - /// The switch lease the caller's box holds: the address its proxied - /// connections come from, and the address a refusal log names it by. + /// The address the caller's proxied connections come from: the switch + /// lease its box holds, or the node's address for a host-address box. lease: Ipv4Addr, /// The caller's session name, for the refusal log. name: String, @@ -713,6 +714,20 @@ pub struct HostnameRegistry { /// the node's own listener at that port keeps answering and the stopped /// box is not silently impersonated by it. stopped: HashSet, + /// Stopped boxes whose host launch has started and has not reached + /// [`Self::mark_running`] yet ([`Self::begin_host_launch`]). The launch + /// can start the box's processes before the host's loop clears `stopped`, + /// so [`Self::has_unleased_callers`] counts a box with this mark as live + /// (NET-070). + launching: HashSet, + /// Each box's current host launch number ([`Self::begin_host_launch`]). + /// A host loop that a newer launch replaced holds an older number, so its + /// late stop mark ([`Self::mark_host_stopped`]) changes nothing (NET-070). + host_launches: HashMap, + /// The last launch number [`Self::begin_host_launch`] gave out. One + /// counter for every box, so a number never repeats, even for a box id + /// that [`Self::forget_own_address`] forgot and a later attach reuses. + last_host_launch: u64, /// box name → the route a host-side proxy forwards its requests on. by_host: HashMap, /// session name → its live registration, for withdrawal on exit. @@ -724,6 +739,17 @@ pub struct HostnameRegistry { /// Recorded by [`Self::register_caller`] at hostname registration, before /// the box has a lease, so the join is ready the moment it attaches. callers: HashMap, + /// The sessions registered as `HostNet` boxes + /// ([`Self::register_host_net`]): they hold no switch lease by design, so + /// a caller fact without a lease is an unreported own-address lease only + /// for a session that is not in this set (NET-070). + host_net: HashSet, + /// The own-address boxes whose attach is running and has not reported + /// its switch lease yet ([`Self::begin_lease_attach`]). Only these make + /// [`Self::has_unreported_lease_callers`] true: a box created and never + /// started, or stopped, has no attach in flight and connects from no + /// lease at all (NET-070). + attaching: HashSet, /// The lease-to-session map that names a caller (NET-070): the switch /// address a proxied request's peer resolves to a session with. Reserved /// switch addresses never appear in it — the switch hands a PTask a lease @@ -767,10 +793,15 @@ impl HostnameRegistry { publish_moves: None, hands: HashMap::new(), stopped: HashSet::new(), + launching: HashSet::new(), + host_launches: HashMap::new(), + last_host_launch: 0, by_host: HashMap::new(), by_session: HashMap::new(), own: HashMap::new(), callers: HashMap::new(), + host_net: HashSet::new(), + attaching: HashSet::new(), by_lease: HashMap::new(), } } @@ -965,6 +996,7 @@ impl HostnameRegistry { /// which runs in the guest, dials those listeners at the guest's own /// `127.0.0.1` (#2748). pub fn register_host_net(&mut self, session_id: SessionId, session_name: &str) -> Hostname { + self.host_net.insert(session_id); let mut route = Route::loopback(session_name, self.node, None); if self.on_switch && let Target::Loopback { dial, .. } = &mut route.target @@ -1041,8 +1073,9 @@ impl HostnameRegistry { /// carry (NET-084) and the two surfaces decide by one rule set (NET-071). /// `None` when the lease names no session — which is what a host-side /// caller (the developer's browser, the daemon's own lanes) is, and what a - /// host-address session is too: neither has a box on the switch, and - /// neither's egress is gated on a direct connection either. + /// host-address session is too: neither has a box on the switch. A + /// host-address box is named by the socket it connects from instead + /// ([`Self::caller_owning`]). #[must_use] pub fn caller_at(&self, lease: Ipv4Addr) -> Option { let id = self.by_lease.get(&lease)?; @@ -1059,6 +1092,108 @@ impl HostnameRegistry { }) } + /// The caller a connection from the node's address stands for + /// (NET-070): the live host-address box `owner`, which owns the + /// connecting socket ([`connecting_box`]), with its egress rules compiled + /// with `from`, the address the connection came from, as their source. + /// A host-address box holds no switch lease, so [`Self::caller_at`] + /// cannot name it. `None` when `owner` has no caller facts, or when it is + /// an own-address box, whose connections come from its lease. The proxy + /// refuses a connection that this returns `None` for: the kernel named a + /// box, so the connection is not a host-side client's, and an unknown box + /// is denied by default. + #[must_use] + pub fn caller_owning(&self, owner: SessionId, from: Ipv4Addr) -> Option { + if self.own.contains_key(&owner) || self.by_lease.values().any(|id| *id == owner) { + return None; + } + let facts = self.callers.get(&owner)?; + Some(Caller { + lease: from, + name: facts.name.clone(), + egress: super::switch::compiled_egress( + Some(&facts.policy), + facts.subnet, + from, + facts.hostname_proxy_port, + ), + }) + } + + /// Whether any live session with caller facts holds no switch lease: a + /// host-address box, or a box whose lease is not reported yet. While + /// none does, no connection from the node's address can be a box's, so + /// the proxy does not look up who owns the connecting socket. A stopped + /// box ([`Self::mark_stopped`]) is not live: it keeps its caller facts + /// until destroy, but it runs nothing that can connect, so it does not + /// count until [`Self::begin_host_launch`] or [`Self::mark_running`]. + /// A launching box with no own address and no lease counts whether or + /// not it has caller facts: an ended box brought up for its destroy hooks + /// can have none, and the proxy refuses its connections as unattributed + /// rather than pass them as no box's (NET-070, design §7.1). + #[must_use] + pub fn has_unleased_callers(&self) -> bool { + let unleased = + |id: &SessionId| !self.own.contains_key(id) && !self.by_lease.values().any(|v| v == id); + self.callers + .keys() + .any(|id| !self.stopped.contains(id) && unleased(id)) + || self.launching.iter().any(unleased) + } + + /// Whether any own-address box is attaching and has not reported its + /// switch lease yet: it sits between [`Self::begin_lease_attach`] at the + /// start of its attach and [`Self::report_own_address`] at the end of it. + /// Its connections come from a lease this node does not hold, so + /// [`connecting_box`] answers [`ConnectingOwner::OffNode`] for them. While + /// this is true, the proxy cannot tell such a peer from a host-side client + /// off the node, and it refuses the peer as unattributed (NET-070, design + /// §7.1). A box that is registered but not attaching does not count: it + /// runs nothing that can connect. + #[must_use] + pub fn has_unreported_lease_callers(&self) -> bool { + self.attaching.iter().any(|id| !self.host_net.contains(id)) + } + + /// Marks `session_id` as an own-address box whose attach has started and + /// whose switch lease is not reported yet (NET-070). The mark ends at + /// [`Self::report_own_address`], at [`Self::end_lease_attach`] when the + /// attach fails or is abandoned, at [`Self::mark_stopped`] and at + /// [`Self::forget_own_address`]. + pub fn begin_lease_attach(&mut self, session_id: SessionId) { + self.attaching.insert(session_id); + } + + /// Marks `session_id` as a box whose host launch has started (NET-070). + /// A resumed box can run processes before its host's loop calls + /// [`Self::mark_running`], so from this mark the box counts as live for + /// [`Self::has_unleased_callers`], stopped or not. The mark ends at + /// [`Self::mark_running`], at [`Self::mark_stopped`], at + /// [`Self::forget_own_address`] and at [`Self::end_host_launch`] when the + /// launch fails. + /// + /// Returns this launch's number, which becomes the box's current launch. + /// The host's loop passes it to [`Self::mark_host_stopped`], so the loop + /// of a host this launch replaced cannot mark the new host's box stopped. + pub fn begin_host_launch(&mut self, session_id: SessionId) -> u64 { + self.last_host_launch += 1; + self.host_launches.insert(session_id, self.last_host_launch); + self.launching.insert(session_id); + self.last_host_launch + } + + /// Ends the launch mark [`Self::begin_host_launch`] set, for a launch + /// that failed (NET-070). + pub fn end_host_launch(&mut self, session_id: SessionId) { + self.launching.remove(&session_id); + } + + /// Ends the attach mark [`Self::begin_lease_attach`] set, for an attach + /// that failed or was abandoned before it reported a lease (NET-070). + pub fn end_lease_attach(&mut self, session_id: SessionId) { + self.attaching.remove(&session_id); + } + /// The own-address publishes standing inside the reserved local range, as /// `(session id, session_name, address, ports)` — the boxes an **absent** /// verdict landing moves onto the `127.0.0.1` interim (NET-123): a hand @@ -1223,6 +1358,7 @@ impl HostnameRegistry { ports: ports.clone(), }; self.by_lease.insert(lease, session_id); + self.attaching.remove(&session_id); // Recorded before the route is derived from it, so the route this // registration installs already targets the lease it is reporting: // [`Self::own_route`] reads the box's lease from this map, and a box @@ -1249,6 +1385,10 @@ impl HostnameRegistry { self.by_lease.remove(&own.lease); } self.callers.remove(&session_id); + self.host_net.remove(&session_id); + self.attaching.remove(&session_id); + self.launching.remove(&session_id); + self.host_launches.remove(&session_id); self.stopped.remove(&session_id); } @@ -1580,6 +1720,7 @@ impl HostnameRegistry { /// state every box is in from finalize, so a caller that never saw a stop /// changes nothing (NET-011). pub fn mark_running(&mut self, session_id: SessionId) { + self.launching.remove(&session_id); if self.stopped.remove(&session_id) { tracing::info!( session_id = %session_id, @@ -1589,6 +1730,27 @@ impl HostnameRegistry { } } + /// Marks a box's host as stopped from the loop of the host that `launch` + /// started ([`Self::begin_host_launch`]), as [`Self::mark_stopped`] does. + /// A loop that a newer launch replaced can end after that launch, so a + /// `launch` older than the box's current launch changes nothing: the box + /// stays live for [`Self::has_unleased_callers`] (NET-070, design §7.1). + pub fn mark_host_stopped(&mut self, session_id: SessionId, launch: u64) { + if let Some(¤t) = self.host_launches.get(&session_id) + && launch < current + { + tracing::debug!( + session_id = %session_id, + launch, + current, + action = "stale-host-stop-ignored", + "a replaced host's loop ended; its stop mark is ignored" + ); + return; + } + self.mark_stopped(session_id); + } + /// Marks a box's host as stopped (NET-128). The name stays **held** — a /// stopped box is never mistaken for one that never existed, so the zone /// never says NXDOMAIN for it — but a name it shares with the node answers @@ -1596,7 +1758,13 @@ impl HostnameRegistry { /// not silently impersonated by a dead box's declaration. A box at its own /// address keeps answering A: that address answers for it alone, and a /// client connecting there is told where the box would be. + /// + /// This mark has no launch guard. A host's loop marks its stop through + /// [`Self::mark_host_stopped`]. pub fn mark_stopped(&mut self, session_id: SessionId) { + // A stopped box has no attach in flight (NET-070). + self.attaching.remove(&session_id); + self.launching.remove(&session_id); if self.stopped.insert(session_id) { tracing::info!( session_id = %session_id, @@ -2815,6 +2983,208 @@ pub(crate) fn warn_publish_collision(address: Ipv4Addr) { ); } +/// What the kernel says owns the client end of a connection to this host +/// (NET-070), as [`connecting_box`] reads it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConnectingOwner { + /// A process in this box's classifier leaf holds the socket, and no + /// holder runs in another box's leaf. + Box(SessionId), + /// Every process that holds the socket runs in no box's leaf, as the + /// developer's browser does. + NoBox, + /// The lookup found no owner: the socket is gone from the tables or + /// held by no process (the client closed it), `/proc` hides a holder or + /// its cgroup from this process, a holder's pid stopped holding the + /// socket while it was read, or holders run in two boxes' leaves. + Unknown, + /// The peer's address is one this node does not hold, so no lookup ran. + /// The peer is a client off the node, such as the host side of a VM's + /// forwarded port, or an own-address box whose lease is not reported + /// yet. The caller decides which ([`HostnameRegistry::has_unreported_lease_callers`]). + OffNode, +} + +/// The box that owns the client end of a TCP connection to this host +/// (NET-070): `peer` is the client's address and `local` the address it +/// connected to, as the accepting side sees them. The kernel's socket tables +/// give the client socket's inode, every process that holds a descriptor for +/// it, and each one's cgroup. A host-address box's processes run in its +/// classifier leaf, `boxes//`, and the leaf's name is the +/// box's id. The socket is the box's when any holder runs in its leaf +/// ([`owner_of_holders`]). [`ConnectingOwner::NoBox`] when every holder runs +/// in no box's leaf, and [`ConnectingOwner::Unknown`] when any step finds +/// nothing or holders run in two boxes' leaves, so a caller can refuse a +/// connection it cannot attribute. +/// +/// A peer whose address this node does not hold is [`ConnectingOwner::OffNode`] +/// with no lookup. A host-address box shares the node's network namespace, +/// so it connects from an address the node holds. A client off the node, +/// such as the host side of a VM's forwarded port that arrives through the +/// switch's gateway, is never in this node's socket tables. Without this +/// rule its lookup finds nothing, and the proxy refuses every host request +/// while a host-address box is live. +#[must_use] +pub fn connecting_box(peer: SocketAddrV4, local: SocketAddrV4) -> ConnectingOwner { + if !node_holds_address(*peer.ip()) { + return ConnectingOwner::OffNode; + } + #[cfg(target_os = "linux")] + { + let Some(inode) = client_socket_inode(peer, local) else { + return ConnectingOwner::Unknown; + }; + let Some(holders) = socket_holders(inode) else { + return ConnectingOwner::Unknown; + }; + // Read each holder's cgroup, then check that the pid still holds the + // socket. A pid that exited and was reused between the two reads + // names another process's cgroup, so it counts as unreadable. + owner_of_holders(holders.into_iter().map(|pid| { + let cgroup = std::fs::read_to_string(format!("/proc/{pid}/cgroup")).ok()?; + holds_socket(pid, inode).then_some(cgroup) + })) + } + #[cfg(not(target_os = "linux"))] + { + let _ = (peer, local); + ConnectingOwner::Unknown + } +} + +/// Whether `ip` is an address this node holds: a loopback address, or one +/// a socket can bind to. A bind to port 0 fails with `EADDRNOTAVAIL` for an +/// address no interface holds. A host that permits non-local binds +/// (`ip_nonlocal_bind`) answers true for every address, so the caller's +/// lookup still runs and fails closed. +fn node_holds_address(ip: Ipv4Addr) -> bool { + ip.is_loopback() || ip.is_unspecified() || std::net::UdpSocket::bind((ip, 0)).is_ok() +} + +/// The kernel's socket-table spelling of an IPv4 socket address: the +/// address as one little-endian word, then the port, both in upper-case hex +/// (`0100007F:1F90` for `127.0.0.1:8080`), and the v4-mapped IPv6 spelling +/// the `tcp6` table prints for the same address. +fn socket_table_keys(addr: SocketAddrV4) -> [String; 2] { + let word = u32::from(*addr.ip()).swap_bytes(); + let port = addr.port(); + [ + format!("{word:08X}:{port:04X}"), + format!("0000000000000000FFFF0000{word:08X}:{port:04X}"), + ] +} + +/// The inode of the socket whose local end is `peer` and whose remote end is +/// `local`: the client end of a connection the accepting side sees from +/// `peer` to `local`. +#[cfg(target_os = "linux")] +fn client_socket_inode(peer: SocketAddrV4, local: SocketAddrV4) -> Option { + let ours = socket_table_keys(peer); + let theirs = socket_table_keys(local); + for table in ["/proc/net/tcp", "/proc/net/tcp6"] { + let Ok(text) = std::fs::read_to_string(table) else { + continue; + }; + if let Some(inode) = socket_row_inode(&text, &ours, &theirs) { + return Some(inode); + } + } + None +} + +/// The inode of the first row of one socket table whose local address is +/// one of `ours` and whose remote address is one of `theirs`. Inode `0` is +/// a socket no process holds (a time-wait row), so it is skipped. +fn socket_row_inode(table: &str, ours: &[String; 2], theirs: &[String; 2]) -> Option { + table.lines().skip(1).find_map(|line| { + let fields: Vec<&str> = line.split_whitespace().collect(); + let (local, remote, inode) = (fields.get(1)?, fields.get(2)?, fields.get(9)?); + if !ours.iter().any(|key| key == local) || !theirs.iter().any(|key| key == remote) { + return None; + } + inode.parse().ok().filter(|inode| *inode != 0) + }) +} + +/// Every process that holds a descriptor for the socket `inode`, in `/proc` +/// order. A socket can have several holders: a forked child keeps its +/// parent's descriptors, and a descriptor can be passed over a Unix socket. +/// `None` when `/proc` cannot be read. +#[cfg(target_os = "linux")] +fn socket_holders(inode: u64) -> Option> { + let holders = std::fs::read_dir("/proc") + .ok()? + .flatten() + .filter_map(|entry| entry.file_name().to_string_lossy().parse::().ok()) + .filter(|pid| holds_socket(*pid, inode)) + .collect(); + Some(holders) +} + +/// Whether process `pid` holds a descriptor for the socket `inode` now. +#[cfg(target_os = "linux")] +fn holds_socket(pid: u32, inode: u64) -> bool { + let target = format!("socket:[{inode}]"); + let Ok(fds) = std::fs::read_dir(format!("/proc/{pid}/fd")) else { + return false; + }; + fds.flatten().any(|fd| { + std::fs::read_link(fd.path()).is_ok_and(|link| link.as_os_str() == target.as_str()) + }) +} + +/// The owner of a socket from its holders' cgroups (NET-070). Each item is +/// one holder's `/proc//cgroup`, or `None` when that read failed or the +/// pid no longer held the socket after it. The socket is a box's when any +/// holder runs in that box's leaf, whatever other holders run outside every +/// box: a process outside the boxes must not lift the box's gate by sharing +/// its socket. [`ConnectingOwner::Unknown`] when there is no holder, when +/// any holder cannot be read, or when holders run in two different boxes' +/// leaves, so the proxy refuses the connection. [`ConnectingOwner::NoBox`] +/// only when every holder runs outside every box. +fn owner_of_holders(holders: impl IntoIterator>) -> ConnectingOwner { + let mut owner = None; + let mut any = false; + for cgroup in holders { + any = true; + let Some(cgroup) = cgroup else { + return ConnectingOwner::Unknown; + }; + if let Some(id) = box_in_cgroup(&cgroup) { + match owner { + Some(seen) if seen != id => return ConnectingOwner::Unknown, + _ => owner = Some(id), + } + } + } + match owner { + Some(id) => ConnectingOwner::Box(id), + None if any => ConnectingOwner::NoBox, + None => ConnectingOwner::Unknown, + } +} + +/// The box whose classifier leaf a process's `/proc//cgroup` names: +/// the path component after `boxes/allow` or `boxes/deny`, read as a box id. +/// `None` for a process in no box's leaf. +fn box_in_cgroup(cgroup: &str) -> Option { + use sandbox2::classifier::BOXES_DIR; + let subtrees = [ + sandbox2::config::Verdict::Allow.dir_name(), + sandbox2::config::Verdict::Deny.dir_name(), + ]; + cgroup.lines().find_map(|line| { + // cgroup2's line is `0::`; a v1 line names a controller. + let path = line.strip_prefix("0::")?; + let parts: Vec<&str> = path.split('/').collect(); + parts.windows(3).find_map(|window| { + (window[0] == BOXES_DIR && subtrees.contains(&window[1])) + .then(|| SessionId::parse_str(window[2]).ok()) + .flatten() + }) + }) +} + /// The wall clock the record stamps grants with, in Unix seconds — the same /// clock a daemon stamps its own start with (the stamp the manager passes /// to [`LoopbackLeaseBook::release_dead_boxes`]), so the sweep's @@ -2909,6 +3279,125 @@ fn listen_local_v4(line: &str, v6: bool) -> Option { mod tests { use super::*; + /// NET-070's socket-owner lookup, step by step. The cgroup a process + /// reports names its box when it runs in a classifier leaf, in either + /// subtree, and names no box otherwise. + #[test] + fn box_in_cgroup_reads_the_classifier_leaf() { + let id = "00000000-0000-0000-0000-0000000000aa"; + let want = SessionId::parse_str(id).ok(); + for subtree in ["allow", "deny"] { + let cgroup = format!("0::/minimald.slice/boxes/{subtree}/{id}\n"); + assert_eq!(box_in_cgroup(&cgroup), want, "{subtree}"); + } + for cgroup in [ + "0::/user.slice/user-1000.slice/session-2.scope\n", + "0::/minimald.slice/boxes/other/00000000-0000-0000-0000-0000000000aa\n", + "0::/minimald.slice/boxes/allow/not-a-box-id\n", + "4:cpu:/minimald.slice/boxes/allow/00000000-0000-0000-0000-0000000000aa\n", + ] { + assert_eq!(box_in_cgroup(cgroup), None, "{cgroup}"); + } + } + + /// NET-070: a socket can have several holders. A holder in a box's leaf + /// makes the socket that box's, whatever holder outside every box shares + /// it and in whatever order `/proc` lists them. Holders in two boxes' + /// leaves, or a holder that cannot be read, make the owner unknown. + #[test] + fn owner_of_holders_prefers_a_box_holder_and_refuses_two_boxes() { + let a = "00000000-0000-0000-0000-0000000000aa"; + let b = "00000000-0000-0000-0000-0000000000bb"; + let in_box = |id: &str| Some(format!("0::/minimal/boxes/deny/{id}\n")); + let outside = || Some("0::/user.slice/user-1000.slice/session-1.scope\n".to_owned()); + let box_a = ConnectingOwner::Box(SessionId::parse_str(a).unwrap()); + + assert_eq!(owner_of_holders([outside(), in_box(a)]), box_a); + assert_eq!(owner_of_holders([in_box(a), outside()]), box_a); + assert_eq!(owner_of_holders([in_box(a), in_box(a)]), box_a); + assert_eq!( + owner_of_holders([in_box(a), outside(), in_box(b)]), + ConnectingOwner::Unknown, + "holders in two boxes' leaves" + ); + assert_eq!( + owner_of_holders([outside(), outside()]), + ConnectingOwner::NoBox + ); + assert_eq!( + owner_of_holders([outside(), None]), + ConnectingOwner::Unknown, + "a holder whose cgroup read failed or whose pid was reused" + ); + assert_eq!( + owner_of_holders(std::iter::empty()), + ConnectingOwner::Unknown, + "no holder" + ); + } + + /// The socket table row of a connection's client end is the one whose + /// local address is the peer and whose remote address is the proxy's, + /// in the v4 table and in the v4-mapped spelling of the v6 table alike. + #[test] + fn socket_row_inode_finds_the_client_end() { + let peer = SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 4), 40000); + let local = SocketAddrV4::new(Ipv4Addr::LOCALHOST, 8080); + let ours = socket_table_keys(peer); + let theirs = socket_table_keys(local); + assert_eq!(ours[0], "0400007F:9C40"); + assert_eq!(theirs[0], "0100007F:1F90"); + let header = " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode"; + let v4 = format!( + "{header}\n 0: 0100007F:1F90 0400007F:9C40 01 00000000:00000000 00:00000000 00000000 1000 0 111 1\n 1: 0400007F:9C40 0100007F:1F90 01 00000000:00000000 00:00000000 00000000 1000 0 222 1\n" + ); + assert_eq!(socket_row_inode(&v4, &ours, &theirs), Some(222)); + let v6 = format!( + "{header}\n 0: 0000000000000000FFFF00000400007F:9C40 0000000000000000FFFF00000100007F:1F90 01 00000000:00000000 00:00000000 00000000 1000 0 333 1\n" + ); + assert_eq!(socket_row_inode(&v6, &ours, &theirs), Some(333)); + let time_wait = format!( + "{header}\n 0: 0400007F:9C40 0100007F:1F90 06 00000000:00000000 00:00000000 00000000 0 0 0 1\n" + ); + assert_eq!(socket_row_inode(&time_wait, &ours, &theirs), None); + } + + /// The kernel half of the lookup, against a real connection: the client + /// socket's inode is found from the accepting side's view, and the + /// process that holds it is this one. + #[cfg(target_os = "linux")] + #[test] + fn connecting_socket_holder_is_found() { + let listener = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let client = std::net::TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (_accepted, peer) = listener.accept().unwrap(); + let (SocketAddr::V4(peer), SocketAddr::V4(local)) = (peer, listener.local_addr().unwrap()) + else { + panic!("an IPv4 connection"); + }; + let inode = client_socket_inode(peer, local).expect("the client socket is in the table"); + assert!(holds_socket(std::process::id(), inode)); + assert!(socket_holders(inode).is_some_and(|pids| pids.contains(&std::process::id()))); + // A test process runs in no box's leaf, so it is no box. + assert_eq!(connecting_box(peer, local), ConnectingOwner::NoBox); + drop(client); + } + + /// A peer at an address this node does not hold is off the node, with no + /// socket lookup: a host-address box connects from the node's own + /// addresses, and an off-node client, such as a VM host's forwarded + /// request through the switch's gateway, is never in the node's tables + /// (NET-070). + #[test] + fn off_node_peer_is_no_box() { + // TEST-NET-1 (RFC 5737): no interface holds it. + let peer = SocketAddrV4::new(Ipv4Addr::new(192, 0, 2, 1), 40000); + let local = SocketAddrV4::new(Ipv4Addr::LOCALHOST, 7654); + assert!(!node_holds_address(*peer.ip())); + assert!(node_holds_address(Ipv4Addr::new(127, 0, 0, 4))); + assert_eq!(connecting_box(peer, local), ConnectingOwner::OffNode); + } + /// A lease an attach path would report, with an `18080:8080` ingress /// declaration behind it. fn leased_ports() -> BTreeMap { diff --git a/crates/minimald/src/net/dns_gate.rs b/crates/minimald/src/net/dns_gate.rs index ecb6ca4004..57d48a277d 100644 --- a/crates/minimald/src/net/dns_gate.rs +++ b/crates/minimald/src/net/dns_gate.rs @@ -57,27 +57,33 @@ //! so the box has no IPv6 or alternative-endpoints path to chase (guest //! IPv6 has no admission path, NET-082) and nothing upstream can answer //! them differently. -//! * **REFUSED for a deny-all box's names outside the zone** (NET-141) — -//! when the box's egress is the deny-all shape (every `allow_*` dimension -//! present and empty, the shape `sessions::EgressPolicy::deny_all` -//! materializes and the classifier places under `deny`), a query toward -//! this box's resolver that asks for any name outside the box zone, of -//! any record type, is answered REFUSED (rcode 5) by the relay itself and -//! never written on to the switch. No name can match an empty -//! `allow_dns_hosts`, so every such query is one NET-141 refuses, and the -//! refusal comes before any onward query: a forwarded query carries data -//! out in the queried name even though the connection after it is -//! dropped. Zone names (NET-072, `host.min.internal` among them) pass -//! for A alone: the switch's resolver answers a zone A itself but sends -//! every other type for a zone name on to the host's resolvers, so a -//! deny-all box's zone query with any question that is not an A is -//! answered NODATA here, its id and questions kept. For a deny-all box -//! "the zone" is the switch's own, narrower match: a name strictly below -//! the apex whose last two labels are exactly the lowercase `min` and -//! `internal`. The apex and `x.MIN.INTERNAL` are forwarded upstream by -//! the switch's resolver, so they are refused here. A box with an allow -//! list or no egress declaration is not refused here: its queries keep -//! the forwarding above, under the zone rules below. +//! * **REFUSED for names the allow list does not match** (NET-141) — when +//! the box declared a non-empty `egress.allow_dns_hosts` (a partial +//! list), or is of the deny-all shape `sessions::EgressPolicy::deny_all` +//! materializes (the empty list with nothing else admitted), a query toward this box's resolver that asks for a name +//! the list does not match and that is outside the box zone, of any +//! record type, is answered REFUSED (rcode 5) by the relay itself and +//! never written on to the switch. The match is the one the ingress leg +//! admits answers by: exact, case-insensitive, a trailing dot ignored. +//! The refusal comes before any onward query: a forwarded query carries +//! data out in the queried name even though the connection after it is +//! dropped. Each refusal writes one warning, rate-limited per box and +//! name, that names the box and the name. Zone names (NET-072, +//! `host.min.internal` among them) are not refused. For a deny-all box +//! they pass for A alone: the switch's resolver answers a zone A itself +//! but sends every other type for a zone name on to the host's +//! resolvers, so a deny-all box's zone query with any question that is +//! not an A is answered NODATA here, its id and questions kept. For a +//! deny-all box "the zone" is the switch's own, narrower match: a name +//! strictly below the apex whose last two labels are exactly the +//! lowercase `min` and `internal`. The apex and `x.MIN.INTERNAL` are +//! forwarded upstream by the switch's resolver, so they are refused here. +//! A box with a partial list reads the zone in any case, and the +//! NXDOMAIN rule below answers those names. A box that declared no names +//! is not refused here: its queries keep the forwarding above, under the +//! zone rules below. Neither is a box whose list is empty but that admits +//! subnets (`--allow-subnets 0.0.0.0/0` alone): `sessions::effective_egress` +//! gives it `allow_dns_hosts = Some([])`, and its names still forward. //! * **NODATA for non-A queries on box-zone names** (NET-006) — a TXT, MX, //! CNAME, SRV or NS query for a `*.min.internal` name is answered empty by //! the relay itself and never written on to the switch, so the box's @@ -352,9 +358,9 @@ pub(crate) struct DnsGate { /// names, which pins nothing (see the module doc). names: Option>, /// Whether the box's egress is the deny-all shape - /// ([`sessions::EgressPolicy::admits_nothing`]): every query it sends for a - /// name outside the box zone is answered REFUSED here and never - /// forwarded (NET-141). + /// ([`sessions::EgressPolicy::admits_nothing`]): its refusal (NET-141) + /// reads the box zone as the switch's own, narrower match, so the apex + /// and a zone suffix not in exact lowercase are refused too. deny_all: bool, /// The box's compiled egress rules: the subnet dimensions the /// intersection reads, and the resolver's address both legs gate on. @@ -399,6 +405,31 @@ pub(crate) struct DnsGate { limiter: Arc, } +/// The rate-limit key prefix of the REFUSED warning for a name a box's allow +/// list does not match (NET-141). +const UNMATCHED_REFUSED_RULE: &str = "dns-unmatched-refused"; + +/// How many rate-limit keys the REFUSED warning spreads a box's names over. +/// The box chooses the names, and the limiter never evicts a key, so the +/// key holds a bucket of the name, not the name: a box that asks for many +/// names grows the limiter by this many keys at most. +const UNMATCHED_REFUSED_BUCKETS: u64 = 256; + +/// The rate-limit key of the REFUSED warning for `name`: one of +/// [`UNMATCHED_REFUSED_BUCKETS`] keys, picked by a hash with a key random to +/// this daemon, so a box cannot pick names that collide with another name. +/// Two names that share a bucket share one line a minute. +fn unmatched_refused_key(name: &str) -> String { + use std::hash::BuildHasher; + static HASHER: std::sync::OnceLock = + std::sync::OnceLock::new(); + let bucket = HASHER + .get_or_init(std::collections::hash_map::RandomState::new) + .hash_one(name) + % UNMATCHED_REFUSED_BUCKETS; + format!("{UNMATCHED_REFUSED_RULE}:{bucket}") +} + /// A DNS wire name in the form the allowlist matches: ASCII-lowercased, /// with the root dot stripped — `github.com` for `GitHub.com.`. fn normalized(name: &str) -> String { @@ -493,6 +524,36 @@ impl DnsGate { || super::dns::is_zone_name(name) } + /// Whether `question` asks for a name NET-141 refuses: the box declared + /// `allow_dns_hosts`, the list does not match the name (the matcher + /// [`Self::allows_name`] admits answers by), and the name is outside the + /// box zone. A deny-all box reads the zone as the switch's own, narrower + /// match ([`answered_on_the_switch`]), so the apex and a zone suffix not + /// in exact lowercase are refused too. Any other box reads the wider + /// match ([`in_zone`]) and leaves those names to the NXDOMAIN rule. A box + /// that declared no names, or declared the empty list without the + /// deny-all shape, is refused nothing here. + fn refuses(&self, question: &Query) -> bool { + // An empty list refuses only in the deny-all shape. A box that + // declared no names but admits subnets (`--allow-subnets 0.0.0.0/0` + // alone) keeps its forwarding. + let Some(names) = self + .names + .as_ref() + .filter(|names| self.deny_all || !names.is_empty()) + else { + return false; + }; + if names.contains(&normalized(&question.name().to_lowercase().to_string())) { + return false; + } + if self.deny_all { + !answered_on_the_switch(question.name()) + } else { + !in_zone(question) + } + } + /// The egress leg, NET-141, NET-006 and NET-136: whether the datagram /// the box sent to `dst` is a query to this box's resolver that the /// relay answers itself, and if so the reply to write back toward the @@ -500,9 +561,11 @@ impl DnsGate { /// forward". Every rule reads every question, not only the first, so a /// name in front cannot carry a second one out. /// - /// In order: a deny-all box's query is answered REFUSED, whatever its - /// record type, when any name it asks is not one the switch answers - /// itself ([`answered_on_the_switch`]; NET-141). Any box's query naming + /// In order: a query from a box with an allow list is answered REFUSED, + /// whatever its record type, when any name it asks is one the list does + /// not match and is outside the box zone ([`Self::refuses`]; NET-141). + /// Each refusal writes one rate-limited warning that names the box and + /// the name. Any box's query naming /// the zone in a form the switch would send upstream — the apex, or a /// zone suffix not in exact lowercase — is answered NXDOMAIN (NET-006). /// Then a query is answered NODATA when any question is an AAAA, HTTPS @@ -544,6 +607,9 @@ impl DnsGate { let (question, rcode) = self.decide(&query)?; let name = normalized(&question.name().to_lowercase().to_string()); let rtype = question.query_type(); + if rcode == ResponseCode::Refused { + self.warn_refused(&name, rtype); + } self.answer(query, rcode, &name, rtype) } @@ -556,9 +622,7 @@ impl DnsGate { let questions = || query.queries.iter(); // REFUSED (NET-141): answered here, so no resolver beyond this one // ever sees the name. - if self.deny_all - && let Some(q) = questions().find(|q| !answered_on_the_switch(q.name())) - { + if let Some(q) = questions().find(|q| self.refuses(q)) { return Some((q.clone(), ResponseCode::Refused)); } // NXDOMAIN (NET-006): a zone name the switch's resolver would send @@ -584,6 +648,27 @@ impl DnsGate { .map(|q| (q.clone(), ResponseCode::NoError)) } + /// The refusal's one rate-limited line (NET-141): the box and the name it + /// asked for that its allow list does not match. Keyed by a bucket of + /// the name ([`unmatched_refused_key`]), so a burst for one name does not + /// silence most others, and the limiter's keys stay bounded. + fn warn_refused(&self, name: &str, rtype: RecordType) { + let key = unmatched_refused_key(name); + if self + .limiter + .should_warn_at(&self.label, &key, Instant::now()) + { + tracing::warn!( + component = COMPONENT, + session_id = %self.label, + name = %name, + query_type = ?rtype, + "answered REFUSED for a name the box's allow_dns_hosts does not match; \ + the query was not forwarded" + ); + } + } + /// The reply the relay writes back for `query` instead of forwarding it: /// `rcode`, every question echoed back, no answers, and the query's id, /// so the box's resolver stack matches the reply to its own query. The @@ -1449,11 +1534,8 @@ pub(crate) mod tests { assert_eq!(out, connect, "{address} resolved from {name} is admitted"); } - // An address resolved for a name the box did not declare is not - // admitted. Under the shipped interim, the query is forwarded and only - // the resulting pin is withheld; the architecture's design would refuse - // the resolution itself. Either way the TCP flow is dropped, so the - // assertion holds for both behaviours. + // A name the box did not declare is refused at the relay (NET-141): + // the query never reaches the switch, and nothing is admitted. let undeclared_query = udp_payload_frame( LEASE, 41000, @@ -1462,28 +1544,15 @@ pub(crate) mod tests { &dns_query("not-in-allowlist.example.", RecordType::A), ); harness.box_end.write_all(&undeclared_query).unwrap(); - let forwarded = - tokio::time::timeout(Duration::from_secs(5), read_framed(&mut harness.switch)) - .await - .expect("query for an undeclared name is forwarded") - .expect("the switch side stays open"); + let reply_frame = read_box_frame(&harness) + .await + .expect("the relay answers the undeclared name itself"); + let (_, payload) = udp_datagram(&reply_frame).expect("the reply is a UDP frame"); assert_eq!( - forwarded, undeclared_query, - "resolution itself is never blocked, only the resulting pin" - ); - let response = dns_response( - "not-in-allowlist.example.", - &[Ipv4Addr::new(198, 51, 100, 7)], + Message::from_vec(payload).unwrap().metadata.response_code, + ResponseCode::Refused, + "an undeclared name is refused, not resolved" ); - let response_frame = udp_payload_frame(RESOLVER, 53, LEASE, 41000, &response); - harness - .switch - .write_all(&wire_frame(&response_frame)) - .await - .unwrap(); - let _ = read_box_frame(&harness) - .await - .expect("the undeclared name's reply passes through"); let undeclared_connect = egress_tcp_frame(LEASE, Ipv4Addr::new(198, 51, 100, 7), 443); let sentinel = arp_frame(LEASE); @@ -2301,11 +2370,11 @@ pub(crate) mod tests { ); // The carve-out is the zone's alone. An ordinary name with no entry - // resolves — the reply passes through — but admits nothing; and a + // is refused at the relay (NET-141) and admits nothing; and a // lookalike that merely *ends in* the zone's words is no more a zone - // name than any other, so the plane refuses its answer too. In both - // cases the connection is refused, the sentinel standing in for - // everything the relay decided before it. + // name than any other, so it is refused too. In both cases the + // connection is refused, the sentinel standing in for everything + // the relay decided before it. for (name, answer) in [ ("example.com.", Ipv4Addr::new(203, 0, 113, 9)), ( @@ -2316,20 +2385,15 @@ pub(crate) mod tests { let query = udp_payload_frame(LEASE, 40001, RESOLVER, 53, &dns_query(name, RecordType::A)); harness.box_end.write_all(&query).unwrap(); - tokio::time::timeout(Duration::from_secs(5), read_framed(&mut harness.switch)) - .await - .expect("the query is forwarded") - .expect("the switch side stays open"); - let response_frame = - udp_payload_frame(RESOLVER, 53, LEASE, 40001, &dns_response(name, &[answer])); - harness - .switch - .write_all(&wire_frame(&response_frame)) - .await - .unwrap(); - let _ = read_box_frame(&harness) + let reply_frame = read_box_frame(&harness) .await - .expect("the reply itself passes through"); + .expect("the relay answers the unlisted name itself"); + let (_, payload) = udp_datagram(&reply_frame).expect("the reply is a UDP frame"); + assert_eq!( + Message::from_vec(payload).unwrap().metadata.response_code, + ResponseCode::Refused, + "{name} is not in the allow list, so it is refused" + ); let denied = egress_tcp_frame(LEASE, answer, 443); let sentinel = arp_frame(LEASE); harness.box_end.write_all(&denied).unwrap(); @@ -2822,6 +2886,209 @@ pub(crate) mod tests { .expect("the switch side stays open"); assert_eq!(forwarded, query, "{name} passes to the resolver"); } + + // The partial allow list: a box allowed only github.com is refused + // example.com at once, in every record type, and the query never + // reaches the switch. github.com and the zone keep forwarding. + let mut harness = spawn_test_relay(&github_only_egress()); + let mut port = 42000; + for name in ["example.com.", "api.github.com.", "github.com.evil."] { + for rtype in [RecordType::A, RecordType::AAAA, RecordType::TXT] { + port += 1; + let query = udp_payload_frame(LEASE, port, RESOLVER, 53, &dns_query(name, rtype)); + harness.box_end.write_all(&query).unwrap(); + let reply_frame = read_box_frame(&harness) + .await + .expect("the relay answers the allow-list box's lookup"); + let (pkt, payload) = udp_datagram(&reply_frame) + .unwrap_or_else(|| panic!("the reply is a UDP frame: {name} {rtype:?}")); + assert_eq!(pkt.src, SocketAddrV4::new(RESOLVER, 53)); + assert_eq!(pkt.dst, SocketAddrV4::new(LEASE, port)); + let reply = Message::from_vec(payload).expect("the reply is a DNS message"); + assert_eq!( + reply.metadata.response_code, + ResponseCode::Refused, + "{name} {rtype:?} is REFUSED for a box allowed only github.com" + ); + assert!(reply.answers.is_empty(), "REFUSED answers nothing"); + + let sentinel = arp_frame(LEASE); + harness.box_end.write_all(&sentinel).unwrap(); + let next = + tokio::time::timeout(Duration::from_secs(5), read_framed(&mut harness.switch)) + .await + .expect("the relay keeps forwarding") + .expect("the switch side stays open"); + assert_eq!( + next, sentinel, + "the refused {name} {rtype:?} query never reached the switch" + ); + } + } + for name in [ + "github.com.", + "GitHub.COM.", + "web.min.internal.", + "host.min.internal.", + ] { + port += 1; + let query = + udp_payload_frame(LEASE, port, RESOLVER, 53, &dns_query(name, RecordType::A)); + harness.box_end.write_all(&query).unwrap(); + let forwarded = + tokio::time::timeout(Duration::from_secs(5), read_framed(&mut harness.switch)) + .await + .expect("an allowed or zone query is forwarded") + .expect("the switch side stays open"); + assert_eq!(forwarded, query, "{name} passes to the resolver"); + } + } + + /// The refusal's diagnostics (NET-141): one warning names the box and + /// the refused name, and a burst of lookups for the same name writes it + /// once. + #[test] + fn unmatched_name_refusal_is_logged_once_per_name() { + let capture = captured_log(); + let gate = gate_for(&github_only_egress()); + let lines = |name: &str| { + capture + .contents() + .lines() + .filter(|line| { + line.contains("answered REFUSED for a name the box's allow_dns_hosts") + && line.contains(&format!("name={name}")) + && line.contains(&format!("session_id={LEASE}")) + }) + .count() + }; + for rtype in [RecordType::A, RecordType::AAAA, RecordType::A] { + let reply = reply_to(&gate, "refused-once.example.", rtype); + assert_eq!(reply.metadata.response_code, ResponseCode::Refused); + } + assert_eq!(lines("refused-once.example"), 1, "one line for the burst"); + // The key is a bucket of the name under a hasher random to the + // process, so a fixed second name shares the first one's bucket on + // one run in 256. Pick a second name the hasher puts in another + // bucket, so the leg is about two keys, every run. + let first = unmatched_refused_key("refused-once.example"); + let other = (0..) + .map(|n| format!("refused-other{n}.example")) + .find(|name| unmatched_refused_key(name) != first) + .expect("some name falls in another of the 256 buckets"); + let reply = reply_to(&gate, &format!("{other}."), RecordType::A); + assert_eq!(reply.metadata.response_code, ResponseCode::Refused); + assert_eq!( + lines(&other), + 1, + "a second name is not silenced by the first" + ); + } + + proptest::proptest! { + /// NET-141's property: for every allow list, every name and every + /// record type, a query for a name the list does not match and that + /// is outside the box zone is answered REFUSED at the relay and never + /// forwarded. A name the list matches, in any case and with or + /// without the root dot, is never refused. The empty list is the + /// deny-all box's. + #[test] + fn unmatched_name_never_forwarded_for_any_allow_set( + patterns in proptest::collection::vec(r"[a-z0-9]{1,6}(\.[a-z0-9]{1,6}){0,2}\.(com|org|dev)", 0..4), + asked in r"[a-zA-Z0-9]{1,6}(\.[a-zA-Z0-9]{1,6}){0,2}\.(com|org|dev|min\.internal|MIN\.INTERNAL)", + pick in proptest::option::of(proptest::prelude::any::()), + upper in proptest::prelude::any::(), + dot in proptest::prelude::any::(), + rtype in proptest::sample::select(vec![ + RecordType::A, RecordType::AAAA, RecordType::TXT, RecordType::MX, + RecordType::HTTPS, RecordType::CNAME, + ]), + ) { + // The list as the box wrote it: with or without the root dot. + let written: Vec = patterns + .iter() + .map(|pattern| if dot { format!("{pattern}.") } else { pattern.clone() }) + .collect(); + let policy = sessions::SessionPolicy { + egress: Some(sessions::EgressPolicy { + allow_protocols: Some(vec![sessions::IpProto::Tcp]), + allow_subnets: Some(Vec::new()), + allow_dns_hosts: Some(written), + deny_subnets: None, + }), + ingress: None, + credentialed_upstream: None, + }; + let gate = gate_for(&policy); + // Half the time the name asked is one of the patterns, in a case + // the matcher must see through. A name on the wire is always + // fully qualified. + let mut name = match (pick, patterns.is_empty()) { + (Some(index), false) => index.get(&patterns).clone(), + _ => asked, + }; + if upper { + name = name.to_ascii_uppercase(); + } + name.push('.'); + let matched = patterns.contains(&normalized(&name)); + // The zone a refusal spares: the switch's own, narrower match for + // the deny-all box, the zone in any case for a partial list. + let parsed = Name::from_ascii(&name).unwrap(); + let zone = if patterns.is_empty() { + answered_on_the_switch(&parsed) + } else { + in_zone(&Query::query(parsed, rtype)) + }; + let rcode = rcode_of(&gate, &multi_query(&[(name.as_str(), rtype)])); + if !matched && !zone { + proptest::prop_assert_eq!( + rcode, + Some(ResponseCode::Refused), + "{} {:?} under {:?} is refused, never forwarded", name, rtype, patterns + ); + } else { + proptest::prop_assert_ne!( + rcode, + Some(ResponseCode::Refused), + "{} {:?} under {:?} is not refused", name, rtype, patterns + ); + } + // Behind a name the list matches, an unmatched name still cannot + // ride the query out. + if let (Some(first), false, false) = (patterns.first(), matched, zone) { + let rcode = rcode_of( + &gate, + &multi_query(&[(format!("{first}.").as_str(), RecordType::A), (name.as_str(), rtype)]), + ); + proptest::prop_assert_eq!(rcode, Some(ResponseCode::Refused)); + } + } + } + + /// An empty name list that is not the deny-all shape refuses nothing: a + /// box created with only `--allow-subnets 0.0.0.0/0` gets + /// `allow_dns_hosts = Some([])`, and its query for an outside name is + /// forwarded, not answered by the gate (NET-141). + #[test] + fn empty_name_list_with_subnets_forwards_outside_names() { + let policy = sessions::SessionPolicy { + egress: Some(sessions::EgressPolicy { + allow_protocols: Some(vec![sessions::IpProto::Tcp]), + allow_subnets: Some(vec!["0.0.0.0/0".into()]), + allow_dns_hosts: Some(Vec::new()), + deny_subnets: None, + }), + ingress: None, + credentialed_upstream: None, + }; + let gate = gate_for(&policy); + let query = dns_query("example.com.", RecordType::A); + assert!( + gate.intercept_query(&SocketAddrV4::new(RESOLVER, 53), &query) + .is_none(), + "the A query for example.com is forwarded" + ); } /// The deny-all refusal at the gate itself: REFUSED for a name outside @@ -2992,9 +3259,9 @@ pub(crate) mod tests { ); } - /// The refusal is the deny-all box's alone: an allow-list box and an - /// allow-all box keep forwarding a name outside the zone in every type - /// but NET-136's, which they keep answering NODATA. + /// An allow-list box keeps forwarding a name its list matches, and an + /// allow-all box keeps forwarding every name outside the zone, in every + /// type but NET-136's, which they keep answering NODATA. #[test] fn allow_list_and_allow_all_boxes_keep_forwarding() { let allow_all = sessions::SessionPolicy { @@ -3002,9 +3269,12 @@ pub(crate) mod tests { ingress: None, credentialed_upstream: None, }; - for policy in [github_only_egress(), allow_all] { + for (policy, names) in [ + (github_only_egress(), &["github.com."][..]), + (allow_all, &["example.com.", "github.com."][..]), + ] { let gate = gate_for(&policy); - for name in ["example.com.", "github.com."] { + for &name in names { for rtype in [RecordType::A, RecordType::TXT] { assert!( gate.intercept_query( @@ -3132,7 +3402,7 @@ pub(crate) mod tests { for gate in forwarding_gates() { for questions in [ [ - ("example.com.", RecordType::A), + ("github.com.", RecordType::A), ("x.min.internal.", RecordType::TXT), ], [ @@ -3405,4 +3675,18 @@ pub(crate) mod tests { "a reply with no question to match pins nothing" ); } + + /// The REFUSED warning's rate-limit keys stay bounded whatever names a + /// box asks for, and one name always maps to one key (NET-141). + #[test] + fn unmatched_name_refusal_keys_stay_bounded() { + let keys: std::collections::HashSet = (0..10_000) + .map(|n| unmatched_refused_key(&format!("name-{n}.example.com"))) + .collect(); + assert!(keys.len() as u64 <= UNMATCHED_REFUSED_BUCKETS); + assert_eq!( + unmatched_refused_key("example.com"), + unmatched_refused_key("example.com") + ); + } } diff --git a/crates/minimald/src/net/provider.rs b/crates/minimald/src/net/provider.rs index 4b546a89c4..dc97fdfe7c 100644 --- a/crates/minimald/src/net/provider.rs +++ b/crates/minimald/src/net/provider.rs @@ -367,6 +367,26 @@ impl OwnAddressReporter { self.runtime_ingress.attached(); } + /// Marks this box as attaching, with its lease not reported yet + /// (NET-070). While the mark stands, the hostname proxy refuses a peer + /// off the node, because it cannot tell this box's connections from one. + /// [`Self::report`] ends the mark, and so does [`Self::attach_ended`]. + pub(crate) fn attach_started(&self) { + self.registry + .write() + .expect("hostname registry lock poisoned") + .begin_lease_attach(self.session_id); + } + + /// Ends the mark [`Self::attach_started`] set, for an attach that failed + /// or was abandoned before it reported a lease (NET-070). + pub(crate) fn attach_ended(&self) { + self.registry + .write() + .expect("hostname registry lock poisoned") + .end_lease_attach(self.session_id); + } + /// The switch lease the attach reported for this box ([`Self::report`]), /// or `None` before an attach reported one. On a VM host the daemon /// builds the box's tap itself after the spawn @@ -808,95 +828,37 @@ impl Network for OwnIpNetwork { /// a self-allocated one otherwise, which only a native switch draws. fn plan(&self) -> PlanFuture<'_> { Box::pin(async move { - let (lease, control, subnet) = { - let mut s = self.switch.lock().await; - let subnet = s.subnet(); - let attach = match (self.box_addresses.as_ref(), self.task_slots.as_deref()) { - (Some(handed), _) => s - .attach_handed(handed.switch_address) - .await - .map_err(NetworkError::new)?, - (None, Some(slots)) => { - s.attach_task_slot(slots).await.map_err(NetworkError::new)? - } - (None, None) => s.attach().await.map_err(NetworkError::new)?, - }; - let control = match s.transport() { - crate::net::SwitchTransport::LocalSpawn => { - ControlChannel::Unix(s.control_socket()) - } - crate::net::SwitchTransport::HostShuttle { cid, port } => { - ControlChannel::Vsock { cid, port } - } - }; - (attach.lease, control, subnet) - }; - - let mechanism = tap_mechanism(&control); - *self.reserved.lock().unwrap() = Some(Reserved { - lease, - subnet, - control, - mechanism, - }); - Ok(own_ip_plan(subnet, lease.ip, mechanism)) + // NET-070: from here until the attach reports the lease, the box's + // connections come from a lease the node does not hold. A failed + // plan never reaches `attach` or `abandon`, so it ends the mark + // itself. + if let Some(own_address) = &self.own_address { + own_address.attach_started(); + } + let planned = self.reserve().await; + if planned.is_err() + && let Some(own_address) = &self.own_address + { + own_address.attach_ended(); + } + planned }) } /// Relay the sandbox's tap onto the switch and apply its ingress. The /// reservation is read, not taken, until the attach has succeeded: a /// failed attach hands the release to [`abandon`](Self::abandon) (017-008). - fn attach(&self, mut spawned: Spawned) -> AttachFuture<'_> { + fn attach(&self, spawned: Spawned) -> AttachFuture<'_> { Box::pin(async move { - let Some(reserved) = self.reserved.lock().unwrap().clone() else { - return Err(NetworkError::new(std::io::Error::other( - "own-IP attach without a plan: nothing was leased", - ))); - }; - - // The mechanism `plan` chose; only the matching branch can succeed. - let tap_fd = match reserved.mechanism { - TapMechanism::InNamespace => { - let Some(fd) = spawned.take_tap_fd() else { - // Report what building one in-namespace needs: which - // capability is missing separates a kernel-config - // problem from a policy one. - let tun = std::path::Path::new("/dev/net/tun").exists(); - let userns = sandbox2::user_namespaces_restriction() - .map_or_else(|| "available".to_string(), |r| r.to_string()); - return Err(NetworkError::new(std::io::Error::other(format!( - "the box's sandbox could not create its network device \ - (/dev/net/tun present: {tun}; user namespaces: {userns})" - )))); - }; - fd - } - TapMechanism::Privileged => { - privileged_tap(reserved.lease, reserved.subnet, spawned.netns_pid()) - .await - .map_err(NetworkError::new)? - } - }; - - let guard = crate::net::gvproxy_network::complete_own_ip_attach( - &self.switch, - tap_fd, - reserved.control, - reserved.lease, - &self.identity, - self.policy.as_ref(), - self.own_address.as_ref(), - self.box_addresses.is_some() - || self - .task_slots - .as_ref() - .is_some_and(|slots| !slots.is_empty()), - ) - .await - .map_err(NetworkError::new)?; - // Committed: the guard owns the release from here. - self.reserved.lock().unwrap().take(); - Ok(Box::new(guard) as Box) + let attached = self.attach_reserved(spawned).await; + // NET-070: a successful attach ended the mark when it reported + // the lease. A failed one ends it here. + if attached.is_err() + && let Some(own_address) = &self.own_address + { + own_address.attach_ended(); + } + attached }) } @@ -907,6 +869,10 @@ impl Network for OwnIpNetwork { /// which is what lets the same box's re-attach re-hand its address. fn abandon(&self) -> AbandonFuture<'_> { Box::pin(async move { + // NET-070: an abandoned launch reports no lease. + if let Some(own_address) = &self.own_address { + own_address.attach_ended(); + } let Some(reserved) = self .reserved .lock() @@ -924,6 +890,100 @@ impl Network for OwnIpNetwork { } } +impl OwnIpNetwork { + /// The body of [`Network::plan`]: leases the address and records the + /// reservation. + async fn reserve(&self) -> Result { + let (lease, control, subnet) = { + let mut s = self.switch.lock().await; + let subnet = s.subnet(); + let attach = match (self.box_addresses.as_ref(), self.task_slots.as_deref()) { + (Some(handed), _) => s + .attach_handed(handed.switch_address) + .await + .map_err(NetworkError::new)?, + (None, Some(slots)) => { + s.attach_task_slot(slots).await.map_err(NetworkError::new)? + } + (None, None) => s.attach().await.map_err(NetworkError::new)?, + }; + let control = match s.transport() { + crate::net::SwitchTransport::LocalSpawn => ControlChannel::Unix(s.control_socket()), + crate::net::SwitchTransport::HostShuttle { cid, port } => { + ControlChannel::Vsock { cid, port } + } + }; + (attach.lease, control, subnet) + }; + + let mechanism = tap_mechanism(&control); + *self.reserved.lock().unwrap() = Some(Reserved { + lease, + subnet, + control, + mechanism, + }); + Ok(own_ip_plan(subnet, lease.ip, mechanism)) + } + + /// The body of [`Network::attach`]: relays the tap and applies the + /// ingress from the reservation `plan` made. + async fn attach_reserved( + &self, + mut spawned: Spawned, + ) -> Result, NetworkError> { + let Some(reserved) = self.reserved.lock().unwrap().clone() else { + return Err(NetworkError::new(std::io::Error::other( + "own-IP attach without a plan: nothing was leased", + ))); + }; + + // The mechanism `plan` chose; only the matching branch can succeed. + let tap_fd = match reserved.mechanism { + TapMechanism::InNamespace => { + let Some(fd) = spawned.take_tap_fd() else { + // Report what building one in-namespace needs: which + // capability is missing separates a kernel-config + // problem from a policy one. + let tun = std::path::Path::new("/dev/net/tun").exists(); + let userns = sandbox2::user_namespaces_restriction() + .map_or_else(|| "available".to_string(), |r| r.to_string()); + return Err(NetworkError::new(std::io::Error::other(format!( + "the box's sandbox could not create its network device \ + (/dev/net/tun present: {tun}; user namespaces: {userns})" + )))); + }; + fd + } + TapMechanism::Privileged => { + privileged_tap(reserved.lease, reserved.subnet, spawned.netns_pid()) + .await + .map_err(NetworkError::new)? + } + }; + + let guard = crate::net::gvproxy_network::complete_own_ip_attach( + &self.switch, + tap_fd, + reserved.control, + reserved.lease, + &self.identity, + self.policy.as_ref(), + self.own_address.as_ref(), + self.box_addresses.is_some() + || self + .task_slots + .as_ref() + .is_some_and(|slots| !slots.is_empty()), + ) + .await + .map_err(NetworkError::new)?; + // Committed: the guard owns the release from here. + self.reserved.lock().unwrap().take(); + Ok(Box::new(guard) as Box) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/minimald/src/net/proxy.rs b/crates/minimald/src/net/proxy.rs index 0d96776629..760f22185f 100644 --- a/crates/minimald/src/net/proxy.rs +++ b/crates/minimald/src/net/proxy.rs @@ -35,7 +35,7 @@ use std::borrow::Cow; use std::collections::BTreeSet; use std::io; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; use std::sync::Arc; use std::time::Duration; @@ -44,6 +44,8 @@ use tokio::io::{ }; use tokio::net::{TcpListener, TcpStream}; +use sessions::SessionId; + use super::dns::HostnameRegistry; use super::policy::Direction; use super::switch::ProxiedRequest; @@ -81,6 +83,19 @@ const HEAD_READ_TIMEOUT: Duration = Duration::from_secs(30); /// the client would wait out the kernel's SYN retries (about two minutes). const UPSTREAM_DIAL_TIMEOUT: Duration = Duration::from_secs(5); +/// How many socket owner lookups (each a walk of `/proc`) run at once across +/// the daemon, for connections to the proxy from the node's address (NET-070). +const OWNER_LOOKUPS: usize = 4; + +/// The slots [`OWNER_LOOKUPS`] counts, shared by every listener the daemon +/// runs, so the blocking pool never holds more than that many walks. +static OWNER_LOOKUP_SLOTS: tokio::sync::Semaphore = + tokio::sync::Semaphore::const_new(OWNER_LOOKUPS); + +/// How long a connection waits for an owner lookup slot before the proxy +/// stops waiting and refuses it as a caller it cannot name. +const OWNER_LOOKUP_WAIT: Duration = Duration::from_secs(5); + /// The host-side lookup the proxy performs for each request: a `Host:`-header /// host (with any `:port` already stripped) to the route its requests forward /// on, or `None` if no live PTask owns it. The host resolver is never consulted. @@ -102,6 +117,25 @@ pub trait HostRoute: Send + Sync + 'static { /// box holds the lease, which is what a host-side peer is (the developer's /// browser, the daemon's own lanes). fn caller_at(&self, lease: Ipv4Addr) -> Option; + + /// The caller a connection from the node's address stands for (NET-070): + /// the live host-address box `owner`, which owns the connecting socket, + /// with its egress rules compiled with `from` as their source. `None` + /// when `owner` is no live host-address box with caller facts. The proxy + /// refuses such a connection as unattributed: the kernel named a box the + /// registry cannot describe, so the request stays gated. + fn caller_owning(&self, owner: SessionId, from: Ipv4Addr) -> Option; + + /// Whether any live caller holds no switch lease. While none does, a + /// connection from the node's address cannot be a box's, and the proxy + /// does not look up who owns the connecting socket. + fn has_unleased_callers(&self) -> bool; + + /// Whether any live caller is an own-address box whose switch lease is + /// not reported yet. While one is, a peer at an address the node does + /// not hold can be that box, and the proxy refuses the peer as + /// unattributed. + fn has_unreported_lease_callers(&self) -> bool; } impl HostRoute for HostnameRegistry { @@ -112,6 +146,18 @@ impl HostRoute for HostnameRegistry { fn caller_at(&self, lease: Ipv4Addr) -> Option { self.caller_at(lease) } + + fn caller_owning(&self, owner: SessionId, from: Ipv4Addr) -> Option { + self.caller_owning(owner, from) + } + + fn has_unleased_callers(&self) -> bool { + self.has_unleased_callers() + } + + fn has_unreported_lease_callers(&self) -> bool { + self.has_unreported_lease_callers() + } } // The daemon shares its live registry behind an `RwLock` (the sessions manager @@ -136,6 +182,27 @@ impl HostRoute for std::sync::RwLock { Err(poisoned) => poisoned.into_inner().caller_at(lease), } } + + fn caller_owning(&self, owner: SessionId, from: Ipv4Addr) -> Option { + match self.read() { + Ok(guard) => guard.caller_owning(owner, from), + Err(poisoned) => poisoned.into_inner().caller_owning(owner, from), + } + } + + fn has_unleased_callers(&self) -> bool { + match self.read() { + Ok(guard) => guard.has_unleased_callers(), + Err(poisoned) => poisoned.into_inner().has_unleased_callers(), + } + } + + fn has_unreported_lease_callers(&self) -> bool { + match self.read() { + Ok(guard) => guard.has_unreported_lease_callers(), + Err(poisoned) => poisoned.into_inner().has_unreported_lease_callers(), + } + } } /// The admit-or-drop decision every proxied request is put to before the @@ -145,6 +212,24 @@ impl HostRoute for std::sync::RwLock { /// the router stays a small `Clone` value handed to every connection task. pub type SharedVerdict = fn(Option<&super::dns::Caller>, &super::dns::Route, u16) -> ProxiedRequest; +/// Names the box that owns the client end of a connection from the node's +/// address: `(peer, local)` as the accepting side sees them. The production +/// lookup is [`super::dns::connecting_box`], which reads the kernel's socket +/// tables and the holder's cgroup. +pub type SocketOwner = fn(SocketAddrV4, SocketAddrV4) -> super::dns::ConnectingOwner; + +/// Who a proxied connection came from, named when the proxy accepts it. +enum CallerLookup { + /// A box, with the egress rules its request is put to. + Box(super::dns::Caller), + /// No box: a host-side client such as the developer's browser. + NoBox, + /// A connection from the node's address whose owner the proxy cannot + /// name while a host-address box is live. It can be that box's, so the + /// proxy refuses it rather than leave it ungated. + Unattributed, +} + /// The shared routing core: maps an HTTP authority (`host` or `host:port`) to /// the upstream socket address a request forwards to. pub struct Router { @@ -152,6 +237,9 @@ pub struct Router { /// The verdict each request is put to (see [`SharedVerdict`]), applied /// before the proxy dials the upstream. verdict: SharedVerdict, + /// Names the box that owns a connection from the node's address (see + /// [`SocketOwner`]). + owner: SocketOwner, } // Manual `Clone` so a `Router` is cheap to hand to each connection task without @@ -161,6 +249,7 @@ impl Clone for Router { Self { table: Arc::clone(&self.table), verdict: self.verdict, + owner: self.owner, } } } @@ -171,7 +260,21 @@ impl Router { /// [`super::switch::proxied_request_verdict`] in every production caller. #[must_use] pub fn new(table: Arc, verdict: SharedVerdict) -> Self { - Self { table, verdict } + Self { + table, + verdict, + owner: super::dns::connecting_box, + } + } + + /// Replaces the socket-owner lookup. A test hook: a test process runs in + /// no box's classifier leaf, so the kernel lookup can never name a box + /// for it. + #[cfg(test)] + #[must_use] + pub(crate) fn with_socket_owner(mut self, owner: SocketOwner) -> Self { + self.owner = owner; + self } /// Routes an HTTP authority to its upstream socket address, or `None` if no @@ -217,6 +320,85 @@ impl Router { fn caller_at(&self, lease: Ipv4Addr) -> Option { self.table.caller_at(lease) } + + /// The caller a connection from `peer` to `local` stands for (NET-070). + /// A peer at a live box's switch lease is that box. Any other peer is + /// the node's address, and the caller is the host-address box that owns + /// the connecting socket, if one does. The socket lookup reads `/proc`, + /// so it runs off the async workers, and it runs only while some live + /// caller holds no switch lease. A lookup that cannot name the socket's + /// holder then is [`CallerLookup::Unattributed`], never no caller: the + /// client can close its socket before the proxy looks, and the gate must + /// not open for that. A socket owned by a box the registry holds no + /// caller facts for is [`CallerLookup::Unattributed`] too: an unknown box + /// is denied by default (design §7.1). Only a socket that no box owns, + /// or a peer at an address the node does not hold, is + /// [`CallerLookup::NoBox`]. The second holds only while no own-address + /// box is attaching and waits for its lease report: such a box connects + /// from a lease the node does not hold, so while one is attaching an + /// off-node peer is [`CallerLookup::Unattributed`]. + async fn caller_for(&self, peer: SocketAddrV4, local: Option) -> CallerLookup { + if let Some(caller) = self.caller_at(*peer.ip()) { + return CallerLookup::Box(caller); + } + if !self.table.has_unleased_callers() { + return CallerLookup::NoBox; + } + let Some(SocketAddr::V4(local)) = local else { + return CallerLookup::Unattributed; + }; + // Each lookup walks `/proc` on the blocking pool, so at most + // OWNER_LOOKUPS of them run at once, daemon-wide. A connection that + // waits past OWNER_LOOKUP_WAIT for a slot is not attributed, and its + // request is refused: a burst of connections cannot fill the pool, + // and it cannot open the gate either. + let Ok(Ok(_slot)) = + tokio::time::timeout(OWNER_LOOKUP_WAIT, OWNER_LOOKUP_SLOTS.acquire()).await + else { + tracing::debug!( + component = "dns-proxy", + %peer, + "no owner lookup slot came free for a proxied connection from the node's address" + ); + return CallerLookup::Unattributed; + }; + let owner = self.owner; + let found = tokio::task::spawn_blocking(move || owner(peer, local)) + .await + .unwrap_or(super::dns::ConnectingOwner::Unknown); + let box_id = match found { + super::dns::ConnectingOwner::Box(box_id) => box_id, + super::dns::ConnectingOwner::NoBox => return CallerLookup::NoBox, + super::dns::ConnectingOwner::OffNode => { + if !self.table.has_unreported_lease_callers() { + return CallerLookup::NoBox; + } + tracing::debug!( + component = "dns-proxy", + %peer, + "a proxied connection from off the node can be a box whose lease is not reported yet" + ); + return CallerLookup::Unattributed; + } + super::dns::ConnectingOwner::Unknown => { + tracing::debug!( + component = "dns-proxy", + %peer, + "found no owner for a proxied connection from the node's address" + ); + return CallerLookup::Unattributed; + } + }; + let caller = self.table.caller_owning(box_id, *peer.ip()); + tracing::debug!( + component = "dns-proxy", + %peer, + %box_id, + caller = caller.as_ref().map_or("none", super::dns::Caller::name), + "named the box that owns a proxied connection from the node's address" + ); + caller.map_or(CallerLookup::Unattributed, CallerLookup::Box) + } } /// Splits an HTTP authority into its host and optional port, handling both the @@ -352,7 +534,8 @@ pub async fn serve(listener: TcpListener, router: Router) -> io let (client, peer) = listener.accept().await?; let router = router.clone(); tokio::spawn(async move { - if let Err(error) = handle_connection_io(client, Some(peer), &router).await { + let local = client.local_addr().ok(); + if let Err(error) = handle_connection_io(client, Some(peer), local, &router).await { tracing::debug!( component = "dns-proxy", %peer, @@ -384,6 +567,8 @@ struct ParsedRequest<'a> { /// core serves the egress proxy's `TcpStream` and any other byte stream a /// caller hands it. `peer` is the client's address when the transport has one /// — what names the caller whose egress rules the request is put to (NET-070). +/// `local` is the address the client connected to: with `peer`, it finds the +/// connecting socket a host-address box owns. /// /// Every refusal — a head that never arrives, an unparseable head, a host no /// live PTask owns, an upstream that will not accept the connection, a request @@ -393,12 +578,26 @@ struct ParsedRequest<'a> { async fn handle_connection_io( mut client: C, peer: Option, + local: Option, router: &Router, ) -> io::Result<()> where C: AsyncRead + AsyncWrite + Unpin, T: HostRoute, { + // Who the request came from (NET-070), named before the head is read, + // while the client still holds its socket: a peer at a live box's switch + // lease is that box, with the egress rules its own outbound frames are + // decided by. A peer at the node's address is the host-address box that + // owns the connecting socket, with that box's egress rules. Any other + // peer — a host-side client such as the developer's browser — is no + // caller, and its egress is ungated here exactly as a direct connection + // from it would be (NET-071). + let lookup = match peer { + Some(SocketAddr::V4(peer)) => router.caller_for(peer, local).await, + _ => CallerLookup::NoBox, + }; + // Bound the head read so a client that connects but never sends a complete // head cannot occupy this task indefinitely. let head = match tokio::time::timeout(HEAD_READ_TIMEOUT, read_head(&mut client)).await { @@ -472,15 +671,26 @@ where let kind = request.kind; let port = port.unwrap_or(DEFAULT_UPSTREAM_PORT); - // Who the request came from (NET-070): a peer at a live box's switch lease - // is that box, with the egress rules its own outbound frames are decided - // by; any other peer — a host-side client, a box with no lease on the - // switch — is no caller, and its egress is ungated here exactly as a - // direct connection from it would be (NET-071). - let caller = peer.and_then(|peer| match peer { - SocketAddr::V4(v4) => router.caller_at(*v4.ip()), - SocketAddr::V6(_) => None, - }); + let caller = match lookup { + CallerLookup::Box(caller) => Some(caller), + CallerLookup::NoBox => None, + CallerLookup::Unattributed => { + let reason = "the proxy cannot tell which box opened this connection"; + tracing::warn!( + component = "dns-proxy", + host, + session = route.session(), + caller = "unknown", + peer = ?peer, + port, + direction = %Direction::Egress, + reason, + status = "403 Forbidden", + "refused a proxied request" + ); + return write_refusal_status(&mut client, "403 Forbidden", reason).await; + } + }; // The shared verdict, before the proxy dials anything: a request the // switch's own gates would refuse is refused here, where the host, the @@ -1954,7 +2164,7 @@ mod tests { let started = tokio::time::Instant::now(); tokio::time::timeout( Duration::from_secs(60), - handle_connection_io(proxy_side, None, &router), + handle_connection_io(proxy_side, None, None, &router), ) .await .expect("the proxy must bound the upstream dial, not wait out the SYN retries") @@ -3266,7 +3476,7 @@ mod tests { let (buf, guard) = capture_logs(); let (mut client, proxy_side) = tokio::io::duplex(1024); client.write_all(head(port).as_bytes()).await.unwrap(); - handle_connection_io(proxy_side, None, &router) + handle_connection_io(proxy_side, None, None, &router) .await .unwrap(); drop(guard); @@ -3865,6 +4075,512 @@ mod tests { } } + /// The node addresses the host-address callers below connect from: one + /// per box, so the stub socket-owner lookup can tell them apart. + const HOST_BOX_ADDR: Ipv4Addr = Ipv4Addr::new(127, 0, 0, 4); + const OPEN_HOST_BOX_ADDR: Ipv4Addr = Ipv4Addr::new(127, 0, 0, 5); + const BROWSER_ADDR: Ipv4Addr = Ipv4Addr::new(127, 0, 0, 6); + const UNATTRIBUTED_ADDR: Ipv4Addr = Ipv4Addr::new(127, 0, 0, 7); + const UNREGISTERED_BOX_ADDR: Ipv4Addr = Ipv4Addr::new(127, 0, 0, 8); + const OFF_NODE_ADDR: Ipv4Addr = Ipv4Addr::new(127, 0, 0, 9); + const HOST_BOX: &str = "00000000-0000-0000-0000-000000000011"; + const OPEN_HOST_BOX: &str = "00000000-0000-0000-0000-000000000012"; + const UNREGISTERED_BOX: &str = "00000000-0000-0000-0000-000000000013"; + + /// The socket-owner lookup a host-address box's connection meets, with + /// the kernel's answer stood in: a test process runs in no box's leaf. + /// The connection from [`HOST_BOX_ADDR`] is owned by the deny-all box, + /// the one from [`OPEN_HOST_BOX_ADDR`] by the box with no egress rules, + /// The one from [`UNATTRIBUTED_ADDR`] has no owner the kernel can name, as + /// a socket the client closed at once has none. The one from + /// [`UNREGISTERED_BOX_ADDR`] is owned by a box with no caller facts in + /// the registry. The one from [`OFF_NODE_ADDR`] comes from an address the + /// node does not hold. Any other connection is owned by no box, as the + /// developer's browser's is. + fn stub_socket_owner( + peer: SocketAddrV4, + local: SocketAddrV4, + ) -> crate::net::dns::ConnectingOwner { + use crate::net::dns::ConnectingOwner; + assert_eq!( + *local.ip(), + Ipv4Addr::LOCALHOST, + "local is the proxy's address" + ); + match *peer.ip() { + HOST_BOX_ADDR => ConnectingOwner::Box(SessionId::parse_str(HOST_BOX).unwrap()), + OPEN_HOST_BOX_ADDR => { + ConnectingOwner::Box(SessionId::parse_str(OPEN_HOST_BOX).unwrap()) + } + UNATTRIBUTED_ADDR => ConnectingOwner::Unknown, + UNREGISTERED_BOX_ADDR => { + ConnectingOwner::Box(SessionId::parse_str(UNREGISTERED_BOX).unwrap()) + } + OFF_NODE_ADDR => ConnectingOwner::OffNode, + _ => ConnectingOwner::NoBox, + } + } + + /// NET-070, design §7.1: an own-address box registers its caller facts + /// at finalize and reports its switch lease only at the end of its + /// attach. In that window its connections come from a lease the node + /// does not hold, so the socket-owner lookup answers off the node. A + /// deny-all box's request to another box in that window is refused, not + /// left ungated. Once the lease is reported, an off-node peer is no box + /// again and its request routes. + #[tokio::test] + async fn proxy_unreported_lease_caller_refused() { + let backend_port = spawn_backend().await; + let subnet = SwitchSubnet::default(); + let web = SessionId::parse_str("00000000-0000-0000-0000-000000000010").unwrap(); + let pending = SessionId::parse_str("00000000-0000-0000-0000-000000000014").unwrap(); + let host_box = SessionId::parse_str(HOST_BOX).unwrap(); + + let mut reg = HostnameRegistry::new(DEFAULT_HOST_ID, true); + reg.report_own_address( + web, + "web", + Ipv4Addr::LOCALHOST, + BTreeMap::from([(18080, backend_port)]), + ); + let deny_all = SessionPolicy { + egress: Some(EgressPolicy::deny_all()), + ingress: None, + credentialed_upstream: None, + }; + // A host-address box keeps the socket-owner lookup running after the + // pending box reports its lease. + reg.register_host_net(host_box, "hostbox"); + reg.register_caller(host_box, "hostbox", &deny_all, subnet, None); + // The own-address caller: caller facts, an attach in flight, no + // reported lease. + reg.register_caller(pending, "pending", &deny_all, subnet, None); + assert!( + !reg.has_unreported_lease_callers(), + "a registered box that is not attaching runs nothing that can connect" + ); + reg.begin_lease_attach(pending); + assert!(reg.has_unreported_lease_callers()); + let table = Arc::new(RwLock::new(reg)); + let router = Router::new(Arc::clone(&table), proxied_request_verdict) + .with_socket_owner(stub_socket_owner); + + let proxy = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let proxy_addr = proxy.local_addr().unwrap(); + tokio::spawn(serve(proxy, router)); + + let refused = proxy_get_from(proxy_addr, OFF_NODE_ADDR, "web.min.internal:18080").await; + assert!( + refused.contains("403 Forbidden"), + "expected an off-node peer to be refused while a lease is unreported, got: {refused}" + ); + + table.write().unwrap().report_own_address( + pending, + "pending", + Ipv4Addr::new(10, 0, 2, 50), + BTreeMap::new(), + ); + assert!(!table.read().unwrap().has_unreported_lease_callers()); + let routed = proxy_get_from(proxy_addr, OFF_NODE_ADDR, "web.min.internal:18080").await; + assert!( + routed.contains("200 OK"), + "expected an off-node peer to route once every lease is reported, got: {routed}" + ); + } + + /// NET-070, design §7.1: an own-address box that registered its caller + /// facts at finalize but never attached, or whose attach failed or whose + /// box stopped, runs nothing that can connect. It does not make the proxy + /// refuse an off-node peer, such as the host side of a VM's forwarded + /// port: that peer is no box and its request routes. + #[tokio::test] + async fn proxy_unattached_lease_caller_not_refused() { + let backend_port = spawn_backend().await; + let subnet = SwitchSubnet::default(); + let web = SessionId::parse_str("00000000-0000-0000-0000-000000000010").unwrap(); + let idle = SessionId::parse_str("00000000-0000-0000-0000-000000000014").unwrap(); + let host_box = SessionId::parse_str(HOST_BOX).unwrap(); + + let mut reg = HostnameRegistry::new(DEFAULT_HOST_ID, true); + reg.report_own_address( + web, + "web", + Ipv4Addr::LOCALHOST, + BTreeMap::from([(18080, backend_port)]), + ); + let deny_all = SessionPolicy { + egress: Some(EgressPolicy::deny_all()), + ingress: None, + credentialed_upstream: None, + }; + // A live host-address box keeps the socket-owner lookup running. + reg.register_host_net(host_box, "hostbox"); + reg.register_caller(host_box, "hostbox", &deny_all, subnet, None); + // Registered at finalize, never attached. + reg.register_caller(idle, "idle", &deny_all, subnet, None); + assert!(!reg.has_unreported_lease_callers()); + // An attach that failed before it reported a lease. + reg.begin_lease_attach(idle); + reg.end_lease_attach(idle); + assert!(!reg.has_unreported_lease_callers()); + // An attach that was in flight when the box stopped. + reg.begin_lease_attach(idle); + reg.mark_stopped(idle); + assert!(!reg.has_unreported_lease_callers()); + let table = Arc::new(RwLock::new(reg)); + let router = Router::new(Arc::clone(&table), proxied_request_verdict) + .with_socket_owner(stub_socket_owner); + + let peer = SocketAddrV4::new(OFF_NODE_ADDR, 40000); + let local = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 3128)); + assert!( + matches!( + router.caller_for(peer, Some(local)).await, + CallerLookup::NoBox + ), + "an off-node peer is no box while no box is attaching" + ); + + let proxy = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let proxy_addr = proxy.local_addr().unwrap(); + tokio::spawn(serve(proxy, router)); + + let routed = proxy_get_from(proxy_addr, OFF_NODE_ADDR, "web.min.internal:18080").await; + assert!( + routed.contains("200 OK"), + "expected an off-node peer to route while no box is attaching, got: {routed}" + ); + } + + /// NET-070 for a host-address caller: a host-address box holds no switch + /// lease, so its request reaches the hostname proxy from the node's + /// address. The proxy names the caller by the box that owns the + /// connecting socket, and puts the request to that box's egress rules, + /// as it does for a switch-lease caller. A deny-all host-address box's + /// request to another box is refused, and the refusal line names it. A + /// host-address box with no egress rules, and a connection no box owns, + /// send the same request and it routes. A connection whose owner the + /// kernel cannot name is refused, not left ungated. So is a connection + /// the kernel names a box for that has no caller facts. + #[tokio::test] + async fn proxy_host_address_caller_egress_denied() { + let backend_port = spawn_backend().await; + let subnet = SwitchSubnet::default(); + let web = SessionId::parse_str("00000000-0000-0000-0000-000000000010").unwrap(); + let host_box = SessionId::parse_str(HOST_BOX).unwrap(); + let open_box = SessionId::parse_str(OPEN_HOST_BOX).unwrap(); + + let mut reg = HostnameRegistry::new(DEFAULT_HOST_ID, true); + // The target: an own-address box publishing one port. + reg.report_own_address( + web, + "web", + Ipv4Addr::LOCALHOST, + BTreeMap::from([(18080, backend_port)]), + ); + // The callers: host-address boxes, as the session actor records them + // (`host_address_box_registers_its_caller_facts` in the box actor's + // tests proves that production registration). Neither ever reports a + // switch lease. + let deny_all = SessionPolicy { + egress: Some(EgressPolicy::deny_all()), + ingress: None, + credentialed_upstream: None, + }; + let no_rules = SessionPolicy { + egress: None, + ingress: None, + credentialed_upstream: None, + }; + for (id, name, policy) in [ + (host_box, "hostbox", deny_all), + (open_box, "openbox", no_rules), + ] { + reg.register_host_net(id, name); + reg.register_caller(id, name, &policy, subnet, None); + } + let router = Router::new(Arc::new(RwLock::new(reg)), proxied_request_verdict) + .with_socket_owner(stub_socket_owner); + + let proxy = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let proxy_addr = proxy.local_addr().unwrap(); + tokio::spawn(serve(proxy, router)); + + let (buf, guard) = capture_logs(); + + let refused = proxy_get_from(proxy_addr, HOST_BOX_ADDR, "web.min.internal:18080").await; + assert!( + refused.contains("403 Forbidden"), + "expected the deny-all host-address box's request to be refused, got: {refused}" + ); + let routed = proxy_get_from(proxy_addr, OPEN_HOST_BOX_ADDR, "web.min.internal:18080").await; + assert!( + routed.contains("200 OK"), + "expected the host-address box with no egress rules to route, got: {routed}" + ); + let routed = proxy_get_from(proxy_addr, BROWSER_ADDR, "web.min.internal:18080").await; + assert!( + routed.contains("200 OK"), + "expected a connection no box owns to route, got: {routed}" + ); + let unattributed = + proxy_get_from(proxy_addr, UNATTRIBUTED_ADDR, "web.min.internal:18080").await; + assert!( + unattributed.contains("403 Forbidden") + && unattributed.contains("the proxy cannot tell which box opened this connection"), + "expected a connection with no named owner to be refused, got: {unattributed}" + ); + let unregistered = + proxy_get_from(proxy_addr, UNREGISTERED_BOX_ADDR, "web.min.internal:18080").await; + assert!( + unregistered.contains("403 Forbidden"), + "expected a connection owned by a box with no caller facts to be refused, got: {unregistered}" + ); + + drop(guard); + let logged = buf.contents(); + let refusals: Vec<&str> = logged + .lines() + .filter(|line| line.contains("network policy violation")) + .collect(); + assert_eq!( + refusals.len(), + 1, + "one policy refusal, the deny-all box's: {logged}" + ); + assert!( + logged + .lines() + .any(|line| line.contains("refused a proxied request") + && line.contains(r#"caller="unknown""#) + && line.contains(&UNATTRIBUTED_ADDR.to_string())), + "expected the unattributed refusal to name the peer, got: {logged}" + ); + assert!( + logged + .lines() + .any(|line| line.contains("refused a proxied request") + && line.contains(&UNREGISTERED_BOX_ADDR.to_string())), + "expected the unregistered box's refusal to name the peer, got: {logged}" + ); + for expected in [ + r#"caller="hostbox""#, + r#"session="web""#, + "direction=egress", + "port=18080", + "the caller's egress policy denies this target", + ] { + assert!( + refusals[0].contains(expected), + "expected the refusal to carry {expected}, got: {logged}" + ); + } + } + + /// NET-070, design §7.1: an ended box brought up for its destroy hooks + /// can launch with no caller facts, when another box holds its name. Its + /// launch alone turns the owner lookup on, so a request from a socket it + /// owns is refused as unattributed, not passed as no box's, and the + /// refusal line names no caller. + #[tokio::test] + async fn proxy_launching_box_without_caller_facts_refused() { + let backend_port = spawn_backend().await; + let web = SessionId::parse_str("00000000-0000-0000-0000-000000000010").unwrap(); + let ended = SessionId::parse_str(UNREGISTERED_BOX).unwrap(); + + let mut reg = HostnameRegistry::new(DEFAULT_HOST_ID, true); + reg.report_own_address( + web, + "web", + Ipv4Addr::LOCALHOST, + BTreeMap::from([(18080, backend_port)]), + ); + reg.begin_host_launch(ended); + assert!( + reg.has_unleased_callers(), + "a launching box with no caller facts turns the owner lookup on" + ); + let router = Router::new(Arc::new(RwLock::new(reg)), proxied_request_verdict) + .with_socket_owner(stub_socket_owner); + + let proxy = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let proxy_addr = proxy.local_addr().unwrap(); + tokio::spawn(serve(proxy, router)); + + let (buf, guard) = capture_logs(); + let refused = + proxy_get_from(proxy_addr, UNREGISTERED_BOX_ADDR, "web.min.internal:18080").await; + assert!( + refused.contains("403 Forbidden"), + "expected the launching box's request to be refused, got: {refused}" + ); + + drop(guard); + let logged = buf.contents(); + assert!( + logged + .lines() + .any(|line| line.contains("refused a proxied request") + && line.contains(r#"caller="unknown""#) + && line.contains(&UNREGISTERED_BOX_ADDR.to_string())), + "expected the refusal to name no caller, got: {logged}" + ); + } + + /// A socket owner that can name no one: a request that reaches the + /// owner lookup with it is refused as unattributed. + fn unknown_socket_owner( + _peer: SocketAddrV4, + _local: SocketAddrV4, + ) -> crate::net::dns::ConnectingOwner { + crate::net::dns::ConnectingOwner::Unknown + } + + /// NET-070: a stopped host-address box keeps its caller facts until + /// destroy, but it runs nothing that can connect. So it does not turn on + /// the owner lookup for connections from the node's address: a request + /// from there routes with no lookup, and a host-side client that closes + /// its socket early is not refused as unattributed. When the box runs + /// again, the lookup is on again. + #[tokio::test] + async fn proxy_stopped_host_address_box_needs_no_owner_lookup() { + let backend_port = spawn_backend().await; + let subnet = SwitchSubnet::default(); + let web = SessionId::parse_str("00000000-0000-0000-0000-000000000010").unwrap(); + let host_box = SessionId::parse_str(HOST_BOX).unwrap(); + + let mut reg = HostnameRegistry::new(DEFAULT_HOST_ID, true); + reg.report_own_address( + web, + "web", + Ipv4Addr::LOCALHOST, + BTreeMap::from([(18080, backend_port)]), + ); + let deny_all = SessionPolicy { + egress: Some(EgressPolicy::deny_all()), + ingress: None, + credentialed_upstream: None, + }; + reg.register_host_net(host_box, "hostbox"); + reg.register_caller(host_box, "hostbox", &deny_all, subnet, None); + assert!( + reg.has_unleased_callers(), + "a running host-address box turns the owner lookup on" + ); + reg.mark_stopped(host_box); + assert!( + !reg.has_unleased_callers(), + "a stopped host-address box does not turn the owner lookup on" + ); + let table = Arc::new(RwLock::new(reg)); + let router = Router::new(Arc::clone(&table), proxied_request_verdict) + .with_socket_owner(unknown_socket_owner); + + let proxy = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let proxy_addr = proxy.local_addr().unwrap(); + tokio::spawn(serve(proxy, router)); + + // The owner lookup answers Unknown for every socket, so a request + // that ran it is refused. This one routes: no lookup ran. + let routed = proxy_get_from(proxy_addr, BROWSER_ADDR, "web.min.internal:18080").await; + assert!( + routed.contains("200 OK"), + "expected a request from the node's address to route with no owner lookup, got: {routed}" + ); + + // A resume starts the host launch before the host's loop clears the + // stopped mark. From the launch mark on, the box counts as live. + table.write().unwrap().begin_host_launch(host_box); + assert!( + table.read().unwrap().has_unleased_callers(), + "a launching host-address box turns the owner lookup on before mark_running" + ); + let launching = proxy_get_from(proxy_addr, BROWSER_ADDR, "web.min.internal:18080").await; + assert!( + launching.contains("403 Forbidden"), + "expected the owner lookup to run for a launching box and refuse an unnamed owner, \ + got: {launching}" + ); + + table.write().unwrap().mark_running(host_box); + assert!( + table.read().unwrap().has_unleased_callers(), + "the host-address box runs again, so the owner lookup is on again" + ); + let refused = proxy_get_from(proxy_addr, BROWSER_ADDR, "web.min.internal:18080").await; + assert!( + refused.contains("403 Forbidden"), + "expected the owner lookup to run again and refuse an unnamed owner, got: {refused}" + ); + } + + /// NET-070: the loop of a replaced host can end after the newer launch's + /// host runs. Its stop mark carries the older launch number, so it does + /// not mark the running box stopped. The box stays live, and a deny-all + /// host-address box's request from the node's address is still refused. + #[tokio::test] + async fn proxy_host_address_caller_egress_denied_after_late_host_stop() { + let backend_port = spawn_backend().await; + let subnet = SwitchSubnet::default(); + let web = SessionId::parse_str("00000000-0000-0000-0000-000000000010").unwrap(); + let host_box = SessionId::parse_str(HOST_BOX).unwrap(); + + let mut reg = HostnameRegistry::new(DEFAULT_HOST_ID, true); + reg.report_own_address( + web, + "web", + Ipv4Addr::LOCALHOST, + BTreeMap::from([(18080, backend_port)]), + ); + let deny_all = SessionPolicy { + egress: Some(EgressPolicy::deny_all()), + ingress: None, + credentialed_upstream: None, + }; + reg.register_host_net(host_box, "hostbox"); + reg.register_caller(host_box, "hostbox", &deny_all, subnet, None); + let table = Arc::new(RwLock::new(reg)); + + let first = table.write().unwrap().begin_host_launch(host_box); + let second = table.write().unwrap().begin_host_launch(host_box); + assert!(second > first, "each launch gets a newer number"); + table.write().unwrap().mark_running(host_box); + + // The first host's loop ends late, after the second host runs. + crate::session_host::NameMarker::new(Arc::clone(&table), host_box, first).mark_stopped(); + assert!( + table.read().unwrap().has_unleased_callers(), + "a replaced host's late stop mark leaves the running box live" + ); + + // The host-address box owns every connecting socket here. + fn host_box_owner( + _peer: SocketAddrV4, + _local: SocketAddrV4, + ) -> crate::net::dns::ConnectingOwner { + crate::net::dns::ConnectingOwner::Box(SessionId::parse_str(HOST_BOX).unwrap()) + } + let router = Router::new(Arc::clone(&table), proxied_request_verdict) + .with_socket_owner(host_box_owner); + let proxy = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let proxy_addr = proxy.local_addr().unwrap(); + tokio::spawn(serve(proxy, router)); + + let refused = proxy_get_from(proxy_addr, BROWSER_ADDR, "web.min.internal:18080").await; + assert!( + refused.contains("403 Forbidden"), + "expected the deny-all host-address box's request to be refused, got: {refused}" + ); + + // The current launch's own stop mark still marks the box stopped. + crate::session_host::NameMarker::new(Arc::clone(&table), host_box, second).mark_stopped(); + assert!( + !table.read().unwrap().has_unleased_callers(), + "the current host's stop mark marks the box stopped" + ); + } + /// NET-070's verify line: a request through the hostname proxy from a /// caller whose egress rules deny the target is refused — put to the same /// pure verdict the caller's own outbound frames meet on the relay, diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index 9a0eac4586..02ba2cbc87 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -1646,6 +1646,17 @@ impl Session { .write() .expect("hostname registry lock poisoned"); reg.register_host_net(record.id, &name); + // NET-070: the hostname proxy decides a host-address box's + // request by its own egress rules. The box holds no switch + // lease, so the proxy finds these facts by the box that owns + // the connecting socket (`caller_owning`). + reg.register_caller( + record.id, + &name, + &record.policy, + subnet, + hostname_proxy_port, + ); } _ => {} } @@ -5085,6 +5096,17 @@ impl Session { .expect("hostname registry lock poisoned") .mark_held_ports_bound(record.id) }); + // NET-070: a resumed box stays `stopped` until its host's loop calls + // `mark_running`, and the launch can start its processes before that. + // The launch mark counts the box as live from here, so the hostname + // proxy looks up who owns a connection from the node's address. + // The launch number guards the stop mark: a host loop that this + // launch replaces cannot mark this launch's box stopped. + let host_launch = self + .hostnames + .write() + .expect("hostname registry lock poisoned") + .begin_host_launch(record.id); let spawn = Box::pin(session_host::Host::spawn( launcher, session_host::HostParams { @@ -5114,6 +5136,7 @@ impl Session { name_marker: Some(session_host::NameMarker::new( Arc::clone(&self.hostnames), record.id, + host_launch, )), }, )); @@ -5126,6 +5149,12 @@ impl Session { None => (None, spawn.await), }; let (host, task, host_ip_enforcement) = spawned.map_err(|e| { + // A failed spawn reaches no host loop, so nothing else ends the + // launch mark (NET-070). + self.hostnames + .write() + .expect("hostname registry lock poisoned") + .end_host_launch(record.id); AttachError::SpawnFailed(with_userns_hint(e, userns_spawn_hint().as_deref())) })?; // The confirmation was a read, not a reservation: the VM host daemon diff --git a/crates/minimald/src/session/tests.rs b/crates/minimald/src/session/tests.rs index f496e50ce4..5814a7bfb8 100644 --- a/crates/minimald/src/session/tests.rs +++ b/crates/minimald/src/session/tests.rs @@ -9913,6 +9913,38 @@ async fn expose_deny_typed_error() { } } +/// NET-070: the session actor registers a host-address box's caller facts +/// through the production path, so the hostname proxy can decide the box's +/// requests from the node's address by its egress rules. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn host_address_box_registers_its_caller_facts() { + let server = TestServer::new().await; + let mut client = server.connect().await; + // The harness default: a host-address box. + let id = create_configured_session(&mut client, "hostcaller", "/uwu").await; + let manager = server.state.sessions_manager().await; + let handle = manager + .get_session(crate::sessions::SessionKeyPredicate::Id(id)) + .await + .unwrap() + .expect("the host-address box resolves"); + let record = handle.record().await.expect("the record reads"); + assert_eq!(record.network, sessions::NetworkMode::HostNet); + + let registry = manager.hostnames(); + let registry = registry.read().expect("registry lock"); + assert!( + registry.has_unleased_callers(), + "a host-address box is a caller with no switch lease" + ); + assert!( + registry + .caller_owning(record.id, std::net::Ipv4Addr::LOCALHOST) + .is_some(), + "the proxy can name the host-address box by the socket it owns" + ); +} + /// A host-address box is answered before its stance is read: its record can /// carry no `dynamic_ingress`, so the stance check alone would deny every /// request — true of the setting, and wrong about the port, which already diff --git a/crates/minimald/src/session_host.rs b/crates/minimald/src/session_host.rs index 37f4d9c54f..a9d077c181 100644 --- a/crates/minimald/src/session_host.rs +++ b/crates/minimald/src/session_host.rs @@ -5086,18 +5086,25 @@ pub(crate) struct NameMarker { registry: std::sync::Arc>, /// The stable id of the session whose box's name is marked. session_id: sessions::SessionId, + /// The host launch this marker's host belongs to + /// ([`crate::net::dns::HostnameRegistry::begin_host_launch`]). The stop + /// mark carries it, so a replaced host cannot mark a newer host's box + /// stopped (NET-070). + launch: u64, } #[cfg(target_os = "linux")] impl NameMarker { - /// Marks one session's box name, in one registry. + /// Marks one session's box name, in one registry, for one host launch. pub(crate) fn new( registry: std::sync::Arc>, session_id: sessions::SessionId, + launch: u64, ) -> Self { Self { registry, session_id, + launch, } } @@ -5115,11 +5122,12 @@ impl NameMarker { /// box is never mistaken for one that never existed — but a name it /// shares with the node answers NODATA until a host runs again, so the /// node's own listener at that port is not spoken for by a dead box. + /// A host that a newer launch replaced changes nothing here. pub(crate) fn mark_stopped(&self) { self.registry .write() .expect("hostname registry lock poisoned") - .mark_stopped(self.session_id); + .mark_host_stopped(self.session_id, self.launch); } } diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index 1d035425b9..edb393ccb7 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -2048,13 +2048,8 @@ proof_own_ip_egress_declared_and_enforced() { # ---- the disallowed NAME, its own assertion ---------------------------- # example.org is outside `allow_dns_hosts`, so its connection must not - # complete. Whether it is refused FAST at resolution (the design outcome: - # the resolver refuses a non-matching name) or TIMES OUT with no reset - # (the current relay leaves the name unpinned and drops the SYN) is NOT - # asserted either way — reconciling those two is the dns_gate work's, not - # this proof's — but the observed outcome is printed, with the curl error - # and the elapsed time, so the transcript names which behaviour this - # build has. + # complete. Its relay answers the name REFUSED (NET-141), so curl must + # fail at resolution, inside 5000 ms, before any connection is tried. name_deny_start_ms="$(now_ms)" mnl session exec "$declare_sid" \ "curl -sS -o /dev/null -w 'HTTP:%{http_code}' --max-time 10 https://example.org" \ @@ -2068,13 +2063,16 @@ proof_own_ip_egress_declared_and_enforced() { cat "$WORK/egress-deny-name.err" 2>/dev/null || true fail fi - name_outcome="fast refusal" + name_outcome="not a resolver refusal" if grep -qi 'could not resolve' "$WORK/egress-deny-name.err" 2>/dev/null; then - name_outcome="fast resolver refusal" - elif [ "$name_elapsed_ms" -ge 6000 ]; then - name_outcome="timeout with no reset" + name_outcome="resolver refusal" fi echo "disallowed-name GET https://example.org -> rc=$name_rc status=${name_status:-} elapsed=${name_elapsed_ms}ms ($name_outcome) curl: ${name_err:-}" + if [ "$name_outcome" != "resolver refusal" ] || [ "$name_elapsed_ms" -ge 5000 ]; then + echo "::error::NET-141: the partial-allow-list box's GET https://example.org did not fail fast at resolution (want 'could not resolve' inside 5000 ms, got $name_outcome in ${name_elapsed_ms}ms) — its relay refuses every name the list does not match" + cat "$WORK/egress-deny-name.err" 2>/dev/null || true + fail + fi else echo "allowed/disallowed-connection WARNING: the declared box could not complete its allowed connection to https://example.com; the connection assertions that need the public internet are skipped as weather warnings" fi From 7302e78d4b1b2cdafd2126e260dbe7b1a556e4c1 Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:40:41 +0000 Subject: [PATCH 13/34] fix(minvmd): bound the in-force baseline's host-alias reach to its fetch ports (#2444) Prerequisite for T102. The node-plane baseline's compiled set admits by subnet alone, so once the baseline phase goes in force the gate's node-plane arm would admit frames to the switch's host alias at every port. The in-force arm now opens the alias only to TCP at the daemon's fetch port and refuses every other port and protocol as the infrastructure drop. The announced arm and every box row's verdict are untouched. The policy text and JSON views spell the alias endpoint with its bound, as 100.64.255.254/32 tcp/443. Backport of #2444. Adapted to the line: the CLI's policy view lives in cmd/session.rs, not cmd/boxes/policy.rs, and boxes:: reads sessions::. (cherry picked from commit 64719084287763446c345ff93644975ea4353e10) --- crates/minimal/src/cmd/session.rs | 4 +- crates/minimal/tests/cli.rs | 47 +++++++---- crates/minvmd/src/net/baseline.rs | 89 ++++++++++++++++++++- crates/minvmd/src/net/egress_gate.rs | 113 ++++++++++++++++++++++++++- 4 files changed, 229 insertions(+), 24 deletions(-) diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index 57e2ea6972..754b8f7a61 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -2658,7 +2658,7 @@ pub fn format_policy( out, " {} {}", entry.category().as_str(), - entry.endpoints().join(", ") + baseline.display_endpoints(entry).join(", ") )?; } } @@ -3125,7 +3125,7 @@ pub fn write_policy_json_noting( .iter() .map(|entry| PolicyBaselineEntryJson { category: entry.category().as_str(), - endpoints: entry.endpoints().to_vec(), + endpoints: baseline.display_endpoints(entry), }) .collect(), }) diff --git a/crates/minimal/tests/cli.rs b/crates/minimal/tests/cli.rs index 70a0b6c9d6..b475f70a36 100644 --- a/crates/minimal/tests/cli.rs +++ b/crates/minimal/tests/cli.rs @@ -3024,24 +3024,41 @@ fn policy_shows_baseline_set() { // The rows are the enumeration the helper carries, one per category — // the set the host-side gate decides the daemon's own frames by once the - // baseline is in force. - let baseline = minvmd::net::NodePlaneBaseline::built_in(fabric); - let rendered_baseline = baseline - .entries() - .iter() - .map(|entry| { - format!( - " {} {}", - entry.category().as_str(), - entry.endpoints().join(", ") - ) - }) - .collect::>() - .join("\n"); + // baseline is in force — each alias endpoint with the in-force arm's + // protocol and port bound beside it (NET-061). + let alias = fabric.host_alias(); + let rendered_baseline = + format!(" registry {alias}/32 tcp/443\n cache {alias}/32 tcp/443\n"); assert!( text.contains(&rendered_baseline), - "the baseline rows are the enumeration's entries, one per category:\n{text}" + "the baseline rows are the enumeration's entries, one per category, \ + with the alias bound:\n{text}" ); + + // The JSON view carries the same bounded endpoints. + let mut out = Vec::new(); + write_policy_json( + &mut out, + &deny_all, + sessions::NetworkMode::OwnIp, + Some(fabric), + Ok(Vec::new()), + &[], + ) + .unwrap(); + let document: Value = serde_json_lenient::from_slice(&out).unwrap(); + let bounded = Value::String(format!("{alias}/32 tcp/443")); + let entries = document["node_plane_baseline"]["entries"] + .as_array() + .expect("the JSON view carries the baseline entries"); + assert_eq!(entries.len(), 2, "registry and cache:\n{document}"); + for entry in entries { + assert_eq!( + entry["endpoints"], + Value::Array(vec![bounded.clone()]), + "the JSON view spells the alias bound:\n{document}" + ); + } // Beside, not instead: the egress block comes first, the baseline set // after it, the ingress block last. let egress_at = text.find("egress").expect("the egress block renders"); diff --git a/crates/minvmd/src/net/baseline.rs b/crates/minvmd/src/net/baseline.rs index 14ac2ce742..b978b88811 100644 --- a/crates/minvmd/src/net/baseline.rs +++ b/crates/minvmd/src/net/baseline.rs @@ -32,7 +32,7 @@ //! deployment's to say, and neither category is ever dropped from the //! enumeration by saying it. -use sessions::core::egress::{EgressRules, Ipv4Cidr}; +use sessions::core::egress::{EgressRules, IPPROTO_TCP, Ipv4Cidr}; use switch::SwitchSubnet; /// A category of the node-plane baseline enumeration: one of the design's @@ -150,9 +150,33 @@ impl NodeBaselinePhase { /// point them — is the change that flips this constant, and the constant is /// the whole cutover: the gate's decision reads it off the baseline it was /// built with ([`NodePlaneBaseline::phase`]), the start-up line logs it, and -/// the tests pin both of its arms, so the flip is one line and nothing else. +/// the tests pin both of its arms, so the flip is one line here, beside the +/// listener that feeds [`REGISTRY_CACHE_FETCH_PORTS`]. pub(crate) const NODE_BASELINE_PHASE: NodeBaselinePhase = NodeBaselinePhase::Announced; +/// The IP protocol the in-force arm opens on the switch's host alias: TCP +/// alone, because the store surface the built-in endpoints name is HTTPS. +/// QUIC is not admitted; carrying it would be a stated choice that adds UDP +/// here. Every other protocol to the alias, ICMP and GRE among them, is the +/// infrastructure drop, whatever its bytes read as a port. +pub(crate) const REGISTRY_CACHE_FETCH_PROTOCOL: u8 = IPPROTO_TCP; + +/// The ports the in-force arm opens on the switch's host alias, over +/// [`REGISTRY_CACHE_FETCH_PROTOCOL`]. The compiled set has no port +/// dimension, so without this bound the alias would be admitted at every +/// port. +/// +/// 443 is the planned port of the helper's store surface on the alias. No +/// listener serves a registry or a cache there yet, and the daemon's own +/// fetches go to `cache.minimal.dev`, not to the alias. When that listener +/// lands, this list must be read from the listener's stack, the way +/// `switch::bep_host::PROXY_PORT` is. `[443]` is the planned port, and no +/// listener serves it. While [`NODE_BASELINE_PHASE`] is +/// [`NodeBaselinePhase::Announced`], only the in-force tests exercise this +/// list. The endpoint-configuration change (#1786) replaces it with the +/// listener's port, or with an empty list if no listener exists. +pub(crate) const REGISTRY_CACHE_FETCH_PORTS: [u16; 1] = [443]; + /// The node-plane baseline set: the categories in force, each with the /// subnets it admits, and the compiled rules the host-side gate decides /// node-plane frames by. @@ -165,6 +189,10 @@ pub struct NodePlaneBaseline { /// The switch's resolver, carried into every recompilation so the /// resolver carve-out survives a re-pointed category. resolver: [u8; 4], + /// The switch's host alias: the address the in-force arm bounds to + /// [`REGISTRY_CACHE_FETCH_PROTOCOL`] at [`REGISTRY_CACHE_FETCH_PORTS`], + /// carried so the policy display spells that bound beside the endpoint. + host_alias: [u8; 4], /// The categories in force, in the order the display spells them. entries: Vec, /// The compiled allow list the gate decides node-plane frames by: every @@ -198,7 +226,9 @@ impl NodePlaneBaseline { /// the enumerated set. /// /// The compiled set allows every IP protocol within the admitted - /// subnets — the categories bound destinations, not protocols — and is + /// subnets — the categories bound destinations, not protocols; the + /// gate's in-force arm bounds the host alias to TCP at + /// [`REGISTRY_CACHE_FETCH_PORTS`] before the set is consulted — and is /// keyed to the in-VM daemon's own address as its lease: the verdict /// rejects any frame wearing another source, so the set decides /// node-plane frames and only those. The resolver's carve-out keeps the @@ -210,6 +240,7 @@ impl NodePlaneBaseline { Self::from_entries( subnet.daemon_ip().octets(), subnet.dns_server().octets(), + host_alias.octets(), vec![ BaselineEntry { category: BaselineCategory::Registry, @@ -251,6 +282,32 @@ impl NodePlaneBaseline { self.phase } + /// One entry's endpoints as the policy display spells them (NET-061): an + /// endpoint naming the host alias carries the in-force arm's bound + /// beside it (`100.64.255.254/32 tcp/443`), because the set has no port + /// dimension and the bound is what in force admits there. Any other + /// endpoint is spelled as the set holds it. + #[must_use] + pub fn display_endpoints(&self, entry: &BaselineEntry) -> Vec { + let alias = format!("{}/32", std::net::Ipv4Addr::from(self.host_alias)); + let ports = REGISTRY_CACHE_FETCH_PORTS + .iter() + .map(u16::to_string) + .collect::>() + .join(","); + entry + .endpoints + .iter() + .map(|endpoint| { + if *endpoint == alias { + format!("{endpoint} tcp/{ports}") + } else { + endpoint.clone() + } + }) + .collect() + } + /// The set as the daemon's start-up line spells it: one /// `category=endpoints` term per category, in the order the categories /// are carried, so the line names the set in force with each entry's @@ -315,10 +372,16 @@ impl NodePlaneBaseline { /// Assembles the set from its parts: the categories carried, and the /// addresses the compiled rules are keyed to. - fn from_entries(node_addr: [u8; 4], resolver: [u8; 4], entries: Vec) -> Self { + fn from_entries( + node_addr: [u8; 4], + resolver: [u8; 4], + host_alias: [u8; 4], + entries: Vec, + ) -> Self { Self { node_addr, resolver, + host_alias, rules: Self::compile_rules(&entries, resolver, node_addr), entries, phase: NODE_BASELINE_PHASE, @@ -523,4 +586,22 @@ mod tests { FrameVerdict::Admit => panic!("a foreign source is not admitted by the set"), } } + + /// NET-061: the policy display spells the in-force arm's bound beside + /// an endpoint naming the host alias, and a re-pointed endpoint as the + /// set holds it. + #[test] + fn display_endpoints_spell_the_alias_bound() { + let baseline = + NodePlaneBaseline::built_in(SUBNET).with_cache(vec!["198.51.100.0/24".to_owned()]); + let alias = format!("{}/32 tcp/443", SUBNET.host_alias()); + assert_eq!( + baseline.display_endpoints(entry(&baseline, BaselineCategory::Registry)), + vec![alias] + ); + assert_eq!( + baseline.display_endpoints(entry(&baseline, BaselineCategory::Cache)), + vec!["198.51.100.0/24".to_owned()] + ); + } } diff --git a/crates/minvmd/src/net/egress_gate.rs b/crates/minvmd/src/net/egress_gate.rs index 675d63d111..49df356e47 100644 --- a/crates/minvmd/src/net/egress_gate.rs +++ b/crates/minvmd/src/net/egress_gate.rs @@ -175,7 +175,9 @@ use tokio::task::{JoinHandle, JoinSet}; use crate::bep_attach::{BoxId, BoxIdText}; use crate::box_registry::{BoxRecord, BoxTable, RowWithdrawal}; -use super::baseline::{NodeBaselinePhase, NodePlaneBaseline}; +use super::baseline::{ + NodeBaselinePhase, NodePlaneBaseline, REGISTRY_CACHE_FETCH_PORTS, REGISTRY_CACHE_FETCH_PROTOCOL, +}; use super::dns_pins; use super::forward_revoke::{self, ForwardedFlows}; @@ -5710,8 +5712,29 @@ fn gate_verdict( // verdict against the enumeration's compiled set — the lease it is keyed // to is the daemon's own address, so the check above and the lease agree // on who may wear it. Announced, the interim node row keeps deciding the - // node plane, and the fall-through reaches it below. + // node plane, and the fall-through reaches it below. The set's verdict + // is by subnet alone, so the alias the built-in endpoints name is the + // arm's to bound: TCP at a fetch port reaches the shared verdict, and + // any other protocol or port is the infrastructure drop, the same + // refusal a box row's alias frames take. The protocol is checked + // because the summary reads a destination port from every protocol's + // bytes. In force, the node plane reaches the alias at TCP 443 only. + // Once the host exposures close by default, that is wider than the + // announced phase's alias reach, which admits no port. The bound must + // also admit the configured host exposures (TCP and UDP) once they + // exist. gominimal/inbox#1144 and gominimal/inbox#1147 own that work. if baseline.phase() == NodeBaselinePhase::InForce && src == baseline.node_addr() { + if let Some(dst) = summary.destination() + && dst == table.subnet().host_alias().octets() + && (summary.protocol() != Some(REGISTRY_CACHE_FETCH_PROTOCOL) + || !REGISTRY_CACHE_FETCH_PORTS.contains(&summary.destination_port())) + { + return Err(GateDrop::Infrastructure { + src, + dst, + dst_port: summary.destination_port(), + }); + } return match egress::verdict(summary, baseline.rules()) { FrameVerdict::Admit => Ok(GateAdmit::Baseline), FrameVerdict::Drop(reason) => Err(GateDrop::Verdict(reason)), @@ -15637,6 +15660,90 @@ mod tests { ); } + /// The in-force arm's own bound on the host alias: the built-in + /// registry and cache endpoints name the alias, and the compiled set's + /// verdict is by subnet alone — no port dimension — so the arm bounds + /// the alias to the ports the daemon's own fetches use. The fetch port + /// stays admitted by the set; a wider port on the alias is the + /// infrastructure drop, the same refusal a box row's alias frames take + /// (design §7.1, NET-062). + #[test] + fn in_force_baseline_bounds_the_host_alias_to_its_fetch_ports() { + let registry = BoxRegistry::new(SUBNET); + registry.register_node_namespace(7654); + let table = registry.table(); + let baseline = NodePlaneBaseline::built_in(SUBNET).in_force(); + let pins = dns_pins::DnsPins::new(SUBNET); + let node = baseline.node_addr(); + let alias = SUBNET.host_alias().octets(); + + let fetch = sessions::core::egress::summarize(&ipv4_frame(node, 6, alias, 443)); + assert_eq!( + gate_verdict(&fetch, None, &table, &baseline, &pins, &ReplyTables::new()), + Ok(GateAdmit::Baseline), + "the daemon's own fetch port on the alias stays admitted by the set" + ); + + let wider = sessions::core::egress::summarize(&ipv4_frame(node, 6, alias, 18081)); + assert!( + matches!( + gate_verdict( + &wider, + None, + &table, + &baseline, + &pins, + &ReplyTables::new() + ), + Err(GateDrop::Infrastructure { dst, dst_port, .. }) + if dst == alias && dst_port == 18081 + ), + "a wider port on the alias is the infrastructure drop, in force" + ); + } + + /// The in-force arm bounds the alias by protocol as well as port: the + /// summary reads a destination port from every protocol's bytes, so an + /// ICMP frame whose checksum reads 443, or a GRE frame (IP protocol 47) + /// whose protocol-type field does, must not pass as a fetch. TCP at 443 + /// is the one admission. + #[test] + fn in_force_baseline_admits_only_tcp_at_the_alias_fetch_port() { + let registry = BoxRegistry::new(SUBNET); + registry.register_node_namespace(7654); + let table = registry.table(); + let baseline = NodePlaneBaseline::built_in(SUBNET).in_force(); + let pins = dns_pins::DnsPins::new(SUBNET); + let node = baseline.node_addr(); + let alias = SUBNET.host_alias().octets(); + let decide = |proto: u8| { + let summary = sessions::core::egress::summarize(&ipv4_frame(node, proto, alias, 443)); + gate_verdict( + &summary, + None, + &table, + &baseline, + &pins, + &ReplyTables::new(), + ) + }; + + assert_eq!( + decide(6), + Ok(GateAdmit::Baseline), + "TCP at the fetch port on the alias is admitted, in force" + ); + for (proto, name) in [(1u8, "ICMP"), (47, "GRE")] { + assert!( + matches!( + decide(proto), + Err(GateDrop::Infrastructure { dst, dst_port: 443, .. }) if dst == alias + ), + "{name} to alias:443 is the infrastructure drop, in force" + ); + } + } + /// NET-130, un-enrolled, at the gate: the node-plane baseline set comes /// from the helper's enumeration, and the gate decides the daemon's own /// frames by it **beside** the boxes' rows — a deny-all box's row does not @@ -15722,7 +15829,7 @@ mod tests { // daemon's shipped cache URL resolves to, whose serving addresses no // subnet list this tree can name — the set says no. let out_of_set = ipv4_frame(node_addr, 6, [8, 8, 8, 8], 443); - let node_marker = ipv4_frame(node_addr, 6, registry_endpoint, 444); + let node_marker = ipv4_frame(node_addr, 6, registry_endpoint, 443); send_frame(&mut h.guest, &out_of_set).await; send_frame(&mut h.guest, &node_marker).await; let seen = expect_frame(&mut h.switch).await; From 7299c15edf37be859bfc64465019ea786b634d70 Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 07:02:52 +0000 Subject: [PATCH 14/34] fix(net): scope the host alias by the host-exposure port list (#2331) Prerequisite for T102. The announced node row carries the frames of every box sharing the guest root namespace, so its reach through the host alias was admitted at every port. The alias now opens only at the host exposures, the inclusive port ranges listed in MINVMD_HOST_EXPOSURE_PORTS, read once at VM start. Unset means no exposure, and every other port is the infrastructure drop for every row. An exposure matches only a TCP or UDP frame, so an ICMP frame never opens the alias. The parser refuses port 0 and names a bad term. The VM lanes export the port their NET-003 check reaches, and add the refused-port half. T102, next, replaces this variable with the [host-exposures] table. Backport of #2331. Adapted to the line: boxes:: reads sessions:: in baseline.rs. (cherry picked from commit 0ce1fd81fc7b29eb3ce7fdcbecc0142459805c72) --- crates/minvmd/src/net.rs | 8 +- crates/minvmd/src/net/baseline.rs | 206 ++++++++++++++++++++++++++- crates/minvmd/src/net/egress_gate.rs | 151 +++++++++++++++----- docs/reference/cli-minvmd.md | 2 + scripts/session-e2e.sh | 88 +++++++++++- 5 files changed, 408 insertions(+), 47 deletions(-) diff --git a/crates/minvmd/src/net.rs b/crates/minvmd/src/net.rs index cc7f738115..b570e3bc2e 100644 --- a/crates/minvmd/src/net.rs +++ b/crates/minvmd/src/net.rs @@ -755,8 +755,12 @@ impl HostGvproxy { // In force, the gate decides a frame from the daemon's own address // by it, beside the boxes' rows; announced — the shipped posture — // the allow-all interim node row still does, and the gate's - // start-up line names the posture deciding. - let baseline = NodePlaneBaseline::built_in(subnet); + // start-up line names the posture deciding. The host exposures the + // alias is opened at for the node row — none unless the deployment + // names ports — are read from the daemon's environment here, on each + // spawn, an unparseable list failing it. + let baseline = NodePlaneBaseline::built_in(subnet) + .with_host_exposure(baseline::host_exposure_ports_from_env()?); let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>(); let (ready_tx, ready_rx) = std::sync::mpsc::channel::>(); diff --git a/crates/minvmd/src/net/baseline.rs b/crates/minvmd/src/net/baseline.rs index b978b88811..92f022d76b 100644 --- a/crates/minvmd/src/net/baseline.rs +++ b/crates/minvmd/src/net/baseline.rs @@ -33,6 +33,7 @@ //! enumeration by saying it. use sessions::core::egress::{EgressRules, IPPROTO_TCP, Ipv4Cidr}; +use std::io; use switch::SwitchSubnet; /// A category of the node-plane baseline enumeration: one of the design's @@ -154,6 +155,76 @@ impl NodeBaselinePhase { /// listener that feeds [`REGISTRY_CACHE_FETCH_PORTS`]. pub(crate) const NODE_BASELINE_PHASE: NodeBaselinePhase = NodeBaselinePhase::Announced; +/// The environment variable the daemon's host-exposure port list is read +/// from, each time the daemon starts the VM's network — from the daemon's +/// own environment, so a change takes a daemon restart, the same shape the +/// hostname proxy port and the egress-default opt-out carry. +pub(crate) const HOST_EXPOSURE_PORTS_ENV: &str = "MINVMD_HOST_EXPOSURE_PORTS"; + +/// The host-exposure port list the daemon starts with, as the deployment +/// spelled it. The design holds the alias default-deny except explicitly +/// configured host exposures (design §7.1), so absent or empty, the list is +/// empty and the alias is refused at every port for every row. The list is +/// comma-separated, each term a port (`4200`) or an inclusive range +/// (`3000-3999`), port 0 excluded; a term that is neither, a reversed range, +/// or a value that is not Unicode fails the start, the error naming the term +/// and the fix. +pub(crate) fn host_exposure_ports_from_env() -> io::Result> { + match std::env::var(HOST_EXPOSURE_PORTS_ENV) { + Ok(raw) => host_exposure_ports_from_raw(Some(&raw)), + Err(std::env::VarError::NotPresent) => host_exposure_ports_from_raw(None), + Err(std::env::VarError::NotUnicode(_)) => Err(io::Error::other(format!( + "{HOST_EXPOSURE_PORTS_ENV} is not valid Unicode. {HOST_EXPOSURE_FIX}" + ))), + } +} + +/// The correction every parse error ends with. +const HOST_EXPOSURE_FIX: &str = "Set it to ports and ranges such as 8080,3000-3999, \ + or set it empty to close host.min.internal."; + +/// Parses one deployment's spelling of the host-exposure port list: every +/// term a port or an inclusive range from 1 to 65535, whitespace trimmed; +/// absent or empty, no exposure at all. +fn host_exposure_ports_from_raw(raw: Option<&str>) -> io::Result> { + let Some(raw) = raw else { + return Ok(Vec::new()); + }; + let port = |text: &str, term: &str| match text.trim().parse::() { + Ok(0) | Err(_) => Err(bad_host_exposure_term(term)), + Ok(port) => Ok(port), + }; + raw.split(',') + .filter(|term| !term.trim().is_empty()) + .map(|term| { + let term = term.trim(); + match term.split_once('-') { + Some((lo, hi)) => { + let (lo, hi) = (port(lo, term)?, port(hi, term)?); + if lo <= hi { + Ok((lo, hi)) + } else { + Err(io::Error::other(format!( + "{HOST_EXPOSURE_PORTS_ENV}: `{term}` is a reversed range; write it \ + low-high, as `{hi}-{lo}`. {HOST_EXPOSURE_FIX}" + ))) + } + } + None => port(term, term).map(|port| (port, port)), + } + }) + .collect() +} + +/// The start-failing error for one unparseable term: the variable, the term +/// as it was read, and the correction. +fn bad_host_exposure_term(term: &str) -> io::Error { + io::Error::other(format!( + "{HOST_EXPOSURE_PORTS_ENV}: `{term}` is not a port from 1 to 65535 or a low-high port \ + range. {HOST_EXPOSURE_FIX}" + )) +} + /// The IP protocol the in-force arm opens on the switch's host alias: TCP /// alone, because the store surface the built-in endpoints name is HTTPS. /// QUIC is not admitted; carrying it would be a stated choice that adds UDP @@ -202,6 +273,14 @@ pub struct NodePlaneBaseline { /// The posture the gate decides the set under: the shipped constant, /// until the store-surface configuration flips it. phase: NodeBaselinePhase, + /// The configured host exposures over the alias, as inclusive port + /// ranges: the ports the node row still reaches the host alias at while + /// the baseline is announced, every other port refused through the + /// alias. Empty unless the deployment names ports. The host-side gate + /// alone reads it: the guest-side verdict's own alias list starts empty + /// and nothing in the run path fills it, so own-address boxes stay + /// refused at every port. + host_exposure: Vec<(u16, u16)>, } impl NodePlaneBaseline { @@ -282,6 +361,46 @@ impl NodePlaneBaseline { self.phase } + /// The host exposures over the alias this gate carries: the inclusive + /// port ranges the announced node row still reaches the host alias at, + /// empty unless the deployment configured ports. + #[must_use] + pub(crate) fn host_exposure_ports(&self) -> &[(u16, u16)] { + &self.host_exposure + } + + /// Points the host exposures at `ports`, replacing the current list: a + /// host names or clears its own exposure surface, and a cleared list is an + /// alias with no opening — refused at every port for every row — not + /// an absent arm. The compiled set is untouched: the list scopes the + /// alias alone. + #[must_use] + pub fn with_host_exposure(mut self, ports: Vec<(u16, u16)>) -> Self { + self.host_exposure = ports; + self + } + + /// The host-exposure list as the daemon's start-up line spells it: one + /// `lo-hi` term per range, a bare port for a single-port term, and + /// `(none)` when the deployment cleared the list. + #[must_use] + pub(crate) fn render_host_exposure(&self) -> String { + if self.host_exposure.is_empty() { + return "(none)".to_string(); + } + self.host_exposure + .iter() + .map(|&(lo, hi)| { + if lo == hi { + lo.to_string() + } else { + format!("{lo}-{hi}") + } + }) + .collect::>() + .join(",") + } + /// One entry's endpoints as the policy display spells them (NET-061): an /// endpoint naming the host alias carries the in-force arm's bound /// beside it (`100.64.255.254/32 tcp/443`), because the set has no port @@ -371,7 +490,9 @@ impl NodePlaneBaseline { } /// Assembles the set from its parts: the categories carried, and the - /// addresses the compiled rules are keyed to. + /// addresses the compiled rules are keyed to. The host exposures start + /// empty, the alias closed; the deployment's spelling replaces them in + /// the run path's builder. fn from_entries( node_addr: [u8; 4], resolver: [u8; 4], @@ -385,6 +506,7 @@ impl NodePlaneBaseline { rules: Self::compile_rules(&entries, resolver, node_addr), entries, phase: NODE_BASELINE_PHASE, + host_exposure: Vec::new(), } } @@ -407,7 +529,10 @@ impl NodePlaneBaseline { #[cfg(test)] mod tests { - use super::{BaselineCategory, BaselineEntry, NodePlaneBaseline}; + use super::{ + BaselineCategory, BaselineEntry, HOST_EXPOSURE_PORTS_ENV, NodePlaneBaseline, + host_exposure_ports_from_raw, + }; use sessions::core::egress::{FrameVerdict, summarize, verdict}; use std::net::Ipv4Addr; use std::str::FromStr; @@ -587,6 +712,83 @@ mod tests { } } + /// The deployment's spelling of the host-exposure list: absent or + /// empty, no exposure — the alias closed; every term a port or an + /// inclusive range from 1 to 65535; an undecodable term, port 0, or a + /// reversed range a surfaced error naming the variable, the term and + /// the fix. + #[test] + fn host_exposure_ports_read_from_the_environment() { + assert!( + host_exposure_ports_from_raw(None).unwrap().is_empty(), + "an unset variable is no exposure: the alias stays closed" + ); + assert_eq!( + host_exposure_ports_from_raw(Some("8080")).unwrap(), + vec![(8080, 8080)], + "a bare port is a single-port range" + ); + assert_eq!( + host_exposure_ports_from_raw(Some(" 3000-3999, 4200 ")).unwrap(), + vec![(3000, 3999), (4200, 4200)], + "terms are inclusive ranges and ports, whitespace trimmed" + ); + assert!( + host_exposure_ports_from_raw(Some("")).unwrap().is_empty(), + "a set-but-empty list is no exposure" + ); + let garbage = host_exposure_ports_from_raw(Some("8080:nope")) + .expect_err("an undecodable term is a surfaced error") + .to_string(); + assert!( + garbage.contains(HOST_EXPOSURE_PORTS_ENV) + && garbage.contains("`8080:nope` is not a port") + && garbage.contains("set it empty to close host.min.internal"), + "an undecodable term names the variable, the term and the fix, got: {garbage}" + ); + for zero in ["0", "0-65535"] { + let error = host_exposure_ports_from_raw(Some(zero)) + .expect_err("port 0 is refused") + .to_string(); + assert!( + error.contains(&format!("`{zero}` is not a port")), + "port 0 names the term, got: {error}" + ); + } + let reversed = host_exposure_ports_from_raw(Some("4000-3000")) + .expect_err("a reversed range is not an inclusive one") + .to_string(); + assert!( + reversed.contains("`4000-3000` is a reversed range") + && reversed.contains("`3000-4000`") + && !reversed.contains("is not a port"), + "a reversed range gets its own message, got: {reversed}" + ); + } + + /// The baseline carries no exposure until the deployment's spelling + /// replaces the list: the builder replaces rather than merges, and a + /// cleared list reaches the accessor empty. + #[test] + fn baseline_carries_the_configured_host_exposures() { + let built_in = NodePlaneBaseline::built_in(SUBNET); + assert!( + built_in.host_exposure_ports().is_empty(), + "the built-in enumeration carries no host exposure" + ); + let re_exposed = built_in.with_host_exposure(vec![(8080, 8080)]); + assert_eq!( + re_exposed.host_exposure_ports(), + &[(8080, 8080)], + "the builder sets the list" + ); + let cleared = re_exposed.with_host_exposure(vec![]); + assert!( + cleared.host_exposure_ports().is_empty(), + "a cleared list is carried empty" + ); + } + /// NET-061: the policy display spells the in-force arm's bound beside /// an endpoint naming the host alias, and a re-pointed endpoint as the /// set holds it. diff --git a/crates/minvmd/src/net/egress_gate.rs b/crates/minvmd/src/net/egress_gate.rs index 49df356e47..fe414de529 100644 --- a/crates/minvmd/src/net/egress_gate.rs +++ b/crates/minvmd/src/net/egress_gate.rs @@ -542,8 +542,14 @@ static INFRASTRUCTURE_RANGES: LazyLock = LazyLock::new(|| /// Whether `dst` lies in the infrastructure deny set as the host frame rule /// holds it ([`INFRASTRUCTURE_RULE`]): a fixed range; the host alias, which /// is default-deny at every port for a box row (design §7.1, NET-062) when -/// `refuse_host_alias` says so; the fabric plane -/// outside `own_block`, the subnet the gate's rows live in — a frame to a +/// `refuse_host_alias` says so, and for the node row's frames too outside +/// the host exposures `host_exposure` carries — the inclusive port ranges +/// the deployment configured, empty unless it named ports, matched against +/// `transport_port`, the TCP or UDP destination port, `None` for any other +/// frame. Box rows stay refused at a configured exposure too: the design +/// holds an exposure as local reach under the box's egress rules (§7.1), +/// and that reach is not built yet; the refusal fails closed, and +/// gominimal/inbox#1144 owns the gap. The fabric plane outside `own_block`, the subnet the gate's rows live in — a frame to a /// sibling or to the daemon inside the node's own block is local reach, /// decided by the row's CIDR rules and the target's ingress, never by this /// rule, and the gateway inside it is the control-surface rule's; or RFC @@ -566,14 +572,19 @@ static INFRASTRUCTURE_RANGES: LazyLock = LazyLock::new(|| /// of every box that shares the guest root namespace (a host-address box, /// the default network mode), and those reach the host alias as own-block /// local reach until the in-force baseline, which names the alias as its -/// registry and cache endpoint, decides the node plane instead. This -/// mirrors the shared verdict, whose own-address refusal is attached to box -/// relays only, never to the daemon relay. +/// registry and cache endpoint, decides the node plane instead. Announced, +/// that reach is scoped: the alias is opened at the configured host +/// exposures alone, and a node-row frame to any other port, or with no TCP +/// or UDP port, is this rule's drop. This mirrors the shared verdict, +/// whose own-address refusal is attached to box relays only, never to the +/// daemon relay. fn infrastructure_destination( dst: [u8; 4], + transport_port: Option, own_block: SwitchSubnet, allow: Option<&[Ipv4Cidr]>, refuse_host_alias: bool, + host_exposure: &[(u16, u16)], ) -> bool { let ranges = &*INFRASTRUCTURE_RANGES; if ranges.fixed.iter().any(|cidr| cidr.contains(dst)) { @@ -582,8 +593,16 @@ fn infrastructure_destination( // The host alias is default-deny (design §7.1, NET-062): a box reaches // the host only through a declared exposure, never by addressing the // alias directly, so the alias is refused under this rule at every port - // whatever the row's rules would say about it. - if refuse_host_alias && dst == own_block.host_alias().octets() { + // whatever the row's rules would say about it. The node row's frames + // — the host-address boxes the interim row carries — reach the alias at + // the configured host exposures alone, and only over TCP or UDP: a frame + // with no transport port (ICMP reads as port 0) fails the opening closed. + let exposed = transport_port.is_some_and(|port| { + host_exposure + .iter() + .any(|&(lo, hi)| lo <= port && port <= hi) + }); + if dst == own_block.host_alias().octets() && (refuse_host_alias || !exposed) { return true; } let in_own_block = (u32::from_be_bytes(dst) & u32::from(own_block.netmask())) @@ -1056,6 +1075,7 @@ impl EgressGate { switch_socket = %switch_sock.display(), unregistered_sources = "dropped", undeclared_box_default = phase.as_str(), + host_exposure_ports = %baseline.render_host_exposure(), "host-side egress gate listening", ); // The node-plane baseline set the gate decides the in-VM daemon's own @@ -5846,14 +5866,41 @@ fn gate_verdict( // row, keyed to the daemon's own address, carries the host-address // boxes' frames, whose `host.min.internal` resolves to the alias, and // keeps own-block local reach to it until the in-force baseline (which - // returned above) decides the node plane. + // returned above) decides the node plane. Announced, that reach is + // scoped by the baseline's configured host exposures, which the arm + // below reads for every row: the box rows' refusal stands at every + // port, and the node row's outside the exposures. The scoping holds only + // while the baseline is announced: in force, the arm above returns first + // and admits the alias only at TCP on `REGISTRY_CACHE_FETCH_PORTS` (443), + // with no host exposures, a gap gominimal/inbox#1144 and + // gominimal/inbox#1147 own. let box_row = src != table.subnet().daemon_ip().octets(); + // A non-first IPv4 fragment carries no readable L4 port — `summarize` + // reports destination port 0 for it — so it fails the exposure match + // and the gate drops it, never admits it: a fragmented UDP datagram to + // an exposed port dies at its later fragments. Reviewer ask + // (CodeRabbit, PR #2331) was to associate later fragments with their + // admitted first fragment, or defer the port check to reassembly. + // Declined: the verdict path is stateless and allocation-free by + // design — every input comes from the one frame being judged — and + // fragment association would add per-flow state here (or reassembly + // elsewhere) to recover a packet whose failure mode is a drop, not an + // exposure: deny-by-default keeps the safe direction. In force, the arm + // above admits no host exposure at all, so the fragment case arises only + // while the baseline is announced. + let transport_port = matches!( + summary.protocol(), + Some(egress::IPPROTO_TCP | egress::IPPROTO_UDP) + ) + .then(|| summary.destination_port()); if let Some(dst) = summary.destination() && infrastructure_destination( dst, + transport_port, table.subnet(), record.egress().allow_subnets(), box_row, + baseline.host_exposure_ports(), ) { return Err(GateDrop::Infrastructure { @@ -15607,15 +15654,21 @@ mod tests { ); } - /// The host alias's default-deny is a box row's alone (design §7.1, - /// NET-062). Announced, the interim node row at the daemon's address - /// carries the frames of every box sharing the guest root namespace — a - /// host-address box, the default network mode, whose `host.min.internal` - /// resolves to the alias — so a node-plane frame to the alias is - /// own-block local reach the row admits, while the same frame from a box - /// lease, even an allow-all one, is the infrastructure drop. + /// The host alias's default-deny is a box row's alone, and the node + /// row's reach through it is scoped by the configured host exposures + /// (design §7.1, NET-062). Announced, the interim node row at the + /// daemon's address carries the frames of every box sharing the guest + /// root namespace — a host-address box, the default network mode, whose + /// `host.min.internal` resolves to the alias. The built-in baseline + /// carries no exposure, so the alias is closed at every port until a + /// deployment names ports; with a list, a node-plane TCP or UDP frame to + /// the alias is own-block local reach the row admits inside the list's + /// ranges, edges included, and the infrastructure drop outside them, + /// while an ICMP frame, which carries no port, is refused even by a list + /// spanning every port. The same frames from a box lease, even an + /// allow-all one, are the infrastructure drop whatever the port. #[test] - fn host_alias_refused_for_box_rows_but_not_the_announced_node_row() { + fn host_alias_refused_for_box_rows_and_scoped_by_the_exposures_for_the_node_row() { let registry = BoxRegistry::new(SUBNET); registry.register_node_namespace(7654); registry.register( @@ -15623,40 +15676,62 @@ mod tests { .with_admitted_ports([8080]), ); let table = registry.table(); - let baseline = NodePlaneBaseline::built_in(SUBNET); - assert_eq!(baseline.phase(), NodeBaselinePhase::Announced); + let built_in = NodePlaneBaseline::built_in(SUBNET); + assert_eq!(built_in.phase(), NodeBaselinePhase::Announced); let pins = dns_pins::DnsPins::new(SUBNET); let summarize = sessions::core::egress::summarize; let alias = SUBNET.host_alias().octets(); + let node = SUBNET.daemon_ip().octets(); - let node_frame = summarize(&ipv4_frame(SUBNET.daemon_ip().octets(), 6, alias, 18081)); - assert_eq!( + let verdict = |src: [u8; 4], proto: u8, port: u16, baseline: &NodePlaneBaseline| { gate_verdict( - &node_frame, + &summarize(&ipv4_frame(src, proto, alias, port)), None, &table, - &baseline, + baseline, &pins, - &ReplyTables::new() - ), + &ReplyTables::new(), + ) + }; + let refused = |result: Result| matches!(result, Err(GateDrop::Infrastructure { dst, .. }) if dst == alias); + + assert!( + refused(verdict(node, 6, 8080, &built_in)), + "with no configured exposure the alias is closed to the node row at every port" + ); + + let exposed = built_in + .clone() + .with_host_exposure(vec![(3000, 3999), (8080, 8080)]); + assert!( + refused(verdict(node, 6, 2999, &exposed)), + "the port below a range's low edge is the infrastructure drop" + ); + for port in [3000, 3999, 8080] { + assert_eq!( + verdict(node, 6, port, &exposed), + Ok(GateAdmit::Row), + "the node row reaches the host alias at exposed port {port}" + ); + } + assert!( + refused(verdict(node, 6, 18081, &exposed)), + "the node row's frame to the host alias outside the exposures is the infrastructure drop" + ); + assert_eq!( + verdict(node, 17, 8080, &exposed), Ok(GateAdmit::Row), - "the announced node row keeps own-block reach to the host alias" + "a UDP frame reaches the host alias at an exposed port" + ); + assert!( + refused(verdict(LEASE, 6, 8080, &exposed)), + "a box row's frame to the host alias is the infrastructure drop at an exposed port" ); - let box_frame = summarize(&ipv4_frame(LEASE, 6, alias, 18081)); + let every_port = built_in.with_host_exposure(vec![(0, u16::MAX)]); assert!( - matches!( - gate_verdict( - &box_frame, - None, - &table, - &baseline, - &pins, - &ReplyTables::new() - ), - Err(GateDrop::Infrastructure { dst, .. }) if dst == alias - ), - "a box row's frame to the host alias is the infrastructure drop" + refused(verdict(node, 1, 0, &every_port)), + "an ICMP frame carries no port and never matches an exposure" ); } diff --git a/docs/reference/cli-minvmd.md b/docs/reference/cli-minvmd.md index 61d101dd13..ff30364b05 100644 --- a/docs/reference/cli-minvmd.md +++ b/docs/reference/cli-minvmd.md @@ -55,6 +55,8 @@ An own-IP box that declares no `egress` section gets the deny-all egress default `minvmd` reads the variable only when it starts, and hands it to the guest at boot. Set it in the environment of the process that starts `minvmd`: `minvmd run`, or the first `min` command that starts the VM for you. Setting it for a later `min` command against a running VM changes nothing. To apply a change, stop `minvmd` (`minvmd stop`) and start it again with the new environment. +`minvmd` closes `host.min.internal` at every port unless `MINVMD_HOST_EXPOSURE_PORTS` lists ports. Set the variable to a comma-separated list of ports and ranges, such as `8080,3000-3999`. A box in the default host-address network mode then reaches the host at those ports and no other. The box's egress rules still apply inside an exposure. An own-address box gets a refusal at every port, whatever the list says. Leave the variable unset or empty to keep `host.min.internal` closed. Each port must lie between 1 and 65535, and each range must run from low to high. Any other value fails the start with an error that names the bad term. `minvmd` reads the variable from its own environment each time it boots the VM, so to apply a change, stop `minvmd` and start it again. + ### `status` ``` diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index edb393ccb7..d48d1aa434 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -358,6 +358,20 @@ case "${1:-}" in ;; esac +# The host exposure the VM-backed daemon opens host.min.internal at. minvmd +# closes the alias at every port unless MINVMD_HOST_EXPOSURE_PORTS lists +# ports, and it reads the list from its own environment at each VM boot, so +# the export must precede the first `min` call, whose autospawn inherits it. +# The list is the one port NET-003 reaches the host's loopback at from a +# host-address box (the proxy proof's PROXY_HOST_PORT) and nothing wider: +# the same proof asserts the alias stays refused at an unlisted port, and +# own-address boxes stay refused at every port, listed or not. A native +# daemon has no alias gate and ignores the variable. +E2E_HOST_EXPOSURE_PORT=18081 +if [ -n "$E2E_VM" ]; then + export MINVMD_HOST_EXPOSURE_PORTS="$E2E_HOST_EXPOSURE_PORT" +fi + # The non-baseline package the sandbox proof adds and then runs. It must be a # real upstream package that is genuinely ABSENT from a fresh shell-stack # sandbox — the `shell` stack composes `base` (bash, coreutils, tar, gzip, …) @@ -408,6 +422,7 @@ PROXY_SEED_DIR="" # seeded by the min.internal proxy proof; removed on teardown PROXY_OWN_SEED_DIR="" # its own-address box's seed; removed on teardown PROXY_HOST_DIR="" # the host-loopback dir that proof serves; removed on teardown PROXY_HOST_SRV_PID="" # the host-loopback server it starts; killed on teardown +PROXY_UNEXPOSED_SRV_PID="" # its listener at a port outside the host exposure; killed on teardown PAR_SEED_DIR="" # seeded by the proxy-parity proof below; removed on teardown PAR_OWN_SEED_DIR="" # its own-address target box's seed; removed on teardown PAR_CALLER_SEED_DIR="" # its denied-caller box's seed; removed on teardown @@ -968,6 +983,9 @@ teardown() { if [ -n "$PROXY_HOST_SRV_PID" ]; then kill "$PROXY_HOST_SRV_PID" 2>/dev/null || true fi + if [ -n "$PROXY_UNEXPOSED_SRV_PID" ]; then + kill "$PROXY_UNEXPOSED_SRV_PID" 2>/dev/null || true + fi [ -n "$PAR_SEED_DIR" ] && rm -rf "$PAR_SEED_DIR" [ -n "$PAR_OWN_SEED_DIR" ] && rm -rf "$PAR_OWN_SEED_DIR" [ -n "$PAR_CALLER_SEED_DIR" ] && rm -rf "$PAR_CALLER_SEED_DIR" @@ -15901,7 +15919,8 @@ proof_min_internal_names_through_proxy() { PROXY_OWN_NAME="e2e-own-proxy" # the own-address box, on switch lanes PROXY_BOX_PORT=18080 # the in-box responders' listen port PROXY_OWN_EXTERNAL_PORT=18082 # ... published externally by --ingress - PROXY_HOST_PORT=18081 # the host-loopback listener (NET-003/004) + PROXY_HOST_PORT="$E2E_HOST_EXPOSURE_PORT" # the host-loopback listener (NET-003/004) + PROXY_UNEXPOSED_PORT=18089 # a host-loopback listener outside the exposure PROXY_DEAD_PORT=19090 # nothing listens: an upstream-refused 502 PROXY_CLOSED_PORT=19091 # not in the own box's ingress map: a 403 PROXY_HOST_ALIAS="100.64.255.254" # the deprecated literal (NET-004) @@ -16116,6 +16135,62 @@ proof_min_internal_names_through_proxy() { # ---- NET-003: host.min.internal from a host-address box ------------------- proxy_assert_host_by_name "$proxy_sid" "NET-003" host + # ---- the host exposure is the only opening (design §7.1) ---------------- + # A VM host's gate opens host.min.internal at the listed exposure alone + # (MINVMD_HOST_EXPOSURE_PORTS, exported at the top), so the same box, the + # same alias, at a port outside the list must not reach the host, even + # with a host-loopback listener answering there. A native host-address box + # shares the host's namespace, with no alias gate to assert. + if [ -n "$E2E_VM" ]; then + ( cd "$PROXY_HOST_DIR" && exec python3 -m http.server "$PROXY_UNEXPOSED_PORT" --bind 127.0.0.1 ) \ + >/dev/null 2>"$WORK/proxy-unexposed-srv.err" & + PROXY_UNEXPOSED_SRV_PID=$! + proxy_unexposed_up=0 + for _ in $(seq 1 40); do + if [ "$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' \ + "http://127.0.0.1:$PROXY_UNEXPOSED_PORT/marker" 2>/dev/null || true)" = "200" ]; then + proxy_unexposed_up=1 + break + fi + sleep 0.25 + done + if [ "$proxy_unexposed_up" -ne 1 ]; then + echo "::error::the unexposed host-loopback http.server never answered on 127.0.0.1:$PROXY_UNEXPOSED_PORT" + cat "$WORK/proxy-unexposed-srv.err" 2>/dev/null || true + fail + fi + # One exec runs both fetches, so a broken exec, a missing curl or a + # failed lookup shows on the exposed fetch and cannot pass as a refusal. + # The exposed port must answer with the marker. The unexposed port must + # fail as a refused connection (curl exit 7) or a timeout (exit 28). + proxy_unexposed_out="$(mnl session exec "$proxy_sid" \ + "curl -sS --max-time 8 http://host.min.internal:$PROXY_HOST_PORT/marker; echo \"exposed_rc=\$?\"; curl -sS --max-time 8 -o /dev/null http://host.min.internal:$PROXY_UNEXPOSED_PORT/marker; echo \"closed_rc=\$?\"" \ + 2>"$WORK/proxy-unexposed.err" || true)" + printf '%s\n' "$proxy_unexposed_out" >"$WORK/proxy-unexposed.out" + proxy_exposed_rc="$(sed -n 's/^.*exposed_rc=\([0-9]*\)$/\1/p' "$WORK/proxy-unexposed.out" | head -n1)" + proxy_unexposed_rc="$(sed -n 's/^closed_rc=\([0-9]*\)$/\1/p' "$WORK/proxy-unexposed.out" | tail -n1)" + if [ "$proxy_exposed_rc" != "0" ] || [[ "$proxy_unexposed_out" != *"$PROXY_HOST_MARKER"* ]]; then + echo "::error::the host-address box did not fetch the marker from host.min.internal at the exposed port $PROXY_HOST_PORT (curl exit '${proxy_exposed_rc:-none}'), so the unexposed-port check proves nothing" + echo "--- exec stdout ---"; cat "$WORK/proxy-unexposed.out" 2>/dev/null || true + echo "--- exec stderr ---"; cat "$WORK/proxy-unexposed.err" 2>/dev/null || true + fail + fi + case "$proxy_unexposed_rc" in + 7 | 28) ;; + 0) + echo "::error::the host-address box reached the host through host.min.internal at port $PROXY_UNEXPOSED_PORT, outside MINVMD_HOST_EXPOSURE_PORTS=$MINVMD_HOST_EXPOSURE_PORTS" + fail + ;; + *) + echo "::error::the fetch from host.min.internal at port $PROXY_UNEXPOSED_PORT failed with curl exit '${proxy_unexposed_rc:-none}', not a refused connection (7) or a timeout (28)" + echo "--- exec stdout ---"; cat "$WORK/proxy-unexposed.out" 2>/dev/null || true + echo "--- exec stderr ---"; cat "$WORK/proxy-unexposed.err" 2>/dev/null || true + fail + ;; + esac + echo "host exposure: host.min.internal:$PROXY_HOST_PORT reached and host.min.internal:$PROXY_UNEXPOSED_PORT refused from the host-address box (curl exit $proxy_unexposed_rc: $(tail -n1 "$WORK/proxy-unexposed.err" 2>/dev/null)) OK" + fi + # ---- NET-001's own-address half: a VM host's lease route ----------------- # Gated like the own-IP proof: MINVMD_GVPROXY_BIN is the one signal that a # switch exists. On every lane that sets it the daemon is in a VM, so its @@ -16182,8 +16257,9 @@ proof_min_internal_names_through_proxy() { # NET-003's own-address half, narrowed to resolution: the box resolves # host.min.internal through the switch zone, to the host alias. Box→host # reach over that alias is default-deny except configured host exposures - # (design §7.1), and this box's switch has none, so the reach is not - # asserted here. TODO(host-exposure): the positive reach arm comes back + # (design §7.1), and an own-address box is refused even at the one + # exposure this run lists, so the reach is not asserted here. + # TODO(host-exposure): the positive reach arm comes back # with a configured exposure (gominimal/inbox#867). proxy_own_resolved="$(mnl session exec "$PROXY_OWN_SID" \ "getent ahostsv4 host.min.internal" 2>"$WORK/proxy-own-resolve.err" \ @@ -16197,7 +16273,9 @@ proof_min_internal_names_through_proxy() { # NET-004: the deprecated literal itself. Box→host reach over the alias # is default-deny except configured host exposures (design §7.1), and - # none is configured, so the box's own relay drops the connect — though + # an own-address box gets none — not even PROXY_HOST_PORT, the one this + # run lists for host-address boxes — so the box's own relay drops the + # connect — though # its egress rules admit every address (0.0.0.0/0 above), so the drop # is the infrastructure deny set's and not the deny-all default's: no # answer and no reset until the timeout, and the relay names the drop @@ -16219,7 +16297,7 @@ proof_min_internal_names_through_proxy() { mnl session exec "$PROXY_OWN_SID" \ "/usr/bin/socat /dev/null TCP:$PROXY_HOST_ALIAS:$PROXY_HOST_PORT,connect-timeout=10" \ >/dev/null 2>"$WORK/proxy-alias.err" && { - echo "::error::NET-004: the own-address box completed a connection to the host alias $PROXY_HOST_ALIAS:$PROXY_HOST_PORT with no host exposure configured" + echo "::error::NET-004: the own-address box completed a connection to the host alias $PROXY_HOST_ALIAS:$PROXY_HOST_PORT, which an own-address box must never reach" fail } proxy_alias_elapsed_ms=$(( $(now_ms) - proxy_alias_start_ms )) From 55163d675bb0c14aafb844c1a86d9bddfb4cca38 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:27:02 +0000 Subject: [PATCH 15/34] fix(minvmd): decide the node plane's own traffic by its baseline set (#2723) T103. The node-plane baseline ships in force. The registry and cache entries name the hosts the daemon's own fetches go to (github.com, cache.minimal.dev), and the node plane's own lookups pin the addresses they resolve to, under a box's window and cap. A frame from the node's own address reaches a pinned address at TCP 443, a subnet entry, the resolver, the host alias at a configured exposure and a resident box's lease, and nothing else. MINVMD_BASELINE_REGISTRY and MINVMD_BASELINE_CACHE re-point the entries. A host-address box sends from the node address too, so the guest classifier writes a box mark into the IPv4 DSCP field. The host gate decides a marked frame by the union of the resident host_ip boxes' rules and drops it when none is resident. The client admits a host_ip box's egress over new host-door verbs before the box runs anything, re-admits it at attach and exec, and retires it when the session goes. Backport of #2723. Lands before T102 here, as on main: T102 edits this commit's baseline and gate code, and its baseline.rs does not apply without it. Adapted to the line: - cmd/boxes/create.rs and vm_host.rs hunks land in cmd/session.rs, dash/rpc.rs in minimal-tui, and the task hunks use the line's config variable. - host_ip_box_with_vm_host and HostIpResidency live in minimal-client's box_registration, since minimal-tui is its own crate here and cannot reach minimal's. minimal-tui reads the record through a new session_record_at there, and its activation admits no declared egress because its policy is the default. - The activation's failure path has no cleanup outcome, so the guard retires on every failed finalize. Main keeps the rules when the destroy may have failed. - DropLimiter stays in egress_gate.rs, since the line has no drop_limiter module. - The line has no ProxyListening or BoxIdMinted replies, and no SessionId::mint, so tests draw their ids from a v4 UUID. boxes:: reads sessions:: throughout. (cherry picked from commit 486ea13f63f2a766cd97c742d5af9f8cd6090ebe) --- crates/minimal-client/src/attach.rs | 44 + crates/minimal-client/src/box_registration.rs | 116 ++ crates/minimal-tui/src/rpc.rs | 50 + crates/minimal/src/cmd/session.rs | 118 +- crates/minimal/src/task.rs | 17 + crates/minimal/tests/cli.rs | 53 +- crates/minimald-rpc/src/lib.rs | 53 + crates/minimald/src/net/classifier.rs | 67 +- crates/minimald/src/rpc.rs | 2 + crates/minvmd/src/box_registry.rs | 288 +++- crates/minvmd/src/control.rs | 34 +- crates/minvmd/src/net/baseline.rs | 610 +++++--- crates/minvmd/src/net/dns_pins.rs | 675 +++++++- crates/minvmd/src/net/egress_gate.rs | 1369 +++++++++++++---- .../tests/guest_classifier_integration.rs | 6 + crates/minvmd/tests/vm_escape_integration.rs | 222 ++- crates/sessions/src/core/egress.rs | 23 + scripts/install-host-classifier.sh | 36 +- scripts/session-e2e.sh | 6 +- 19 files changed, 3164 insertions(+), 625 deletions(-) diff --git a/crates/minimal-client/src/attach.rs b/crates/minimal-client/src/attach.rs index 3714874aa9..1c29d774b8 100644 --- a/crates/minimal-client/src/attach.rs +++ b/crates/minimal-client/src/attach.rs @@ -533,6 +533,50 @@ pub fn hold_box_name_beside( ); } +/// Admits (`resident`) or retires a `host_ip` box's egress policy at the +/// VM host daemon whose control socket is `control_sock` (NET-085). A +/// `host_ip` box leaves the VM from the node's own address, and the host +/// gate decides its frames by the policies admitted here: a box whose +/// policy is not admitted reaches nothing outside the VM. Keyed by the +/// session `session_id`, so an unnamed box and a task box admit the same +/// way. Blocking and best-effort: a failed exchange warns and returns +/// `false`. A daemon that predates the verbs refuses them, and its gate +/// predates the decision too. +pub fn host_ip_box_at( + control_sock: &Path, + session_id: sessions::SessionId, + egress: Option<&sessions::EgressPolicy>, + resident: bool, +) -> bool { + let (verb, operation) = if resident { + ( + minimald_rpc::BoxControlRequest::AdmitHostIpBox(minimald_rpc::HostIpBoxRequest { + session_id, + egress: egress.cloned(), + }), + "admit", + ) + } else { + ( + minimald_rpc::BoxControlRequest::RetireHostIpBox( + minimald_rpc::RetireHostIpBoxRequest { session_id }, + ), + "retire", + ) + }; + let failure = match host_control(control_sock, &verb) { + Ok((minimald_rpc::BoxControlReply::HostIpBox { .. }, _)) => return true, + Ok((other, _)) => format!("another verb's reply: {other:?}"), + Err(error) => format!("{error:#}"), + }; + tracing::warn!( + session = %session_id, + operation, + "the host_ip box's egress {operation} could not be made ({failure})" + ); + false +} + /// Withdraws a box's host row (T66) on the VM host daemon whose control /// socket is `control_sock`, once the session that registered it is gone: /// the creator presents the name and the pair the registration handed back, diff --git a/crates/minimal-client/src/box_registration.rs b/crates/minimal-client/src/box_registration.rs index 6035c2a8eb..758495814e 100644 --- a/crates/minimal-client/src/box_registration.rs +++ b/crates/minimal-client/src/box_registration.rs @@ -283,6 +283,12 @@ pub async fn register_box_with_vm_host( read for {switch_address}; the registration did not happen" ) } + minimald_rpc::BoxControlReply::HostIpBox { .. } => { + anyhow::bail!( + "the VM host daemon answered the box registration with the reply to another \ + request; the registration did not happen" + ) + } minimald_rpc::BoxControlReply::Audited { .. } => { anyhow::bail!( "the VM host daemon answered the box registration with an audit report's \ @@ -740,6 +746,116 @@ async fn reminting_autogen( } } +/// The session's record over a connection of its own, bounded by +/// [`BOX_CONTROL_TIMEOUT`], for a caller that holds only the daemon's socket: +/// `None` once the session is gone. +pub async fn session_record_at( + sock: &Path, + id: sessions::SessionId, +) -> anyhow::Result> { + tokio::time::timeout(BOX_CONTROL_TIMEOUT, async { + let mut client = crate::Client::connect(sock).await?; + let resp = client + .oneshot_rpc::( + minimald_rpc::GetSessionRecordRequest::Id(id), + ) + .await + .context("GetSessionRecord RPC failed")?; + anyhow::Ok(resp.record) + }) + .await + .map_err(|_| anyhow::anyhow!("no session lookup answer within {BOX_CONTROL_TIMEOUT:?}"))? +} + +/// Admits (`resident`) or retires a `host_ip` box's egress policy on the +/// VM host daemon (NET-085), off the async worker and bounded by +/// [`BOX_CONTROL_TIMEOUT`]. The host gate decides a frame a `host_ip` box +/// sends by the policies admitted there, so a box is admitted before it +/// runs anything and retired once it is gone. Returns whether the VM host +/// daemon answered: `false` for no VM host daemon to ask, a refusal, or no +/// answer (each warned on). +pub async fn host_ip_box_with_vm_host( + control_sock: Option, + id: sessions::SessionId, + egress: Option, + resident: bool, +) -> bool { + let Some(sock_path) = control_sock else { + return false; + }; + let exchange = tokio::task::spawn_blocking(move || { + crate::attach::host_ip_box_at(&sock_path, id, egress.as_ref(), resident) + }); + match tokio::time::timeout(BOX_CONTROL_TIMEOUT, exchange).await { + Ok(Ok(answered)) => answered, + Ok(Err(error)) => { + tracing::warn!(%error, "the host_ip box's egress exchange thread failed"); + false + } + Err(_) => { + tracing::warn!( + session = %id, + "the host_ip box's egress exchange had no answer within {BOX_CONTROL_TIMEOUT:?}" + ); + false + } + } +} + +/// The `host_ip` box an activation admitted on the VM host daemon +/// (NET-085), retired when the activation ends without the session: drop +/// it without [`Self::keep`] and the box's policy is retired, so a session +/// the activation's cleanup destroyed widens no gate. Retiring is blocking +/// and best-effort, as the admit is. +pub struct HostIpResidency { + control_sock: std::path::PathBuf, + id: sessions::SessionId, + kept: bool, +} + +impl HostIpResidency { + /// Admits the box's `egress` for session `id` on the VM host daemon + /// whose control socket is `control_sock`, when the box is a `host_ip` + /// box. `control_sock` is `None` on a host that is not VM-backed. The + /// guard is `None` there, for another network mode, and when the admit + /// was not answered, since then nothing needs retiring. + pub async fn admit( + control_sock: Option, + id: sessions::SessionId, + network: sessions::NetworkMode, + egress: Option<&sessions::EgressPolicy>, + ) -> Option { + if network != sessions::NetworkMode::HostNet { + return None; + } + let control_sock = control_sock?; + host_ip_box_with_vm_host(Some(control_sock.clone()), id, egress.cloned(), true) + .await + .then_some(Self { + control_sock, + id, + kept: false, + }) + } + + /// Keeps the admitted policy: the session stays, so its destroy retires + /// it (`release_held_box_name`). The policy also stays while the box + /// is `stopped` or `exited`, so the gate's union can include a box with + /// no live process until its destroy. + /// owns this case. + pub fn keep(&mut self) { + self.kept = true; + } +} + +impl Drop for HostIpResidency { + fn drop(&mut self) { + if !self.kept { + crate::attach::host_ip_box_at(&self.control_sock, self.id, None, false); + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/minimal-tui/src/rpc.rs b/crates/minimal-tui/src/rpc.rs index 61b511c8f0..9f25d59b7d 100644 --- a/crates/minimal-tui/src/rpc.rs +++ b/crates/minimal-tui/src/rpc.rs @@ -10,6 +10,9 @@ use std::path::{Path, PathBuf}; use anyhow::Context as _; use minimal_client::Client; +use minimal_client::box_registration::{ + HostIpResidency, host_ip_box_with_vm_host, session_record_at, +}; use minimald_rpc::{ CreateSession, CreateSessionRequest, DestroySession, DestroySessionRequest, Errorable, GetEffectiveSessionPolicy, GetEffectiveSessionPolicyRequest, GetSessionRecord, @@ -356,6 +359,30 @@ pub async fn prepare_attach( record: Option, ) -> anyhow::Result> { minimal_client::box_registration::resume_box_row(sock, record.as_ref()).await?; + // A `host_ip` box's egress, re-admitted as `min session attach` re-admits + // it: a VM host daemon that restarted holds none, and its gate then drops + // the box's frames outside the VM (NET-085). Idempotent. The record was + // read before the admit, so a session destroyed in between has its rules + // retired here once a second lookup finds it gone. + if let Some(record) = record.as_ref() + && record.network == NetworkMode::HostNet + && host_ip_box_with_vm_host( + minimal_client::box_registration::vm_host_control_sock(sock), + record.id, + record.policy.egress.clone(), + true, + ) + .await + && let Ok(None) = session_record_at(sock, record.id).await + { + host_ip_box_with_vm_host( + minimal_client::box_registration::vm_host_control_sock(sock), + record.id, + None, + false, + ) + .await; + } Ok(record.and_then(|record| record.box_addresses)) } @@ -364,6 +391,16 @@ pub async fn prepare_attach( /// registered is withdrawn with the pair off its record, and a `host_ip` /// box's zone hold is released. A session that held neither owes nothing. async fn settle_destroyed_box(sock: &Path, id: SessionId, record: &sessions::Record) { + // A `host_ip` box's egress goes with its session, named or not (NET-085). + if record.network == NetworkMode::HostNet && record.box_addresses.is_none() { + host_ip_box_with_vm_host( + minimal_client::box_registration::vm_host_control_sock(sock), + id, + None, + false, + ) + .await; + } let Some(name) = record.name.as_deref() else { return; }; @@ -535,6 +572,16 @@ pub async fn activate( } let id = created.id; let egress_deny_all_default = undeclared_own_ip && deny_all_default_binds(&created); + // A `host_ip` box's egress, admitted on the VM host daemon before the box + // runs anything, as `min session activate` admits it (NET-085). The guard + // retires it if the flow below fails. + let mut host_ip_residency = HostIpResidency::admit( + minimal_client::box_registration::vm_host_control_sock(sock), + id, + network, + None, + ) + .await; // The registration's lease is held across the flow and committed only // once the session is active: until then a dashboard that dies leaves @@ -623,6 +670,9 @@ pub async fn activate( // its name and be reaped at the next daemon restart anyway. match flow { Ok(package_check_skipped) => { + if let Some(residency) = host_ip_residency.as_mut() { + residency.keep(); + } // The session is active: a `host_ip` box holds its name in the // zone (NODATA) the way `min session activate` holds it, read // off the record so an autogen name is the one held. diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index 754b8f7a61..b772ac1c38 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -48,11 +48,13 @@ fn activate_refusal(error: anyhow::Error) -> anyhow::Error { // The box registration exchange with the VM host daemon (T66) — the // register, its lease, the withdrawal and the resume — is the client // library's, shared with `min dash` so the two front-ends cannot drift. -pub(crate) use minimal_client::box_registration::withdraw_box_row; use minimal_client::box_registration::{ BOX_CONTROL_TIMEOUT, control_request_with_vm_host, finalize_holding_lease, register_box_reminting_autogen, resume_box_row, }; +pub(crate) use minimal_client::box_registration::{ + HostIpResidency, host_ip_box_with_vm_host, withdraw_box_row, +}; #[cfg(test)] use minimal_client::box_registration::{ RegisteredWithVmHost, register_box_for_activation, register_box_with_vm_host, @@ -238,6 +240,9 @@ pub(crate) async fn release_held_box_name( if box_addresses.is_some() { return; } + // A `host_ip` box's egress goes with its session, named or not. A box + // of another mode admitted none, and a retire of it is the goal state. + host_ip_box_with_vm_host(control_sock.clone(), id, None, false).await; let Some(name) = name else { return }; hold_box_name_with_vm_host(control_sock, name, Some(id), false).await; } @@ -315,6 +320,7 @@ pub(crate) async fn release_held_name_after_attach( .await; } None => { + host_ip_box_with_vm_host(control_sock_beside(sock), id, None, false).await; hold_box_name_with_vm_host(control_sock_beside(sock), name, Some(id), false).await; } }, @@ -364,6 +370,27 @@ pub(crate) async fn rehold_held_name_before_attach( if record.network != sessions::NetworkMode::HostNet { return; } + // The box's egress, re-admitted before the attach or the exec for the + // same reason the name is re-held: a VM host daemon that restarted + // holds no `host_ip` box's policy, and its gate then drops every frame + // the box sends outside the VM (NET-085). Idempotent. + // + // The record was read before the admit, so a destroy in between can + // retire the box first and leave its rules standing for a gone session. + // The session is looked up again once the admit is made, and a session + // gone by then has its rules retired, as the name re-hold below undoes + // its hold. A destroy that lands after that lookup retires them itself. + if host_ip_box_with_vm_host( + control_sock_beside(sock), + id, + record.policy.egress.clone(), + true, + ) + .await + && let Ok(None) = attached_session_record(sock, id).await + { + host_ip_box_with_vm_host(control_sock_beside(sock), id, None, false).await; + } if record.box_addresses.is_some() { return; } @@ -1236,6 +1263,18 @@ pub(crate) async fn activate_session( ); } let id = created.id; + // A `host_ip` box's egress, admitted on the VM host daemon before the + // box runs anything (NET-085): the host gate decides every frame the box + // sends from the node's own address by the admitted policies, and drops + // a box's frames that none admits. The guard retires it unless the + // session outlives this activation. + let mut host_ip_residency = HostIpResidency::admit( + control_sock.clone(), + id, + config.network, + config.policy.egress.as_ref(), + ) + .await; // NET-079: the host's verdict on whether it can decide a host-address // box's egress per box, spelled by the daemon — the side that read the @@ -1651,6 +1690,9 @@ pub(crate) async fn activate_session( // (the attach hand-off below and the user's own session are fair game // for interrupts, but not this cleanup). drop(interrupt_guard); + if let Some(residency) = host_ip_residency.as_mut() { + residency.keep(); + } // The `host_ip` interim: the box shares the node's own row, so its name // is held in the zone with no row behind it and answers NODATA instead @@ -6007,10 +6049,25 @@ mod tests { sock_path: std::path::PathBuf, reply: String, ) -> std::sync::Arc>> { + fake_vm_host_with_residency(sock_path, reply).await.0 + } + + /// [`fake_vm_host`], with the `host_ip` box admits and retires (NET-085) + /// kept apart: the second list holds them, each answered with the + /// residency reply, so the first holds every other verb in its order. + async fn fake_vm_host_with_residency( + sock_path: std::path::PathBuf, + reply: String, + ) -> ( + std::sync::Arc>>, + std::sync::Arc>>, + ) { use tokio::io::{AsyncBufReadExt as _, AsyncWriteExt as _}; let requests = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let residency = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); let listener = tokio::net::UnixListener::bind(&sock_path).unwrap(); let seen = std::sync::Arc::clone(&requests); + let seen_residency = std::sync::Arc::clone(&residency); tokio::spawn(async move { loop { let (stream, _) = match listener.accept().await { @@ -6022,9 +6079,36 @@ mod tests { if lines.read_line(&mut line).await.is_err() { return; } - seen.lock().unwrap().push(line.trim().to_string()); let mut writer = lines.into_inner(); - if writer.write_all(reply.as_bytes()).await.is_err() { + let parsed = + serde_json_lenient::from_str::(line.trim()); + let answer = match parsed { + Ok( + request @ (minimald_rpc::BoxControlRequest::AdmitHostIpBox(_) + | minimald_rpc::BoxControlRequest::RetireHostIpBox(_)), + ) => { + let (session_id, resident) = match &request { + minimald_rpc::BoxControlRequest::AdmitHostIpBox(admit) => { + (admit.session_id, true) + } + minimald_rpc::BoxControlRequest::RetireHostIpBox(retire) => { + (retire.session_id, false) + } + _ => unreachable!("matched above"), + }; + seen_residency.lock().unwrap().push(request); + serde_json_lenient::to_string(&minimald_rpc::BoxControlReply::HostIpBox { + session_id, + resident, + }) + .unwrap() + } + _ => { + seen.lock().unwrap().push(line.trim().to_string()); + reply.clone() + } + }; + if writer.write_all(answer.as_bytes()).await.is_err() { return; } if writer.write_all(b"\n").await.is_err() { @@ -6032,7 +6116,7 @@ mod tests { } } }); - requests + (requests, residency) } /// T66: an own-address box on a VM-backed host registers with the VM @@ -7060,7 +7144,7 @@ mod tests { let dir = tempfile::TempDir::new().unwrap(); let provider_dir = dir.path().join("providers").join("local-minvmd0"); std::fs::create_dir_all(&provider_dir).unwrap(); - let requests = fake_vm_host( + let (requests, residency) = fake_vm_host_with_residency( provider_dir.join("control.sock"), r#"{"name":"task-build-9c1e","held":false}"#.to_string(), ) @@ -7088,6 +7172,14 @@ mod tests { Some(id), "the release names its session" ); + // The run's end retires the task box's egress too (NET-085). + assert_eq!( + *residency.lock().unwrap(), + vec![minimald_rpc::BoxControlRequest::RetireHostIpBox( + minimald_rpc::RetireHostIpBoxRequest { session_id: id } + )], + "the run's end retires the host_ip box's egress" + ); } /// The `host_ip` hold's attach-exit side: the shell-exit prompt's @@ -7235,7 +7327,7 @@ mod tests { async fn a_gone_sessions_rehold_hands_the_name_back_to_its_new_holder() { let dir = tempfile::TempDir::new().unwrap(); let ssh_sock = dir.path().join("ssh.sock"); - let requests = fake_vm_host( + let (requests, residency) = fake_vm_host_with_residency( dir.path().join(minvmd::control::CONTROL_SOCK_FILE), r#"{"name":"web","held":true}"#.to_string(), ) @@ -7291,6 +7383,20 @@ mod tests { ], "the gone session's re-hold, then the name handed back to the live one" ); + // The re-hold re-admits the box's egress first (NET-085), by the + // session the record names. The session is gone, so the second + // lookup finds nothing and the admit is retired again. + let residency = residency.lock().unwrap(); + assert!( + matches!( + residency.as_slice(), + [ + minimald_rpc::BoxControlRequest::AdmitHostIpBox(admit), + minimald_rpc::BoxControlRequest::RetireHostIpBox(retire), + ] if admit.session_id == gone && retire.session_id == gone + ), + "an admit for the gone session, then its retire: {residency:?}" + ); } /// The undo of a gone session's re-hold, by what the lookup of its name diff --git a/crates/minimal/src/task.rs b/crates/minimal/src/task.rs index 6e430755cc..306ab9e1d0 100644 --- a/crates/minimal/src/task.rs +++ b/crates/minimal/src/task.rs @@ -745,6 +745,20 @@ pub async fn cmd_task_run(global: &GlobalArgs, args: TaskRunArgs) -> Result<(), // finalize, with the record still unfinalized and reapable. minimal_client::ensure_version_reported(created.daemon_version.as_deref())?; let id = created.id; + // The task box is a `host_ip` box: its egress is admitted on the VM + // host daemon before it runs anything (NET-085), or the host gate drops + // every frame it sends outside the VM. The guard retires it on an early + // return; the run's end retires it through `release_task_name_hold`. + let mut host_ip_residency = crate::HostIpResidency::admit( + crate::vm_host_control_sock( + crate::daemon_provider_kind(global), + global.minimal_dir.as_deref(), + ), + id, + config.network, + config.policy.egress.as_ref(), + ) + .await; let session_name = config .name .clone() @@ -925,6 +939,9 @@ pub async fn cmd_task_run(global: &GlobalArgs, args: TaskRunArgs) -> Result<(), if args.keep { eprintln!("Session {session_name} kept — attach with: min session attach {session_name}"); } + if let Some(residency) = host_ip_residency.as_mut() { + residency.keep(); + } release_task_name_hold( crate::vm_host_control_sock( crate::daemon_provider_kind(global), diff --git a/crates/minimal/tests/cli.rs b/crates/minimal/tests/cli.rs index b475f70a36..59fade710d 100644 --- a/crates/minimal/tests/cli.rs +++ b/crates/minimal/tests/cli.rs @@ -2964,10 +2964,10 @@ async fn policy_shows_unset_egress_as_named_default() { /// rules (NET-130): the helper's built-in enumeration of the categories the /// in-VM daemon's own traffic may reach, one row per category, on an /// own-address box, headed by the posture the host-side gate decides the -/// daemon's own fetches under — announced, the shipped posture, the run -/// path's allow-all interim node row still decides those fetches, so the -/// rows are what the set will bound, not what bounds them today; in force, -/// it decides them whatever the box's own declaration resolves to. The set +/// daemon's own fetches under — in force, the shipped posture: the set is +/// what bounds Minimal's own downloads in the VM, whatever the box's own +/// declaration resolves to, and each entry names the host those downloads +/// go to. The set /// is shown where the fabric the helper gates is named — the microVM /// backend's plan — and nowhere it is not: a host-address box shares its /// host's namespace and has no switch fabric, and a backend with no @@ -3008,34 +3008,35 @@ fn policy_shows_baseline_set() { "the box's own rules are shown:\n{text}" ); // The posture is spelled beside the set, the way the daemon's start-up - // line spells it. The shipped posture is announced — the run path's - // allow-all interim node row still decides the daemon's own fetches — so - // the display must not present the rows as what bounds them. The flip of - // `NODE_BASELINE_PHASE` updates this assertion with the rest of the - // cutover. + // line spells it. The shipped posture is in force: the set is what + // bounds Minimal's own downloads in the VM. A frame from the node + // address that wears the box mark is decided by the resident host_ip + // boxes' rules, so no caveat rides beside the posture, and the line + // names no design term. assert!( text.contains( "destinations for Minimal's own downloads in this VM (not this box's rules): \ - announced (node row's allow-all interim)" + in force\n" ), "the helper's baseline set is shown beside the box's rules, headed by the \ - gate's announced posture:\n{text}" + gate's in-force posture:\n{text}" + ); + assert!( + !text.contains("interim") && !text.contains("bypass"), + "the posture names no design term and no caveat:\n{text}" ); // The rows are the enumeration the helper carries, one per category — - // the set the host-side gate decides the daemon's own frames by once the - // baseline is in force — each alias endpoint with the in-force arm's - // protocol and port bound beside it (NET-061). - let alias = fabric.host_alias(); - let rendered_baseline = - format!(" registry {alias}/32 tcp/443\n cache {alias}/32 tcp/443\n"); + // the set the host-side gate decides the daemon's own frames by — each + // named by the host the daemon's own downloads go to (NET-061). + let rendered_baseline = " registry github.com\n cache cache.minimal.dev\n"; assert!( - text.contains(&rendered_baseline), + text.contains(rendered_baseline), "the baseline rows are the enumeration's entries, one per category, \ - with the alias bound:\n{text}" + each naming its host:\n{text}" ); - // The JSON view carries the same bounded endpoints. + // The JSON view carries the same posture and endpoints. let mut out = Vec::new(); write_policy_json( &mut out, @@ -3047,16 +3048,20 @@ fn policy_shows_baseline_set() { ) .unwrap(); let document: Value = serde_json_lenient::from_slice(&out).unwrap(); - let bounded = Value::String(format!("{alias}/32 tcp/443")); + assert_eq!( + document["node_plane_baseline"]["phase"], + Value::String("in force".to_string()), + "the JSON view carries the in-force posture:\n{document}" + ); let entries = document["node_plane_baseline"]["entries"] .as_array() .expect("the JSON view carries the baseline entries"); assert_eq!(entries.len(), 2, "registry and cache:\n{document}"); - for entry in entries { + for (entry, host) in entries.iter().zip(["github.com", "cache.minimal.dev"]) { assert_eq!( entry["endpoints"], - Value::Array(vec![bounded.clone()]), - "the JSON view spells the alias bound:\n{document}" + Value::Array(vec![Value::String(host.to_string())]), + "the JSON view names each entry's host:\n{document}" ); } // Beside, not instead: the egress block comes first, the baseline set diff --git a/crates/minimald-rpc/src/lib.rs b/crates/minimald-rpc/src/lib.rs index 4bdf3203e2..a2d7792cb4 100644 --- a/crates/minimald-rpc/src/lib.rs +++ b/crates/minimald-rpc/src/lib.rs @@ -776,6 +776,38 @@ pub struct HoldBoxNameRequest { pub session_id: Option, } +/// A `host_ip` box's egress policy, for the VM host's gate to hold while +/// the box is resident ([`BoxControlRequest::AdmitHostIpBox`], NET-085). +/// +/// A `host_ip` box leaves the VM from the node's own address, so it has no +/// row of its own. The guest marks its frames, and root in the guest can +/// write the same mark on any frame. The host gate therefore decides a +/// marked frame from the node address by the union of the rules this verb +/// registers, and drops it when no `host_ip` box is resident. Keyed by the +/// session id, so an unnamed box and a task box register the same way. +/// Answered with [`BoxControlReply::HostIpBox`]; idempotent, and a second +/// admit for the same session replaces the policy the first one held. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct HostIpBoxRequest { + /// The session the box runs as. + pub session_id: sessions::SessionId, + /// The box's egress policy, as the session's create declared it. + /// Absent is the egress default's to fill, the same meaning the create + /// request's policy carries. The host fills it under its own phase. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub egress: Option, +} + +/// The session whose `host_ip` box left the VM host's resident set +/// ([`BoxControlRequest::RetireHostIpBox`]): its rules admit nothing from +/// then on. Answered with [`BoxControlReply::HostIpBox`]; a session the set +/// does not hold is the goal state already holding. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct RetireHostIpBoxRequest { + /// The session the box ran as. + pub session_id: sessions::SessionId, +} + /// What side of the in-VM daemon reported a runtime-admitted port (NET-045, /// NET-138): the fixed fact the host's log line and audit copy name, so a /// tail can tell an expose decision from an answered ask from a listen @@ -1288,6 +1320,16 @@ pub enum BoxControlRequest { /// [`BoxControlReply::NameHeld`]; a name no hold kept is the goal /// state already holding. ReleaseBoxName(HoldBoxNameRequest), + /// Hold a resident `host_ip` box's egress policy on the host + /// ([`HostIpBoxRequest`], NET-085): the gate decides a box-marked frame + /// from the node address by the union of the policies held. Served on + /// the host's control socket only, answered with + /// [`BoxControlReply::HostIpBox`]. + AdmitHostIpBox(HostIpBoxRequest), + /// Drop the policy [`BoxControlRequest::AdmitHostIpBox`] holds for a + /// session: the box stopped or was destroyed. Answered with + /// [`BoxControlReply::HostIpBox`]. + RetireHostIpBox(RetireHostIpBoxRequest), /// The in-VM daemon's report that one of its boxes published a port at /// runtime (NET-138), carried on the daemon's own control channel: the /// host records it in the row only within the grant the host-side @@ -1654,6 +1696,17 @@ pub enum BoxControlReply { /// The verb succeeded: a withdrawal's echo of the pair the row went by, /// and a registration's answer on a daemon that predates box ids. Addresses(BoxAddresses), + /// An admit or a retire of a `host_ip` box's policy was answered: the + /// session back, `resident` saying which — `true` the gate holds the + /// box's policy now, `false` it holds none for the session. The + /// `resident` marker is this shape's own field, so no other reply + /// decodes as it. + HostIpBox { + /// The session the admit or the retire was about. + session_id: sessions::SessionId, + /// The marker: whether the gate holds the box's policy now. + resident: bool, + }, /// A hold or a release was answered: the name back, `held` saying /// which — `true` a hold the table now keeps, `false` a release that /// freed one (or found none). diff --git a/crates/minimald/src/net/classifier.rs b/crates/minimald/src/net/classifier.rs index c4863a08b2..0d02fd8118 100644 --- a/crates/minimald/src/net/classifier.rs +++ b/crates/minimald/src/net/classifier.rs @@ -1494,6 +1494,11 @@ pub fn render_guest_ruleset_over( .arg(params.node_plane_address.to_string()) .arg("--ct-mark-mask") .arg(format!("0x{:08x}", params.ct_mark_mask)) + // The guest's one address carries both the boxes cohort and the + // daemon's own traffic, so the VM host's gate reads the sender's + // cgroup from this DSCP mark instead of from the source address. + .arg("--cohort-dscp") + .arg(sessions::core::egress::BOX_COHORT_DSCP.to_string()) .env_clear() .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -4712,12 +4717,15 @@ mod tests { // own parameters — a VM-backed guest's carve-out is the gateway's // resolver. let print_mount = standin_mount(); + let dscp = sessions::core::egress::BOX_COHORT_DSCP.to_string(); let printed = run_step( &print_mount, &[ "--print-ruleset", GUEST_RESOLVER_FLAGS[0], GUEST_RESOLVER_FLAGS[1], + "--cohort-dscp", + &dscp, ], None, ); @@ -4748,7 +4756,20 @@ mod tests { // posture lifts is the only privilege it does — so this lane // asserts the one text on root lanes too, not only where a // person's account happens to be the one that ran sudo. - let installed = run_install_with(&mount, &GUEST_RESOLVER_FLAGS, stub.path()); + // The guest's render names the box mark, so the install is handed + // the same flag and renders the same one text. + let installed = run_install_with( + &mount, + &[ + GUEST_RESOLVER_FLAGS[0], + GUEST_RESOLVER_FLAGS[1], + GUEST_RESOLVER_FLAGS[2], + GUEST_RESOLVER_FLAGS[3], + "--cohort-dscp", + &dscp, + ], + stub.path(), + ); assert!( installed.status.success(), "the install lays out its tree and hands its one transaction to nft: {}{}", @@ -5422,6 +5443,43 @@ mod tests { ); } + /// The guest's table writes the sender's cgroup into every packet it + /// sends: the boxes cohort's connections leave with the box mark, and + /// every other packet with the field cleared, so the VM host's gate + /// can tell a box's frame from the daemon's own at the one address + /// both leave from. A native install, which takes no `--cohort-dscp`, + /// renders no such chain and leaves the field alone. + #[test] + fn guest_table_marks_the_box_cohort_in_dscp() { + let guest_ip = IpAddr::V4(crate::net::SwitchSubnet::default().daemon_ip()); + let ruleset = String::from_utf8(guest_ruleset(guest_ip, guest_ip)) + .expect("the guest's render is text"); + let dscp = sessions::core::egress::BOX_COHORT_DSCP; + let marks = chain_rules(&ruleset, "mark_cohort"); + assert_eq!(marks.len(), 2, "the chain holds two rules: {marks:?}"); + assert!( + ruleset.contains( + "chain mark_cohort {\n type filter hook postrouting priority mangle;" + ), + "the mark is a postrouting filter chain, which sees every packet:\n{ruleset}" + ); + assert!( + marks[0].ends_with(&format!("ip dscp set {dscp}")) && marks[0].contains("=="), + "the cohort's connections take the box mark: {marks:?}" + ); + assert!( + marks[1].ends_with("ip dscp set 0") && marks[1].contains("!="), + "every other packet leaves with the field cleared: {marks:?}" + ); + + let mount = standin_mount(); + let native = run_step(&mount, &["--print-ruleset"], None); + assert!( + !String::from_utf8_lossy(&native.stdout).contains("mark_cohort"), + "a native install renders no DSCP chain" + ); + } + /// NET-079: the guest loads the installer's own render, not a second /// spelling of it kept in step. The daemon compiles the installer in /// from the one file — `include_str!`, never a copy staged anywhere a @@ -5454,6 +5512,9 @@ mod tests { "the harness pair the step's own tests render", ), ] { + // The guest's render also names the box mark, which a native + // install never takes, so the step is handed the same flag. + let dscp = sessions::core::egress::BOX_COHORT_DSCP.to_string(); let step = step_command_over( &mount, &[ @@ -5462,6 +5523,8 @@ mod tests { GUEST_RESOLVER_FLAGS[1], GUEST_RESOLVER_FLAGS[2], GUEST_RESOLVER_FLAGS[3], + "--cohort-dscp", + &dscp, ], None, &cohort.to_string(), @@ -5536,6 +5599,8 @@ mod tests { guest_ip.to_string(), "--ct-mark-mask".to_string(), format!("0x{:08x}", GUEST_CT_MARK_MASK), + "--cohort-dscp".to_string(), + sessions::core::egress::BOX_COHORT_DSCP.to_string(), ], "every parameter arrives as argv after --print-ruleset, none \ interpolated into the script the daemon pipes" diff --git a/crates/minimald/src/rpc.rs b/crates/minimald/src/rpc.rs index 8d469ae309..d97b33c109 100644 --- a/crates/minimald/src/rpc.rs +++ b/crates/minimald/src/rpc.rs @@ -964,6 +964,8 @@ async fn serve_answerer_control( | BoxControlRequest::Withdraw(_) | BoxControlRequest::HoldBoxName(_) | BoxControlRequest::ReleaseBoxName(_) + | BoxControlRequest::AdmitHostIpBox(_) + | BoxControlRequest::RetireHostIpBox(_) | BoxControlRequest::AdmitPort(_) | BoxControlRequest::WithdrawPort(_) | BoxControlRequest::ReadRow(_) diff --git a/crates/minvmd/src/box_registry.rs b/crates/minvmd/src/box_registry.rs index b73db5572f..81d97cc5f4 100644 --- a/crates/minvmd/src/box_registry.rs +++ b/crates/minvmd/src/box_registry.rs @@ -2780,6 +2780,11 @@ pub struct BoxRegistry { /// frees only that session's hold, never a newer session's under the /// same name. held_names: Arc>>>, + /// The resident `host_ip` boxes' compiled egress, by session + /// ([`Self::admit_host_ip_box`], NET-085), shared by every clone and + /// every [`BoxTable`]: the union the gate decides a box-marked frame + /// from the node address by. + host_ip_boxes: Arc>, /// The held names reloaded from [`HELD_NAMES_FILE`] that no hold has /// re-made since, each with when it was reloaded: dropped once /// [`HELD_NAME_RECONFIRM_BOUND`] has passed @@ -2902,6 +2907,7 @@ impl Clone for BoxRegistry { rows: self.rows.clone(), stopped: Arc::clone(&self.stopped), held_names: Arc::clone(&self.held_names), + host_ip_boxes: Arc::clone(&self.host_ip_boxes), unconfirmed_holds: Arc::clone(&self.unconfirmed_holds), table_pings: Arc::clone(&self.table_pings), withdrawal_reports: self.withdrawal_reports.clone(), @@ -2942,6 +2948,7 @@ impl BoxRegistry { rows: Arc::new(RwLock::new(BTreeMap::new())), stopped: Arc::new(RwLock::new(BTreeSet::new())), held_names: Arc::new(RwLock::new(BTreeMap::new())), + host_ip_boxes: Arc::new(RwLock::new(HostIpBoxes::default())), unconfirmed_holds: Arc::new(Mutex::new(BTreeMap::new())), table_pings: Arc::new(Mutex::new(Vec::new())), withdrawal_reports: reports, @@ -5305,6 +5312,7 @@ impl BoxRegistry { pub fn table(&self) -> BoxTable { BoxTable { rows: Arc::clone(&self.rows), + host_ip_boxes: Arc::clone(&self.host_ip_boxes), subnet: self.subnet, withdrawal_reports: self.withdrawal_reports.clone(), row_withdrawals: Arc::clone(&self.row_withdrawals), @@ -5312,8 +5320,238 @@ impl BoxRegistry { row_epoch: Arc::clone(&self.row_epoch), } } + + /// Holds a resident `host_ip` box's egress for the gate (NET-085): the + /// session's policy, filled under this registry's egress default the + /// way the guest daemon fills it, and compiled with the node's own + /// address as its lease, the one source a `host_ip` box's frames carry. + /// A second admit for the same session with rules equal to the held + /// ones changes nothing and logs nothing: the cohort record, and so + /// every resident box's pins and flows, stay. A second admit with other + /// rules replaces the first. Returns `true`: the gate holds the box's + /// rules once this returns. + /// + /// A real change of the resident set rebuilds the cohort record: a + /// box admitted with new or changed rules, or a box retired + /// ([`Self::retire_host_ip_box`]). The pin table then starts a new + /// entry, so every resident box's name pins and retained flows go, and + /// each box gets them back when it resolves its names again. + /// + /// The policy is the host-side client's, carried over the host's + /// control socket, never anything the guest says. One info line names + /// the session and the count of resident boxes. + /// + /// Only a client of this host retires the rules + /// ([`Self::retire_host_ip_box`]). Four cases leave them in + /// `host_ip_boxes` while the box has no live process: + /// + /// - A destroy that no client of this host issues, such as the in-VM + /// daemon's reap. The rules stay until the VM host daemon restarts. + /// - A retire that fails. The rules stay until the VM host daemon + /// restarts. + /// - A box that the in-VM daemon sets to `stopped` at its start + /// (`stop_orphaned_running_records` in `minimald`'s `box_manager`, + /// BOX-154). The rules stay in the union while no process runs. + /// - A box that an attach leaves `stopped` or `exited` through Keep + /// (`HostIpResidency::keep` in the `min` CLI). The rules stay until + /// the box is destroyed. + /// + /// In each case the union the gate decides a box-marked frame by can + /// include a box that is not live. A resume (BOX-025) does not admit a + /// box again. After a VM host daemon restart, a resumed `host_ip` box + /// gets no egress until it is created again. + /// owns closing these + /// cases. + pub fn admit_host_ip_box( + &self, + session_id: sessions::SessionId, + declared: Option<&EgressPolicy>, + ) -> bool { + let policy = match sessions::effective_egress( + declared, + sessions::NetworkMode::HostNet, + self.egress_default_phase, + self.egress_deny_all_opt_out, + ) { + sessions::EffectiveEgress::Declared(policy) => Some(policy), + sessions::EffectiveEgress::DenyAll => Some(EgressPolicy::deny_all()), + sessions::EffectiveEgress::AllowAll => None, + }; + let node = self.subnet.daemon_ip().octets(); + let resolver = self.subnet.dns_server().octets(); + let rules = HostIpBoxRules { + session_id, + egress: EgressRules::from_policy(policy.as_ref(), resolver, node), + dns_hosts: policy + .as_ref() + .and_then(|policy| policy.allow_dns_hosts.clone()) + .unwrap_or_default(), + allow_subnets: policy + .as_ref() + .and_then(|policy| policy.allow_subnets.clone()), + }; + let mut resident = self + .host_ip_boxes + .write() + .expect("the resident host_ip boxes' lock is never held across a panic"); + if resident + .boxes + .get(&session_id) + .is_some_and(|held| **held == rules) + { + // An attach re-admits with the rules the set already holds. The + // cohort record stays the same `Arc`, so the pins and flows every + // resident box took under it stay too. + return true; + } + resident.boxes.insert(session_id, Arc::new(rules)); + resident.cohort = HostIpBoxes::cohort_record(&resident.boxes, self.subnet); + let count = resident.boxes.len(); + drop(resident); + tracing::info!( + session = %session_id, + resident_host_ip_boxes = count, + "holding the host_ip box's egress rules; box-marked frames from the node \ + address are decided by the resident boxes' rules" + ); + true + } + + /// Drops the egress [`Self::admit_host_ip_box`] holds for a session: + /// its box stopped or was destroyed, so its rules admit nothing from + /// now on. A session the set does not hold is the goal state already + /// holding. Returns whether a box's rules were dropped. + pub fn retire_host_ip_box(&self, session_id: sessions::SessionId) -> bool { + let mut resident = self + .host_ip_boxes + .write() + .expect("the resident host_ip boxes' lock is never held across a panic"); + let removed = resident.boxes.remove(&session_id).is_some(); + if removed { + resident.cohort = HostIpBoxes::cohort_record(&resident.boxes, self.subnet); + } + let count = resident.boxes.len(); + drop(resident); + if removed { + tracing::info!( + session = %session_id, + resident_host_ip_boxes = count, + "dropped the host_ip box's egress rules" + ); + } + removed + } } +/// One resident `host_ip` box's egress, compiled on the host +/// ([`BoxRegistry::admit_host_ip_box`], NET-085): the rules a box-marked +/// frame from the node address is decided by, beside every other resident +/// box's. +#[derive(Debug, PartialEq, Eq)] +pub struct HostIpBoxRules { + session_id: sessions::SessionId, + egress: EgressRules, + dns_hosts: Vec, + allow_subnets: Option>, +} + +impl HostIpBoxRules { + /// The session the box runs as. + #[must_use] + pub fn session_id(&self) -> sessions::SessionId { + self.session_id + } + + /// The box's compiled egress rules, leased to the node's own address. + #[must_use] + pub fn egress(&self) -> &EgressRules { + &self.egress + } + + /// Whether the box declared DNS hosts: its undeclared destinations are + /// then the host-side DNS admission table's to decide. + #[must_use] + pub fn resolves_names(&self) -> bool { + !self.dns_hosts.is_empty() + } +} + +/// The resident `host_ip` boxes ([`BoxRegistry::admit_host_ip_box`]) and the +/// one row-shaped record their declared names pin under. +#[derive(Debug, Default)] +struct HostIpBoxes { + boxes: BTreeMap>, + /// The record the host-side DNS admission table pins the resident + /// boxes' declared names under: every resident box's DNS hosts and + /// allowed subnets, keyed to the node's own address. `None` when no + /// resident box declared a DNS host, so nothing can pin for them. + /// Rebuilt on every change, so a pin taken for a box that left never + /// outlives the record it was taken under. + cohort: Option>, +} + +impl HostIpBoxes { + /// The pin record for `boxes`: `None` when no box declared a DNS host. + fn cohort_record( + boxes: &BTreeMap>, + subnet: SwitchSubnet, + ) -> Option> { + let mut dns_hosts: Vec = Vec::new(); + let mut allow_subnets: Vec = Vec::new(); + for rules in boxes.values().filter(|rules| rules.resolves_names()) { + for host in &rules.dns_hosts { + if !dns_hosts.contains(host) { + dns_hosts.push(host.clone()); + } + } + for subnet in rules.allow_subnets.iter().flatten() { + if !allow_subnets.contains(subnet) { + allow_subnets.push(subnet.clone()); + } + } + } + if dns_hosts.is_empty() { + return None; + } + let node = subnet.daemon_ip(); + let policy = EgressPolicy { + allow_subnets: Some(allow_subnets.clone()), + allow_dns_hosts: Some(dns_hosts.clone()), + allow_protocols: None, + deny_subnets: None, + }; + Some(Arc::new(BoxRecord { + name: HOST_IP_COHORT_NAME.to_string(), + box_id: crate::bep_attach::mint_box_id(), + switch_addr: node, + loopback_addr: node, + admitted_ports: Vec::new(), + declared_names: Vec::new(), + egress: EgressRules::from_policy( + Some(&policy), + subnet.dns_server().octets(), + node.octets(), + ), + resolves_names: true, + dns_hosts, + deny_all: false, + credentialed_upstream: false, + dynamic_ingress: DynamicIngress::Deny, + dynamic_range: None, + runtime_ports: Mutex::new(RowRuntime::default()), + egress_allow_list: allow_subnets, + attributed: AtomicBool::new(false), + liveness: Mutex::new(RowLiveness::default()), + task_row_of: None, + task_addrs: Vec::new(), + })) + } +} + +/// The name the resident `host_ip` boxes' pin record carries in the DNS +/// admission table's log lines. +const HOST_IP_COHORT_NAME: &str = "host_ip boxes"; + /// The run of published loopback addresses a box may take inside `slice`: /// the slice clamped to the reserved local range's interior. The range's /// network address, `.1` and its broadcast address are never a box's — a @@ -5406,6 +5644,7 @@ fn in_reserved_local_range(addr: Ipv4Addr) -> bool { #[derive(Debug, Clone)] pub struct BoxTable { rows: Arc>, + host_ip_boxes: Arc>, subnet: SwitchSubnet, withdrawal_reports: std::sync::mpsc::Sender, row_withdrawals: RowWithdrawals, @@ -5603,6 +5842,33 @@ impl BoxTable { self.subnet.first_ptask() <= addr && addr <= self.subnet.last_ptask() } + /// The resident `host_ip` boxes' rules, in session order: the union a + /// box-marked frame from the node address is decided by (NET-085). + /// Empty when no `host_ip` box is resident, and the gate then drops + /// every such frame. + #[must_use] + pub fn host_ip_boxes(&self) -> Vec> { + self.host_ip_boxes + .read() + .expect("the resident host_ip boxes' lock is never held across a panic") + .boxes + .values() + .cloned() + .collect() + } + + /// The record the resident `host_ip` boxes' declared names pin under in + /// the DNS admission table, keyed to the node's own address. `None` + /// when no resident box declared a DNS host. + #[must_use] + pub fn host_ip_cohort(&self) -> Option> { + self.host_ip_boxes + .read() + .expect("the resident host_ip boxes' lock is never held across a panic") + .cohort + .clone() + } + /// Every published row, in switch-address order. #[must_use] pub fn rows(&self) -> Vec> { @@ -8492,7 +8758,14 @@ mod tests { // proves the whole lot was decided before the comparison. let undeclared = ipv4_frame(lease, 6, [203, 0, 113, 7], 443); let unknown = ipv4_frame([100, 64, 0, 99], 6, [10, 1, 2, 3], 80); - let node_frame = ipv4_frame(SUBNET.daemon_ip().octets(), 6, [10, 1, 2, 3], 80); + // The node's frame is one the in-force baseline set admits: its + // own lookup, through the resolver carve-out. + let node_frame = ipv4_frame( + SUBNET.daemon_ip().octets(), + 17, + SUBNET.dns_server().octets(), + 53, + ); let foreign_arp = arp_frame([203, 0, 113, 7]); let marker = ipv4_frame(lease, 6, [10, 1, 2, 3], 80); for frame in [&undeclared, &unknown, &node_frame, &foreign_arp] { @@ -8506,7 +8779,7 @@ mod tests { assert_eq!( expect_frame(&mut harness.switch).await, node_frame, - "the node namespace's frame is decided by its own row" + "the node namespace's frame is decided by the node-plane baseline set" ); assert_eq!( expect_frame(&mut harness.switch).await, @@ -8813,7 +9086,14 @@ mod tests { // that proves the whole lot was decided before the comparison. let undeclared = ipv4_frame(lease, 6, [203, 0, 113, 7], 443); let unknown = ipv4_frame([100, 64, 0, 99], 6, [10, 1, 2, 3], 80); - let node_frame = ipv4_frame(SUBNET.daemon_ip().octets(), 6, [10, 1, 2, 3], 80); + // The node's frame is one the in-force baseline set admits: its + // own lookup, through the resolver carve-out. + let node_frame = ipv4_frame( + SUBNET.daemon_ip().octets(), + 17, + SUBNET.dns_server().octets(), + 53, + ); let foreign_arp = arp_frame([203, 0, 113, 7]); let marker = ipv4_frame(lease, 6, [10, 1, 2, 3], 80); for frame in [&undeclared, &unknown, &node_frame, &foreign_arp] { @@ -8827,7 +9107,7 @@ mod tests { assert_eq!( expect_frame(&mut harness.switch).await, node_frame, - "the node namespace's frame is decided by its own row" + "the node namespace's frame is decided by the node-plane baseline set" ); assert_eq!( expect_frame(&mut harness.switch).await, diff --git a/crates/minvmd/src/control.rs b/crates/minvmd/src/control.rs index b826df8e36..c8e051cf8e 100644 --- a/crates/minvmd/src/control.rs +++ b/crates/minvmd/src/control.rs @@ -967,6 +967,29 @@ fn serve_request( let reply = order.apply(|| release_box_name(boxes, &request.name, request.session_id)); write_reply(stream, &reply) } + // A `host_ip` box's policy changes no zone table, so it takes no + // ticket. Only the host's door serves it: the guest must never say + // what a marked frame may reach (NET-085). + (BoxControlRequest::AdmitHostIpBox(request), ControlDoor::Host) => { + let resident = boxes.admit_host_ip_box(request.session_id, request.egress.as_ref()); + write_reply( + stream, + &BoxControlReply::HostIpBox { + session_id: request.session_id, + resident, + }, + ) + } + (BoxControlRequest::RetireHostIpBox(request), ControlDoor::Host) => { + boxes.retire_host_ip_box(request.session_id); + write_reply( + stream, + &BoxControlReply::HostIpBox { + session_id: request.session_id, + resident: false, + }, + ) + } (BoxControlRequest::AnswererStatus, ControlDoor::Host) => { // The read-only status answers both host facts the CLI's // surfaces read (T93): the answerer's own state, always — @@ -1341,6 +1364,8 @@ fn request_verb(request: &BoxControlRequest) -> &'static str { BoxControlRequest::Withdraw(_) => "withdraw", BoxControlRequest::HoldBoxName(_) => "hold_box_name", BoxControlRequest::ReleaseBoxName(_) => "release_box_name", + BoxControlRequest::AdmitHostIpBox(_) => "admit_host_ip_box", + BoxControlRequest::RetireHostIpBox(_) => "retire_host_ip_box", BoxControlRequest::AnswererStatus => "answerer_status", BoxControlRequest::AdmitPort(_) => "admit_port", BoxControlRequest::WithdrawPort(_) => "withdraw_port", @@ -3504,7 +3529,7 @@ mod tests { reply for {port}/{proto:?}" ) } - other @ BoxControlReply::RowStanding { .. } => { + other @ (BoxControlReply::RowStanding { .. } | BoxControlReply::HostIpBox { .. }) => { panic!("a box verb is never answered with a row-standing read, got {other:?}") } other @ BoxControlReply::Audited { .. } => { @@ -3983,7 +4008,7 @@ mod tests { BoxControlReply::PortRecorded { .. } => { panic!("a withdrawal echoes the pair it went by, never a port report") } - other @ BoxControlReply::RowStanding { .. } => { + other @ (BoxControlReply::RowStanding { .. } | BoxControlReply::HostIpBox { .. }) => { panic!("a box verb is never answered with a row-standing read, got {other:?}") } other @ BoxControlReply::Audited { .. } => { @@ -4103,7 +4128,8 @@ mod tests { BoxControlReply::PortRecorded { .. } => { panic!("a foreign pair's withdrawal must be refused, got a port report") } - other @ BoxControlReply::RowStanding { .. } => { + other @ (BoxControlReply::RowStanding { .. } + | BoxControlReply::HostIpBox { .. }) => { panic!("a box verb is never answered with a row-standing read, got {other:?}") } other @ BoxControlReply::Audited { .. } => { @@ -4176,7 +4202,7 @@ mod tests { BoxControlReply::PortRecorded { .. } => { panic!("a repeat withdrawal echoes the pair, never a port report") } - other @ BoxControlReply::RowStanding { .. } => { + other @ (BoxControlReply::RowStanding { .. } | BoxControlReply::HostIpBox { .. }) => { panic!("a box verb is never answered with a row-standing read, got {other:?}") } other @ BoxControlReply::Audited { .. } => { diff --git a/crates/minvmd/src/net/baseline.rs b/crates/minvmd/src/net/baseline.rs index 92f022d76b..fd3d0b6a59 100644 --- a/crates/minvmd/src/net/baseline.rs +++ b/crates/minvmd/src/net/baseline.rs @@ -14,25 +14,32 @@ //! source identity the set's verdict accepts. //! //! The host-side egress gate decides a frame from the daemon's address by -//! the compiled set ([`NodePlaneBaseline::rules`]) beside the boxes' rows: a -//! deny-all box cannot clip the daemon's own fetches, because the set is -//! decided before the table is consulted, and a frame wearing the daemon's -//! address to reach somewhere else is bounded by the enumeration, not by the -//! interim row (NET-085). Whether the gate decides by the set yet is the -//! baseline's phase ([`NodeBaselinePhase`], [`NODE_BASELINE_PHASE`]) — -//! announced until the node plane's store surfaces are configured, the same -//! cutover shape the unregistered-source phase carries -//! ([`crate::net::egress_gate::UNREGISTERED_SOURCE_PHASE`]). +//! this set beside the boxes' rows: a deny-all box cannot clip the daemon's +//! own fetches, because the set is decided before the box's row is +//! consulted, and a frame wearing the daemon's address to reach somewhere +//! else is bounded by the enumeration, not by the interim row (NET-085). +//! The gate decides by the set from VM start ([`NODE_BASELINE_PHASE`] is +//! [`NodeBaselinePhase::InForce`]). //! -//! The built-in registry and cache endpoints name the helper's answer -//! address on the fabric — the switch's host alias, the one address in this -//! tree's address plan where a store surface the helper itself carries -//! answers — because an un-enrolled host's own supply is the helper's to -//! carry. Which registry and which cache the node plane uses is the -//! deployment's to say, and neither category is ever dropped from the -//! enumeration by saying it. - -use sessions::core::egress::{EgressRules, IPPROTO_TCP, Ipv4Cidr}; +//! The registry and cache entries are named endpoints: the hosts the +//! daemon's own fetches go to ([`DAEMON_REGISTRY_ENDPOINT`], +//! [`DAEMON_CACHE_ENDPOINT`]). A name is not an address, so the set admits a +//! name's addresses only as the node plane's own lookups resolve them: the +//! host-side DNS admission table pins each answer against the entry that +//! named it ([`crate::net::dns_pins`]), under the same window and per-name +//! cap a box's pins hold. An endpoint spelled as a subnet (`a.b.c.d/n`) is +//! admitted as it stands. Which registry and which cache the node plane +//! uses is the deployment's to say (`MINVMD_BASELINE_REGISTRY`, +//! `MINVMD_BASELINE_CACHE`), every change is logged, and neither category is +//! ever dropped from the enumeration by saying it. +//! +//! A frame from the node address that wears the box mark is not the node +//! plane's: the gate decides it by the union of the resident `host_ip` +//! boxes' rules, and drops it when no `host_ip` box is resident +//! ([`crate::net::egress_gate`]). The set never decides a marked frame, and +//! the node row never does either. + +use sessions::core::egress::{EgressRules, Ipv4Cidr}; use std::io; use switch::SwitchSubnet; @@ -63,10 +70,12 @@ impl BaselineCategory { } } -/// One category's entry in the enumeration: the category and the subnets it -/// admits, spelled the way an egress policy declares subnets (`a.b.c.d/n`) — -/// the same spelling the compiled rules parse from, so the set a host sees -/// in its log and its policy display is the set the gate decides by. +/// One category's entry in the enumeration: the category and the endpoints +/// it admits. An endpoint is a host name (`cache.minimal.dev`), whose +/// addresses the node plane's own lookups pin, or a subnet spelled the way an +/// egress policy declares one (`a.b.c.d/n`), admitted as it stands. The set +/// a host sees in its log and its policy display is the set the gate decides +/// by. #[derive(Debug, Clone, PartialEq, Eq)] pub struct BaselineEntry { category: BaselineCategory, @@ -80,53 +89,55 @@ impl BaselineEntry { self.category } - /// The entry's endpoints, as the CIDR spellings the compiled set parses - /// from. + /// The entry's endpoints, as the host names and the subnet spellings + /// the entry was configured with. #[must_use] pub fn endpoints(&self) -> &[String] { &self.endpoints } - /// The entry's endpoints in the matchable form the compiled rules hold. - /// The spellings are the constructors' own, so each parses; one that - /// does not is skipped the way an egress policy's subnet declarations - /// are compiled ([`EgressRules::from_policy`]). + /// The entry's subnet endpoints in the matchable form the compiled rules + /// hold. An endpoint that does not parse as a subnet is a host name, and + /// [`Self::names`] carries it instead. fn cidrs(&self) -> Vec { self.endpoints .iter() .filter_map(|cidr| Ipv4Cidr::parse(cidr)) .collect() } + + /// The entry's host-name endpoints, normalized to the form DNS names are + /// matched in (ASCII-lowercased, the root dot stripped): the names whose + /// answers the node plane's own lookups pin against this entry. + pub(crate) fn names(&self) -> impl Iterator + '_ { + self.endpoints + .iter() + .filter(|endpoint| Ipv4Cidr::parse(endpoint).is_none()) + .map(|name| { + let lowered = name.trim().to_ascii_lowercase(); + lowered.strip_suffix('.').unwrap_or(&lowered).to_string() + }) + .filter(|name| !name.is_empty()) + } } /// The posture the node plane's own traffic is decided under: whether the /// host-side gate decides a frame from the in-VM daemon's address by this -/// enumeration's compiled set, or leaves the node plane to the run path's -/// interim node row while the set is announced. -/// -/// Shipped announced, for the one thing the enumeration cannot fix alone: -/// the built-in registry and cache endpoints name the helper's answer -/// address on the fabric, and the guest daemon's shipped cache endpoint is -/// a public service whose serving addresses no subnet list this tree can -/// name. A gate deciding by the built-in set before the node plane's store -/// surfaces are configured would sever the daemon's own fetches — the -/// traffic the enumeration exists to keep passing (NET-080). The flip is -/// [`NODE_BASELINE_PHASE`], one constant, the same shape the -/// unregistered-source cutover carries -/// ([`crate::net::egress_gate::UNREGISTERED_SOURCE_PHASE`]). +/// enumeration, or leaves the node plane to the run path's interim node row. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum NodeBaselinePhase { /// The enumeration is carried, logged at VM start, and shown beside the - /// box's rules; the node plane's frames are still decided by the interim - /// node row. - Announced, - /// The gate decides node-plane frames by the enumeration's compiled set. + /// box's rules; the node plane's frames are decided by the interim node + /// row. /// /// Constructed today only by the tests that pin the phase's other arm - /// ([`NodePlaneBaseline::in_force`]) — the arm the store-surface - /// configuration change flips [`NODE_BASELINE_PHASE`] onto. The gate's - /// decision reads the arm by path, so the variant is carried live even - /// unconstructed; no dead-code expectation sits on it. + /// ([`NodePlaneBaseline::announced`]). The gate's decision reads the arm + /// by path, so the variant is carried live even unconstructed. + Announced, + /// The gate decides node-plane frames by the enumeration: the addresses + /// the node plane's own lookups pinned for an entry, the entries' subnet + /// endpoints, the host alias at the configured host exposures, and the + /// resolver carve-out. InForce, } @@ -134,26 +145,115 @@ impl NodeBaselinePhase { /// The phase as the value the gate's start-up line logs and the policy /// display spells beside the set: one spelling for the posture, so a /// host reading its policy output reads the same terms its daemon log - /// wrote at VM start, and a host running the announced interim can tell - /// it is. + /// wrote at VM start. pub fn as_str(self) -> &'static str { match self { Self::Announced => "announced (node row's allow-all interim)", - Self::InForce => "in force (decided by the baseline set)", + Self::InForce => "in force", } } } -/// The phase this build ships: announced, because the node plane's store -/// surfaces are not yet configured to the endpoints the built-in enumeration -/// names. The endpoint-configuration change (#1786) — the deployment's -/// registry and cache, answering where `with_registry` and `with_cache` -/// point them — is the change that flips this constant, and the constant is -/// the whole cutover: the gate's decision reads it off the baseline it was -/// built with ([`NodePlaneBaseline::phase`]), the start-up line logs it, and -/// the tests pin both of its arms, so the flip is one line here, beside the -/// listener that feeds [`REGISTRY_CACHE_FETCH_PORTS`]. -pub(crate) const NODE_BASELINE_PHASE: NodeBaselinePhase = NodeBaselinePhase::Announced; +/// The phase this build ships: in force. The registry and cache entries name +/// the hosts the daemon's own fetches go to, and the node plane's own lookups +/// pin their addresses, so the gate deciding by the set keeps those fetches +/// passing (NET-080) and bounds everything else the node's address sends +/// (NET-085). The gate's decision reads the phase off the baseline it was +/// built with ([`NodePlaneBaseline::phase`]), and the start-up line logs it. +pub(crate) const NODE_BASELINE_PHASE: NodeBaselinePhase = NodeBaselinePhase::InForce; + +/// The registry the daemon's own fetches resolve packages against: the host +/// of the package source repository a project's `[upstream]` names (`op`'s +/// default, `https://github.com/gominimal/pkgs`). Before a box launches, the +/// in-VM daemon git-fetches that repository at its locked commit to build +/// the package graph, so the fetch is node-plane traffic and the host must +/// be a member. The package index and the package content the daemon reads +/// next are objects in the cache ([`DAEMON_CACHE_ENDPOINT`]). Nothing on a +/// VM host configures another location for the guest daemon, so the +/// daemon's configured registry is this default. A deployment re-points the +/// category with [`BASELINE_REGISTRY_ENV`]. +pub const DAEMON_REGISTRY_ENDPOINT: &str = "github.com"; + +/// The cache the daemon's own fetches read package content from: the host of +/// its default remote cache read location (`mctx`'s +/// `DEFAULT_REMOTE_CACHE_READ_URL`, `https://cache.minimal.dev/`). A +/// deployment re-points the category with [`BASELINE_CACHE_ENV`]. +pub const DAEMON_CACHE_ENDPOINT: &str = "cache.minimal.dev"; + +/// The environment variable that re-points the registry category: a +/// comma-separated list of host names (`pkgs.example.com`) and subnets +/// (`a.b.c.d/n`). Absent, the category keeps [`DAEMON_REGISTRY_ENDPOINT`]. +/// Set and empty, the category stays carried and admits nothing. Read from +/// the reading process's own environment each time the set is built. +pub(crate) const BASELINE_REGISTRY_ENV: &str = "MINVMD_BASELINE_REGISTRY"; + +/// The environment variable that re-points the cache category, spelled the +/// way [`BASELINE_REGISTRY_ENV`] is. Absent, the category keeps +/// [`DAEMON_CACHE_ENDPOINT`]. +pub(crate) const BASELINE_CACHE_ENV: &str = "MINVMD_BASELINE_CACHE"; + +/// One category's endpoints as the deployment spelled them in `var`, or +/// `None` when the variable is absent. A value that is not Unicode keeps the +/// built-in endpoints and logs a warning that names the variable. +fn baseline_endpoints_from_env(var: &str) -> Option> { + match std::env::var(var) { + Ok(raw) => Some(baseline_endpoints_from_raw(var, &raw)), + Err(std::env::VarError::NotPresent) => None, + Err(std::env::VarError::NotUnicode(_)) => { + tracing::warn!( + variable = var, + "the node-plane baseline entry's variable is not valid Unicode; the built-in \ + endpoints stay in force", + ); + None + } + } +} + +/// Parses one category's endpoint list: comma-separated terms, whitespace +/// trimmed, empty terms skipped. A term with a `/` must be an IPv4 subnet, +/// and any other term must be a host name of letters, digits, `-` and `.`. +/// A term that is neither is skipped with a warning that names it, so a typo +/// narrows the set and never widens it. A bare IPv4 address is not a host +/// name: it is skipped with a warning that names its `/32` subnet form. +fn baseline_endpoints_from_raw(var: &str, raw: &str) -> Vec { + raw.split(',') + .map(str::trim) + .filter(|term| !term.is_empty()) + .filter(|term| { + if term.parse::().is_ok() { + tracing::warn!( + variable = var, + term = %term, + "skipped a node-plane baseline endpoint that is a bare IPv4 address; \ + write it as the subnet {term}/32", + ); + return false; + } + let valid = if term.contains('/') { + Ipv4Cidr::parse(term).is_some() + } else { + term.trim_end_matches('.').split('.').all(|label| { + !label.is_empty() + && label.len() <= 63 + && label + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-') + }) + }; + if !valid { + tracing::warn!( + variable = var, + term = %term, + "skipped a node-plane baseline endpoint that is neither a host name nor an \ + IPv4 subnet", + ); + } + valid + }) + .map(str::to_string) + .collect() +} /// The environment variable the daemon's host-exposure port list is read /// from, each time the daemon starts the VM's network — from the daemon's @@ -225,31 +325,8 @@ fn bad_host_exposure_term(term: &str) -> io::Error { )) } -/// The IP protocol the in-force arm opens on the switch's host alias: TCP -/// alone, because the store surface the built-in endpoints name is HTTPS. -/// QUIC is not admitted; carrying it would be a stated choice that adds UDP -/// here. Every other protocol to the alias, ICMP and GRE among them, is the -/// infrastructure drop, whatever its bytes read as a port. -pub(crate) const REGISTRY_CACHE_FETCH_PROTOCOL: u8 = IPPROTO_TCP; - -/// The ports the in-force arm opens on the switch's host alias, over -/// [`REGISTRY_CACHE_FETCH_PROTOCOL`]. The compiled set has no port -/// dimension, so without this bound the alias would be admitted at every -/// port. -/// -/// 443 is the planned port of the helper's store surface on the alias. No -/// listener serves a registry or a cache there yet, and the daemon's own -/// fetches go to `cache.minimal.dev`, not to the alias. When that listener -/// lands, this list must be read from the listener's stack, the way -/// `switch::bep_host::PROXY_PORT` is. `[443]` is the planned port, and no -/// listener serves it. While [`NODE_BASELINE_PHASE`] is -/// [`NodeBaselinePhase::Announced`], only the in-force tests exercise this -/// list. The endpoint-configuration change (#1786) replaces it with the -/// listener's port, or with an empty list if no listener exists. -pub(crate) const REGISTRY_CACHE_FETCH_PORTS: [u16; 1] = [443]; - /// The node-plane baseline set: the categories in force, each with the -/// subnets it admits, and the compiled rules the host-side gate decides +/// endpoints it admits, and the compiled rules the host-side gate decides /// node-plane frames by. #[derive(Debug, Clone, PartialEq, Eq)] pub struct NodePlaneBaseline { @@ -260,77 +337,77 @@ pub struct NodePlaneBaseline { /// The switch's resolver, carried into every recompilation so the /// resolver carve-out survives a re-pointed category. resolver: [u8; 4], - /// The switch's host alias: the address the in-force arm bounds to - /// [`REGISTRY_CACHE_FETCH_PROTOCOL`] at [`REGISTRY_CACHE_FETCH_PORTS`], - /// carried so the policy display spells that bound beside the endpoint. - host_alias: [u8; 4], /// The categories in force, in the order the display spells them. entries: Vec, /// The compiled allow list the gate decides node-plane frames by: every - /// category's endpoints, every IP protocol within them, keyed to the - /// daemon's own address as its lease. + /// category's subnet endpoints, every IP protocol within them, keyed to + /// the daemon's own address as its lease. A named endpoint is not in it: + /// its addresses are the node plane's own pins. rules: EgressRules, - /// The posture the gate decides the set under: the shipped constant, - /// until the store-surface configuration flips it. + /// The posture the gate decides the set under: the shipped constant. phase: NodeBaselinePhase, /// The configured host exposures over the alias, as inclusive port - /// ranges: the ports the node row still reaches the host alias at while - /// the baseline is announced, every other port refused through the - /// alias. Empty unless the deployment names ports. The host-side gate - /// alone reads it: the guest-side verdict's own alias list starts empty - /// and nothing in the run path fills it, so own-address boxes stay - /// refused at every port. + /// ranges: the ports the node plane reaches the host alias at, every + /// other port refused through the alias. Empty unless the deployment + /// names ports. The host-side gate alone reads it: the guest-side + /// verdict's own alias list starts empty and nothing in the run path + /// fills it, so own-address boxes stay refused at every port. host_exposure: Vec<(u16, u16)>, } impl NodePlaneBaseline { /// The built-in enumeration for one switch fabric. The categories are - /// the design's, in the order the display spells them; the built-in - /// endpoints come from the fabric's own address plan: + /// the design's, in the order the display spells them, and each names + /// the host the daemon's own fetches go to: + /// + /// - `registry`: [`DAEMON_REGISTRY_ENDPOINT`], where the daemon fetches + /// the package source repository its package graph is built from. + /// - `cache`: [`DAEMON_CACHE_ENDPOINT`], where the daemon reads the + /// package index and package content. /// - /// - `registry` and `cache`: the helper's answer address on the fabric - /// (the switch's host alias) — the one address the plan holds for a - /// store surface the helper itself carries. Which registry and which - /// cache the node plane uses is the deployment's to say: - /// [`with_registry`](Self::with_registry) and - /// [`with_cache`](Self::with_cache) replace these endpoints within - /// their category, and a category is replaced, never absent. + /// Which registry and which cache the node plane uses is the + /// deployment's to say: [`BASELINE_REGISTRY_ENV`] and + /// [`BASELINE_CACHE_ENV`], when set, replace these endpoints within their + /// category through [`with_registry`](Self::with_registry) and + /// [`with_cache`](Self::with_cache), and each change is logged. A category + /// is replaced, never absent. The VM host daemon reads them at VM start, + /// and the policy display reads them from its own environment, so the two + /// show one set when both run with the same environment. /// /// The set carries no row for the switch fabric itself: the categories /// are the design's named service planes, and the plane the node lives - /// on is no allowance of its own — a node-plane frame aimed at a box is - /// the target box's ingress rules to decide, so a row naming the box - /// leases would hand whoever wears the node's address every box's - /// listener, the reach the escape bound (NET-085) exists to keep inside - /// the enumerated set. + /// on is no allowance of its own. A node-plane frame aimed at a box's + /// lease is the target box's ingress to decide, and the gate's in-force + /// arm passes it on to the switch for that decision. /// /// The compiled set allows every IP protocol within the admitted - /// subnets — the categories bound destinations, not protocols; the - /// gate's in-force arm bounds the host alias to TCP at - /// [`REGISTRY_CACHE_FETCH_PORTS`] before the set is consulted — and is - /// keyed to the in-VM daemon's own address as its lease: the verdict - /// rejects any frame wearing another source, so the set decides - /// node-plane frames and only those. The resolver's carve-out keeps the - /// node resolving under the declaration, the same carve-out a box's - /// rules carry. + /// subnets, and is keyed to the in-VM daemon's own address as its lease: + /// the verdict rejects any frame wearing another source, so the set + /// decides node-plane frames and only those. The resolver's carve-out + /// keeps the node resolving, which is how its named endpoints get pinned. #[must_use] pub fn built_in(subnet: SwitchSubnet) -> Self { - let host_alias = subnet.host_alias(); - Self::from_entries( + let mut baseline = Self::from_entries( subnet.daemon_ip().octets(), subnet.dns_server().octets(), - host_alias.octets(), vec![ BaselineEntry { category: BaselineCategory::Registry, - endpoints: vec![format!("{host_alias}/32")], + endpoints: vec![DAEMON_REGISTRY_ENDPOINT.to_string()], }, BaselineEntry { category: BaselineCategory::Cache, - endpoints: vec![format!("{host_alias}/32")], + endpoints: vec![DAEMON_CACHE_ENDPOINT.to_string()], }, ], - ) + ); + if let Some(endpoints) = baseline_endpoints_from_env(BASELINE_REGISTRY_ENV) { + baseline = baseline.with_registry(endpoints); + } + if let Some(endpoints) = baseline_endpoints_from_env(BASELINE_CACHE_ENV) { + baseline = baseline.with_cache(endpoints); + } + baseline } /// The in-VM daemon's own address on the fabric: the one source identity @@ -353,17 +430,38 @@ impl NodePlaneBaseline { &self.rules } - /// The posture the gate decides the set under — announced, the shipped - /// posture, the run path's allow-all interim node row still decides the - /// node plane's frames; in force, the compiled set does. + /// The posture the gate decides the set under. In force, the shipped + /// posture: the set decides the node plane's frames. Announced: the run + /// path's allow-all interim node row does. #[must_use] pub fn phase(&self) -> NodeBaselinePhase { self.phase } + /// The host-name endpoints of every entry, each with the categories that + /// name it, in the order the entries carry them: the names whose answers + /// the node plane's own lookups pin ([`crate::net::dns_pins`]). One name + /// can serve two categories, the built-in registry and cache among them. + #[must_use] + pub(crate) fn named_endpoints(&self) -> Vec<(String, Vec)> { + let mut named: Vec<(String, Vec)> = Vec::new(); + for entry in &self.entries { + for name in entry.names() { + match named.iter_mut().find(|(known, _)| *known == name) { + Some((_, categories)) if !categories.contains(&entry.category) => { + categories.push(entry.category); + } + Some(_) => {} + None => named.push((name, vec![entry.category])), + } + } + } + named + } + /// The host exposures over the alias this gate carries: the inclusive - /// port ranges the announced node row still reaches the host alias at, - /// empty unless the deployment configured ports. + /// port ranges the node plane reaches the host alias at, empty unless + /// the deployment configured ports. #[must_use] pub(crate) fn host_exposure_ports(&self) -> &[(u16, u16)] { &self.host_exposure @@ -401,41 +499,29 @@ impl NodePlaneBaseline { .join(",") } - /// One entry's endpoints as the policy display spells them (NET-061): an - /// endpoint naming the host alias carries the in-force arm's bound - /// beside it (`100.64.255.254/32 tcp/443`), because the set has no port - /// dimension and the bound is what in force admits there. Any other - /// endpoint is spelled as the set holds it. + /// One entry's endpoints as the policy display spells them (NET-061): + /// each host name and each subnet as the entry holds it. #[must_use] pub fn display_endpoints(&self, entry: &BaselineEntry) -> Vec { - let alias = format!("{}/32", std::net::Ipv4Addr::from(self.host_alias)); - let ports = REGISTRY_CACHE_FETCH_PORTS - .iter() - .map(u16::to_string) - .collect::>() - .join(","); - entry - .endpoints - .iter() - .map(|endpoint| { - if *endpoint == alias { - format!("{endpoint} tcp/{ports}") - } else { - endpoint.clone() - } - }) - .collect() + entry.endpoints.clone() } /// The set as the daemon's start-up line spells it: one /// `category=endpoints` term per category, in the order the categories /// are carried, so the line names the set in force with each entry's - /// category. + /// category. An entry with no endpoint is spelled `(none)`. #[must_use] pub(crate) fn render(&self) -> String { self.entries .iter() - .map(|entry| format!("{}={}", entry.category.as_str(), entry.endpoints.join(","))) + .map(|entry| { + let endpoints = if entry.endpoints.is_empty() { + "(none)".to_string() + } else { + entry.endpoints.join(",") + }; + format!("{}={endpoints}", entry.category.as_str()) + }) .collect::>() .join(", ") } @@ -443,8 +529,9 @@ impl NodePlaneBaseline { /// Points the registry category at `endpoints`, replacing the built-in: /// which registry the node plane uses is the host's to configure, within /// the category. The category stays carried — an empty list is a - /// registry that admits nothing, not an absent one (NET-130) — and the - /// compiled set is rebuilt from every category's endpoints. + /// registry that admits nothing, not an absent one (NET-130) — the + /// compiled set is rebuilt from every category's endpoints, and the + /// change is logged. #[must_use] pub fn with_registry(mut self, endpoints: Vec) -> Self { self.replace(BaselineCategory::Registry, endpoints); @@ -453,39 +540,48 @@ impl NodePlaneBaseline { /// Points the cache category at `endpoints`, the way /// [`with_registry`](Self::with_registry) points the registry: replaced - /// within its category, never absent, the compiled set rebuilt. + /// within its category, never absent, the compiled set rebuilt, the + /// change logged. #[must_use] pub fn with_cache(mut self, endpoints: Vec) -> Self { self.replace(BaselineCategory::Cache, endpoints); self } - /// Pins the phase's other arm ([`NodeBaselinePhase::InForce`]): the gate - /// decides node-plane frames by the compiled set. The arm the - /// store-surface configuration change flips the shipped constant onto — - /// built here so the in-force behavior has its proof before the flip - /// lands, rather than tests to rewrite after it. + /// Pins the phase's other arm ([`NodeBaselinePhase::Announced`]): the + /// run path's interim node row decides node-plane frames. Kept so the + /// tests can pin both arms of the phase. #[cfg(test)] - pub(crate) fn in_force(mut self) -> Self { - self.phase = NodeBaselinePhase::InForce; + pub(crate) fn announced(mut self) -> Self { + self.phase = NodeBaselinePhase::Announced; self } - /// Re-points one category's endpoints and rebuilds the compiled set. - /// A category the enumeration does not carry is added rather than - /// skipped, so configuring a category can never make it absent. + /// Re-points one category's endpoints, rebuilds the compiled set, and + /// logs the change with the endpoints before and after it. A category + /// the enumeration does not carry is added rather than skipped, so + /// configuring a category can never make it absent. fn replace(&mut self, category: BaselineCategory, endpoints: Vec) { - match self + let previous = match self .entries .iter_mut() .find(|entry| entry.category == category) { - Some(entry) => entry.endpoints = endpoints, - None => self.entries.push(BaselineEntry { - category, - endpoints, - }), - } + Some(entry) => std::mem::replace(&mut entry.endpoints, endpoints.clone()), + None => { + self.entries.push(BaselineEntry { + category, + endpoints: endpoints.clone(), + }); + Vec::new() + } + }; + tracing::info!( + entry = category.as_str(), + previous = %previous.join(","), + endpoints = %endpoints.join(","), + "re-pointed a node-plane baseline entry", + ); self.rules = Self::compile_rules(&self.entries, self.resolver, self.node_addr); } @@ -493,16 +589,10 @@ impl NodePlaneBaseline { /// addresses the compiled rules are keyed to. The host exposures start /// empty, the alias closed; the deployment's spelling replaces them in /// the run path's builder. - fn from_entries( - node_addr: [u8; 4], - resolver: [u8; 4], - host_alias: [u8; 4], - entries: Vec, - ) -> Self { + fn from_entries(node_addr: [u8; 4], resolver: [u8; 4], entries: Vec) -> Self { Self { node_addr, resolver, - host_alias, rules: Self::compile_rules(&entries, resolver, node_addr), entries, phase: NODE_BASELINE_PHASE, @@ -510,8 +600,8 @@ impl NodePlaneBaseline { } } - /// Compiles the categories' endpoints into the allow list the gate - /// decides node-plane frames by: every category's subnets, every IP + /// Compiles the categories' subnet endpoints into the allow list the + /// gate decides node-plane frames by: every category's subnets, every IP /// protocol within them, the daemon's own address as the lease the /// verdict checks the frame's source against. fn compile_rules( @@ -530,8 +620,10 @@ impl NodePlaneBaseline { #[cfg(test)] mod tests { use super::{ - BaselineCategory, BaselineEntry, HOST_EXPOSURE_PORTS_ENV, NodePlaneBaseline, - host_exposure_ports_from_raw, + BASELINE_CACHE_ENV, BASELINE_REGISTRY_ENV, BaselineCategory, BaselineEntry, + DAEMON_CACHE_ENDPOINT, DAEMON_REGISTRY_ENDPOINT, HOST_EXPOSURE_PORTS_ENV, + NodeBaselinePhase, NodePlaneBaseline, baseline_endpoints_from_env, + baseline_endpoints_from_raw, host_exposure_ports_from_raw, }; use sessions::core::egress::{FrameVerdict, summarize, verdict}; use std::net::Ipv4Addr; @@ -588,31 +680,77 @@ mod tests { /// NET-130: on a host un-enrolled, the node-plane baseline set comes /// from the helper's built-in enumeration — and the enumeration keeps - /// the registry and the cache categories, configurable as to which - /// registry and which cache, never absent: re-pointing one leaves the - /// other carried with its built-in endpoints, and the compiled set - /// carries the replacement beside the survivors. + /// the registry and the cache categories, each named by the host the + /// daemon's own fetches go to, configurable as to which registry and + /// which cache, never absent: re-pointing one leaves the other carried + /// with its built-in endpoints, every re-pointing is logged, and the + /// compiled set carries the replacement beside the survivors. #[test] fn baseline_enumeration_always_carries_registry_and_cache() { + let (log, _guard) = crate::net::egress_gate::test_support::capture_log(); let baseline = NodePlaneBaseline::built_in(SUBNET); // The built-in enumeration carries every category — the registry and - // the cache, the two the design names — each with an endpoint to - // admit. + // the cache, the two the design names — each a named endpoint: the + // host the daemon's own fetches go to, never an address the fabric + // plan made up. assert_eq!(baseline.entries().len(), 2); - assert!( - !entry(&baseline, BaselineCategory::Registry) - .endpoints() - .is_empty() + assert_eq!( + entry(&baseline, BaselineCategory::Registry).endpoints(), + &[DAEMON_REGISTRY_ENDPOINT.to_string()] ); - assert!( - !entry(&baseline, BaselineCategory::Cache) - .endpoints() - .is_empty() + assert_eq!( + entry(&baseline, BaselineCategory::Cache).endpoints(), + &[DAEMON_CACHE_ENDPOINT.to_string()] + ); + // Both names are the ones the node plane's own lookups pin against, + // each against its own category: the package source repository's + // host for the registry, the cache's host for the cache. + assert_eq!( + baseline.named_endpoints(), + vec![ + ("github.com".to_string(), vec![BaselineCategory::Registry]), + ( + "cache.minimal.dev".to_string(), + vec![BaselineCategory::Cache] + ), + ] ); // And it carries the in-VM daemon's own address, the one source // identity the set's verdict accepts. assert_eq!(baseline.node_addr(), SUBNET.daemon_ip().octets()); + // The shipped posture is in force: the set decides the node plane's + // own frames, and a marked frame is the resident `host_ip` boxes' + // to decide, never the node row's. The posture carries no caveat. + assert_eq!(baseline.phase(), NodeBaselinePhase::InForce); + assert_eq!(baseline.phase().as_str(), "in force"); + + // The deployment's spelling of an entry: host names and subnets, + // whitespace trimmed, and a term that is neither skipped, so a typo + // narrows the set and never widens it. A bare address is skipped + // too: its subnet form is `/32`. Set and empty, the category + // is carried with no endpoint. + assert_eq!( + baseline_endpoints_from_raw( + BASELINE_REGISTRY_ENV, + " pkgs.example.com , 203.0.113.0/24,,bad name,10.0.0.0/99,a..b,203.0.113.9 " + ), + vec!["pkgs.example.com".to_string(), "203.0.113.0/24".to_string()] + ); + assert!(baseline_endpoints_from_raw(BASELINE_CACHE_ENV, "").is_empty()); + let emptied = baseline + .clone() + .with_cache(baseline_endpoints_from_raw(BASELINE_CACHE_ENV, " ")); + assert!( + entry(&emptied, BaselineCategory::Cache) + .endpoints() + .is_empty(), + "an emptied cache is carried and admits nothing, never absent" + ); + // Absent, the built-in endpoints stand: this run sets neither + // variable, which is how the built-in assertions above hold. + assert_eq!(baseline_endpoints_from_env(BASELINE_REGISTRY_ENV), None); + assert_eq!(baseline_endpoints_from_env(BASELINE_CACHE_ENV), None); // Which registry the node plane uses is the host's to say. The // registry category is replaced within itself, and the cache is @@ -646,6 +784,29 @@ mod tests { registry_endpoints, ); + // Every change is logged, naming the entry and its endpoints before + // and after. + let logged = log.contents(); + for line in [ + "entry=\"registry\" previous=github.com endpoints=10.9.0.0/16", + "entry=\"cache\" previous=cache.minimal.dev endpoints=10.8.0.0/16", + ] { + assert!( + logged.contains(line) && logged.contains("re-pointed a node-plane baseline entry"), + "each re-pointing is logged ({line}), got: {logged}" + ); + } + + // An empty list keeps the category, admitting nothing: never absent. + let emptied = re_cached.clone().with_registry(Vec::new()); + assert_eq!(emptied.entries().len(), 2); + assert!( + entry(&emptied, BaselineCategory::Registry) + .endpoints() + .is_empty() + ); + assert_eq!(emptied.render(), "registry=(none), cache=10.8.0.0/16"); + // The compiled set decides by what the categories carry: the // replaced registry's subnets are admitted beside the surviving // cache's, and a destination no category names is not. @@ -667,45 +828,28 @@ mod tests { ); } - /// The built-in set's compiled rules decide node-plane frames by the - /// enumeration alone: the daemon's own address may reach the categories' - /// endpoints and nothing else, and a frame wearing another source is - /// rejected by the lease the set is keyed to. + /// The built-in set's compiled rules carry no address of their own: the + /// named endpoints are admitted by the node plane's pins, not by a + /// subnet, so a frame from the daemon's own address to any destination + /// is the undeclared drop here and the pins decide it at the gate. A + /// frame wearing another source is rejected by the lease the set is + /// keyed to. #[test] - fn baseline_rules_admit_the_enumerated_categories_alone() { + fn baseline_rules_admit_no_address_but_the_resolver() { let baseline = NodePlaneBaseline::built_in(SUBNET); let node_addr = baseline.node_addr(); - // The registry and cache endpoints — the helper's answer address on - // the fabric — are admitted from the daemon's own address. - let summary = summarize(&tcp_frame( - node_addr, - first_endpoint(&baseline, BaselineCategory::Registry), - 443, - )); - assert!(matches!( - verdict(&summary, baseline.rules()), - FrameVerdict::Admit - )); - - // A destination no category names is dropped as undeclared — the - // gap the announced phase covers until the store surfaces are - // configured (see `NodeBaselinePhase`). let summary = summarize(&tcp_frame(node_addr, [8, 8, 8, 8], 443)); match verdict(&summary, baseline.rules()) { FrameVerdict::Drop(reason) => { assert_eq!(reason.rule(), "egress-undeclared-subnet") } - FrameVerdict::Admit => panic!("an out-of-set destination is not admitted"), + FrameVerdict::Admit => panic!("the compiled set admits no address of its own"), } // A frame wearing another source is rejected by the lease the set // is keyed to: the set decides node-plane frames, and only those. - let summary = summarize(&tcp_frame( - [100, 64, 0, 9], - first_endpoint(&baseline, BaselineCategory::Registry), - 443, - )); + let summary = summarize(&tcp_frame([100, 64, 0, 9], [8, 8, 8, 8], 443)); match verdict(&summary, baseline.rules()) { FrameVerdict::Drop(reason) => assert_eq!(reason.rule(), "egress-foreign-source"), FrameVerdict::Admit => panic!("a foreign source is not admitted by the set"), @@ -789,17 +933,15 @@ mod tests { ); } - /// NET-061: the policy display spells the in-force arm's bound beside - /// an endpoint naming the host alias, and a re-pointed endpoint as the - /// set holds it. + /// NET-061: the policy display spells each endpoint as the set holds + /// it, a host name as a name and a subnet as a subnet. #[test] - fn display_endpoints_spell_the_alias_bound() { + fn display_endpoints_spell_the_entries() { let baseline = NodePlaneBaseline::built_in(SUBNET).with_cache(vec!["198.51.100.0/24".to_owned()]); - let alias = format!("{}/32 tcp/443", SUBNET.host_alias()); assert_eq!( baseline.display_endpoints(entry(&baseline, BaselineCategory::Registry)), - vec![alias] + vec!["github.com".to_owned()] ); assert_eq!( baseline.display_endpoints(entry(&baseline, BaselineCategory::Cache)), diff --git a/crates/minvmd/src/net/dns_pins.rs b/crates/minvmd/src/net/dns_pins.rs index e76ad3015e..f822b551ed 100644 --- a/crates/minvmd/src/net/dns_pins.rs +++ b/crates/minvmd/src/net/dns_pins.rs @@ -72,6 +72,22 @@ //! queries REFUSED first, as the precision copy honest clients meet; this //! is the deciding copy, outside the VM. //! +//! ## The node plane's own lookups (NET-130, NET-085) +//! +//! The node-plane baseline set names the registry and the cache by host +//! name ([`crate::net::baseline`]), and a name is not an address. So the +//! table holds one more entry, for the in-VM daemon's own address, once the +//! gate adopts an in-force baseline ([`DnsPins::adopt_baseline`]): its names +//! are the baseline's named endpoints, and each answer the node plane's own +//! lookups receive for one of them pins against that entry, under the same +//! three reply conditions, the same window, the same per-name cap and the +//! same flow retention a box's pins hold. The answers pass the same +//! rebinding intersection, with the baseline's subnet endpoints as the +//! allowance, so an answer inside the host or the fabric never pins. Every +//! pin and every re-pin says so at `info`, naming the entry, the addresses +//! and the lookup's source. The node entry is the node's for the VM's +//! lifetime: no relay retires the node's address. +//! //! ## Lifetime //! //! An entry is born from its row, keyed by the row's switch address, and @@ -112,6 +128,7 @@ use sessions::core::egress::{ use switch::SwitchSubnet; use crate::box_registry::{BoxRecord, BoxTable}; +use crate::net::baseline::{BaselineCategory, BaselineEntry, NodeBaselinePhase, NodePlaneBaseline}; use crate::net::egress_gate::DropLimiter; /// Ethernet II header length: destination MAC (6) + source MAC (6) + @@ -403,6 +420,53 @@ struct QueryKey { name: String, } +/// The node plane's grant, adopted from an in-force baseline: the names its +/// own lookups may pin, each with the baseline entries that name it, and the +/// subnet endpoints the rebinding intersection reads as the allowance. +/// Carried behind an `Arc` so the node's entry can prove it was built from +/// the grant the table holds now ([`DnsPins::adopt_baseline`]). +struct NodePlaneNames { + /// The in-VM daemon's own address: the one row this grant applies to. + node_addr: [u8; 4], + /// The baseline's named endpoints, normalized, each with the categories + /// that name it. + names: HashMap>, + /// The baseline's subnet endpoints: the allowance an answer inside + /// private space must fall in to pin. + allow: Vec, + /// The entries the grant was adopted from, compared at each adoption so + /// an unchanged baseline keeps the node's pins. + entries: Vec, +} + +impl NodePlaneNames { + /// The grant an in-force baseline gives the node plane's own lookups. + fn of(baseline: &NodePlaneBaseline) -> Self { + Self { + node_addr: baseline.node_addr(), + names: baseline.named_endpoints().into_iter().collect(), + allow: baseline + .rules() + .allow_subnets() + .map(<[egress::Ipv4Cidr]>::to_vec) + .unwrap_or_default(), + entries: baseline.entries().to_vec(), + } + } + + /// The entries that name `name`, as the pin line spells them + /// (`registry,cache`), or `None` for a name no entry names. + fn entries_naming(&self, name: &str) -> Option { + self.names.get(name).map(|categories| { + categories + .iter() + .map(|category| category.as_str()) + .collect::>() + .join(",") + }) + } +} + /// One box's host-side admission state, built from its row and keyed by the /// row's switch address: the names the row declared (normalized to the form /// DNS names are matched in), the infrastructure deny set its answers are @@ -442,6 +506,12 @@ struct BoxPins { /// admitted it — the per-name cap counts by owner — and the instant its /// window ends. admitted: Mutex>, + /// The node plane's grant, for the entry built for the in-VM daemon's + /// own row once the gate adopted an in-force baseline: its names join + /// [`Self::names`], its subnets are the rebinding allowance, and every + /// pin it takes is logged against the entry that named it. `None` for + /// every box's entry. + node_plane: Option>, /// The box's flow table: the flows it opened through a pin — established /// while the window held, retained past it until the flow ends, each /// with the instant its last frame rode it (the idle bound's clock) — @@ -487,13 +557,26 @@ impl BoxPins { /// Builds one box's admission state from its row and the switch's /// address plan — the names the row declared, the infrastructure set the /// intersection subtracts, and the tables the replies fill. - fn built(record: &Arc, subnet: SwitchSubnet) -> Self { + /// + /// `node_plane` is the node plane's grant when `record` is the in-VM + /// daemon's own row and the gate adopted an in-force baseline: its names + /// join the row's own. + fn built( + record: &Arc, + subnet: SwitchSubnet, + node_plane: Option>, + ) -> Self { Self { record: Arc::clone(record), names: record .allow_dns_hosts() .iter() .map(|host| normalized(host)) + .chain( + node_plane + .iter() + .flat_map(|grant| grant.names.keys().cloned()), + ) .collect(), infrastructure: InfrastructureDenySet::new( record.egress().resolver(), @@ -501,6 +584,7 @@ impl BoxPins { ), outstanding: Mutex::new(HashMap::new()), admitted: Mutex::new(HashMap::new()), + node_plane, flows: Mutex::new(FlowTable::new()), at_cap_admitted: AtomicU64::new(0), sweeps: AtomicU64::new(0), @@ -696,10 +780,17 @@ impl BoxPins { // carve-out — an ordinary name is the only kind that can pin here — // so the same answer is refused like any other that resolved into // the plane, and the reach is the same: none. + // The node plane's own names are the baseline's grant, not the + // node row's: their allowance is the baseline's subnet endpoints, so + // an answer in private space pins only where an entry names it. + let allow = match &self.node_plane { + Some(grant) if grant.names.contains_key(&asked) => Some(grant.allow.as_slice()), + _ => self.record.egress().allow_subnets(), + }; let split = egress::rebinding_intersection( &answers, false, - self.record.egress().allow_subnets(), + allow, self.record.egress().deny_subnets(), &self.infrastructure, ); @@ -752,6 +843,7 @@ impl BoxPins { .filter(|admission| &*admission.name == name) .count(); let mut admitted_now = Vec::new(); + let mut refreshed = 0usize; let mut over_cap = Vec::new(); for address in addresses { if let Some(existing) = admitted.get_mut(address) { @@ -759,6 +851,7 @@ impl BoxPins { // under whichever owner first admitted the address. existing.expires = expires; admitted_now.push(*address); + refreshed += 1; continue; } if held >= DNS_MAX_ADDRESSES_PER_NAME { @@ -796,6 +889,42 @@ impl BoxPins { if admitted_now.is_empty() { return; } + let addresses: Vec = admitted_now.iter().copied().map(Ipv4Addr::from).collect(); + // The node plane's own lookup for a baseline entry: one line per pin + // and per re-pin, naming the entry, the addresses, the lookup's + // source and the resolver that answered it — the line a bundle's + // daemon log reads the node plane's reach by. + if let Some(entry) = self + .node_plane + .as_ref() + .and_then(|grant| grant.entries_naming(name)) + { + let source = self.record.switch_addr(); + let resolver = Ipv4Addr::from(self.record.egress().resolver()); + let window_secs = DNS_ADMISSION_WINDOW.as_secs(); + if refreshed == admitted_now.len() { + tracing::info!( + entry = %entry, + name, + ?addresses, + lookup_source = %source, + resolver = %resolver, + window_secs, + "re-pinned a node-plane baseline entry's addresses from the node's own lookup" + ); + } else { + tracing::info!( + entry = %entry, + name, + ?addresses, + lookup_source = %source, + resolver = %resolver, + window_secs, + "pinned a node-plane baseline entry's addresses from the node's own lookup" + ); + } + return; + } if !self.logged_first_pin.swap(true, Ordering::Relaxed) { tracing::info!( switch_addr = %self.record.switch_addr(), @@ -806,7 +935,6 @@ impl BoxPins { lookups received" ); } - let addresses: Vec = admitted_now.iter().copied().map(Ipv4Addr::from).collect(); tracing::debug!( switch_addr = %self.record.switch_addr(), namespace = %self.record.name(), @@ -984,6 +1112,18 @@ struct Inner { /// logged again. Cleared beside the entries at [`DnsPins::retire`], /// the same lifetime. refused: Mutex>>, + /// The node plane's grant, adopted from the gate's in-force baseline + /// ([`DnsPins::adopt_baseline`]): `None` until one is adopted, and + /// `None` again for an announced baseline, whose node row decides the + /// node plane instead. + node_plane: Mutex>>, + /// The resident `host_ip` boxes' entry (NET-085): built from the + /// registry's record of their declared names + /// ([`BoxTable::host_ip_cohort`]), fed by the node address's own + /// lookups, and read only for a box-marked frame. Kept apart from the + /// node row's entry, so a pin a box's name took never admits the node + /// plane's own frames, and a baseline pin never admits a box's. + host_ip: Mutex>>, /// How many frames the gate admitted because a live pin named the /// destination (the observability counter: what the host-side decision /// passed). @@ -1005,6 +1145,8 @@ impl DnsPins { subnet, boxes: Mutex::new(HashMap::new()), refused: Mutex::new(HashMap::new()), + node_plane: Mutex::new(None), + host_ip: Mutex::new(None), admitted_by_pin: AtomicU64::new(0), refused_for_want_of_pin: AtomicU64::new(0), }), @@ -1030,7 +1172,8 @@ impl DnsPins { /// make, and a forged reply wearing whatever address the row named /// would pin through it. fn entry(&self, record: &Arc) -> Option> { - if record.allow_dns_hosts().is_empty() { + let node_plane = self.node_plane_of(record); + if record.allow_dns_hosts().is_empty() && node_plane.is_none() { return None; } let plan_resolver = self.inner.subnet.dns_server().octets(); @@ -1044,16 +1187,80 @@ impl DnsPins { .boxes .lock() .expect("the admission table's lock is held only across this lookup"); + let same_grant = |entry: &BoxPins| match (&entry.node_plane, &node_plane) { + (None, None) => true, + (Some(held), Some(now)) => Arc::ptr_eq(held, now), + _ => false, + }; match boxes.get(&key) { - Some(entry) if Arc::ptr_eq(&entry.record, record) => Some(Arc::clone(entry)), + Some(entry) if Arc::ptr_eq(&entry.record, record) && same_grant(entry) => { + Some(Arc::clone(entry)) + } _ => { - let entry = Arc::new(BoxPins::built(record, self.inner.subnet)); + let entry = Arc::new(BoxPins::built(record, self.inner.subnet, node_plane)); boxes.insert(key, Arc::clone(&entry)); Some(entry) } } } + /// The node plane's grant for `record`: the adopted grant when `record` + /// is the in-VM daemon's own row, `None` for every other row. + fn node_plane_of(&self, record: &Arc) -> Option> { + self.inner + .node_plane + .lock() + .expect("the node-plane grant's lock is held only across this read") + .as_ref() + .filter(|grant| grant.node_addr == record.switch_addr().octets()) + .map(Arc::clone) + } + + /// Adopts the gate's node-plane baseline: in force, its named endpoints + /// become the names the node plane's own lookups pin against, each + /// against the entry that named it (NET-130); announced, the node plane + /// pins nothing, because its interim row decides it. An unchanged + /// baseline keeps the grant the node's pins were taken under, and a + /// changed one replaces it, so the node's next lookup starts a fresh + /// entry and no address an old entry named survives the change. + pub(crate) fn adopt_baseline(&self, baseline: &NodePlaneBaseline) { + let mut held = self + .inner + .node_plane + .lock() + .expect("the node-plane grant's lock is held only across this update"); + if baseline.phase() != NodeBaselinePhase::InForce { + *held = None; + return; + } + if held.as_ref().is_some_and(|grant| { + grant.node_addr == baseline.node_addr() && grant.entries == baseline.entries() + }) { + return; + } + *held = Some(Arc::new(NodePlaneNames::of(baseline))); + } + + /// The in-force arm's pin check for a frame from the in-VM daemon's own + /// address: whether a live pin the node plane's own lookups took for a + /// baseline entry names `dst`, or the flow such a pin established still + /// holds it. The baseline is adopted first, so the gate's decision and + /// the pins always read the same grant. A table with no node row pins + /// nothing for the node, and the frame is refused. + pub(crate) fn admits_node_frame( + &self, + table: &BoxTable, + baseline: &NodePlaneBaseline, + dst: [u8; 4], + pkt: Option<&L4Packet>, + now: Instant, + ) -> bool { + self.adopt_baseline(baseline); + table + .by_source(baseline.node_addr()) + .is_some_and(|record| self.admits_frame(&record, dst, pkt, now)) + } + /// Says a refused row's resolver mismatch, once per row: the row whose /// resolver is not the plan's own is logged the first time the table sees /// it, naming the box, the resolver the row names and the plan's own — @@ -1112,6 +1319,11 @@ impl DnsPins { if pkt.src.port() != DNS_PORT { return; } + if *pkt.dst.ip() == table.subnet().daemon_ip() + && let Some(entry) = self.host_ip_entry(table) + { + entry.observe(pkt, datagram, limiter, now); + } let Some(record) = table.by_source(pkt.dst.ip().octets()) else { return; }; @@ -1132,11 +1344,18 @@ impl DnsPins { /// leg keeps. A query over TCP is never recorded — the UDP datagram is /// the one DNS path v1 carries and the one a reply can pin by — and a /// query from a source no row holds records nothing either. + /// + /// `box_cohort_marked` is the query frame's own mark. A query from the + /// node address records into exactly one entry, so the name split + /// matches the address split (NET-085): a marked query into the + /// resident `host_ip` boxes' entry, and an unmarked one into the node + /// row's entry. A reply pins only in the entry that recorded its query. pub(crate) fn observe_query( &self, table: &BoxTable, pkt: &L4Packet, datagram: &[u8], + box_cohort_marked: bool, now: Instant, ) { if pkt.proto != IPPROTO_UDP @@ -1145,6 +1364,12 @@ impl DnsPins { { return; } + if *pkt.src.ip() == table.subnet().daemon_ip() && box_cohort_marked { + if let Some(entry) = self.host_ip_entry(table) { + entry.record_query(pkt, datagram, now); + } + return; + } let Some(record) = table.by_source(pkt.src.ip().octets()) else { return; }; @@ -1154,6 +1379,62 @@ impl DnsPins { entry.record_query(pkt, datagram, now); } + /// The resident `host_ip` boxes' entry, built from the registry's + /// record of their declared names on first sight and rebuilt when that + /// record changes: a box that leaves takes its names' pins with it. + /// `None` when no resident box declared a DNS host, or when the + /// record's resolver is not the plan's own. + fn host_ip_entry(&self, table: &BoxTable) -> Option> { + let mut held = self + .inner + .host_ip + .lock() + .expect("the host_ip entry's lock is held only across this lookup"); + let Some(record) = table.host_ip_cohort() else { + *held = None; + return None; + }; + let plan_resolver = self.inner.subnet.dns_server().octets(); + if record.egress().resolver() != plan_resolver { + *held = None; + drop(held); + self.refuse_resolver(&record, plan_resolver); + return None; + } + match held.as_ref() { + Some(entry) if Arc::ptr_eq(&entry.record, &record) => Some(Arc::clone(entry)), + _ => { + let entry = Arc::new(BoxPins::built(&record, self.inner.subnet, None)); + *held = Some(Arc::clone(&entry)); + Some(entry) + } + } + } + + /// The pin arm for a box-marked frame from the node address (NET-085): + /// whether a live pin a resident `host_ip` box's declared name took + /// names `dst`, or the flow such a pin established still holds it. + /// Counted with every other pin decision. + pub(crate) fn admits_host_ip_frame( + &self, + table: &BoxTable, + dst: [u8; 4], + pkt: Option<&L4Packet>, + now: Instant, + ) -> bool { + let admitted = self + .host_ip_entry(table) + .is_some_and(|entry| entry.admits(dst, pkt, now)); + if admitted { + self.inner.admitted_by_pin.fetch_add(1, Ordering::Relaxed); + } else { + self.inner + .refused_for_want_of_pin + .fetch_add(1, Ordering::Relaxed); + } + admitted + } + /// The verdict's pin arm: whether the gate may lift the row's /// undeclared-destination drop for the frame whose destination is `dst` /// and whose L4 addressing was `pkt` — a live pin names the destination, @@ -1381,11 +1662,14 @@ pub(crate) mod tests { use hickory_proto::rr::rdata::{A, AAAA, CNAME, HTTPS, SVCB}; use hickory_proto::rr::{Name, RData, Record, RecordType}; use sessions::EgressPolicy; - use sessions::core::egress::{DNS_ADMISSION_WINDOW, DNS_FLOW_IDLE_CAP, DNS_MAX_FLOWS_PER_BOX}; + use sessions::core::egress::{ + DNS_ADMISSION_WINDOW, DNS_FLOW_IDLE_CAP, DNS_MAX_ADDRESSES_PER_NAME, DNS_MAX_FLOWS_PER_BOX, + }; use switch::SwitchSubnet; use super::{DnsPins, IPPROTO_TCP, TCP_FIN, deny_all_refusal, udp_datagram}; use crate::box_registry::{BoxRegistration, BoxRegistry, BoxTable}; + use crate::net::baseline::{DAEMON_CACHE_ENDPOINT, NodePlaneBaseline}; use crate::net::egress_gate::DropLimiter; /// The address plan every registry below is built for — the same plan the @@ -1585,6 +1869,19 @@ pub(crate) mod tests { /// sockets: from the box, on its lookup's own source port, to the plan's /// resolver, in the datagram shape the relay hands the table. fn lookup(pins: &DnsPins, table: &BoxTable, lease: [u8; 4], name: &str, now: Instant) { + lookup_marked(pins, table, lease, name, false, now); + } + + /// [`lookup`] with the query frame's box-cohort mark spelled out: a + /// marked query from the node address is a resident `host_ip` box's. + fn lookup_marked( + pins: &DnsPins, + table: &BoxTable, + lease: [u8; 4], + name: &str, + marked: bool, + now: Instant, + ) { let query = udp_payload_frame( Ipv4Addr::from(lease), 40000, @@ -1594,7 +1891,7 @@ pub(crate) mod tests { ); let (pkt, datagram) = udp_datagram(&query) .expect("the test's query frame parses as the egress leg parses it"); - pins.observe_query(table, &pkt, datagram, now); + pins.observe_query(table, &pkt, datagram, marked, now); } /// Observes one reply the switch returned toward `lease`, answering @@ -1672,6 +1969,368 @@ pub(crate) mod tests { observe_datagram(pins, table, lease, reply, limiter, now); } + /// Drives one of the node plane's own lookups — the in-VM daemon's query + /// from its own address to the plan's resolver, then the reply the + /// resolver returned toward it — through both legs of the table, after + /// adopting `baseline` the way the gate does at VM start. + pub(crate) fn node_resolves( + pins: &DnsPins, + table: &BoxTable, + baseline: &NodePlaneBaseline, + name: &str, + answers: &[Ipv4Addr], + now: Instant, + ) { + pins.adopt_baseline(baseline); + let node = baseline.node_addr(); + lookup(pins, table, node, name, now); + observe(pins, table, node, name, answers, &DropLimiter::new(), now); + } + + /// Drives one lookup a resident `host_ip` box makes from the node + /// address: the query leaves box-marked, and the reply returns to the + /// node address. + fn host_ip_resolves( + pins: &DnsPins, + table: &BoxTable, + name: &str, + answers: &[Ipv4Addr], + now: Instant, + ) { + let node = SUBNET.daemon_ip().octets(); + lookup_marked(pins, table, node, name, true, now); + observe(pins, table, node, name, answers, &DropLimiter::new(), now); + } + + /// NET-085: a resident `host_ip` box's declared names pin in an entry + /// of their own, fed by the node address's lookups. Its pins admit a + /// box-marked frame and never the node plane's own frames, a baseline + /// pin never admits a box-marked frame, and the box's retire takes its + /// pins with it. + #[test] + fn host_ip_box_names_pin_apart_from_the_node_plane() { + let registry = BoxRegistry::new(SUBNET); + registry.register_node_namespace(7654); + let table = registry.table(); + let baseline = NodePlaneBaseline::built_in(SUBNET); + let pins = DnsPins::new(SUBNET); + let now = Instant::now(); + let session = sessions::SessionId::parse_str(&uuid::Uuid::new_v4().to_string()).unwrap(); + assert!(registry.admit_host_ip_box( + session, + Some(&EgressPolicy { + allow_protocols: None, + allow_subnets: Some(Vec::new()), + allow_dns_hosts: Some(vec!["example.com".to_string()]), + deny_subnets: None, + }), + )); + let site = Ipv4Addr::new(203, 0, 113, 50); + assert!( + !pins.admits_host_ip_frame(&table, site.octets(), None, now), + "nothing is pinned before a lookup" + ); + host_ip_resolves(&pins, &table, "example.com", &[site], now); + assert!( + pins.admits_host_ip_frame(&table, site.octets(), None, now), + "the box's declared name pins its answer for a box-marked frame" + ); + assert!( + !pins.admits_node_frame(&table, &baseline, site.octets(), None, now), + "a box's name never admits the node plane's own frames" + ); + let cache = Ipv4Addr::new(203, 0, 113, 10); + node_resolves( + &pins, + &table, + &baseline, + DAEMON_CACHE_ENDPOINT, + &[cache], + now, + ); + assert!(pins.admits_node_frame(&table, &baseline, cache.octets(), None, now)); + assert!( + !pins.admits_host_ip_frame(&table, cache.octets(), None, now), + "a baseline pin never admits a box-marked frame" + ); + assert!(registry.retire_host_ip_box(session)); + assert!( + !pins.admits_host_ip_frame(&table, site.octets(), None, now), + "a retired box's pins go with it" + ); + } + + /// NET-085: an attach re-admits its `host_ip` box with the rules the set + /// already holds. That changes nothing: the cohort record stays the same + /// `Arc`, so a pin another resident box took under it stays. + #[test] + fn host_ip_readmit_with_equal_rules_keeps_the_cohort_pins() { + let registry = BoxRegistry::new(SUBNET); + registry.register_node_namespace(7654); + let table = registry.table(); + let pins = DnsPins::new(SUBNET); + let now = Instant::now(); + let policy = |host: &str| EgressPolicy { + allow_protocols: None, + allow_subnets: Some(Vec::new()), + allow_dns_hosts: Some(vec![host.to_string()]), + deny_subnets: None, + }; + let box_a = sessions::SessionId::parse_str(&uuid::Uuid::new_v4().to_string()).unwrap(); + let box_b = sessions::SessionId::parse_str(&uuid::Uuid::new_v4().to_string()).unwrap(); + assert!(registry.admit_host_ip_box(box_a, Some(&policy("a.example.com")))); + assert!(registry.admit_host_ip_box(box_b, Some(&policy("b.example.com")))); + let site = Ipv4Addr::new(203, 0, 113, 60); + host_ip_resolves(&pins, &table, "b.example.com", &[site], now); + assert!(pins.admits_host_ip_frame(&table, site.octets(), None, now)); + let before = table.host_ip_cohort().expect("both boxes declare a name"); + + assert!(registry.admit_host_ip_box(box_a, Some(&policy("a.example.com")))); + let after = table.host_ip_cohort().expect("both boxes declare a name"); + assert!( + Arc::ptr_eq(&before, &after), + "a re-admit with equal rules keeps the cohort record" + ); + assert!( + pins.admits_host_ip_frame(&table, site.octets(), None, now), + "a re-admit with equal rules keeps the other box's pin" + ); + + assert!(registry.admit_host_ip_box(box_a, Some(&policy("c.example.com")))); + let changed = table.host_ip_cohort().expect("both boxes declare a name"); + assert!( + !Arc::ptr_eq(&before, &changed), + "a re-admit with other rules rebuilds the cohort record" + ); + } + + /// NET-085: the node address's lookups split by the query frame's mark, + /// the same way its frames split. A marked query for a baseline name + /// never pins for the node plane, and an unmarked query for a name a + /// resident `host_ip` box declared never pins for that box. + #[test] + fn node_address_queries_pin_only_in_the_entry_their_mark_names() { + let registry = BoxRegistry::new(SUBNET); + registry.register_node_namespace(7654); + let table = registry.table(); + let baseline = NodePlaneBaseline::built_in(SUBNET); + let pins = DnsPins::new(SUBNET); + let now = Instant::now(); + assert!(registry.admit_host_ip_box( + sessions::SessionId::parse_str(&uuid::Uuid::new_v4().to_string()).unwrap(), + Some(&EgressPolicy { + allow_protocols: None, + allow_subnets: Some(Vec::new()), + allow_dns_hosts: Some(vec!["example.com".to_string()]), + deny_subnets: None, + }), + )); + pins.adopt_baseline(&baseline); + let cache = Ipv4Addr::new(203, 0, 113, 10); + host_ip_resolves(&pins, &table, DAEMON_CACHE_ENDPOINT, &[cache], now); + assert!( + !pins.admits_node_frame(&table, &baseline, cache.octets(), None, now), + "a marked query for a baseline name never pins for the node plane" + ); + let site = Ipv4Addr::new(203, 0, 113, 50); + node_resolves(&pins, &table, &baseline, "example.com", &[site], now); + assert!( + !pins.admits_host_ip_frame(&table, site.octets(), None, now), + "an unmarked query for a host_ip box's name never pins for that box" + ); + } + + /// NET-130 and NET-085's node-plane half: the node plane's own lookups + /// for a baseline entry pin the addresses they resolved to against that + /// entry, and nothing else does. Before any lookup no address admits; + /// after the daemon's own lookup of the cache's name, its answers do and + /// only they; a name no entry names pins nothing, an answer inside + /// private space pins nothing, a reply to no query pins nothing, a box's + /// lookup of the same name pins nothing for the node, and the pins hold + /// for the shared window under the shared per-name cap. Every pin and + /// every re-pin is one `info` line naming the entry, the addresses and + /// the lookup's source. + #[test] + fn baseline_entry_pins_the_daemon_lookup_answers() { + let (log, _guard) = crate::net::egress_gate::test_support::capture_log(); + let registry = BoxRegistry::new(SUBNET); + registry.register_node_namespace(7654); + // A box that declared the same name: its pins are its own. + dns_box( + ®istry, + "weather", + LEASE, + vec![DAEMON_CACHE_ENDPOINT.to_string()], + Vec::new(), + ); + let table = registry.table(); + let baseline = NodePlaneBaseline::built_in(SUBNET); + let pins = DnsPins::new(SUBNET); + let now = Instant::now(); + let node_admits = |dst: Ipv4Addr, at: Instant| { + pins.admits_node_frame(&table, &baseline, dst.octets(), None, at) + }; + let cache = Ipv4Addr::new(203, 0, 113, 10); + + // Nothing is pinned before the node's own lookup. + assert!( + !node_admits(cache, now), + "no address is pinned before a lookup" + ); + + // The daemon's own lookup of the cache's name pins its answers, and + // only them. + node_resolves( + &pins, + &table, + &baseline, + DAEMON_CACHE_ENDPOINT, + &[cache], + now, + ); + assert!(node_admits(cache, now), "the lookup's answer is pinned"); + assert!( + !node_admits(Ipv4Addr::new(203, 0, 113, 11), now), + "an address the answer did not name is not pinned" + ); + wait_for_line(&log, "pinned a node-plane baseline entry's addresses"); + let logged = log.contents(); + assert!( + logged.contains("entry=cache") + && logged.contains("addresses=[203.0.113.10]") + && logged.contains("lookup_source=100.64.255.253") + && logged.contains(&format!("resolver={}", SUBNET.dns_server())), + "the pin line names the entry, the addresses and the lookup's source, got: {logged}" + ); + + // The same lookup again re-pins, and says so. + node_resolves( + &pins, + &table, + &baseline, + DAEMON_CACHE_ENDPOINT, + &[cache], + now, + ); + assert!( + log.contents() + .contains("re-pinned a node-plane baseline entry's addresses"), + "a re-pin is logged, got: {}", + log.contents() + ); + + // A name no entry names pins nothing for the node. + let other = Ipv4Addr::new(198, 51, 100, 20); + node_resolves(&pins, &table, &baseline, "example.com", &[other], now); + assert!( + !node_admits(other, now), + "a name no entry names pins nothing" + ); + + // An answer in private space is the rebinding intersection's refusal. + let private = Ipv4Addr::new(10, 1, 2, 3); + node_resolves( + &pins, + &table, + &baseline, + DAEMON_CACHE_ENDPOINT, + &[private], + now, + ); + assert!(!node_admits(private, now), "a private answer never pins"); + + // A reply that answers no query of the node's pins nothing. + let unasked = Ipv4Addr::new(203, 0, 113, 30); + observe( + &pins, + &table, + baseline.node_addr(), + DAEMON_CACHE_ENDPOINT, + &[unasked], + &DropLimiter::new(), + now, + ); + assert!( + !node_admits(unasked, now), + "an unsolicited reply pins nothing" + ); + + // A box's lookup of the same name pins for the box, never the node. + let boxed = Ipv4Addr::new(203, 0, 113, 40); + resolve( + &pins, + &table, + LEASE, + DAEMON_CACHE_ENDPOINT, + &[boxed], + &DropLimiter::new(), + now, + ); + assert!( + !node_admits(boxed, now), + "a box's answer is not the node's pin" + ); + + // The shared window: past it, the pin admits no new flow. + assert!( + !node_admits(cache, now + DNS_ADMISSION_WINDOW), + "the pin holds for the shared window and no longer" + ); + + // The shared per-name cap: answers past it are refused. + let many: Vec = (1..=40).map(|n| Ipv4Addr::new(198, 51, 100, n)).collect(); + let later = now + DNS_ADMISSION_WINDOW + Duration::from_secs(1); + node_resolves( + &pins, + &table, + &baseline, + DAEMON_CACHE_ENDPOINT, + &many, + later, + ); + let pinned = many + .iter() + .filter(|addr| node_admits(**addr, later)) + .count(); + assert_eq!(pinned, DNS_MAX_ADDRESSES_PER_NAME, "the per-name cap holds"); + + // An announced baseline pins nothing: the node row decides there. + let announced = NodePlaneBaseline::built_in(SUBNET).announced(); + pins.adopt_baseline(&announced); + let fresh = Ipv4Addr::new(203, 0, 113, 50); + lookup( + &pins, + &table, + baseline.node_addr(), + DAEMON_CACHE_ENDPOINT, + later, + ); + observe( + &pins, + &table, + baseline.node_addr(), + DAEMON_CACHE_ENDPOINT, + &[fresh], + &DropLimiter::new(), + later, + ); + pins.adopt_baseline(&baseline); + assert!( + !node_admits(fresh, later), + "a lookup under an announced baseline pins nothing for the node" + ); + } + + /// Panics unless `log` holds `needle`: the table logs synchronously, so + /// the line is there by the time the call that wrote it returns. + fn wait_for_line(log: &crate::net::egress_gate::test_support::CaptureWriter, needle: &str) { + let logged = log.contents(); + assert!( + logged.contains(needle), + "expected `{needle}` in the log, got: {logged}" + ); + } + /// The table admits exactly the answers the box's own lookup received, /// and nothing else. Before any reply nothing admits; after one, the /// answers do — and only they: an address the answer did not name is diff --git a/crates/minvmd/src/net/egress_gate.rs b/crates/minvmd/src/net/egress_gate.rs index fe414de529..f06a9a9a7a 100644 --- a/crates/minvmd/src/net/egress_gate.rs +++ b/crates/minvmd/src/net/egress_gate.rs @@ -175,9 +175,7 @@ use tokio::task::{JoinHandle, JoinSet}; use crate::bep_attach::{BoxId, BoxIdText}; use crate::box_registry::{BoxRecord, BoxTable, RowWithdrawal}; -use super::baseline::{ - NodeBaselinePhase, NodePlaneBaseline, REGISTRY_CACHE_FETCH_PORTS, REGISTRY_CACHE_FETCH_PROTOCOL, -}; +use super::baseline::{NodeBaselinePhase, NodePlaneBaseline}; use super::dns_pins; use super::forward_revoke::{self, ForwardedFlows}; @@ -438,6 +436,32 @@ const DROP_WARN_MAX_TRACKED_PAIRS: usize = 1024; /// simply is not one the host published. const UNKNOWN_SOURCE_RULE: &str = "egress-unknown-source"; +/// The rule name for a frame from the in-VM daemon's own address, under the +/// in-force node-plane baseline, toward an address no baseline entry pinned +/// (NET-085, NET-130), or toward a pinned address at another protocol or +/// port: the node plane reaches the addresses its own lookups resolved a +/// registry or cache name to at TCP 443, the entries' subnets, the host +/// alias at the configured host exposures, the resolver and the boxes' +/// leases, and nothing else. +const NODE_PLANE_UNPINNED_RULE: &str = "egress-node-plane-unpinned"; + +/// The IP protocol a pinned baseline address is reached over: TCP alone, +/// because the registry and the cache the daemon fetches from serve HTTPS +/// (`https://github.com/gominimal/pkgs`, `https://cache.minimal.dev/`). A +/// pin names an address, and a registry or cache name often resolves to a +/// shared CDN address, so without this bound the pin would open that address +/// at every protocol and port. QUIC is not admitted: carrying it would be a +/// stated choice that adds UDP here. An endpoint spelled as a subnet is the +/// deployment's own allowance and is decided by the set's compiled rules, +/// without this bound. +const NODE_PLANE_FETCH_PROTOCOL: u8 = egress::IPPROTO_TCP; + +/// The ports a pinned baseline address is reached at, over +/// [`NODE_PLANE_FETCH_PROTOCOL`]: the HTTPS port the daemon's registry and +/// cache fetches use. Any other port to a pinned address is +/// [`NODE_PLANE_UNPINNED_RULE`]'s drop, the same as an address no pin names. +const NODE_PLANE_FETCH_PORTS: [u16; 1] = [443]; + /// The rule name for a frame headed to the switch's own address — the plan's /// gateway — that is not a resolver query: TCP or UDP to the resolver's port /// is the one carve-out the frame rules admit there, and everything else — @@ -1083,11 +1107,15 @@ impl EgressGate { // bundle's daemon log tail holds to see which categories the node // plane may reach and under which posture the gate decides them // (NET-130). + // The pins the node plane's own lookups take are adopted here, from + // the same baseline the gate decides by, so the two never disagree. + pins.adopt_baseline(&baseline); tracing::info!( baseline = %baseline.render(), node_addr = %Ipv4Addr::from(baseline.node_addr()), node_baseline_phase = baseline.phase().as_str(), - "node-plane baseline set at the gate", + "node-plane baseline set at the gate; each entry's addresses are pinned from the \ + node's own lookups", ); // One limiter for the whole gate: a guest that reconnects must not // reset the rate window its drops are counted in. @@ -4825,6 +4853,19 @@ async fn relay_frames_to_switch( } continue; } + Err(GateDrop::NodePlaneUnpinned { src, dst, dst_port }) => { + limiter.warn_node_plane_unpinned(src, dst, dst_port); + continue; + } + Err(GateDrop::HostIpUnbounded { + src, + dst, + dst_port, + resident, + }) => { + limiter.warn_host_ip_unbounded(src, dst, dst_port, resident); + continue; + } Err(GateDrop::Verdict(reason @ DropReason::UndeclaredSubnet { dst, .. })) => { // The row's own rules refused the destination, and the line // names it beside the source and the rule: the destination @@ -4858,7 +4899,9 @@ async fn relay_frames_to_switch( // frameless and unpublishable for the rest of the VM's life, over a // shuttle close its control path rode out. let src = match admitted { - GateAdmit::Baseline | GateAdmit::Row | GateAdmit::ProxyLane => summary.source(), + GateAdmit::Baseline | GateAdmit::Row | GateAdmit::ProxyLane | GateAdmit::HostIpBox => { + summary.source() + } }; if let Some(src) = src && src != baseline.node_addr() @@ -4944,7 +4987,13 @@ async fn relay_frames_to_switch( // the resolver's port — and only for a frame the queue took, which // is why this sits after the enqueue. if let Some((query, datagram)) = dns { - pins.observe_query(table, &query, datagram, Instant::now()); + pins.observe_query( + table, + &query, + datagram, + summary.box_cohort_marked(), + Instant::now(), + ); } } } @@ -5094,8 +5143,20 @@ enum GateAdmit { /// the row's own fact; nothing the guest says grants one, and nothing in /// it opens the address's other ports or protocols. ProxyLane, + /// A resident `host_ip` box's rules admitted a frame from the node + /// address that wears the box mark (NET-085, design §4.3 rule 2). The + /// guest's table marks a `host_ip` box's connections, and root in the + /// guest can mark any frame, so the mark only says which rules decide + /// the frame: the union of the resident `host_ip` boxes' rules, which + /// the host-side client registered ([`BoxTable::host_ip_boxes`]). + HostIpBox, } +/// The rule a box-marked frame from the node address drops under when no +/// resident `host_ip` box's rules admit it, or when no `host_ip` box is +/// resident: the rate-limit key and the drop line's `rule_matched`. +pub(crate) const HOST_IP_BOXES_RULE: &str = "egress-host-ip-boxes"; + /// How long a bare SYN the ingress leg delivered at a port no publish dials /// stays refusable ([`ReplyTables::observe_refusable`]): the refusal follows /// the SYN at once, and a retransmitted SYN notes the connect afresh, so the @@ -5652,16 +5713,104 @@ fn reply_tuple_of(pkt: &dns_pins::L4Packet) -> egress::FlowTuple { ) } +/// The in-force node-plane decision for a frame from the in-VM daemon's own +/// address (NET-085, NET-130). The frame is admitted to: +/// +/// - an address the node plane's own lookups pinned for a baseline entry's +/// name, or a flow such a pin established ([`dns_pins::DnsPins::admits_node_frame`]), +/// at [`NODE_PLANE_FETCH_PROTOCOL`] and [`NODE_PLANE_FETCH_PORTS`] only; +/// - a subnet a baseline entry names, and the resolver carve-out, both by the +/// set's compiled rules; +/// - the host alias at the configured host exposures, over TCP or UDP; +/// - a box's lease the registry holds, which the target box's ingress +/// decides on the switch's side, the reach the node's hostname proxy and +/// its answers to the boxes ride. +/// +/// Everything else is refused. The infrastructure deny set is decided first, +/// the Box Egress Proxy's address is refused because no category names it, +/// and a destination no rule admits and no pin names is +/// [`NODE_PLANE_UNPINNED_RULE`]'s drop. +fn node_plane_verdict( + summary: &FrameSummary, + l4: Option<&dns_pins::L4Packet>, + table: &BoxTable, + baseline: &NodePlaneBaseline, + pins: &dns_pins::DnsPins, +) -> Result { + let Some(src) = summary.source() else { + return Err(GateDrop::Verdict(family_drop(summary.family()))); + }; + // ARP and every frame with no destination address: the set's verdict + // decides them, the way a row's decides a box's. + let Some(dst) = summary.destination() else { + return match egress::verdict(summary, baseline.rules()) { + FrameVerdict::Admit => Ok(GateAdmit::Baseline), + FrameVerdict::Drop(reason) => Err(GateDrop::Verdict(reason)), + }; + }; + let dst_port = summary.destination_port(); + let subnet = table.subnet(); + if dst == subnet.box_egress_proxy_address().octets() { + return Err(GateDrop::ProxyLane { src, dst, dst_port }); + } + let transport_port = matches!( + summary.protocol(), + Some(egress::IPPROTO_TCP | egress::IPPROTO_UDP) + ) + .then_some(dst_port); + if infrastructure_destination( + dst, + transport_port, + subnet, + baseline.rules().allow_subnets(), + false, + baseline.host_exposure_ports(), + ) { + return Err(GateDrop::Infrastructure { src, dst, dst_port }); + } + // Past the infrastructure check, the alias is at a configured host + // exposure: the one opening the alias has. + if dst == subnet.host_alias().octets() { + return Ok(GateAdmit::Baseline); + } + // A box's lease: the target's ingress decides it on the switch's side. + if dst != src + && table + .by_source(dst) + .is_some_and(|record| record.switch_addr().octets() == dst) + { + return Ok(GateAdmit::Baseline); + } + match egress::verdict(summary, baseline.rules()) { + FrameVerdict::Admit => Ok(GateAdmit::Baseline), + FrameVerdict::Drop(DropReason::UndeclaredSubnet { .. }) + if summary.protocol() == Some(NODE_PLANE_FETCH_PROTOCOL) + && NODE_PLANE_FETCH_PORTS.contains(&dst_port) + && pins.admits_node_frame(table, baseline, dst, l4, Instant::now()) => + { + Ok(GateAdmit::Baseline) + } + FrameVerdict::Drop(DropReason::UndeclaredSubnet { .. }) => { + Err(GateDrop::NodePlaneUnpinned { src, dst, dst_port }) + } + FrameVerdict::Drop(reason) => Err(GateDrop::Verdict(reason)), + } +} + /// The gate's admit-or-drop decision for one frame summary against the /// host-side table (NET-081): pure — a function of the summary, the table, /// the baseline set, the phase, the DNS admission table and the reply-flow /// tables, nothing else — and deliberately separate from the relay loop that /// applies it, the same discipline the shared verdict keeps. /// -/// The frame's source address is the whole of the routing: the node plane's -/// own address is decided by the baseline set ([`NodePlaneBaseline`], the -/// helper's enumeration of the categories the in-VM daemon's traffic may -/// reach, NET-130), the published namespace that holds any other source +/// The frame's source address is the routing, with one split at the node +/// address: the node plane's own frames there are decided by the baseline +/// set ([`NodePlaneBaseline`], the helper's enumeration of the categories +/// the in-VM daemon's traffic may reach, NET-130), and a frame the guest's +/// classifier marked as a box's (`BOX_COHORT_DSCP`, written from the +/// sender's cgroup) is decided by the union of the resident `host_ip` +/// boxes' rules ([`host_ip_verdict`]), never by the node row. The +/// published namespace that holds any other source /// supplies the rules its frames are decided by, and an address neither /// covers is the phase's to decide — [`UnregisteredSourcePhase`] carries what /// that means and why. The node plane is decided before the table because the @@ -5728,38 +5877,6 @@ fn gate_verdict( let Some(src) = summary.source() else { return Err(GateDrop::Verdict(family_drop(summary.family()))); }; - // The node plane's own frames, when the baseline is in force: the shared - // verdict against the enumeration's compiled set — the lease it is keyed - // to is the daemon's own address, so the check above and the lease agree - // on who may wear it. Announced, the interim node row keeps deciding the - // node plane, and the fall-through reaches it below. The set's verdict - // is by subnet alone, so the alias the built-in endpoints name is the - // arm's to bound: TCP at a fetch port reaches the shared verdict, and - // any other protocol or port is the infrastructure drop, the same - // refusal a box row's alias frames take. The protocol is checked - // because the summary reads a destination port from every protocol's - // bytes. In force, the node plane reaches the alias at TCP 443 only. - // Once the host exposures close by default, that is wider than the - // announced phase's alias reach, which admits no port. The bound must - // also admit the configured host exposures (TCP and UDP) once they - // exist. gominimal/inbox#1144 and gominimal/inbox#1147 own that work. - if baseline.phase() == NodeBaselinePhase::InForce && src == baseline.node_addr() { - if let Some(dst) = summary.destination() - && dst == table.subnet().host_alias().octets() - && (summary.protocol() != Some(REGISTRY_CACHE_FETCH_PROTOCOL) - || !REGISTRY_CACHE_FETCH_PORTS.contains(&summary.destination_port())) - { - return Err(GateDrop::Infrastructure { - src, - dst, - dst_port: summary.destination_port(), - }); - } - return match egress::verdict(summary, baseline.rules()) { - FrameVerdict::Admit => Ok(GateAdmit::Baseline), - FrameVerdict::Drop(reason) => Err(GateDrop::Verdict(reason)), - }; - } // NET-040's answer half: a frame whose five-tuple exactly reverses a live // inbound flow this gate's ingress leg recorded — the connection a // published port received, delivered by this gate itself — passes @@ -5824,6 +5941,146 @@ fn gate_verdict( return Err(GateDrop::SwitchControlSurface { src, dst_port }); } } + // The node plane's own frames, when the baseline is in force (NET-085, + // NET-130): decided by the baseline set, never by the interim node row. + // The arm sits after the reply record, so the answers to connections the + // node's published ports received still pass, and after the + // control-surface check, so the switch's own ports stay refused. + // The node address is not the node plane's alone: a host-address box, + // the default network mode, leaves from it too. What tells them apart + // is the sender's cgroup, which the guest's classifier writes into the + // frame as `BOX_COHORT_DSCP` for a connection a box opened and clears + // on every other packet. An unmarked frame is the daemon's own, and the + // baseline set decides it, so a guest whose classifier never loaded + // fails closed to the baseline. + // The mark is no proof of a box: root in the guest can write it on any + // frame (NET-085). So a marked frame is decided by the union of the + // resident `host_ip` boxes' rules, which the host-side client + // registered, and dropped when no `host_ip` box is resident (design + // §4.3 rule 2). The node row never decides it. A process that writes + // the mark reaches only what a resident box's own rules admit. + if baseline.phase() == NodeBaselinePhase::InForce && src == baseline.node_addr() { + if summary.box_cohort_marked() { + return host_ip_verdict(summary, l4, table, baseline, pins, src); + } + return node_plane_verdict(summary, l4, table, baseline, pins); + } + row_verdict(summary, l4, table, baseline, pins, src) +} + +/// The decision for a box-marked frame from the node address under the +/// in-force baseline (NET-085, design §4.3 rule 2): admitted when one +/// resident `host_ip` box's rules admit it, and dropped under +/// [`HOST_IP_BOXES_RULE`] otherwise, and always when no `host_ip` box is +/// resident. +/// +/// Each resident box decides the frame the way a row decides its own: the +/// infrastructure deny set first, with the box's own `allow_subnets` as the +/// RFC 1918 allowance and the node's own block reach a `host_ip` box keeps, +/// then the box's compiled rules, then the DNS admission table's pins for +/// the names the resident boxes declared. The Box Egress Proxy's listener is +/// decided by the node row's lane declaration, as it was for every frame the +/// node row carried, because the proxy's attachment for the node address is +/// the node row's. +fn host_ip_verdict( + summary: &FrameSummary, + l4: Option<&dns_pins::L4Packet>, + table: &BoxTable, + baseline: &NodePlaneBaseline, + pins: &dns_pins::DnsPins, + src: [u8; 4], +) -> Result { + let dst = summary.destination(); + let dst_port = summary.destination_port(); + let unbounded = |resident: usize| GateDrop::HostIpUnbounded { + src, + dst: dst.unwrap_or_default(), + dst_port, + resident, + }; + let resident = table.host_ip_boxes(); + if resident.is_empty() { + return Err(unbounded(0)); + } + if let Some(dst) = dst + && dst == table.subnet().box_egress_proxy_address().octets() + { + let declared = table + .by_source(src) + .is_some_and(|record| record.declares_credentialed_upstream()); + let listener = egress::ProxyListener::at(dst); + if egress::proxy_lane_admits(declared, listener, summary) { + return Ok(GateAdmit::ProxyLane); + } + return Err(GateDrop::ProxyLane { src, dst, dst_port }); + } + let transport_port = matches!( + summary.protocol(), + Some(egress::IPPROTO_TCP | egress::IPPROTO_UDP) + ) + .then_some(dst_port); + let mut infrastructure = false; + // Whether a name-resolving box left the frame to the pins. The pins are + // one shared cohort entry, so the gate asks them once, after the loop, + // and the frame counts once. + let mut ask_pins = false; + for rules in &resident { + if let Some(dst) = dst + && infrastructure_destination( + dst, + transport_port, + table.subnet(), + rules.egress().allow_subnets(), + false, + baseline.host_exposure_ports(), + ) + { + infrastructure = true; + continue; + } + match egress::verdict(summary, rules.egress()) { + FrameVerdict::Admit => return Ok(GateAdmit::HostIpBox), + FrameVerdict::Drop(DropReason::UndeclaredSubnet { .. }) if rules.resolves_names() => { + ask_pins = true; + } + FrameVerdict::Drop(_) => {} + } + } + if ask_pins && pins.admits_host_ip_frame(table, dst.unwrap_or_default(), l4, Instant::now()) { + return Ok(GateAdmit::HostIpBox); + } + // Every resident box refused the frame. A destination every box's + // check put in the infrastructure set keeps that drop's own rule, so + // the line a host reads names the set rather than the boxes' rules. + if infrastructure + && let Some(dst) = dst + && resident.iter().all(|rules| { + infrastructure_destination( + dst, + transport_port, + table.subnet(), + rules.egress().allow_subnets(), + false, + baseline.host_exposure_ports(), + ) + }) + { + return Err(GateDrop::Infrastructure { src, dst, dst_port }); + } + Err(unbounded(resident.len())) +} + +/// The rest of [`gate_verdict`]'s order for a frame the node-plane arm did +/// not decide: the source's row, then the infrastructure set, then the proxy +/// lane, then the row's rules, then the pin arm. +fn row_verdict( + summary: &FrameSummary, + l4: Option<&dns_pins::L4Packet>, + table: &BoxTable, + baseline: &NodePlaneBaseline, + pins: &dns_pins::DnsPins, + src: [u8; 4], +) -> Result { // No namespace holds the source — NET-081's failure case, held // unconditionally, under no phase (NET-085, T89 #1925): an address no // namespace holds never leaves the VM, toward every destination alike, @@ -5865,15 +6122,12 @@ fn gate_verdict( // The host-alias refusal is a box row's alone: the node namespace's // row, keyed to the daemon's own address, carries the host-address // boxes' frames, whose `host.min.internal` resolves to the alias, and - // keeps own-block local reach to it until the in-force baseline (which - // returned above) decides the node plane. Announced, that reach is - // scoped by the baseline's configured host exposures, which the arm - // below reads for every row: the box rows' refusal stands at every - // port, and the node row's outside the exposures. The scoping holds only - // while the baseline is announced: in force, the arm above returns first - // and admits the alias only at TCP on `REGISTRY_CACHE_FETCH_PORTS` (443), - // with no host exposures, a gap gominimal/inbox#1144 and - // gominimal/inbox#1147 own. + // keeps own-block local reach to it while the baseline is announced + // (in force, the node-plane arm above returned first, with the same + // scoping). That reach is scoped by the baseline's configured host + // exposures, which the arm below reads for every row: the box rows' + // refusal stands at every port, and the node row's outside the + // exposures. let box_row = src != table.subnet().daemon_ip().octets(); // A non-first IPv4 fragment carries no readable L4 port — `summarize` // reports destination port 0 for it — so it fails the exposure match @@ -5885,9 +6139,9 @@ fn gate_verdict( // design — every input comes from the one frame being judged — and // fragment association would add per-flow state here (or reassembly // elsewhere) to recover a packet whose failure mode is a drop, not an - // exposure: deny-by-default keeps the safe direction. In force, the arm - // above admits no host exposure at all, so the fragment case arises only - // while the baseline is announced. + // exposure: deny-by-default keeps the safe direction. The in-force + // node-plane arm reads the exposures the same way, with the same + // fragment case. let transport_port = matches!( summary.protocol(), Some(egress::IPPROTO_TCP | egress::IPPROTO_UDP) @@ -5936,9 +6190,8 @@ fn gate_verdict( // protocol, neither of which the declaration ever opened, and the node // row under the announced baseline, which is the allow-all the in-force // enumeration replaces. - // The in-force baseline set needs no arm of its own: it never admits - // the address for any category, so its compiled rules refuse the frame - // where they refuse any undeclared destination. + // Under the in-force baseline the node plane never reaches this arm: + // its own arm above refuses the address under the same rule. if let Some(dst) = summary.destination() && dst == table.subnet().box_egress_proxy_address().octets() { @@ -6060,6 +6313,31 @@ enum GateDrop { /// The port the frame named there, `0` when it carried none. dst_port: u16, }, + /// The frame wore the in-VM daemon's own address under the in-force + /// baseline and named a destination no baseline entry pinned and no + /// other node-plane opening admits ([`NODE_PLANE_UNPINNED_RULE`]). + NodePlaneUnpinned { + /// The daemon's own address, the source the frame wore. + src: [u8; 4], + /// The destination no pin named. + dst: [u8; 4], + /// The port the frame named there, `0` when it carried none. + dst_port: u16, + }, + /// The frame wore the node's own address and the box mark, and no + /// resident `host_ip` box's rules admit it: the box mark is no proof of + /// a box, so nothing else may (NET-085, [`HOST_IP_BOXES_RULE`]). + HostIpUnbounded { + /// The node's own address, the source the frame wore. + src: [u8; 4], + /// The destination no resident box admits, `0.0.0.0` for a frame + /// with none. + dst: [u8; 4], + /// The port the frame named there, `0` when it carried none. + dst_port: u16, + /// How many `host_ip` boxes were resident: `0` when none was. + resident: usize, + }, } impl GateDrop { @@ -6073,6 +6351,8 @@ impl GateDrop { Self::UnregisteredSource { .. } => UNREGISTERED_SOURCE_RULE, Self::SwitchControlSurface { .. } => SWITCH_CONTROL_RULE, Self::ProxyLane { .. } => PROXY_LANE_RULE, + Self::NodePlaneUnpinned { .. } => NODE_PLANE_UNPINNED_RULE, + Self::HostIpUnbounded { .. } => HOST_IP_BOXES_RULE, } } } @@ -6469,6 +6749,91 @@ impl DropLimiter { } } + /// Emits the drop's line for one box-marked frame from the node address + /// that no resident `host_ip` box admits ([`HOST_IP_BOXES_RULE`]): the + /// same rate limit every drop's line answers to, naming the address, + /// the destination, the port and how many `host_ip` boxes were + /// resident, so a bundle's reader can tell a frame no box admits from + /// one sent while no `host_ip` box ran. Returns whether a line was + /// written. + fn warn_host_ip_unbounded( + &self, + src: [u8; 4], + dst: [u8; 4], + dst_port: u16, + resident: usize, + ) -> bool { + match self.should_warn_at(Some(src), HOST_IP_BOXES_RULE, Instant::now()) { + WarnDecision::Silent => false, + WarnDecision::Named if resident == 0 => { + tracing::warn!( + source = %Ipv4Addr::from(src), + destination = %Ipv4Addr::from(dst), + port = %if dst_port == 0 { "none".to_string() } else { dst_port.to_string() }, + resident_host_ip_boxes = resident, + rule_matched = HOST_IP_BOXES_RULE, + "dropped a box-marked frame from the node address: no host_ip box is \ + resident, so no box's rules admit it", + ); + true + } + WarnDecision::Named => { + tracing::warn!( + source = %Ipv4Addr::from(src), + destination = %Ipv4Addr::from(dst), + port = %if dst_port == 0 { "none".to_string() } else { dst_port.to_string() }, + resident_host_ip_boxes = resident, + rule_matched = HOST_IP_BOXES_RULE, + "dropped a box-marked frame from the node address toward a destination \ + no resident host_ip box's rules admit", + ); + true + } + WarnDecision::Overflow => { + tracing::warn!( + rule_matched = HOST_IP_BOXES_RULE, + "dropped box-marked frames from more distinct source addresses than the \ + gate keeps a window per source for; one line per rule covers the rest", + ); + true + } + } + } + + /// Emits the drop's line for one node-plane frame refused for want of a + /// pin ([`NODE_PLANE_UNPINNED_RULE`]): the same rate limit every drop's + /// line answers to, naming the daemon's address, the destination and the + /// port, so a bundle's reader learns which reach the node plane was + /// refused and that no baseline entry's lookup had pinned it. Returns + /// whether a line was written. + fn warn_node_plane_unpinned(&self, src: [u8; 4], dst: [u8; 4], dst_port: u16) -> bool { + match self.should_warn_at(Some(src), NODE_PLANE_UNPINNED_RULE, Instant::now()) { + WarnDecision::Silent => false, + WarnDecision::Named => { + tracing::warn!( + source = %Ipv4Addr::from(src), + destination = %Ipv4Addr::from(dst), + port = %if dst_port == 0 { "none".to_string() } else { dst_port.to_string() }, + rule_matched = NODE_PLANE_UNPINNED_RULE, + "dropped a node-plane frame leaving the VM toward an address no baseline \ + entry pinned, or toward a pinned address at a port other than TCP 443; \ + the node plane reaches the registry and the cache at TCP 443 on the \ + addresses its own lookups resolved them to", + ); + true + } + WarnDecision::Overflow => { + tracing::warn!( + rule_matched = NODE_PLANE_UNPINNED_RULE, + "dropped node-plane frames toward unpinned addresses from more distinct \ + source addresses than the gate keeps a window per source for; one line \ + per rule covers the rest", + ); + true + } + } + } + /// Emits the drop's line for one frame the row's own rules refused toward /// an undeclared subnet: the same rate limit every drop's line answers /// to, and naming the destination address and the port the frame gave @@ -7349,6 +7714,13 @@ pub(crate) mod test_support { frame } + /// `frame` as the guest's classifier leaves a box's packet: the IPv4 + /// DSCP field set to [`sessions::core::egress::BOX_COHORT_DSCP`]. + pub(crate) fn box_cohort_frame(mut frame: Vec) -> Vec { + frame[15] = sessions::core::egress::BOX_COHORT_DSCP << 2; + frame + } + /// An Ethernet II ARP frame announcing `spa` as its sender protocol /// address — the address an ARP frame's source is read from. pub(crate) fn arp_frame(spa: [u8; 4]) -> Vec { @@ -7397,24 +7769,24 @@ mod tests { use tokio::sync::mpsc; use super::test_support::{ - CaptureWriter, DEADLINE, arp_frame, capture_log, connect_control, connect_over, - expect_frame, expect_silence, expect_teardown, expose_request, gate_connected, - gate_connected_with_phase, gate_over, gate_over_control, gate_over_shipped, gate_over_with, - gate_over_with_node_baseline, ipv4_frame, ipv6_frame, read_within, send_frame, - unexpose_request, wait_for_log, + CaptureWriter, DEADLINE, arp_frame, box_cohort_frame, capture_log, connect_control, + connect_over, expect_frame, expect_silence, expect_teardown, expose_request, + gate_connected, gate_connected_with_phase, gate_over, gate_over_control, gate_over_shipped, + gate_over_with, gate_over_with_node_baseline, ipv4_frame, ipv6_frame, read_within, + send_frame, unexpose_request, wait_for_log, }; use super::{ AcceptFailure, CONNECT_REQUEST, CONTROL_VERBS, ControlVerb, DROP_WARN_MAX_TRACKED_PAIRS, - DROP_WARN_MIN_INTERVAL, DropLimiter, EgressGate, GateAdmit, GateDrop, GuestSource, - GuestSpeak, HANDSHAKE_TIMEOUT, MALFORMED_PUBLISH_RULE, MAX_HEAD, MAX_LIVE_RELAYS, - MAX_NAMED_TARGET, PROXY_LANE_RULE, PublishedForwards, Record, ReplyTables, + DROP_WARN_MIN_INTERVAL, DropLimiter, EgressGate, FrameSummary, GateAdmit, GateDrop, + GuestSource, GuestSpeak, HANDSHAKE_TIMEOUT, HOST_IP_BOXES_RULE, MALFORMED_PUBLISH_RULE, + MAX_HEAD, MAX_LIVE_RELAYS, MAX_NAMED_TARGET, PublishedForwards, Record, ReplyTables, UNDECLARED_PUBLISH_RECORD_RULE, UNDECLARED_RETRACT_RULE, UNKNOWN_PUBLISH_ADDRESS_RULE, UNREGISTERED_LIVE_LEASE_RULE, UNREGISTERED_PUBLISH_RULE, UNREGISTERED_SOURCE_PHASE, UNREGISTERED_SOURCE_RULE, UnregisteredSourcePhase, WarnDecision, accept_loop, dns_pins, gate_verdict, max_frame, render_record, serve_connection, }; use crate::box_registry::{BoxRegistration, BoxRegistry, BoxTable}; - use crate::net::baseline::{BaselineCategory, NodeBaselinePhase, NodePlaneBaseline}; + use crate::net::baseline::{NodeBaselinePhase, NodePlaneBaseline}; /// The default switch subnet, the plan every registry below is built for. const SUBNET: SwitchSubnet = switch::DEFAULT_SUBNET; @@ -9443,7 +9815,8 @@ mod tests { /// is refused without a flow behind it, and so is a frame from the /// answerer port the record's tuple never named. The resolver carve-out /// is unchanged — a UDP query to the resolver's port still falls through - /// to the row's rules and passes, whatever the table holds. + /// to the node-plane baseline set's rules and passes, whatever the table + /// holds. #[test] fn gate_verdict_refuses_gateway_dial_from_admitted_port_without_inbound_flow() { let registry = BoxRegistry::new(SUBNET); @@ -9586,8 +9959,8 @@ mod tests { ); assert_eq!( decide(&replies, &query), - Ok(GateAdmit::Row), - "the resolver carve-out falls through to the row's rules, as before" + Ok(GateAdmit::Baseline), + "the resolver carve-out falls through to the node-plane baseline set's rules" ); } @@ -12501,10 +12874,9 @@ mod tests { /// is on no lane — its baseline set is the enumeration of the categories /// NET-130 admits, and no category names the proxy. So the set never /// admits the address: with the baseline in force a node frame to the - /// proxy is dropped by the compiled set like any other undeclared - /// destination, and while the interim allow-all node row still decides - /// the node plane, that row declares no lane either, so the proxy rule - /// drops it there too — the shipped arm and the flip hold one posture. + /// proxy is the proxy rule's drop, and under the announced baseline the + /// interim node row declares no lane either, so the proxy rule drops it + /// there too. Either posture holds one refusal. #[test] fn baseline_set_never_admits_the_proxy_address() { let registry = BoxRegistry::new(SUBNET); @@ -12522,8 +12894,7 @@ mod tests { for endpoint in entry.endpoints() { let address = endpoint .split_once('/') - .expect("an endpoint is spelled `a.b.c.d/n`") - .0; + .map_or(endpoint.as_str(), |(a, _)| a); assert_ne!( address, Ipv4Addr::from(proxy).to_string(), @@ -12534,36 +12905,27 @@ mod tests { } } - // In force: the node's frame to the proxy is decided by the compiled - // set — and refused, at the proxy's port and over UDP too, because - // the address is inside no category's endpoints. - let in_force = baseline.clone().in_force(); + // In force, the shipped posture: the node's frame to the proxy is + // refused under the proxy rule, at the proxy's port and over UDP too. + assert_eq!(baseline.phase(), NodeBaselinePhase::InForce); for (proto, port) in [(6, 8118), (17, 8118)] { let frame = sessions::core::egress::summarize(&ipv4_frame(node, proto, proxy, port)); - match gate_verdict(&frame, None, &table, &in_force, &pins, &ReplyTables::new()) { - Ok(GateAdmit::Baseline) => panic!( - "the baseline set admitted the proxy's address at proto {proto} port {port}" - ), - Ok(admitted) => panic!( - "the in-force gate admitted a node frame to the proxy by another way: \ - {admitted:?}" - ), - Err(drop) => assert!( - drop.rule() != PROXY_LANE_RULE, - "the in-force set refuses the proxy's address as an undeclared \ - destination, not by the lane rule, got {}", - drop.rule(), + assert!( + matches!( + gate_verdict(&frame, None, &table, &baseline, &pins, &ReplyTables::new()), + Err(GateDrop::ProxyLane { .. }) ), - } + "the in-force set refuses the proxy's address at proto {proto} port {port}" + ); } - // Announced, the shipped interim: the node row — allow-all, and the - // row that decides the node plane until the flip — declares no lane, - // so the proxy rule drops the same frame. Either posture, no reach. + // Announced: the node row — allow-all — declares no lane, so the + // proxy rule drops the same frame. Either posture, no reach. + let announced = baseline.clone().announced(); let frame = sessions::core::egress::summarize(&ipv4_frame(node, 6, proxy, 8118)); assert!( matches!( - gate_verdict(&frame, None, &table, &baseline, &pins, &ReplyTables::new()), + gate_verdict(&frame, None, &table, &announced, &pins, &ReplyTables::new()), Err(GateDrop::ProxyLane { .. }) ), "the interim node row carries no lane, so its frame to the proxy is the proxy \ @@ -15093,13 +15455,13 @@ mod tests { /// attempt, grouped by spoofed source, in the gate's drop-line shape, so /// the bound a spoof bought can be read straight off the output. /// - /// Where the shipped posture differs from the bound's shape the test - /// pins both arms, the way the relay tests do: while the node baseline - /// is announced the node row is allow-all — the gap the baseline - /// phase's flip (#1786) closes. An in-plan source no row holds is not a - /// posture any more (NET-085): it drops under the unregistered rule in - /// every list, the frame half having no interim, so the record's two - /// halves vary the baseline alone. + /// The shipped posture is the in-force baseline: a spoof of the node's + /// own address reaches the addresses the node plane's own lookups pinned + /// for a baseline entry, the resolver, and nothing else. The announced + /// arm, the allow-all node row the flip closed, is pinned beside it so + /// the gap it was stays visible. An in-plan source no row holds drops + /// under the unregistered rule in every list (NET-085), so the record's + /// two halves vary the baseline alone. #[test] fn spoofed_source_bounded_to_resident_union() { /// One spoofed attempt as the record holds it: the frame wore, and @@ -15130,6 +15492,9 @@ mod tests { /// The row's declared credentialed lane admitted the frame at /// the proxy's address (NET-134): the admission no rules made. AdmittedByProxyLane, + /// A resident `host_ip` box's rules admitted a box-marked frame + /// from the node address. + AdmittedByHostIpBox, /// The gate dropped the frame, by the rule whose name the drop /// warning carries. Dropped(&'static str), @@ -15177,6 +15542,7 @@ mod tests { Ok(GateAdmit::Row) if src == baseline.node_addr() => Verdict::AdmittedByNodeRow, Ok(GateAdmit::Row) => Verdict::AdmittedByRow, Ok(GateAdmit::ProxyLane) => Verdict::AdmittedByProxyLane, + Ok(GateAdmit::HostIpBox) => Verdict::AdmittedByHostIpBox, Err(drop) => Verdict::Dropped(drop.rule()), }; Attempt { @@ -15215,11 +15581,12 @@ mod tests { // attempt is decided by the rows and the baseline set. let pins = dns_pins::DnsPins::new(SUBNET); let node = baseline.node_addr(); - // The node-baseline posture this build ships: announced, the - // allow-all node row. The frame half's own posture has no arms - // (NET-085) — an unregistered in-plan source drops in every list — - // so the record's second list varies the baseline alone. - let shipped = NodeBaselinePhase::Announced; + // The node-baseline posture this build ships: in force. The frame + // half's own posture has no arms (NET-085) — an unregistered in-plan + // source drops in every list — so the record's second list varies + // the baseline alone. + let shipped = NodeBaselinePhase::InForce; + assert_eq!(baseline.phase(), shipped); let resolver = SUBNET.dns_server().octets(); let web_only = [10, 1, 2, 3]; let db_only = [192, 0, 2, 5]; @@ -15229,32 +15596,26 @@ mod tests { // is a destination, this is a source, and the two share nothing but a // TEST-NET spelling — one is not where the other goes. let out_of_plan_src = [198, 51, 100, 7]; - // The node plane's own endpoint, read off the enumeration the way the - // helper spells it (`a.b.c.d/n`), not hardcoded: the in-force bound's - // admitted destination is the set's, not this test's. - let baseline_endpoint = baseline - .entries() - .iter() - .find(|entry| entry.category() == BaselineCategory::Registry) - .expect("the enumeration carries the registry") - .endpoints() - .first() - .expect("the registry carries an endpoint") - .split_once('/') - .expect("an endpoint is spelled `a.b.c.d/n`") - .0 - .parse::() - .expect("an endpoint's address parses") - .octets(); + // The node plane's own endpoint: the address its own lookup of the + // cache's name resolved to, pinned against the baseline's entry the + // way the gate's ingress leg pins it. + let baseline_endpoint = [203, 0, 113, 10]; + dns_pins::tests::node_resolves( + &pins, + &table, + &baseline, + crate::net::baseline::DAEMON_CACHE_ENDPOINT, + &[Ipv4Addr::from(baseline_endpoint)], + std::time::Instant::now(), + ); // Every spoofed attempt, decided and recorded: the tuples are // (baseline posture, source, protocol, destination, port), and each // is summarized, decided by [`gate_verdict`], and recorded exactly - // as the gate's relay would hand the frame over. The in-force world - // is the one the baseline flip creates — the compiled baseline set - // binding beside the rows — built here so the in-force arms have - // their proof before the flip lands. - let baseline_in_force = baseline.clone().in_force(); + // as the gate's relay would hand the frame over. The announced world + // is the one the flip closed — the allow-all node row — kept so the + // gap stays pinned beside its replacement. + let baseline_announced = baseline.clone().announced(); let shipped_attempts: Vec = [ // web's address, spoofed: decided by web's own row — the lab // subnet over TCP, the resolver carve-out — and nowhere else, not @@ -15278,10 +15639,10 @@ mod tests { (shipped, [100, 64, 0, 11], 6, web_only, 80), (shipped, [100, 64, 0, 11], 17, resolver, 53), (shipped, [100, 64, 0, 11], 6, resolver, 53), - // The node plane's own address, shipped: the allow-all interim - // node row admits even the outside destination — the gap the - // baseline phase's flip (#1786) closes, into the in-force arm - // the in-force list pins. The node row stands in for the + // The node plane's own address, shipped: bounded by the baseline + // set, decided beside the boxes' rows, so a spoof of the + // daemon's address buys the pinned endpoint and the resolver, + // not a box's row and not the outside. The node row stands in for the // `host_ip` cohort's identity today (NET-078: outside the box // host the cohort is one source), so a spoof of it exercises the // row a cohort-wide identity will be decided by; once the cohort @@ -15290,6 +15651,9 @@ mod tests { // `host_ip` members, and that a revoked member's destinations no // longer admit. (shipped, node, 6, outside, 443), + (shipped, node, 6, baseline_endpoint, 443), + (shipped, node, 6, web_only, 80), + (shipped, node, 17, resolver, 53), // An in-plan address no row holds: dropped under the // unregistered rule, under no phase — NET-085's drop is the // frame half's unconditional posture, so this list and the @@ -15302,28 +15666,31 @@ mod tests { .into_iter() .map(|(_, src, proto, dst, port)| decide(&table, &baseline, &pins, src, proto, dst, port)) .collect(); - let in_force_attempts: Vec = [ - // The node plane's own address, under the in-force baseline set: - // bounded by the enumeration, decided beside the boxes' rows, and - // so a spoof of the daemon's address buys the set, not a box's - // row. - (NodeBaselinePhase::InForce, node, 6, baseline_endpoint, 443), - (NodeBaselinePhase::InForce, node, 6, web_only, 80), - (NodeBaselinePhase::InForce, node, 6, outside, 443), - // The unrowed in-plan source under the in-force baseline: the + let announced_attempts: Vec = [ + // The node plane's own address under the announced baseline: the + // allow-all interim node row admits even the outside + // destination, the gap the in-force arm closed. + (NodeBaselinePhase::Announced, node, 6, outside, 443), + // The unrowed in-plan source under the announced baseline: the // same unregistered drop the shipped list pins — the baseline's // phase does not reach the frame half either (NET-085). - (NodeBaselinePhase::InForce, stray, 6, web_only, 80), - (NodeBaselinePhase::InForce, out_of_plan_src, 6, web_only, 80), + (NodeBaselinePhase::Announced, stray, 6, web_only, 80), + ( + NodeBaselinePhase::Announced, + out_of_plan_src, + 6, + web_only, + 80, + ), ] .into_iter() .map(|(_, src, proto, dst, port)| { - decide(&table, &baseline_in_force, &pins, src, proto, dst, port) + decide(&table, &baseline_announced, &pins, src, proto, dst, port) }) .collect(); let attempts: Vec = shipped_attempts .into_iter() - .chain(in_force_attempts) + .chain(announced_attempts) .collect(); // The record, one line per attempt, grouped by spoofed source, in @@ -15363,21 +15730,16 @@ mod tests { // spoofed frame lands inside the union — a resident box's declared // subnet over its declared protocol, or the resolver carve-out every // row and the baseline set share — and the baseline set's admit is - // inside the enumeration's endpoints (or the same carve-out). The - // two interim arms are the documented gaps, asserted as such below, - // so they are not bound here but named. + // an address the node plane's own lookup pinned (or the same + // carve-out). The announced node row is the documented gap, asserted + // as such below, so it is not bound here but named. let in_union = |dst: [u8; 4], proto: u8, port: u16| { let web = Ipv4Cidr::parse("10.0.0.0/8").expect("web's subnet parses"); let db = Ipv4Cidr::parse("192.0.2.0/24").expect("db's subnet parses"); (proto == 6 && (web.contains(dst) || db.contains(dst))) || (proto == 17 && dst == resolver && port == 53) }; - let baseline_set: Vec = baseline - .entries() - .iter() - .flat_map(|entry| entry.endpoints().iter()) - .map(|endpoint| Ipv4Cidr::parse(endpoint).expect("a baseline endpoint parses")) - .collect(); + let baseline_set = [baseline_endpoint]; for attempt in &attempts { match attempt.verdict { Verdict::AdmittedByRow => assert!( @@ -15388,22 +15750,88 @@ mod tests { attempt.port, ), Verdict::AdmittedByBaseline => assert!( - baseline_set.iter().any(|cidr| cidr.contains(attempt.dst)) + baseline_set.contains(&attempt.dst) || (attempt.proto == 17 && attempt.dst == resolver && attempt.port == 53), "a spoof wearing the node's address reached {}:{} outside the baseline set", Ipv4Addr::from(attempt.dst), attempt.port, ), - // The shipped allow-all node row's admits are the gap, - // pinned to its in-force replacement by the verdict lookups + // The announced allow-all node row's admits are the gap, + // pinned beside its in-force replacement by the verdict lookups // below. A proxy-lane admit is none of this test's attempts // — no attempt names the proxy's address — and the arm // keeps the match exhaustive for the ones that will. - Verdict::AdmittedByNodeRow | Verdict::AdmittedByProxyLane | Verdict::Dropped(_) => { - } + Verdict::AdmittedByNodeRow + | Verdict::AdmittedByProxyLane + | Verdict::AdmittedByHostIpBox + | Verdict::Dropped(_) => {} } } + // The box-marked spoof (NET-085): root in the VM writes the box mark + // on a frame from the node address. The mark proves no box, so the + // union of the resident `host_ip` boxes' rules decides the frame, + // never the node row and never the baseline set. With no `host_ip` + // box resident, every marked frame drops, the pinned baseline + // endpoint included. + let marked = |dst: [u8; 4], port: u16| { + let frame = sessions::core::egress::summarize(&box_cohort_frame(ipv4_frame( + node, 6, dst, port, + ))); + assert!(frame.box_cohort_marked()); + gate_verdict(&frame, None, &table, &baseline, &pins, &ReplyTables::new()) + }; + let unbounded = |dst: [u8; 4], resident: usize| GateDrop::HostIpUnbounded { + src: node, + dst, + dst_port: 443, + resident, + }; + for dst in [outside, baseline_endpoint] { + assert_eq!( + marked(dst, 443), + Err(unbounded(dst, 0)), + "a marked spoof to {} with no host_ip box resident drops under \ + {HOST_IP_BOXES_RULE}", + Ipv4Addr::from(dst), + ); + } + // One `host_ip` box resident, admitting TCP to TEST-NET-2 alone: a + // marked frame there passes under the box's rules, and a marked + // spoof anywhere else drops, the outside and the baseline endpoint + // both. + let session = sessions::SessionId::parse_str(&uuid::Uuid::new_v4().to_string()).unwrap(); + let host_ip_reach = [198, 51, 100, 9]; + assert!(registry.admit_host_ip_box( + session, + Some(&EgressPolicy { + allow_protocols: Some(vec![sessions::IpProto::Tcp]), + allow_subnets: Some(vec!["198.51.100.0/24".to_string()]), + allow_dns_hosts: None, + deny_subnets: None, + }), + )); + assert_eq!( + marked(host_ip_reach, 443), + Ok(GateAdmit::HostIpBox), + "a marked frame a resident host_ip box admits passes" + ); + for dst in [outside, baseline_endpoint] { + assert_eq!( + marked(dst, 443), + Err(unbounded(dst, 1)), + "a marked spoof to {} that no resident host_ip box admits drops", + Ipv4Addr::from(dst), + ); + } + // The box leaves: its reach goes with it. + assert!(registry.retire_host_ip_box(session)); + assert_eq!( + marked(host_ip_reach, 443), + Err(unbounded(host_ip_reach, 0)), + "a retired host_ip box's rules admit nothing" + ); + // The universal the table makes cheap, beside the membership loop // above — NET-085 as a bound, not a list. Two halves: // @@ -15534,32 +15962,39 @@ mod tests { Verdict::Dropped("egress-undeclared-subnet"), ); - // The node plane's own address, under the in-force set: the - // enumeration's endpoint admitted, everything else dropped — the + // The node plane's own address, under the in-force set: the pinned + // endpoint and the resolver admitted, everything else dropped — the // bound a spoof of the daemon's address buys, beside the boxes' rows. + // web's subnet is private space, so the infrastructure rule refuses + // it before the pins are asked; the outside address is refused for + // want of a pin. assert_eq!( - verdict_of(NodeBaselinePhase::InForce, node, 6, baseline_endpoint, 443), + verdict_of(shipped, node, 6, baseline_endpoint, 443), Verdict::AdmittedByBaseline, ); assert_eq!( - verdict_of(NodeBaselinePhase::InForce, node, 6, web_only, 80), - Verdict::Dropped("egress-undeclared-subnet"), + verdict_of(shipped, node, 17, resolver, 53), + Verdict::AdmittedByBaseline, ); assert_eq!( - verdict_of(NodeBaselinePhase::InForce, node, 6, outside, 443), - Verdict::Dropped("egress-undeclared-subnet"), + verdict_of(shipped, node, 6, web_only, 80), + Verdict::Dropped("egress-infrastructure-destination"), + ); + assert_eq!( + verdict_of(shipped, node, 6, outside, 443), + Verdict::Dropped("egress-node-plane-unpinned"), ); - // The allow-all node row's arm, with the in-force arm it becomes: the - // node row admits the outside destination today, and the baseline - // phase's flip (#1786) turns it into the drop above. The unrowed + // The allow-all node row's arm, with the in-force arm that replaced + // it: the announced node row admits the outside destination, and the + // in-force arm turns it into the drop above. The unrowed // in-plan source drops under the unregistered rule under both // baseline postures — NET-085's drop is unconditional, so no arm // waits on a flip for it — and the out-of-plan source stays rule // 0's under both, so nothing borrows the plan's infrastructure as a // source. assert_eq!( - verdict_of(shipped, node, 6, outside, 443), + verdict_of(NodeBaselinePhase::Announced, node, 6, outside, 443), Verdict::AdmittedByNodeRow, ); assert_eq!( @@ -15567,7 +16002,7 @@ mod tests { Verdict::Dropped("egress-unregistered-source"), ); assert_eq!( - verdict_of(NodeBaselinePhase::InForce, stray, 6, web_only, 80), + verdict_of(NodeBaselinePhase::Announced, stray, 6, web_only, 80), Verdict::Dropped("egress-unregistered-source"), ); assert_eq!( @@ -15575,7 +16010,13 @@ mod tests { Verdict::Dropped("egress-unknown-source"), ); assert_eq!( - verdict_of(NodeBaselinePhase::InForce, out_of_plan_src, 6, web_only, 80), + verdict_of( + NodeBaselinePhase::Announced, + out_of_plan_src, + 6, + web_only, + 80 + ), Verdict::Dropped("egress-unknown-source"), ); @@ -15598,72 +16039,268 @@ mod tests { )); } - /// The shipped arm of the node-plane decision: the phase this build - /// ships is announced, and a frame wearing the in-VM daemon's own - /// address is decided by the run path's allow-all interim node row — - /// the row [`BoxRegistry::register_node_namespace`] publishes, the way - /// the run path registers it at boot — yielding [`GateAdmit::Row`] from - /// that row, never [`GateAdmit::Baseline`]. The baseline set's arm is - /// the store-surface configuration's to switch on (#1786), and the - /// in-force arm's proof is - /// [`unenrolled_baseline_set_from_helper_enumeration`]'s. + /// NET-085 and NET-130, the shipped arm of the node-plane decision: the + /// phase this build ships is in force, and a frame wearing the in-VM + /// daemon's own address is admitted only to an address a baseline + /// entry's lookup pinned, the resolver carve-out, the host alias at a + /// configured exposure and a box's lease — never by the interim node + /// row. Everything else is the drop for want of a pin, the + /// infrastructure drop or the proxy rule's. #[test] - fn node_plane_source_decided_by_the_node_row_while_announced() { + fn node_plane_frame_admitted_only_to_pinned_baseline_address() { let registry = BoxRegistry::new(SUBNET); - // The run path's own registration: the allow-all interim node row at - // the daemon's address, registered at VM boot (cmd/run.rs) with the - // proxy port the boot line hands the guest daemon. + // The run path's own registration: the interim node row at the + // daemon's address, registered at VM boot with the proxy port the + // boot line hands the guest daemon — the row whose allow-all the + // in-force arm no longer consults. registry.register_node_namespace(7654); + // A resident own-address box, whose lease the node's hostname proxy + // dials. + registry.register( + BoxRegistration::new("web", Ipv4Addr::from(LEASE), Ipv4Addr::LOCALHOST) + .with_admitted_ports([8080]), + ); let table = registry.table(); - // Built without [`NodePlaneBaseline::in_force`], so this is the - // phase the build ships. - let baseline = NodePlaneBaseline::built_in(SUBNET); + let baseline = NodePlaneBaseline::built_in(SUBNET).with_host_exposure(vec![(4200, 4200)]); assert_eq!( baseline.phase(), - NodeBaselinePhase::Announced, - "the shipped posture is announced, the arm this test pins" + NodeBaselinePhase::InForce, + "the shipped posture is in force, the arm this test pins" ); - // The pin table, empty as the run path hands it: the node row - // declares no names. let pins = dns_pins::DnsPins::new(SUBNET); + let node = SUBNET.daemon_ip().octets(); + let decide = |proto: u8, dst: [u8; 4], port: u16| { + gate_verdict( + &sessions::core::egress::summarize(&ipv4_frame(node, proto, dst, port)), + None, + &table, + &baseline, + &pins, + &ReplyTables::new(), + ) + }; + let cache = [203, 0, 113, 10]; - // A node-plane frame to a destination no category of the enumeration - // names — the public store the shipped cache URL resolves to. The - // interim node row decides it, and the row allows all, so the frame - // is admitted by [`GateAdmit::Row`]: the enumeration's compiled set, - // which admits no such destination, decided nothing here. - let summarize = sessions::core::egress::summarize; - let node_frame = summarize(&ipv4_frame( - SUBNET.daemon_ip().octets(), - 6, - [8, 8, 8, 8], - 443, + // Before the node's own lookup, the cache's address is not pinned: + // the frame is the drop for want of a pin, whatever the node row's + // allow-all would have said. + assert_eq!( + decide(6, cache, 443), + Err(GateDrop::NodePlaneUnpinned { + src: node, + dst: cache, + dst_port: 443 + }), + "an address no lookup pinned is refused" + ); + + // The resolver carve-out: the node resolves its entries' names. + assert_eq!( + decide(17, SUBNET.dns_server().octets(), 53), + Ok(GateAdmit::Baseline), + "the node reaches the resolver" + ); + + // The node's own lookup of the cache's name pins the answer, and the + // frame to it is admitted by the baseline set. + dns_pins::tests::node_resolves( + &pins, + &table, + &baseline, + crate::net::baseline::DAEMON_CACHE_ENDPOINT, + &[Ipv4Addr::from(cache)], + Instant::now(), + ); + assert_eq!( + decide(6, cache, 443), + Ok(GateAdmit::Baseline), + "the pinned address is admitted" + ); + // The pin opens the address at TCP 443 alone: another TCP port, UDP + // to the same port, and ICMP are the drop for want of a pin, so a + // shared address a registry name resolves to is not open at every + // port. + for (proto, port) in [(6, 22), (6, 80), (17, 443), (1, 0)] { + assert_eq!( + decide(proto, cache, port), + Err(GateDrop::NodePlaneUnpinned { + src: node, + dst: cache, + dst_port: port + }), + "the pinned address is refused at protocol {proto} port {port}" + ); + } + + // A neighbour of the pinned address, and a public address no lookup + // named, stay refused for want of a pin. + for dst in [[203, 0, 113, 11], [8, 8, 8, 8]] { + assert!( + matches!(decide(6, dst, 443), Err(GateDrop::NodePlaneUnpinned { .. })), + "{} is refused: no baseline entry pinned it", + Ipv4Addr::from(dst) + ); + } + // A name no entry names pins nothing for the node. + let other = [198, 51, 100, 20]; + dns_pins::tests::node_resolves( + &pins, + &table, + &baseline, + "example.com", + &[Ipv4Addr::from(other)], + Instant::now(), + ); + assert!( + matches!( + decide(6, other, 443), + Err(GateDrop::NodePlaneUnpinned { .. }) + ), + "a lookup of a name no entry names admits nothing" + ); + + // The host alias at a configured exposure is admitted, and at any + // other port it is the infrastructure drop. + let alias = SUBNET.host_alias().octets(); + assert_eq!(decide(6, alias, 4200), Ok(GateAdmit::Baseline)); + assert!(matches!( + decide(6, alias, 443), + Err(GateDrop::Infrastructure { dst, .. }) if dst == alias + )); + + // A resident box's lease: the target's ingress decides it. + assert_eq!(decide(6, LEASE, 8080), Ok(GateAdmit::Baseline)); + + // Private space and the switch's own ports stay refused. + assert!(matches!( + decide(6, [10, 1, 2, 3], 443), + Err(GateDrop::Infrastructure { .. }) + )); + assert!(matches!( + decide(6, SUBNET.dns_server().octets(), 80), + Err(GateDrop::SwitchControlSurface { .. }) )); + } + + /// The node address carries two senders: the guest daemon and every + /// `host_ip` box, the default network mode. The guest's classifier + /// writes the sender's cgroup into the frame as the box mark. A marked + /// frame at the node address is decided by the resident `host_ip` + /// boxes' rules, which the host-side client registered, so a default + /// session keeps its outbound network (NET-107). Root in the guest can + /// write the mark too, so with no `host_ip` box resident the same frame + /// drops (NET-085). + #[test] + fn box_marked_node_frame_decided_by_resident_host_ip_boxes() { + let registry = BoxRegistry::new(SUBNET); + registry.register_node_namespace(7654); + let table = registry.table(); + let baseline = NodePlaneBaseline::built_in(SUBNET); + assert_eq!(baseline.phase(), NodeBaselinePhase::InForce); + let pins = dns_pins::DnsPins::new(SUBNET); + let node = SUBNET.daemon_ip().octets(); + let outside = [8, 8, 8, 8]; + let frame = box_cohort_frame(ipv4_frame(node, 6, outside, 443)); + let summary = sessions::core::egress::summarize(&frame); + assert!(summary.box_cohort_marked()); + let decide = |summary: &FrameSummary| { + gate_verdict(summary, None, &table, &baseline, &pins, &ReplyTables::new()) + }; assert_eq!( + decide(&summary), + Err(GateDrop::HostIpUnbounded { + src: node, + dst: outside, + dst_port: 443, + resident: 0, + }), + "with no host_ip box resident, a marked frame drops under {HOST_IP_BOXES_RULE}" + ); + // A `host_ip` box with no `egress` section: the default leaves a box + // that shares the node's address allow-all, so its marked frame to + // the outside passes under its rules. + let session = sessions::SessionId::parse_str(&uuid::Uuid::new_v4().to_string()).unwrap(); + assert!(registry.admit_host_ip_box(session, None)); + assert_eq!( + decide(&summary), + Ok(GateAdmit::HostIpBox), + "a marked frame at the node address is the resident host_ip boxes' to decide" + ); + // The box's verdict still keeps the floors every row keeps: the + // link-local metadata service and the switch's own ports stay + // refused. + let metadata = box_cohort_frame(ipv4_frame(node, 6, [169, 254, 169, 254], 80)); + assert!(matches!( gate_verdict( - &node_frame, + &sessions::core::egress::summarize(&metadata), None, &table, &baseline, &pins, &ReplyTables::new() ), - Ok(GateAdmit::Row), - "announced, the interim node row decides the node plane's frames — \ - never the baseline set" - ); + Err(GateDrop::Infrastructure { .. }) + )); + let control = box_cohort_frame(ipv4_frame(node, 6, SUBNET.dns_server().octets(), 80)); + assert!(matches!( + gate_verdict( + &sessions::core::egress::summarize(&control), + None, + &table, + &baseline, + &pins, + &ReplyTables::new() + ), + Err(GateDrop::SwitchControlSurface { .. }) + )); + } + + /// The daemon's own frame at the node address carries no box mark, and + /// the baseline set still judges it: an address no baseline entry + /// pinned is refused. A DSCP value other than the box mark is not a box + /// either, so the baseline decides a frame the guest left unmarked. + #[test] + fn daemon_frame_at_node_address_still_judged_by_baseline() { + let registry = BoxRegistry::new(SUBNET); + registry.register_node_namespace(7654); + let table = registry.table(); + let baseline = NodePlaneBaseline::built_in(SUBNET); + let pins = dns_pins::DnsPins::new(SUBNET); + let node = SUBNET.daemon_ip().octets(); + let outside = [8, 8, 8, 8]; + let mut other_dscp = ipv4_frame(node, 6, outside, 443); + other_dscp[15] = 0b10_1110 << 2; + for frame in [ipv4_frame(node, 6, outside, 443), other_dscp] { + let summary = sessions::core::egress::summarize(&frame); + assert!(!summary.box_cohort_marked()); + assert_eq!( + gate_verdict( + &summary, + None, + &table, + &baseline, + &pins, + &ReplyTables::new() + ), + Err(GateDrop::NodePlaneUnpinned { + src: node, + dst: outside, + dst_port: 443 + }), + "the daemon's own frame is the baseline's to decide" + ); + } } /// The host alias's default-deny is a box row's alone, and the node - /// row's reach through it is scoped by the configured host exposures - /// (design §7.1, NET-062). Announced, the interim node row at the - /// daemon's address carries the frames of every box sharing the guest - /// root namespace — a host-address box, the default network mode, whose - /// `host.min.internal` resolves to the alias. The built-in baseline + /// plane's reach through it is scoped by the configured host exposures + /// (design §7.1, NET-062). The daemon's address carries the frames of + /// every box sharing the guest root namespace — a host-address box, the + /// default network mode, whose `host.min.internal` resolves to the + /// alias — and the in-force baseline decides them. The built-in baseline /// carries no exposure, so the alias is closed at every port until a /// deployment names ports; with a list, a node-plane TCP or UDP frame to - /// the alias is own-block local reach the row admits inside the list's - /// ranges, edges included, and the infrastructure drop outside them, + /// the alias is admitted by the baseline set inside the list's ranges, edges included, and the infrastructure drop outside them, /// while an ICMP frame, which carries no port, is refused even by a list /// spanning every port. The same frames from a box lease, even an /// allow-all one, are the infrastructure drop whatever the port. @@ -15677,7 +16314,7 @@ mod tests { ); let table = registry.table(); let built_in = NodePlaneBaseline::built_in(SUBNET); - assert_eq!(built_in.phase(), NodeBaselinePhase::Announced); + assert_eq!(built_in.phase(), NodeBaselinePhase::InForce); let pins = dns_pins::DnsPins::new(SUBNET); let summarize = sessions::core::egress::summarize; let alias = SUBNET.host_alias().octets(); @@ -15697,7 +16334,7 @@ mod tests { assert!( refused(verdict(node, 6, 8080, &built_in)), - "with no configured exposure the alias is closed to the node row at every port" + "with no configured exposure the alias is closed to the node plane at every port" ); let exposed = built_in @@ -15710,17 +16347,17 @@ mod tests { for port in [3000, 3999, 8080] { assert_eq!( verdict(node, 6, port, &exposed), - Ok(GateAdmit::Row), - "the node row reaches the host alias at exposed port {port}" + Ok(GateAdmit::Baseline), + "the node plane reaches the host alias at exposed port {port}" ); } assert!( refused(verdict(node, 6, 18081, &exposed)), - "the node row's frame to the host alias outside the exposures is the infrastructure drop" + "the node plane's frame to the host alias outside the exposures is the infrastructure drop" ); assert_eq!( verdict(node, 17, 8080, &exposed), - Ok(GateAdmit::Row), + Ok(GateAdmit::Baseline), "a UDP frame reaches the host alias at an exposed port" ); assert!( @@ -15735,85 +16372,37 @@ mod tests { ); } - /// The in-force arm's own bound on the host alias: the built-in - /// registry and cache endpoints name the alias, and the compiled set's - /// verdict is by subnet alone — no port dimension — so the arm bounds - /// the alias to the ports the daemon's own fetches use. The fetch port - /// stays admitted by the set; a wider port on the alias is the - /// infrastructure drop, the same refusal a box row's alias frames take - /// (design §7.1, NET-062). - #[test] - fn in_force_baseline_bounds_the_host_alias_to_its_fetch_ports() { - let registry = BoxRegistry::new(SUBNET); - registry.register_node_namespace(7654); - let table = registry.table(); - let baseline = NodePlaneBaseline::built_in(SUBNET).in_force(); - let pins = dns_pins::DnsPins::new(SUBNET); - let node = baseline.node_addr(); - let alias = SUBNET.host_alias().octets(); - - let fetch = sessions::core::egress::summarize(&ipv4_frame(node, 6, alias, 443)); - assert_eq!( - gate_verdict(&fetch, None, &table, &baseline, &pins, &ReplyTables::new()), - Ok(GateAdmit::Baseline), - "the daemon's own fetch port on the alias stays admitted by the set" - ); - - let wider = sessions::core::egress::summarize(&ipv4_frame(node, 6, alias, 18081)); - assert!( - matches!( - gate_verdict( - &wider, - None, - &table, - &baseline, - &pins, - &ReplyTables::new() - ), - Err(GateDrop::Infrastructure { dst, dst_port, .. }) - if dst == alias && dst_port == 18081 - ), - "a wider port on the alias is the infrastructure drop, in force" - ); - } - - /// The in-force arm bounds the alias by protocol as well as port: the - /// summary reads a destination port from every protocol's bytes, so an - /// ICMP frame whose checksum reads 443, or a GRE frame (IP protocol 47) - /// whose protocol-type field does, must not pass as a fetch. TCP at 443 - /// is the one admission. + /// The in-force arm opens the host alias at the configured host + /// exposures and nowhere else: the registry and the cache are named + /// endpoints now, so no fetch port is held open on the alias, and with + /// no exposure TCP at 443 is the infrastructure drop. With 443 exposed, + /// TCP and UDP pass, and a frame with no transport port — ICMP, or GRE + /// whose bytes read as 443 — never matches the exposure. #[test] - fn in_force_baseline_admits_only_tcp_at_the_alias_fetch_port() { + fn in_force_baseline_opens_the_alias_at_the_host_exposures_alone() { let registry = BoxRegistry::new(SUBNET); registry.register_node_namespace(7654); let table = registry.table(); - let baseline = NodePlaneBaseline::built_in(SUBNET).in_force(); + let built_in = NodePlaneBaseline::built_in(SUBNET); let pins = dns_pins::DnsPins::new(SUBNET); - let node = baseline.node_addr(); + let node = built_in.node_addr(); let alias = SUBNET.host_alias().octets(); - let decide = |proto: u8| { + let decide = |proto: u8, baseline: &NodePlaneBaseline| { let summary = sessions::core::egress::summarize(&ipv4_frame(node, proto, alias, 443)); - gate_verdict( - &summary, - None, - &table, - &baseline, - &pins, - &ReplyTables::new(), - ) + gate_verdict(&summary, None, &table, baseline, &pins, &ReplyTables::new()) }; + let refused = |result: Result| matches!(result, Err(GateDrop::Infrastructure { dst, dst_port: 443, .. }) if dst == alias); - assert_eq!( - decide(6), - Ok(GateAdmit::Baseline), - "TCP at the fetch port on the alias is admitted, in force" + assert!( + refused(decide(6, &built_in)), + "with no exposure, TCP at 443 on the alias is the infrastructure drop" ); + let exposed = built_in.clone().with_host_exposure(vec![(443, 443)]); + assert_eq!(decide(6, &exposed), Ok(GateAdmit::Baseline)); + assert_eq!(decide(17, &exposed), Ok(GateAdmit::Baseline)); for (proto, name) in [(1u8, "ICMP"), (47, "GRE")] { assert!( - matches!( - decide(proto), - Err(GateDrop::Infrastructure { dst, dst_port: 443, .. }) if dst == alias - ), + refused(decide(proto, &exposed)), "{name} to alias:443 is the infrastructure drop, in force" ); } @@ -15822,13 +16411,15 @@ mod tests { /// NET-130, un-enrolled, at the gate: the node-plane baseline set comes /// from the helper's enumeration, and the gate decides the daemon's own /// frames by it **beside** the boxes' rows — a deny-all box's row does not - /// clip the daemon's own registry and cache fetches (NET-080), decided - /// before the table is consulted — and a destination the enumeration does - /// not name is dropped: the gap the announced phase covers until the - /// node plane's store surfaces are configured (`NodeBaselinePhase`). + /// clip the daemon's own registry and cache fetches (NET-080) — through + /// the relay's two legs: the daemon's own lookup of the cache's name + /// reaches the switch, the reply pins its answer on the way back, and the + /// fetch to that address is admitted, while an address no lookup pinned + /// is dropped with a line naming the drop for want of a pin. #[tokio::test] async fn unenrolled_baseline_set_from_helper_enumeration() { let registry = BoxRegistry::new(SUBNET); + registry.register_node_namespace(7654); // A deny-all box: no declared subnets, so nothing but the resolver // carve-out passes its row — the box the node plane's own fetches // must not be bound by. @@ -15841,39 +16432,36 @@ mod tests { deny_subnets: None, }), ); - let baseline = NodePlaneBaseline::built_in(SUBNET).in_force(); + let baseline = NodePlaneBaseline::built_in(SUBNET); let node_addr = baseline.node_addr(); - let registry_endpoint = baseline - .entries() - .iter() - .find(|entry| entry.category() == BaselineCategory::Registry) - .expect("the enumeration carries the registry") - .endpoints() - .first() - .expect("the registry carries an endpoint") - .split_once('/') - .expect("an endpoint is spelled `a.b.c.d/n`") - .0 - .parse::() - .expect("an endpoint's address parses") - .octets(); + let cache_name = crate::net::baseline::DAEMON_CACHE_ENDPOINT; + let cache_addr = Ipv4Addr::new(203, 0, 113, 10); let mut h = gate_over_with_node_baseline(registry, UNREGISTERED_SOURCE_PHASE, baseline).await; - // VM start says what the node plane may reach, one term per category: - // the line a diagnostic bundle's daemon log tail carries. + // VM start says what the node plane may reach, one term per category, + // and that the set is in force: the line a diagnostic bundle's daemon + // log tail carries. No interim rides beside it: a box-marked frame + // from the node address is the resident host_ip boxes' to decide. wait_for_log(&h.log, "node-plane baseline set").await; let logged = h.log.contents(); - for category in ["registry=", "cache="] { + for term in [ + "registry=github.com", + "cache=cache.minimal.dev", + "node_baseline_phase=\"in force\"", + "node_addr=100.64.255.253", + ] { assert!( - logged.contains(category), - "the start-up line names the baseline entries by category, got: {logged}" + logged.contains(term), + "the start-up line carries {term}, got: {logged}" + ); + } + for gone in ["box_marked_node_frames", "can still bypass"] { + assert!( + !logged.contains(gone), + "the start-up line names no interim ({gone}), got: {logged}" ); } - assert!( - logged.contains("node_addr=100.64.255.253"), - "the start-up line names the in-VM daemon's own address, got: {logged}" - ); // The box's own frame, to somewhere its deny-all declaration does not // name: dropped, where it stands. The marker behind it is the one @@ -15889,27 +16477,116 @@ mod tests { "the deny-all box's undeclared frame never reached the switch; the marker did" ); - // The node plane's own frame, to the enumeration's registry endpoint: - // decided by the baseline set beside the box's row — admitted, and - // the switch sees it exactly as it was sent. - let node_frame = ipv4_frame(node_addr, 6, registry_endpoint, 443); - send_frame(&mut h.guest, &node_frame).await; - let seen = expect_frame(&mut h.switch).await; + // Before the daemon's own lookup the cache's address is not pinned: + // the fetch is dropped, and the marker behind it — the daemon's own + // query for the cache's name, the resolver carve-out — arrives. + let early_fetch = ipv4_frame(node_addr, 6, cache_addr.octets(), 443); + send_frame(&mut h.guest, &early_fetch).await; + let query = dns_pins::tests::udp_payload_frame( + Ipv4Addr::from(node_addr), + 40000, + SUBNET.dns_server(), + 53, + &dns_pins::tests::dns_query(cache_name), + ); + send_frame(&mut h.guest, &query).await; + assert_eq!( + expect_frame(&mut h.switch).await, + query, + "the unpinned fetch never reached the switch; the daemon's own query did" + ); + wait_for_log(&h.log, "egress-node-plane-unpinned").await; + let logged = h.log.contents(); + assert!( + logged.contains("source=100.64.255.253") + && logged.contains("destination=203.0.113.10") + && logged.contains("no baseline entry pinned"), + "the drop line names a node-plane frame refused for want of a pin, got: {logged}" + ); + + // The resolver's reply, on its way back to the daemon, pins the + // answer against the cache's entry. + let reply = dns_pins::tests::udp_payload_frame( + SUBNET.dns_server(), + 53, + Ipv4Addr::from(node_addr), + 40000, + &dns_pins::tests::dns_response(cache_name, &[cache_addr]), + ); + send_frame(&mut h.switch, &reply).await; assert_eq!( - seen, node_frame, + expect_frame(&mut h.guest).await, + reply, + "the reply reaches the daemon in full" + ); + wait_for_log(&h.log, "pinned a node-plane baseline entry's addresses").await; + let logged = h.log.contents(); + assert!( + logged.contains("entry=cache") + && logged.contains("addresses=[203.0.113.10]") + && logged.contains("lookup_source=100.64.255.253"), + "the pin line names the entry, the addresses and the lookup's source, got: {logged}" + ); + + // The daemon's own fetch, to the address its lookup pinned: admitted + // beside the deny-all box's row, exactly as it was sent. + let fetch = ipv4_frame(node_addr, 6, cache_addr.octets(), 443); + send_frame(&mut h.guest, &fetch).await; + assert_eq!( + expect_frame(&mut h.switch).await, + fetch, "the daemon's own fetch reaches the switch as it was sent" ); - // To an address no category names — the public store the guest - // daemon's shipped cache URL resolves to, whose serving addresses no - // subnet list this tree can name — the set says no. + // The registry's leg: the package source repository a box launch + // git-fetches before its package graph exists. The daemon's own + // lookup of the registry's name pins its answer against the registry + // entry, and the fetch to it is admitted — the fetch whose drop hung + // a box launch inside the VM. + let registry_name = crate::net::baseline::DAEMON_REGISTRY_ENDPOINT; + let registry_addr = Ipv4Addr::new(203, 0, 113, 20); + let registry_query = dns_pins::tests::udp_payload_frame( + Ipv4Addr::from(node_addr), + 40001, + SUBNET.dns_server(), + 53, + &dns_pins::tests::dns_query(registry_name), + ); + send_frame(&mut h.guest, ®istry_query).await; + assert_eq!( + expect_frame(&mut h.switch).await, + registry_query, + "the daemon's own query for the registry's name reaches the switch" + ); + let registry_reply = dns_pins::tests::udp_payload_frame( + SUBNET.dns_server(), + 53, + Ipv4Addr::from(node_addr), + 40001, + &dns_pins::tests::dns_response(registry_name, &[registry_addr]), + ); + send_frame(&mut h.switch, ®istry_reply).await; + assert_eq!( + expect_frame(&mut h.guest).await, + registry_reply, + "the registry's reply reaches the daemon in full" + ); + wait_for_log(&h.log, "entry=registry").await; + let registry_fetch = ipv4_frame(node_addr, 6, registry_addr.octets(), 443); + send_frame(&mut h.guest, ®istry_fetch).await; + assert_eq!( + expect_frame(&mut h.switch).await, + registry_fetch, + "the daemon's own fetch from the registry reaches the switch as it was sent" + ); + + // To an address no lookup pinned, the set says no. let out_of_set = ipv4_frame(node_addr, 6, [8, 8, 8, 8], 443); - let node_marker = ipv4_frame(node_addr, 6, registry_endpoint, 443); send_frame(&mut h.guest, &out_of_set).await; - send_frame(&mut h.guest, &node_marker).await; - let seen = expect_frame(&mut h.switch).await; + send_frame(&mut h.guest, &fetch).await; assert_eq!( - seen, node_marker, + expect_frame(&mut h.switch).await, + fetch, "the daemon's out-of-set frame never reached the switch; the marker did" ); expect_silence(&mut h.switch).await; @@ -17751,14 +18428,21 @@ mod tests { let mut h = gate_over(registry).await; // Node-plane traffic on the relay — the in-VM daemon's own frames, - // admitted by the node's row — beside the box's declared frame, the + // admitted by the node-plane baseline set — beside the box's declared frame, the // traffic the same connection attributes to it. - let node_frame = ipv4_frame(SUBNET.daemon_ip().octets(), 6, [10, 1, 2, 3], 80); + // The node's frame is one the in-force baseline set admits: its own + // lookup, through the resolver carve-out. + let node_frame = ipv4_frame( + SUBNET.daemon_ip().octets(), + 17, + SUBNET.dns_server().octets(), + 53, + ); send_frame(&mut h.guest, &node_frame).await; assert_eq!( expect_frame(&mut h.switch).await, node_frame, - "the node's frame reaches the switch by its row" + "the node's frame reaches the switch by the node-plane baseline set" ); let box_frame = ipv4_frame(LEASE, 6, [10, 1, 2, 3], 80); send_frame(&mut h.guest, &box_frame).await; @@ -17789,7 +18473,7 @@ mod tests { ); // And the reach survives: the node's next connection carries its - // frames again, admitted by the row no report retired. + // frames again, decided for the row no report retired. let (mut guest, mut switch) = connect_over(&h).await; send_frame(&mut guest, &node_frame).await; assert_eq!( @@ -17824,12 +18508,19 @@ mod tests { // One node-sourced frame, one lease-run frame, both admitted — the // traffic the relay attributes to the connection that carried it. - let node_frame = ipv4_frame(SUBNET.daemon_ip().octets(), 6, [10, 1, 2, 3], 80); + // The node's frame is one the in-force baseline set admits: its own + // lookup, through the resolver carve-out. + let node_frame = ipv4_frame( + SUBNET.daemon_ip().octets(), + 17, + SUBNET.dns_server().octets(), + 53, + ); send_frame(&mut h.guest, &node_frame).await; assert_eq!( expect_frame(&mut h.switch).await, node_frame, - "the node's frame reaches the switch by its row" + "the node's frame reaches the switch by the node-plane baseline set" ); let box_frame = ipv4_frame(LEASE, 6, [10, 1, 2, 3], 80); send_frame(&mut h.guest, &box_frame).await; diff --git a/crates/minvmd/tests/guest_classifier_integration.rs b/crates/minvmd/tests/guest_classifier_integration.rs index cb8fd919d8..d031865ed6 100644 --- a/crates/minvmd/tests/guest_classifier_integration.rs +++ b/crates/minvmd/tests/guest_classifier_integration.rs @@ -1654,6 +1654,10 @@ fn native_ruleset_digest() -> String { // fails this digest, not a guest's table). let listener_addr = "127.0.0.1"; let listener_port = "7656"; + // The guest's render also names the box mark its table writes into a + // box's packets, so the VM host's gate can tell a box's frame from the + // daemon's own at the one address both leave from. + let cohort_dscp = sessions::core::egress::BOX_COHORT_DSCP.to_string(); let out = Command::new("bash") .arg(&script) .args([ @@ -1670,6 +1674,8 @@ fn native_ruleset_digest() -> String { &identity.to_string(), "--ct-mark-mask", GUEST_CT_MARK_MASK_ARG, + "--cohort-dscp", + &cohort_dscp, ]) .env("MINIMAL_OVERRIDE_CGROUP_MOUNTINFO", &mountinfo) .output() diff --git a/crates/minvmd/tests/vm_escape_integration.rs b/crates/minvmd/tests/vm_escape_integration.rs index 98b56b8597..82cf7ebba9 100644 --- a/crates/minvmd/tests/vm_escape_integration.rs +++ b/crates/minvmd/tests/vm_escape_integration.rs @@ -885,6 +885,10 @@ struct SpoofedFlow { dst_port: u16, src_port: u16, marker: String, + /// The DSCP value the flow's IPv4 frames carry: `0` for an unmarked + /// frame, [`sessions::core::egress::BOX_COHORT_DSCP`] for a frame that + /// wears the box mark. + dscp: u8, } const TCP_SYN: u8 = 0x02; @@ -914,6 +918,7 @@ const ARP_REPLY: u16 = 2; would repeat the flow's own fields" )] fn tcp_frame( + dscp: u8, src_mac: [u8; 6], src_ip: Ipv4Addr, src_port: u16, @@ -942,7 +947,7 @@ fn tcp_frame( // IPv4 header: version 4, IHL 5, no options, total length, checksum. let total_len = (20 + tcp.len()) as u16; let mut ip = Vec::with_capacity(20); - ip.extend_from_slice(&[0x45, 0x00]); + ip.extend_from_slice(&[0x45, dscp << 2]); ip.extend_from_slice(&total_len.to_be_bytes()); ip.extend_from_slice(&[0x00, 0x01]); // id ip.extend_from_slice(&[0x00, 0x00]); // no fragment flags @@ -1288,6 +1293,7 @@ fn spoofed_flow( write_frame(&mut sock, &claim).map_err(|e| format!("write ARP claim: {e}"))?; if !sent_syn { let syn = tcp_frame( + flow.dscp, flow.src_mac, flow.src, flow.src_port, @@ -1319,6 +1325,7 @@ fn spoofed_flow( Some(Incoming::SynAck(synack_seq)) => { // The handshake's last leg: ACK, then the marker on a push. let ack = tcp_frame( + flow.dscp, flow.src_mac, flow.src, flow.src_port, @@ -1331,6 +1338,7 @@ fn spoofed_flow( ); write_frame(&mut sock, &ack).map_err(|e| format!("write ACK: {e}"))?; let push = tcp_frame( + flow.dscp, flow.src_mac, flow.src, flow.src_port, @@ -1559,6 +1567,18 @@ impl HostListener { log.join("; ") } + /// How many connections the listener accepted so far. The switch's NAT + /// dials the listener when a SYN arrives, before it answers the SYN, so + /// a new accept proves a SYN left the VM whatever happens to its reply. + fn accepted(&self) -> usize { + self.log + .lock() + .expect("log lock is held only across this read") + .iter() + .filter(|entry| entry.starts_with("a connection was accepted")) + .count() + } + /// Whether a marker whose text contains `needle` has arrived yet. fn seen_any(&self, needle: &str) -> bool { self.seen @@ -1591,6 +1611,53 @@ impl Drop for HostListener { } } +/// Admits (`resident`) or retires a `host_ip` box's egress on the VM host +/// daemon's control socket, the exchange `min session activate` makes for a +/// `host_ip` box (NET-085): one request line in, one reply line out. +fn host_ip_box( + control_sock: &Path, + session_id: sessions::SessionId, + egress: Option, + resident: bool, +) -> Result<(), String> { + use std::io::BufRead as _; + let request = if resident { + minimald_rpc::BoxControlRequest::AdmitHostIpBox(minimald_rpc::HostIpBoxRequest { + session_id, + egress, + }) + } else { + minimald_rpc::BoxControlRequest::RetireHostIpBox(minimald_rpc::RetireHostIpBoxRequest { + session_id, + }) + }; + let mut line = serde_json_lenient::to_string(&request) + .map_err(|e| format!("serialize the host_ip box request: {e}"))?; + line.push('\n'); + let mut stream = UnixStream::connect(control_sock) + .map_err(|e| format!("connect to the control socket: {e}"))?; + stream + .set_read_timeout(Some(Duration::from_secs(30))) + .map_err(|e| format!("set the control socket's read timeout: {e}"))?; + stream + .write_all(line.as_bytes()) + .map_err(|e| format!("write the host_ip box request: {e}"))?; + let mut reply = String::new(); + std::io::BufReader::new(stream) + .read_line(&mut reply) + .map_err(|e| format!("read the host_ip box reply: {e}"))?; + match serde_json_lenient::from_str(reply.trim()) + .map_err(|e| format!("parse the host_ip box reply {reply:?}: {e}"))? + { + minimald_rpc::BoxControlReply::HostIpBox { + resident: answered, .. + } if answered == resident => Ok(()), + other => Err(format!( + "the VM host answered the host_ip box request with {other:?}" + )), + } +} + /// One recorded spoof attempt: what the frame wore, where it was headed, and /// the verdict its arrival gave it — the observability the bound is read /// through. @@ -1754,6 +1821,7 @@ async fn vm_escape_bounded_to_resident_union() { dst_port: port, src_port: 40_000 + u16::from(src.octets()[3]), marker: format!("spoof-{src}-arrived"), + dscp: 0, }; // The one arm this posture leaves: refused before any frame leaves // the VM — silence at the listener, and the unregistered-source line @@ -1807,6 +1875,7 @@ async fn vm_escape_bounded_to_resident_union() { dst_port: port, src_port: 40_000, marker: "spoof-203.0.113.7-arrived".to_string(), + dscp: 0, }; let verdict = match spoofed_flow(&guest.gate_sock, &flow, DROP_DEADLINE) { Ok(_) => panic!( @@ -1864,6 +1933,156 @@ async fn vm_escape_bounded_to_resident_union() { // unobservable frames; the record above is what the VM boundary can say // today. + // The box-marked spoof from the node's own address (NET-085): root in + // the VM writes the box mark on a frame from the node address. The mark + // proves no box, so the gate decides the frame by the union of the + // resident `host_ip` boxes' rules and drops it when no `host_ip` box is + // resident. Sent last: the flow's ARP claim announces the node's address + // from the spoofer's MAC, and the guest daemon's own traffic does not + // need the switch after this point. + let node = switch::DEFAULT_SUBNET.daemon_ip(); + let marked_flow = |tag: &str, src_port: u16| SpoofedFlow { + src: node, + src_mac: [0x02, 0x50, 0x64, 0x00, 0xfd, 0x01], + dst: host, + dst_port: port, + src_port, + marker: format!("marked-{tag}-arrived"), + dscp: sessions::core::egress::BOX_COHORT_DSCP, + }; + // The drop line is rate-limited per source and rule, so only the first + // refusal is sure to write one: a later refusal inside the window is + // read off the listener's silence alone. + let refused_marked = |flow: &SpoofedFlow, resident: usize, line_due: bool| -> String { + match spoofed_flow(&guest.gate_sock, flow, DROP_DEADLINE) { + Ok(marker_ack) => panic!( + "vm_escape_integration: a box-marked spoof from the node address \ + reached {host}:{port} with {resident} host_ip box(es) resident and \ + none admitting it; the gate must drop it (NET-085) [{marker_ack}]" + ), + Err(e) => { + assert!( + !listener.seen_any(&flow.marker), + "the marked spoof was refused at the gate, but its marker reached \ + the host listener; the listener saw: {}", + listener.report() + ); + assert!( + !line_due + || guest + .log_lines(&["egress-host-ip-boxes"]) + .iter() + .any(|line| { + line.contains(&format!("source={node}")) + && line.contains(&format!("resident_host_ip_boxes={resident}")) + }), + "the gate dropped the marked spoof without its host_ip line naming \ + the node address and {resident} resident box(es); the gate said: {}", + gate_account(&guest) + ); + format!("marked spoof refused with {resident} host_ip box(es) resident [{e}]") + } + } + }; + // No `host_ip` box resident: dropped. + let verdict = refused_marked(&marked_flow("none-resident", 41_001), 0, true); + attempts.push(Attempt { + source: node, + destination: host, + port, + verdict, + }); + // One `host_ip` box resident whose rules do not name the listener: + // still dropped. + let elsewhere = sessions::SessionId::parse_str(&uuid::Uuid::new_v4().to_string()).unwrap(); + host_ip_box( + &guest.control_sock, + elsewhere, + Some(sessions::EgressPolicy { + allow_protocols: Some(vec![sessions::IpProto::Tcp]), + allow_subnets: Some(vec!["192.0.2.0/24".to_string()]), + allow_dns_hosts: None, + deny_subnets: None, + }), + true, + ) + .expect("admitting a host_ip box elsewhere"); + let verdict = refused_marked(&marked_flow("not-admitted", 41_002), 1, false); + attempts.push(Attempt { + source: node, + destination: host, + port, + verdict, + }); + // A second `host_ip` box whose rules name the listener: the marked frame + // passes under that box's rules, and its SYN reaches the listener. + // + // The spoofer cannot complete the handshake, and the gate is not why. + // The in-VM daemon holds the node address with its own MAC, and its own + // traffic keeps the switch's neighbor entry for that address reachable. + // An unsolicited ARP reply does not override a reachable entry, so the + // SYN-ACK goes to the daemon's connection, where the VM's kernel answers + // it, and never to the spoofer's. A real host_ip box shares the daemon's + // address and MAC, so its replies take that same path. What the gate + // decides is the outbound frame, and the NAT's dial at the listener is + // the proof that the SYN left the VM: the NAT dials the backend before + // it answers the SYN. + let reaching = sessions::SessionId::parse_str(&uuid::Uuid::new_v4().to_string()).unwrap(); + host_ip_box( + &guest.control_sock, + reaching, + Some(sessions::EgressPolicy { + allow_protocols: Some(vec![sessions::IpProto::Tcp]), + allow_subnets: Some(vec![host_cidr.clone()]), + allow_dns_hosts: None, + deny_subnets: None, + }), + true, + ) + .expect("admitting a host_ip box that reaches the listener"); + let admitted = marked_flow("admitted", 41_003); + let accepted_before = listener.accepted(); + // The flow ends at its deadline without a SYN-ACK: the reply goes to the + // in-VM daemon (see above). A completed flow is fine too. + let outcome = match spoofed_flow(&guest.gate_sock, &admitted, Duration::from_secs(10)) { + Ok(marker_ack) => format!("the flow completed [{marker_ack}]"), + Err(e) => format!("the reply went to the in-VM daemon [{e}]"), + }; + let dialed_end = Instant::now() + Duration::from_secs(10); + while listener.accepted() <= accepted_before && Instant::now() < dialed_end { + std::thread::sleep(Duration::from_millis(100)); + } + assert!( + listener.accepted() > accepted_before, + "vm_escape_integration: a box-marked frame a resident host_ip box admits \ + never reached {host}:{port}: the listener accepted no new connection \ + ({outcome}); the listener saw: {}; the gate said: {}", + listener.report(), + gate_account(&guest) + ); + assert!( + !guest + .log_lines(&["egress-host-ip-boxes"]) + .iter() + .any(|line| line.contains("resident_host_ip_boxes=2")), + "the gate dropped the admitted marked frame under the host_ip rule; \ + the gate said: {}", + gate_account(&guest) + ); + attempts.push(Attempt { + source: node, + destination: host, + port, + verdict: format!( + "marked frame admitted by a resident host_ip box's rules, and its SYN \ + reached the listener ({outcome})" + ), + }); + for session in [elsewhere, reaching] { + host_ip_box(&guest.control_sock, session, None, false) + .expect("retiring the test's host_ip boxes"); + } + // The record, then the host-side gate's own lines for the spoofed // sources: the attempt table and the drop lines beside it, so the // bound can be read off the test output. @@ -1879,6 +2098,7 @@ async fn vm_escape_bounded_to_resident_union() { "egress-unregistered-source", "egress-unknown-source", "egress-undeclared-subnet", + "egress-host-ip-boxes", ]) { eprintln!(" {line}"); } diff --git a/crates/sessions/src/core/egress.rs b/crates/sessions/src/core/egress.rs index 0b73c07fe3..baae4bca93 100644 --- a/crates/sessions/src/core/egress.rs +++ b/crates/sessions/src/core/egress.rs @@ -120,6 +120,17 @@ pub const PROXY_LISTENER_PORT: u16 = 8118; /// ([`proxy_lane_admits`]). pub const PROXY_LISTENER_PROTOCOL: u8 = 6; +/// The DSCP value a VM guest's classifier writes on every IPv4 packet of a +/// connection a box's cgroup opened, so the host-side gate can tell a box's +/// frame from the guest daemon's own when both leave from the node address. +/// The guest's table clears the field on every other packet, so an honest +/// sender outside the boxes cohort never wears the value. Root in the guest +/// can write any frame, the mark included, so the mark is no proof of a +/// box: the host gate bounds a marked frame from the node address by the +/// rules it decides the cohort's frames by. It is in RFC 2474's pool 2 +/// (`xxxx11`), the codepoints kept for local use. +pub const BOX_COHORT_DSCP: u8 = 0b11_0111; + /// The Box Egress Proxy's listener on the switch this box attaches to — /// the destination a credentialed lane's frames are admitted to, and the /// only one (NET-134): the listener, never the address, because what a @@ -500,6 +511,8 @@ pub struct FrameSummary { /// [`DNS_PORT`], so the missing port fails the resolver carve-out /// closed rather than opening it. dst_port: u16, + /// The IPv4 header's DSCP field, `0` when the frame carries none. + dscp: u8, } impl FrameSummary { @@ -535,6 +548,13 @@ impl FrameSummary { pub fn destination_port(&self) -> u16 { self.dst_port } + + /// Whether a VM guest's classifier marked the frame as a box's own: + /// an IPv4 frame whose DSCP field is [`BOX_COHORT_DSCP`]. + #[must_use] + pub fn box_cohort_marked(&self) -> bool { + self.family == FrameFamily::Ipv4 && self.dscp == BOX_COHORT_DSCP + } } /// Extracts the verdict's inputs from one Ethernet II frame. Pure and @@ -549,6 +569,7 @@ pub fn summarize(frame: &[u8]) -> FrameSummary { dst: None, proto: None, dst_port: 0, + dscp: 0, }; if frame.len() < ETH_HDR { return none(FrameFamily::Truncated); @@ -582,6 +603,7 @@ pub fn summarize(frame: &[u8]) -> FrameSummary { dst: None, proto: None, dst_port: 0, + dscp: 0, }; } ETHERTYPE_IPV6 => return none(FrameFamily::Ipv6), @@ -610,6 +632,7 @@ pub fn summarize(frame: &[u8]) -> FrameSummary { dst: Some([ip[16], ip[17], ip[18], ip[19]]), proto: Some(ip[9]), dst_port, + dscp: ip[1] >> 2, } } diff --git a/scripts/install-host-classifier.sh b/scripts/install-host-classifier.sh index 5068d39322..3fd0aa4968 100755 --- a/scripts/install-host-classifier.sh +++ b/scripts/install-host-classifier.sh @@ -21,6 +21,7 @@ # [--answerer-address ADDR] [--answerer-port PORT] # [--ct-mark-mask 0x30000000] # [--cohort-address ADDR --node-plane-address ADDR] +# [--cohort-dscp 1-63] # VM guest only # sudo scripts/install-host-classifier.sh --pid PID # sudo scripts/install-host-classifier.sh --user NAME --place-listener SOCK # sudo scripts/install-host-classifier.sh --uninstall @@ -172,6 +173,11 @@ answerer_port=7656 # cgroup matches, translated to nothing. cohort_address= node_plane_address= +# The DSCP value a VM guest writes on a box's packets, so the VM host's +# gate can tell a box's frame from the guest daemon's own when both leave +# from the guest's one address. Only the guest's own render passes it. A +# native host renders no DSCP rule and leaves the field alone. +cohort_dscp= # The unix socket --place-listener reads the daemon's pid from: every # process of the delegated account holding it is placed in the daemon leaf. listener= @@ -221,6 +227,17 @@ while [ $# -gt 0 ]; do node_plane_address=$2 shift 2 ;; + --cohort-dscp) + [ $# -ge 2 ] || die "--cohort-dscp needs a DSCP value" + case "$2" in ''|*[!0-9]*) + die "--cohort-dscp needs a numeric DSCP value, got: $2" ;; + esac + if [ "$2" -lt 1 ] || [ "$2" -gt 63 ]; then + die "--cohort-dscp needs a DSCP value from 1 to 63, got: $2" + fi + cohort_dscp=$2 + shift 2 + ;; --ct-mark-mask) [ $# -ge 2 ] || die "--ct-mark-mask needs a mask" ct_mark_mask=$2 @@ -680,6 +697,14 @@ cgroup_level() { # nothing is translated, and the chain stands so an association can add # the reserved addresses to the same two marks without a reinstall. # +# With --cohort-dscp, a VM guest's render adds the mark_cohort chain. It +# runs at postrouting as a filter chain, so it sees every packet, not only +# a connection's first. A packet of a connection the boxes cohort opened +# leaves with the given DSCP value. Every other packet leaves with the +# field cleared, so no process outside the cohort can wear the value. The +# VM host's gate reads the value to decide a frame from the guest's one +# address by the box's rules, and an unmarked frame by the node plane's. +# # The ipv6_out chain refuses every IPv6 connection a box in the boxes # subtree opens to anything but loopback, whatever the box declares. A # host-address box's allow list is decided by IPv4 address rules only, and @@ -703,6 +728,15 @@ render_ruleset() { ct mark and $mask_hex == $cohort_hex oifname != \"lo\" snat ip to $cohort_address ct mark and $mask_hex == $node_hex oifname != \"lo\" snat ip to $node_plane_address" fi + dscp_chain= + if [ -n "$cohort_dscp" ]; then + dscp_chain=" + chain mark_cohort { + type filter hook postrouting priority mangle; policy accept; + ct mark and $mask_hex == $cohort_hex ip dscp set $cohort_dscp + ct mark and $mask_hex != $cohort_hex ip dscp set 0 + }" + fi carve_out_rule=" ip daddr $answerer_address udp dport $answerer_port accept" dstnat_chain=" @@ -745,7 +779,7 @@ table inet $TABLE_NAME { } chain postrouting { type nat hook postrouting priority srcnat; policy accept;$snat_rules - } + }$dscp_chain } RULES } diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index d48d1aa434..4fa704d8ae 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -15151,9 +15151,9 @@ ESU_LISTENER_EOF esu_fail fi if ! grep -q 'own downloads in this VM' <<<"$esu_policy" \ - || ! grep -q 'registry 100.64.255.254/32' <<<"$esu_policy" \ - || ! grep -q 'cache 100.64.255.254/32' <<<"$esu_policy"; then - echo "::error::box $esu_box's policy does not show the node-plane baseline set beside its rules (the registry and cache categories at the host alias)" + || ! grep -q 'registry github.com' <<<"$esu_policy" \ + || ! grep -q 'cache cache.minimal.dev' <<<"$esu_policy"; then + echo "::error::box $esu_box's policy does not show the node-plane baseline set beside its rules (the registry github.com and the cache cache.minimal.dev)" esu_fail fi done From 5dddc5e43830ecd2a9a0ca94332268382ba29da7 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:20:11 +0000 Subject: [PATCH 16/34] fix(minvmd): refuse flows to host.min.internal outside the design's openings (#2871) T102. Behaviour change, for the release notes: host-address boxes reach only ports opened in [host-exposures]. Before this, a host-address box reached the host through host.min.internal at every port. Own-address boxes stay refused end to end: the host gate would admit a configured exposure under their rules, but the guest still refuses them. The egress gate refuses a flow to the host alias unless a port in the current [host-exposures] table of the client config.toml opens it, and the row's own rules then decide the flow. Each entry is { proto, port }, with proto defaulting to tcp. The node plane alone also reaches the ports its baseline endpoints open at the alias. minvmd rereads the table when the file changes, so an entry opens or closes its port without a restart. A missing or unparseable file fails closed. An entry at a Minimal listener's port is refused and logged. Each refused flow logs under the host-alias rule with the row and the port. This retires MINVMD_HOST_EXPOSURE_PORTS, and minvmd warns if it is still set. Backport of #2871. Lands after T103 here, as on main: its baseline and gate edits sit on that commit. T147 #2956, the project-file warning, stays out of 0.7.3. A project's [host-exposures] table still opens nothing, because the reader takes only the client file. Adapted to the line: - The client config lives in sessions/src/client/config.rs, and host_exposures joins it without main's [secrets] and [secret-store-rules] sections. - boxes:: reads sessions::, tests draw session ids from a v4 UUID, and DropLimiter stays in egress_gate.rs. (cherry picked from commit 964a71b577bfc7c0c460d9dff72ba9d8f632c746) --- crates/minvmd/src/net.rs | 39 +- crates/minvmd/src/net/baseline.rs | 295 +++--- crates/minvmd/src/net/egress_gate.rs | 1176 +++++++++++++++++++---- crates/minvmd/src/net/host_exposures.rs | 613 ++++++++++++ crates/sessions/src/client/config.rs | 47 + docs/reference/cli-minvmd.md | 2 +- scripts/session-e2e.sh | 34 +- 7 files changed, 1810 insertions(+), 396 deletions(-) create mode 100644 crates/minvmd/src/net/host_exposures.rs diff --git a/crates/minvmd/src/net.rs b/crates/minvmd/src/net.rs index b570e3bc2e..5df220f4c2 100644 --- a/crates/minvmd/src/net.rs +++ b/crates/minvmd/src/net.rs @@ -94,6 +94,7 @@ pub use shuttle::{VSOCK_GVPROXY_SHUTTLE_PORT, resolve_gate_sock, resolve_switch_ #[cfg_attr(not(minvmd_libkrun), allow(unused_imports))] pub(crate) use stale_gvproxy::reap_stale_gvproxy; mod baseline; +pub(crate) mod host_exposures; pub use baseline::{NodeBaselinePhase, NodePlaneBaseline}; pub use switch::bep_host::{BepBoxSource, BepWire}; pub use switch::{BepDevice, BepDeviceEnds, BepHost, BepPeer}; @@ -609,6 +610,25 @@ async fn monitor_switch_datapath( } } +/// The environment variable that listed the host exposures before the +/// `[host-exposures]` table of the client `config.toml` replaced it. +const RETIRED_HOST_EXPOSURE_ENV: &str = "MINVMD_HOST_EXPOSURE_PORTS"; + +/// Logs a warning when the retired host-exposure variable is set. `minvmd` +/// ignores its value, and only the `[host-exposures]` table opens a port. +fn warn_retired_host_exposure_env(value: Option) -> bool { + let Some(value) = value else { + return false; + }; + tracing::warn!( + variable = RETIRED_HOST_EXPOSURE_ENV, + value = %value.to_string_lossy(), + "ignored MINVMD_HOST_EXPOSURE_PORTS: host.min.internal opens only the ports \ + that the [host-exposures] table of the client config.toml lists" + ); + true +} + /// Deliver `signal` to the process referred to by `pidfd` via /// `pidfd_send_signal(2)`. A pidfd is bound to the exact process instance /// opened at spawn, so this call returns `ESRCH` after the process exits @@ -756,11 +776,16 @@ impl HostGvproxy { // by it, beside the boxes' rows; announced — the shipped posture — // the allow-all interim node row still does, and the gate's // start-up line names the posture deciding. The host exposures the - // alias is opened at for the node row — none unless the deployment - // names ports — are read from the daemon's environment here, on each - // spawn, an unparseable list failing it. + // alias is opened at — none unless the client `config.toml` names + // ports in `[host-exposures]` — are the live table: the gate reads + // it again when the file changes, so an added entry opens its port + // and a removed one closes it, with no restart (NET-161). + // The retired variable no longer opens a port. Say so in the daemon + // log, so an operator who still sets it sees why the alias stays + // closed at its ports (NET-159). + warn_retired_host_exposure_env(std::env::var_os(RETIRED_HOST_EXPOSURE_ENV)); let baseline = NodePlaneBaseline::built_in(subnet) - .with_host_exposure(baseline::host_exposure_ports_from_env()?); + .with_host_exposures(host_exposures::HostExposures::client_config()); let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>(); let (ready_tx, ready_rx) = std::sync::mpsc::channel::>(); @@ -1000,6 +1025,12 @@ mod tests { use std::sync::Mutex; use tracing_subscriber::fmt::MakeWriter; + #[test] + fn retired_host_exposure_env_is_warned_when_set() { + assert!(!warn_retired_host_exposure_env(None)); + assert!(warn_retired_host_exposure_env(Some("8080".into()))); + } + fn spawn_sleep() -> Child { // A long-lived child to stand in for gvproxy in supervision tests. Command::new("sleep") diff --git a/crates/minvmd/src/net/baseline.rs b/crates/minvmd/src/net/baseline.rs index fd3d0b6a59..9a7065fa2d 100644 --- a/crates/minvmd/src/net/baseline.rs +++ b/crates/minvmd/src/net/baseline.rs @@ -39,8 +39,9 @@ //! ([`crate::net::egress_gate`]). The set never decides a marked frame, and //! the node row never does either. +use crate::net::host_exposures::{Holder, HostExposure, HostExposures, PortRange}; use sessions::core::egress::{EgressRules, Ipv4Cidr}; -use std::io; +use std::sync::Arc; use switch::SwitchSubnet; /// A category of the node-plane baseline enumeration: one of the design's @@ -255,75 +256,17 @@ fn baseline_endpoints_from_raw(var: &str, raw: &str) -> Vec { .collect() } -/// The environment variable the daemon's host-exposure port list is read -/// from, each time the daemon starts the VM's network — from the daemon's -/// own environment, so a change takes a daemon restart, the same shape the -/// hostname proxy port and the egress-default opt-out carry. -pub(crate) const HOST_EXPOSURE_PORTS_ENV: &str = "MINVMD_HOST_EXPOSURE_PORTS"; - -/// The host-exposure port list the daemon starts with, as the deployment -/// spelled it. The design holds the alias default-deny except explicitly -/// configured host exposures (design §7.1), so absent or empty, the list is -/// empty and the alias is refused at every port for every row. The list is -/// comma-separated, each term a port (`4200`) or an inclusive range -/// (`3000-3999`), port 0 excluded; a term that is neither, a reversed range, -/// or a value that is not Unicode fails the start, the error naming the term -/// and the fix. -pub(crate) fn host_exposure_ports_from_env() -> io::Result> { - match std::env::var(HOST_EXPOSURE_PORTS_ENV) { - Ok(raw) => host_exposure_ports_from_raw(Some(&raw)), - Err(std::env::VarError::NotPresent) => host_exposure_ports_from_raw(None), - Err(std::env::VarError::NotUnicode(_)) => Err(io::Error::other(format!( - "{HOST_EXPOSURE_PORTS_ENV} is not valid Unicode. {HOST_EXPOSURE_FIX}" - ))), - } -} +/// The name every box and the node plane resolve the host alias by. +pub(crate) const HOST_ALIAS_NAME: &str = "host.min.internal"; -/// The correction every parse error ends with. -const HOST_EXPOSURE_FIX: &str = "Set it to ports and ranges such as 8080,3000-3999, \ - or set it empty to close host.min.internal."; +/// The IP protocol the node plane reaches a baseline endpoint at the host +/// alias over: TCP, the protocol of the registry and cache fetches. +pub(crate) const ALIAS_FETCH_PROTOCOL: u8 = sessions::core::egress::IPPROTO_TCP; -/// Parses one deployment's spelling of the host-exposure port list: every -/// term a port or an inclusive range from 1 to 65535, whitespace trimmed; -/// absent or empty, no exposure at all. -fn host_exposure_ports_from_raw(raw: Option<&str>) -> io::Result> { - let Some(raw) = raw else { - return Ok(Vec::new()); - }; - let port = |text: &str, term: &str| match text.trim().parse::() { - Ok(0) | Err(_) => Err(bad_host_exposure_term(term)), - Ok(port) => Ok(port), - }; - raw.split(',') - .filter(|term| !term.trim().is_empty()) - .map(|term| { - let term = term.trim(); - match term.split_once('-') { - Some((lo, hi)) => { - let (lo, hi) = (port(lo, term)?, port(hi, term)?); - if lo <= hi { - Ok((lo, hi)) - } else { - Err(io::Error::other(format!( - "{HOST_EXPOSURE_PORTS_ENV}: `{term}` is a reversed range; write it \ - low-high, as `{hi}-{lo}`. {HOST_EXPOSURE_FIX}" - ))) - } - } - None => port(term, term).map(|port| (port, port)), - } - }) - .collect() -} - -/// The start-failing error for one unparseable term: the variable, the term -/// as it was read, and the correction. -fn bad_host_exposure_term(term: &str) -> io::Error { - io::Error::other(format!( - "{HOST_EXPOSURE_PORTS_ENV}: `{term}` is not a port from 1 to 65535 or a low-high port \ - range. {HOST_EXPOSURE_FIX}" - )) -} +/// The ports the node plane reaches a baseline endpoint at the host alias +/// at: the HTTPS port of the registry and cache fetches. The gate pins these +/// to its own fetch ports, so the two cannot drift. +pub(crate) const ALIAS_FETCH_PORTS: [u16; 1] = [443]; /// The node-plane baseline set: the categories in force, each with the /// endpoints it admits, and the compiled rules the host-side gate decides @@ -346,13 +289,17 @@ pub struct NodePlaneBaseline { rules: EgressRules, /// The posture the gate decides the set under: the shipped constant. phase: NodeBaselinePhase, - /// The configured host exposures over the alias, as inclusive port - /// ranges: the ports the node plane reaches the host alias at, every - /// other port refused through the alias. Empty unless the deployment - /// names ports. The host-side gate alone reads it: the guest-side - /// verdict's own alias list starts empty and nothing in the run path - /// fills it, so own-address boxes stay refused at every port. - host_exposure: Vec<(u16, u16)>, + /// The host alias on this fabric: the address [`Self::alias_ports`] + /// opens for the node plane. + alias: [u8; 4], + /// The ports the set's own endpoints open at the host alias, for the + /// node-plane identity alone ([`Self::alias_ports`]). + alias_ports: Vec, + /// The configured host exposures over the alias: the current + /// `[host-exposures]` table of the client `config.toml`. The gate opens + /// the alias at these ports for every row, and the row's own rules + /// decide the flow from there. + host_exposure: HostExposures, } impl NodePlaneBaseline { @@ -390,6 +337,7 @@ impl NodePlaneBaseline { let mut baseline = Self::from_entries( subnet.daemon_ip().octets(), subnet.dns_server().octets(), + subnet.host_alias().octets(), vec![ BaselineEntry { category: BaselineCategory::Registry, @@ -459,44 +407,47 @@ impl NodePlaneBaseline { named } - /// The host exposures over the alias this gate carries: the inclusive - /// port ranges the node plane reaches the host alias at, empty unless - /// the deployment configured ports. + /// The host exposures the current table opens at the alias while the + /// listeners in `held` hold their ports, each a protocol and a port: + /// empty unless the client `config.toml` names entries in + /// `[host-exposures]`. The gate reads them at each flow (NET-161, + /// NET-162). + #[must_use] + pub(crate) fn host_exposure_ports(&self, held: &[Holder]) -> Arc<[HostExposure]> { + self.host_exposure.current(held) + } + + /// The ports the set's own endpoints open at the host alias, over + /// [`ALIAS_FETCH_PROTOCOL`]: [`ALIAS_FETCH_PORTS`] when an entry names + /// the alias, as a subnet that covers it or as `host.min.internal`, and + /// none otherwise. The gate reads this set for the node-plane identity + /// alone. A box row and the host-address cohort never get it. + #[must_use] + pub(crate) fn alias_ports(&self) -> &[PortRange] { + &self.alias_ports + } + + /// Points the host exposures at a fixed list, replacing the current + /// table: a cleared list is an alias with no opening. The compiled set + /// is untouched: the list scopes the alias alone. #[must_use] - pub(crate) fn host_exposure_ports(&self) -> &[(u16, u16)] { - &self.host_exposure + pub fn with_host_exposure(self, ports: Vec) -> Self { + self.with_host_exposures(HostExposures::fixed(ports)) } - /// Points the host exposures at `ports`, replacing the current list: a - /// host names or clears its own exposure surface, and a cleared list is an - /// alias with no opening — refused at every port for every row — not - /// an absent arm. The compiled set is untouched: the list scopes the - /// alias alone. + /// Points the host exposures at a live table, so the gate decides each + /// flow by the table as it is at that flow. #[must_use] - pub fn with_host_exposure(mut self, ports: Vec<(u16, u16)>) -> Self { - self.host_exposure = ports; + pub fn with_host_exposures(mut self, exposures: HostExposures) -> Self { + self.host_exposure = exposures; self } - /// The host-exposure list as the daemon's start-up line spells it: one - /// `lo-hi` term per range, a bare port for a single-port term, and - /// `(none)` when the deployment cleared the list. + /// The current host-exposure table as the daemon log spells it: one + /// `proto/port` term per exposure, and `(none)` for no exposure. #[must_use] pub(crate) fn render_host_exposure(&self) -> String { - if self.host_exposure.is_empty() { - return "(none)".to_string(); - } - self.host_exposure - .iter() - .map(|&(lo, hi)| { - if lo == hi { - lo.to_string() - } else { - format!("{lo}-{hi}") - } - }) - .collect::>() - .join(",") + self.host_exposure.render() } /// One entry's endpoints as the policy display spells them (NET-061): @@ -583,20 +534,44 @@ impl NodePlaneBaseline { "re-pointed a node-plane baseline entry", ); self.rules = Self::compile_rules(&self.entries, self.resolver, self.node_addr); + self.alias_ports = Self::compile_alias_ports(&self.entries, self.alias); } /// Assembles the set from its parts: the categories carried, and the /// addresses the compiled rules are keyed to. The host exposures start - /// empty, the alias closed; the deployment's spelling replaces them in - /// the run path's builder. - fn from_entries(node_addr: [u8; 4], resolver: [u8; 4], entries: Vec) -> Self { + /// empty, the alias closed; the run path hands the live table in. + fn from_entries( + node_addr: [u8; 4], + resolver: [u8; 4], + alias: [u8; 4], + entries: Vec, + ) -> Self { Self { node_addr, resolver, rules: Self::compile_rules(&entries, resolver, node_addr), + alias, + alias_ports: Self::compile_alias_ports(&entries, alias), entries, phase: NODE_BASELINE_PHASE, - host_exposure: Vec::new(), + host_exposure: HostExposures::fixed(Vec::new()), + } + } + + /// The set's own openings at the host alias: [`ALIAS_FETCH_PORTS`] when + /// an entry names the alias, as a subnet that covers it or by the name + /// [`HOST_ALIAS_NAME`], and none otherwise. The node plane reaches a + /// registry or a cache on the host this way, and only at the fetch + /// ports. + fn compile_alias_ports(entries: &[BaselineEntry], alias: [u8; 4]) -> Vec { + let names_alias = entries.iter().any(|entry| { + entry.cidrs().iter().any(|cidr| cidr.contains(alias)) + || entry.names().any(|name| name == HOST_ALIAS_NAME) + }); + if names_alias { + ALIAS_FETCH_PORTS.iter().map(|&port| (port, port)).collect() + } else { + Vec::new() } } @@ -619,11 +594,11 @@ impl NodePlaneBaseline { #[cfg(test)] mod tests { + use super::HostExposure; use super::{ BASELINE_CACHE_ENV, BASELINE_REGISTRY_ENV, BaselineCategory, BaselineEntry, - DAEMON_CACHE_ENDPOINT, DAEMON_REGISTRY_ENDPOINT, HOST_EXPOSURE_PORTS_ENV, - NodeBaselinePhase, NodePlaneBaseline, baseline_endpoints_from_env, - baseline_endpoints_from_raw, host_exposure_ports_from_raw, + DAEMON_CACHE_ENDPOINT, DAEMON_REGISTRY_ENDPOINT, NodeBaselinePhase, NodePlaneBaseline, + baseline_endpoints_from_env, baseline_endpoints_from_raw, }; use sessions::core::egress::{FrameVerdict, summarize, verdict}; use std::net::Ipv4Addr; @@ -833,11 +808,41 @@ mod tests { /// subnet, so a frame from the daemon's own address to any destination /// is the undeclared drop here and the pins decide it at the gate. A /// frame wearing another source is rejected by the lease the set is - /// keyed to. + /// keyed to. The set opens the host alias only when an entry names it, + /// and then at the fetch ports alone (NET-156). #[test] - fn baseline_rules_admit_no_address_but_the_resolver() { + fn baseline_rules_admit_the_enumerated_categories_alone() { let baseline = NodePlaneBaseline::built_in(SUBNET); let node_addr = baseline.node_addr(); + let alias = SUBNET.host_alias().octets(); + assert!( + baseline.alias_ports().is_empty(), + "the built-in categories name no endpoint at the host alias" + ); + let summary = summarize(&tcp_frame(node_addr, alias, 443)); + assert!( + matches!(verdict(&summary, baseline.rules()), FrameVerdict::Drop(_)), + "the compiled set admits nothing at the host alias" + ); + let at_alias = baseline + .clone() + .with_cache(vec![format!("{}/32", Ipv4Addr::from(alias))]); + assert_eq!( + at_alias.alias_ports(), + &[(443, 443)], + "a cache at the host alias opens the fetch port alone" + ); + let by_name = baseline + .clone() + .with_registry(vec!["Host.Min.Internal.".to_string()]); + assert_eq!(by_name.alias_ports(), &[(443, 443)]); + let elsewhere = baseline + .clone() + .with_cache(vec!["203.0.113.0/24".to_string()]); + assert!( + elsewhere.alias_ports().is_empty(), + "a subnet that does not cover the alias opens nothing there" + ); let summary = summarize(&tcp_frame(node_addr, [8, 8, 8, 8], 443)); match verdict(&summary, baseline.rules()) { @@ -856,60 +861,6 @@ mod tests { } } - /// The deployment's spelling of the host-exposure list: absent or - /// empty, no exposure — the alias closed; every term a port or an - /// inclusive range from 1 to 65535; an undecodable term, port 0, or a - /// reversed range a surfaced error naming the variable, the term and - /// the fix. - #[test] - fn host_exposure_ports_read_from_the_environment() { - assert!( - host_exposure_ports_from_raw(None).unwrap().is_empty(), - "an unset variable is no exposure: the alias stays closed" - ); - assert_eq!( - host_exposure_ports_from_raw(Some("8080")).unwrap(), - vec![(8080, 8080)], - "a bare port is a single-port range" - ); - assert_eq!( - host_exposure_ports_from_raw(Some(" 3000-3999, 4200 ")).unwrap(), - vec![(3000, 3999), (4200, 4200)], - "terms are inclusive ranges and ports, whitespace trimmed" - ); - assert!( - host_exposure_ports_from_raw(Some("")).unwrap().is_empty(), - "a set-but-empty list is no exposure" - ); - let garbage = host_exposure_ports_from_raw(Some("8080:nope")) - .expect_err("an undecodable term is a surfaced error") - .to_string(); - assert!( - garbage.contains(HOST_EXPOSURE_PORTS_ENV) - && garbage.contains("`8080:nope` is not a port") - && garbage.contains("set it empty to close host.min.internal"), - "an undecodable term names the variable, the term and the fix, got: {garbage}" - ); - for zero in ["0", "0-65535"] { - let error = host_exposure_ports_from_raw(Some(zero)) - .expect_err("port 0 is refused") - .to_string(); - assert!( - error.contains(&format!("`{zero}` is not a port")), - "port 0 names the term, got: {error}" - ); - } - let reversed = host_exposure_ports_from_raw(Some("4000-3000")) - .expect_err("a reversed range is not an inclusive one") - .to_string(); - assert!( - reversed.contains("`4000-3000` is a reversed range") - && reversed.contains("`3000-4000`") - && !reversed.contains("is not a port"), - "a reversed range gets its own message, got: {reversed}" - ); - } - /// The baseline carries no exposure until the deployment's spelling /// replaces the list: the builder replaces rather than merges, and a /// cleared list reaches the accessor empty. @@ -917,18 +868,18 @@ mod tests { fn baseline_carries_the_configured_host_exposures() { let built_in = NodePlaneBaseline::built_in(SUBNET); assert!( - built_in.host_exposure_ports().is_empty(), + built_in.host_exposure_ports(&[]).is_empty(), "the built-in enumeration carries no host exposure" ); - let re_exposed = built_in.with_host_exposure(vec![(8080, 8080)]); + let re_exposed = built_in.with_host_exposure(vec![HostExposure::tcp(8080)]); assert_eq!( - re_exposed.host_exposure_ports(), - &[(8080, 8080)], + &*re_exposed.host_exposure_ports(&[]), + &[HostExposure::tcp(8080)], "the builder sets the list" ); let cleared = re_exposed.with_host_exposure(vec![]); assert!( - cleared.host_exposure_ports().is_empty(), + cleared.host_exposure_ports(&[]).is_empty(), "a cleared list is carried empty" ); } diff --git a/crates/minvmd/src/net/egress_gate.rs b/crates/minvmd/src/net/egress_gate.rs index f06a9a9a7a..771d558a1a 100644 --- a/crates/minvmd/src/net/egress_gate.rs +++ b/crates/minvmd/src/net/egress_gate.rs @@ -175,9 +175,10 @@ use tokio::task::{JoinHandle, JoinSet}; use crate::bep_attach::{BoxId, BoxIdText}; use crate::box_registry::{BoxRecord, BoxTable, RowWithdrawal}; -use super::baseline::{NodeBaselinePhase, NodePlaneBaseline}; +use super::baseline::{self, NodeBaselinePhase, NodePlaneBaseline}; use super::dns_pins; use super::forward_revoke::{self, ForwardedFlows}; +use super::host_exposures; /// The HTTP request that upgrades a control-socket connection into a raw /// frame stream. The guest shuttle writes this head before its first frame; @@ -462,6 +463,14 @@ const NODE_PLANE_FETCH_PROTOCOL: u8 = egress::IPPROTO_TCP; /// [`NODE_PLANE_UNPINNED_RULE`]'s drop, the same as an address no pin names. const NODE_PLANE_FETCH_PORTS: [u16; 1] = [443]; +// The baseline set opens the host alias at the node plane's fetch ports, over +// its fetch protocol, and at nothing else. +const _: () = assert!( + NODE_PLANE_FETCH_PORTS.len() == baseline::ALIAS_FETCH_PORTS.len() + && NODE_PLANE_FETCH_PORTS[0] == baseline::ALIAS_FETCH_PORTS[0] + && NODE_PLANE_FETCH_PROTOCOL == baseline::ALIAS_FETCH_PROTOCOL +); + /// The rule name for a frame headed to the switch's own address — the plan's /// gateway — that is not a resolver query: TCP or UDP to the resolver's port /// is the one carve-out the frame rules admit there, and everything else — @@ -563,21 +572,180 @@ static INFRASTRUCTURE_RANGES: LazyLock = LazyLock::new(|| } }); -/// Whether `dst` lies in the infrastructure deny set as the host frame rule -/// holds it ([`INFRASTRUCTURE_RULE`]): a fixed range; the host alias, which -/// is default-deny at every port for a box row (design §7.1, NET-062) when -/// `refuse_host_alias` says so, and for the node row's frames too outside -/// the host exposures `host_exposure` carries — the inclusive port ranges -/// the deployment configured, empty unless it named ports, matched against -/// `transport_port`, the TCP or UDP destination port, `None` for any other -/// frame. Box rows stay refused at a configured exposure too: the design -/// holds an exposure as local reach under the box's egress rules (§7.1), -/// and that reach is not built yet; the refusal fails closed, and -/// gominimal/inbox#1144 owns the gap. The fabric plane outside `own_block`, the subnet the gate's rows live in — a frame to a -/// sibling or to the daemon inside the node's own block is local reach, -/// decided by the row's CIDR rules and the target's ingress, never by this -/// rule, and the gateway inside it is the control-surface rule's; or RFC -/// 1918 space the row's `allow_subnets` does not cover. +/// The rule name for a frame to the host alias, `host.min.internal`, at a +/// port no opening the source's row has covers (NET-156, design §7.1). The +/// drop line names the row ([`AliasRow`]) and the port beside it. +const HOST_ALIAS_RULE: &str = "host-alias"; + +/// The row a frame to the host alias is decided for: the in-VM daemon's own +/// traffic (`node`), the host-address boxes that share the guest's root +/// namespace (`cohort`, the frames the node address carries for them), or +/// one box with its own address (`box`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AliasRow { + Node, + Cohort, + Box, +} + +impl AliasRow { + /// The row as the drop line spells it. + fn as_str(self) -> &'static str { + match self { + Self::Node => "node", + Self::Cohort => "cohort", + Self::Box => "box", + } + } +} + +/// The openings the host alias has for one row. The alias is default-deny +/// (design §7.1, NET-156, NET-158): every row reaches it at the configured +/// host exposures, the current `[host-exposures]` table (NET-161), and the +/// row's own rules decide the flow from there. A box row reaches an +/// exposure only when the box names the host, by `host.min.internal` in +/// `egress.allow_dns_hosts` or by an `allow_subnets` that covers the alias +/// (NET-160). The node-plane identity alone also reaches the alias at the +/// ports the baseline set's own endpoints open there +/// ([`NodePlaneBaseline::alias_ports`]). An exposure opens its port for its +/// own protocol alone. +/// +/// Minimal's own listeners and sockets are refused ahead of the exposure +/// openings, at each flow (NET-162): the hostname proxy at the node row's +/// assigned port, the box-name answerer, the Box Egress Proxy's listener, +/// and the baseline endpoints at the alias for every row but the node's. +/// So a listener that binds an exposed port later shadows the entry from +/// the next flow on ([`host_exposures::HostExposures::current`]). +/// +/// The design's two other host-side openings are not at the alias: the +/// resolver's port is on the gateway address ([`SWITCH_CONTROL_RULE`]'s +/// arm), and a credentialed lane's steered flow goes to the Box Egress +/// Proxy's own address ([`PROXY_LANE_RULE`]'s arm). So a frame to the alias +/// at the resolver's port or the proxy's port is refused here like any other. +#[derive(Clone, Copy)] +struct AliasOpenings<'a> { + row: AliasRow, + baseline: &'a NodePlaneBaseline, + table: &'a BoxTable, + /// Whether the row names the host, the box's half of an exposure + /// (NET-160). Always `true` for the node and the cohort rows. + names_host: bool, +} + +impl<'a> AliasOpenings<'a> { + fn new(row: AliasRow, baseline: &'a NodePlaneBaseline, table: &'a BoxTable) -> Self { + Self { + row, + baseline, + table, + names_host: true, + } + } + + /// The openings for one box row, whose exposures need the box to name + /// the host (NET-160). + fn for_box(baseline: &'a NodePlaneBaseline, table: &'a BoxTable, record: &BoxRecord) -> Self { + let alias = table.subnet().host_alias().octets(); + Self { + names_host: names_host_by_name(record.allow_dns_hosts()) + || record + .egress() + .allow_subnets() + .is_none_or(|list| list.iter().any(|cidr| cidr.contains(alias))), + ..Self::new(AliasRow::Box, baseline, table) + } + } + + /// Minimal's own listeners and sockets at the alias as of this flow + /// (NET-162), the hostname proxy's port read from the node row the + /// host registered before boot. + fn listeners(&self) -> Vec { + let node = self + .table + .by_source(self.table.subnet().daemon_ip().octets()); + host_exposures::minimal_listeners( + node.as_deref().map_or(&[][..], BoxRecord::admitted_ports), + self.baseline.alias_ports(), + baseline::ALIAS_FETCH_PROTOCOL, + ) + } + + /// Whether the row reaches the alias with this transport, the frame's + /// IP protocol and destination port when it is TCP or UDP. A frame with + /// no transport port (ICMP, a later fragment) fails every opening closed. + /// The table is read here, so only a frame to the alias pays for it. + fn admit(&self, transport: Option<(u8, u16)>) -> bool { + let Some((protocol, port)) = transport else { + return false; + }; + let within = + |ranges: &[(u16, u16)]| ranges.iter().any(|&(lo, hi)| lo <= port && port <= hi); + if self.row == AliasRow::Node + && protocol == baseline::ALIAS_FETCH_PROTOCOL + && within(self.baseline.alias_ports()) + { + // The baseline set's ports at the alias are the node plane's + // alone (NET-158). Another protocol at the same port is not a + // baseline fetch, so the table decides it. + return true; + } + let held = self.listeners(); + // The table holds its entries against the listeners as of this + // flow, so it records a listener that shadows an entry even when + // the check below refuses the frame. + let exposures = self.baseline.host_exposure_ports(&held); + if held.iter().any(|holder| holder.holds_port(protocol, port)) { + // Minimal's own listener, refused ahead of the exposures at + // each flow (NET-162), even when a table entry names the port. + return false; + } + self.names_host + && exposures + .iter() + .any(|exposure| exposure.admits(protocol, port)) + } +} + +/// Whether `allow_dns_hosts` names the host alias, `host.min.internal`, +/// as the box's half of a host exposure (NET-160). +fn names_host_by_name(allow_dns_hosts: &[String]) -> bool { + allow_dns_hosts.iter().any(|host| { + host.trim_end_matches('.') + .eq_ignore_ascii_case(baseline::HOST_ALIAS_NAME) + }) +} + +/// Which infrastructure rule refuses a destination. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum InfrastructureHit { + /// The infrastructure deny set ([`INFRASTRUCTURE_RULE`]). + Set, + /// The host alias outside the row's openings ([`HOST_ALIAS_RULE`]). + HostAlias, +} + +/// The frame's IP protocol and destination port when it is TCP or UDP, the +/// transport the alias's openings are matched against. +fn alias_transport(summary: &FrameSummary) -> Option<(u8, u16)> { + summary + .protocol() + .filter(|&protocol| matches!(protocol, egress::IPPROTO_TCP | egress::IPPROTO_UDP)) + .map(|protocol| (protocol, summary.destination_port())) +} + +/// Which rule, if any, refuses `dst` as infrastructure, as the host frame +/// rule holds it ([`INFRASTRUCTURE_RULE`], [`HOST_ALIAS_RULE`]): a fixed +/// range; the host alias outside the openings `alias` grants the row; the +/// fabric plane outside `own_block`, the subnet the gate's rows live in — a +/// frame to a sibling or to the daemon inside the node's own block is local +/// reach, decided by the row's CIDR rules and the target's ingress, never by +/// this rule, and the gateway inside it is the control-surface rule's; or +/// RFC 1918 space the row's `allow_subnets` does not cover. +/// +/// `transport` is the frame's protocol and port when it is TCP or UDP +/// ([`alias_transport`]), `None` for any other frame. A non-first IPv4 +/// fragment carries no readable port and reads as port 0, so it fails every +/// alias opening: the stateless verdict fails such a frame closed. /// /// `allow` is the row's compiled `allow_subnets`, `None` when the dimension /// is undeclared — allow-all, the shipped default (03-spec R2.1) — which @@ -591,51 +759,30 @@ static INFRASTRUCTURE_RANGES: LazyLock = LazyLock::new(|| /// allowing the range, so neither a name rule nor a pin can become a way /// around leaving it undeclared. /// -/// `refuse_host_alias` is `false` for the node namespace's row alone: the -/// interim node row, keyed to the daemon's own address, carries the frames -/// of every box that shares the guest root namespace (a host-address box, -/// the default network mode), and those reach the host alias as own-block -/// local reach until the in-force baseline, which names the alias as its -/// registry and cache endpoint, decides the node plane instead. Announced, -/// that reach is scoped: the alias is opened at the configured host -/// exposures alone, and a node-row frame to any other port, or with no TCP -/// or UDP port, is this rule's drop. This mirrors the shared verdict, -/// whose own-address refusal is attached to box relays only, never to the -/// daemon relay. fn infrastructure_destination( dst: [u8; 4], - transport_port: Option, + transport: Option<(u8, u16)>, own_block: SwitchSubnet, allow: Option<&[Ipv4Cidr]>, - refuse_host_alias: bool, - host_exposure: &[(u16, u16)], -) -> bool { + alias: AliasOpenings<'_>, +) -> Option { let ranges = &*INFRASTRUCTURE_RANGES; if ranges.fixed.iter().any(|cidr| cidr.contains(dst)) { - return true; + return Some(InfrastructureHit::Set); } - // The host alias is default-deny (design §7.1, NET-062): a box reaches - // the host only through a declared exposure, never by addressing the - // alias directly, so the alias is refused under this rule at every port - // whatever the row's rules would say about it. The node row's frames - // — the host-address boxes the interim row carries — reach the alias at - // the configured host exposures alone, and only over TCP or UDP: a frame - // with no transport port (ICMP reads as port 0) fails the opening closed. - let exposed = transport_port.is_some_and(|port| { - host_exposure - .iter() - .any(|&(lo, hi)| lo <= port && port <= hi) - }); - if dst == own_block.host_alias().octets() && (refuse_host_alias || !exposed) { - return true; + // The host alias is default-deny (design §7.1, NET-156): it opens only + // at the row's openings, whatever the row's rules say about it. + if dst == own_block.host_alias().octets() { + return (!alias.admit(transport)).then_some(InfrastructureHit::HostAlias); } let in_own_block = (u32::from_be_bytes(dst) & u32::from(own_block.netmask())) == u32::from(own_block.network()); if ranges.plane.contains(dst) && !in_own_block { - return true; + return Some(InfrastructureHit::Set); } - ranges.rfc1918.iter().any(|cidr| cidr.contains(dst)) - && !allow.is_none_or(|list| list.iter().any(|cidr| cidr.contains(dst))) + (ranges.rfc1918.iter().any(|cidr| cidr.contains(dst)) + && !allow.is_none_or(|list| list.iter().any(|cidr| cidr.contains(dst)))) + .then_some(InfrastructureHit::Set) } /// The rule name for a frame headed to the Box Egress Proxy's address that @@ -4823,6 +4970,10 @@ async fn relay_frames_to_switch( limiter.warn_infrastructure(src, dst, dst_port); continue; } + Err(GateDrop::HostAlias { src, dst_port, row }) => { + limiter.warn_host_alias(src, dst_port, row); + continue; + } Err(GateDrop::ProxyLane { src, dst, dst_port }) => { // The plan's diagnostics line wants the refusal to name the // box, not only its address: the row the frame's source @@ -5721,7 +5872,8 @@ fn reply_tuple_of(pkt: &dns_pins::L4Packet) -> egress::FlowTuple { /// at [`NODE_PLANE_FETCH_PROTOCOL`] and [`NODE_PLANE_FETCH_PORTS`] only; /// - a subnet a baseline entry names, and the resolver carve-out, both by the /// set's compiled rules; -/// - the host alias at the configured host exposures, over TCP or UDP; +/// - the host alias at the configured host exposures, over TCP or UDP, and +/// at the ports the set's own endpoints open there; /// - a box's lease the registry holds, which the target box's ingress /// decides on the switch's side, the reach the node's hostname proxy and /// its answers to the boxes ride. @@ -5753,23 +5905,27 @@ fn node_plane_verdict( if dst == subnet.box_egress_proxy_address().octets() { return Err(GateDrop::ProxyLane { src, dst, dst_port }); } - let transport_port = matches!( - summary.protocol(), - Some(egress::IPPROTO_TCP | egress::IPPROTO_UDP) - ) - .then_some(dst_port); - if infrastructure_destination( + match infrastructure_destination( dst, - transport_port, + alias_transport(summary), subnet, baseline.rules().allow_subnets(), - false, - baseline.host_exposure_ports(), + AliasOpenings::new(AliasRow::Node, baseline, table), ) { - return Err(GateDrop::Infrastructure { src, dst, dst_port }); + Some(InfrastructureHit::Set) => { + return Err(GateDrop::Infrastructure { src, dst, dst_port }); + } + Some(InfrastructureHit::HostAlias) => { + return Err(GateDrop::HostAlias { + src, + dst_port, + row: AliasRow::Node, + }); + } + None => {} } - // Past the infrastructure check, the alias is at a configured host - // exposure: the one opening the alias has. + // Past the infrastructure check, the alias is at one of the node's + // openings there: a host exposure or the baseline set's own port. if dst == subnet.host_alias().octets() { return Ok(GateAdmit::Baseline); } @@ -6014,11 +6170,20 @@ fn host_ip_verdict( } return Err(GateDrop::ProxyLane { src, dst, dst_port }); } - let transport_port = matches!( - summary.protocol(), - Some(egress::IPPROTO_TCP | egress::IPPROTO_UDP) - ) - .then_some(dst_port); + let transport = alias_transport(summary); + let openings = AliasOpenings::new(AliasRow::Cohort, baseline, table); + // The alias's openings are the cohort's, the same for every resident + // box, so the alias is decided once, before the boxes' rules. + if let Some(dst) = dst + && dst == table.subnet().host_alias().octets() + && !openings.admit(transport) + { + return Err(GateDrop::HostAlias { + src, + dst_port, + row: AliasRow::Cohort, + }); + } let mut infrastructure = false; // Whether a name-resolving box left the frame to the pins. The pins are // one shared cohort entry, so the gate asks them once, after the loop, @@ -6028,12 +6193,12 @@ fn host_ip_verdict( if let Some(dst) = dst && infrastructure_destination( dst, - transport_port, + transport, table.subnet(), rules.egress().allow_subnets(), - false, - baseline.host_exposure_ports(), + openings, ) + .is_some() { infrastructure = true; continue; @@ -6057,12 +6222,12 @@ fn host_ip_verdict( && resident.iter().all(|rules| { infrastructure_destination( dst, - transport_port, + transport, table.subnet(), rules.egress().allow_subnets(), - false, - baseline.host_exposure_ports(), + openings, ) + .is_some() }) { return Err(GateDrop::Infrastructure { src, dst, dst_port }); @@ -6119,49 +6284,48 @@ fn row_verdict( // now — the source check above returns before any other source — so the // set applies to every box-plane packet, and no row's rules concede the // fabric. - // The host-alias refusal is a box row's alone: the node namespace's - // row, keyed to the daemon's own address, carries the host-address - // boxes' frames, whose `host.min.internal` resolves to the alias, and - // keeps own-block local reach to it while the baseline is announced - // (in force, the node-plane arm above returned first, with the same - // scoping). That reach is scoped by the baseline's configured host - // exposures, which the arm below reads for every row: the box rows' - // refusal stands at every port, and the node row's outside the - // exposures. - let box_row = src != table.subnet().daemon_ip().octets(); - // A non-first IPv4 fragment carries no readable L4 port — `summarize` - // reports destination port 0 for it — so it fails the exposure match - // and the gate drops it, never admits it: a fragmented UDP datagram to - // an exposed port dies at its later fragments. Reviewer ask - // (CodeRabbit, PR #2331) was to associate later fragments with their - // admitted first fragment, or defer the port check to reassembly. - // Declined: the verdict path is stateless and allocation-free by - // design — every input comes from the one frame being judged — and - // fragment association would add per-flow state here (or reassembly - // elsewhere) to recover a packet whose failure mode is a drop, not an - // exposure: deny-by-default keeps the safe direction. The in-force - // node-plane arm reads the exposures the same way, with the same - // fragment case. - let transport_port = matches!( - summary.protocol(), - Some(egress::IPPROTO_TCP | egress::IPPROTO_UDP) - ) - .then(|| summary.destination_port()); - if let Some(dst) = summary.destination() - && infrastructure_destination( + // The host alias opens at the configured host exposures for every row, + // and the row's own rules decide the flow from there (design §7.1). The + // node namespace's row, keyed to the daemon's own address, carries the + // host-address boxes' frames under the announced baseline, so it is the + // cohort here and never gets the baseline set's own openings. A + // non-first IPv4 fragment reads as port 0 and fails the openings + // closed: the verdict path is stateless, and a drop is the safe + // direction. + let row = if src == table.subnet().daemon_ip().octets() { + AliasRow::Cohort + } else { + AliasRow::Box + }; + // A box row reaches an exposure only when it names the host (NET-160). + let openings = match row { + AliasRow::Box => AliasOpenings::for_box(baseline, table, &record), + _ => AliasOpenings::new(row, baseline, table), + }; + if let Some(dst) = summary.destination() { + match infrastructure_destination( dst, - transport_port, + alias_transport(summary), table.subnet(), record.egress().allow_subnets(), - box_row, - baseline.host_exposure_ports(), - ) - { - return Err(GateDrop::Infrastructure { - src, - dst, - dst_port: summary.destination_port(), - }); + openings, + ) { + Some(InfrastructureHit::Set) => { + return Err(GateDrop::Infrastructure { + src, + dst, + dst_port: summary.destination_port(), + }); + } + Some(InfrastructureHit::HostAlias) => { + return Err(GateDrop::HostAlias { + src, + dst_port: summary.destination_port(), + row, + }); + } + None => {} + } } // The Box Egress Proxy's listener, decided beside every rule (NET-134): // the proxy is a credentialed lane's infrastructure — the one host-side @@ -6240,6 +6404,18 @@ fn row_verdict( { Ok(GateAdmit::Row) } + // The alias passed its openings above, so the frame is at a host + // exposure. A box that names `host.min.internal` reaches it by the + // name, as a pin admits an answer to a declared name, while its + // deny rules and protocols still decide first (NET-160). + FrameVerdict::Drop(reason) + if matches!(reason, DropReason::UndeclaredSubnet { .. }) + && row == AliasRow::Box + && summary.destination() == Some(table.subnet().host_alias().octets()) + && names_host_by_name(record.allow_dns_hosts()) => + { + Ok(GateAdmit::Row) + } FrameVerdict::Drop(reason) => Err(GateDrop::Verdict(reason)), } } @@ -6298,6 +6474,16 @@ enum GateDrop { /// The port the frame named there, `0` when it carried none. dst_port: u16, }, + /// The frame named the host alias at a port no opening of its row + /// covers (NET-156, [`HOST_ALIAS_RULE`]). + HostAlias { + /// The source address the frame wore. + src: [u8; 4], + /// The port the frame named at the alias, `0` when it carried none. + dst_port: u16, + /// The row the alias's openings were decided for. + row: AliasRow, + }, /// The frame named the Box Egress Proxy's address and the lane does not /// admit it (NET-134): its source carries no credentialed lane, or the /// frame is headed somewhere at the address the declaration never @@ -6347,6 +6533,7 @@ impl GateDrop { match self { Self::Verdict(reason) => reason.rule(), Self::Infrastructure { .. } => INFRASTRUCTURE_RULE, + Self::HostAlias { .. } => HOST_ALIAS_RULE, Self::UnknownSource { .. } => UNKNOWN_SOURCE_RULE, Self::UnregisteredSource { .. } => UNREGISTERED_SOURCE_RULE, Self::SwitchControlSurface { .. } => SWITCH_CONTROL_RULE, @@ -6638,6 +6825,36 @@ impl DropLimiter { } } + /// Emits the warning for one frame to the host alias outside its row's + /// openings (NET-156): the same rate limit a drop's line answers to, + /// keyed by the source and the rule, and naming the row and the port. + /// Returns whether a line was written. + fn warn_host_alias(&self, src: [u8; 4], dst_port: u16, row: AliasRow) -> bool { + match self.should_warn_at(Some(src), HOST_ALIAS_RULE, Instant::now()) { + WarnDecision::Silent => false, + WarnDecision::Named => { + tracing::warn!( + source = %Ipv4Addr::from(src), + row = row.as_str(), + port = dst_port, + rule_matched = HOST_ALIAS_RULE, + "dropped a frame to host.min.internal at a port that no host exposure \ + opens for this row", + ); + true + } + WarnDecision::Overflow => { + tracing::warn!( + rule_matched = HOST_ALIAS_RULE, + "dropped frames to host.min.internal from more distinct source addresses \ + than the gate keeps a window per source for; one line per rule covers \ + the rest", + ); + true + } + } + } + /// Emits the warning for one frame headed to the Box Egress Proxy's /// address that the lane does not admit (NET-134) — a source that /// carries no credentialed lane, or a frame at the address that is not @@ -7776,14 +7993,14 @@ mod tests { send_frame, unexpose_request, wait_for_log, }; use super::{ - AcceptFailure, CONNECT_REQUEST, CONTROL_VERBS, ControlVerb, DROP_WARN_MAX_TRACKED_PAIRS, - DROP_WARN_MIN_INTERVAL, DropLimiter, EgressGate, FrameSummary, GateAdmit, GateDrop, - GuestSource, GuestSpeak, HANDSHAKE_TIMEOUT, HOST_IP_BOXES_RULE, MALFORMED_PUBLISH_RULE, - MAX_HEAD, MAX_LIVE_RELAYS, MAX_NAMED_TARGET, PublishedForwards, Record, ReplyTables, - UNDECLARED_PUBLISH_RECORD_RULE, UNDECLARED_RETRACT_RULE, UNKNOWN_PUBLISH_ADDRESS_RULE, - UNREGISTERED_LIVE_LEASE_RULE, UNREGISTERED_PUBLISH_RULE, UNREGISTERED_SOURCE_PHASE, - UNREGISTERED_SOURCE_RULE, UnregisteredSourcePhase, WarnDecision, accept_loop, dns_pins, - gate_verdict, max_frame, render_record, serve_connection, + AcceptFailure, AliasRow, CONNECT_REQUEST, CONTROL_VERBS, ControlVerb, + DROP_WARN_MAX_TRACKED_PAIRS, DROP_WARN_MIN_INTERVAL, DropLimiter, EgressGate, FrameSummary, + GateAdmit, GateDrop, GuestSource, GuestSpeak, HANDSHAKE_TIMEOUT, HOST_IP_BOXES_RULE, + MALFORMED_PUBLISH_RULE, MAX_HEAD, MAX_LIVE_RELAYS, MAX_NAMED_TARGET, PublishedForwards, + Record, ReplyTables, UNDECLARED_PUBLISH_RECORD_RULE, UNDECLARED_RETRACT_RULE, + UNKNOWN_PUBLISH_ADDRESS_RULE, UNREGISTERED_LIVE_LEASE_RULE, UNREGISTERED_PUBLISH_RULE, + UNREGISTERED_SOURCE_PHASE, UNREGISTERED_SOURCE_RULE, UnregisteredSourcePhase, WarnDecision, + accept_loop, dns_pins, gate_verdict, max_frame, render_record, serve_connection, }; use crate::box_registry::{BoxRegistration, BoxRegistry, BoxTable}; use crate::net::baseline::{NodeBaselinePhase, NodePlaneBaseline}; @@ -12401,9 +12618,9 @@ mod tests { /// and neither does the widest allowance. The node's own block is the /// carve-out for a sibling and the daemon: the same CIDR row reaches a /// sibling, local reach the row's rules and the target's ingress decide. - /// The host alias is not local reach — it is default-deny at every port - /// (design §7.1, NET-062), refused under this rule whatever the row's - /// rules admit. RFC 1918 + /// The host alias is not local reach — it is default-deny outside the + /// host exposures (design §7.1, NET-156), refused under its own rule + /// (`host-alias`), whose line names the row and the port. RFC 1918 /// is the one exemption: a `10.0.0.0/8` destination is refused for the /// name-declaring row, whose declared `allow_subnets` does not cover it /// — the frame its deferral would otherwise have passed to the guest — @@ -12501,13 +12718,14 @@ mod tests { // The drops say so under the rule, naming the destination and the // port, one line per source: the first refused frame of each. wait_for_log(&h.log, "egress-infrastructure-destination").await; + wait_for_log(&h.log, "rule_matched=\"host-alias\"").await; let logged = h.log.contents(); for needle in [ "source=100.64.0.9", "destination=10.1.2.3", "port=443", "source=100.64.0.10", - "destination=100.64.255.254", + "row=\"box\"", "source=100.64.0.11", "destination=169.254.169.254", "source=100.64.0.12", @@ -16061,7 +16279,8 @@ mod tests { .with_admitted_ports([8080]), ); let table = registry.table(); - let baseline = NodePlaneBaseline::built_in(SUBNET).with_host_exposure(vec![(4200, 4200)]); + let baseline = NodePlaneBaseline::built_in(SUBNET) + .with_host_exposure(vec![crate::net::host_exposures::HostExposure::tcp(4200)]); assert_eq!( baseline.phase(), NodeBaselinePhase::InForce, @@ -16160,12 +16379,16 @@ mod tests { ); // The host alias at a configured exposure is admitted, and at any - // other port it is the infrastructure drop. + // other port it is the host-alias drop. let alias = SUBNET.host_alias().octets(); assert_eq!(decide(6, alias, 4200), Ok(GateAdmit::Baseline)); assert!(matches!( decide(6, alias, 443), - Err(GateDrop::Infrastructure { dst, .. }) if dst == alias + Err(GateDrop::HostAlias { + dst_port: 443, + row: AliasRow::Node, + .. + }) )); // A resident box's lease: the target's ingress decides it. @@ -16292,120 +16515,663 @@ mod tests { } } - /// The host alias's default-deny is a box row's alone, and the node - /// plane's reach through it is scoped by the configured host exposures - /// (design §7.1, NET-062). The daemon's address carries the frames of - /// every box sharing the guest root namespace — a host-address box, the - /// default network mode, whose `host.min.internal` resolves to the - /// alias — and the in-force baseline decides them. The built-in baseline - /// carries no exposure, so the alias is closed at every port until a - /// deployment names ports; with a list, a node-plane TCP or UDP frame to - /// the alias is admitted by the baseline set inside the list's ranges, edges included, and the infrastructure drop outside them, - /// while an ICMP frame, which carries no port, is refused even by a list - /// spanning every port. The same frames from a box lease, even an - /// allow-all one, are the infrastructure drop whatever the port. + /// The gate's verdict for one frame `src` sends to the host alias, with + /// the box mark when `marked`. + fn alias_verdict( + table: &BoxTable, + baseline: &NodePlaneBaseline, + src: [u8; 4], + marked: bool, + proto: u8, + port: u16, + ) -> Result { + let alias = SUBNET.host_alias().octets(); + let frame = ipv4_frame(src, proto, alias, port); + let frame = if marked { + box_cohort_frame(frame) + } else { + frame + }; + gate_verdict( + &sessions::core::egress::summarize(&frame), + None, + table, + baseline, + &dns_pins::DnsPins::new(SUBNET), + &ReplyTables::new(), + ) + } + + /// A box with its own address, an allow-all one included, reaches the + /// host alias at a configured host exposure and nowhere else (NET-156). + /// Its own rules decide the exposure: a deny-all box reaches nothing. + /// The alias is not where the resolver or the Box Egress Proxy listen, + /// so the alias at their ports is refused, while the resolver on the + /// gateway and a lane box's flow to the proxy's own address still pass. + /// The drop is the `host-alias` rule's and names the `box` row. #[test] - fn host_alias_refused_for_box_rows_and_scoped_by_the_exposures_for_the_node_row() { + fn host_alias_refused_outside_exposures_for_boxes() { let registry = BoxRegistry::new(SUBNET); registry.register_node_namespace(7654); registry.register( BoxRegistration::new("bare", Ipv4Addr::from(LEASE), Ipv4Addr::LOCALHOST) .with_admitted_ports([8080]), ); + let locked = [100, 64, 0, 11]; + registry.register( + BoxRegistration::new("locked", Ipv4Addr::from(locked), Ipv4Addr::LOCALHOST) + .with_egress_policy(EgressPolicy { + allow_protocols: None, + allow_subnets: None, + allow_dns_hosts: None, + deny_subnets: Some(vec!["0.0.0.0/0".to_string()]), + }), + ); + let laned = [100, 64, 0, 12]; + registry.register( + BoxRegistration::new("laned", Ipv4Addr::from(laned), Ipv4Addr::LOCALHOST) + .with_credentialed_upstream(sessions::CredentialedUpstream::default()), + ); let table = registry.table(); - let built_in = NodePlaneBaseline::built_in(SUBNET); - assert_eq!(built_in.phase(), NodeBaselinePhase::InForce); - let pins = dns_pins::DnsPins::new(SUBNET); - let summarize = sessions::core::egress::summarize; - let alias = SUBNET.host_alias().octets(); - let node = SUBNET.daemon_ip().octets(); - - let verdict = |src: [u8; 4], proto: u8, port: u16, baseline: &NodePlaneBaseline| { - gate_verdict( - &summarize(&ipv4_frame(src, proto, alias, port)), - None, - &table, - baseline, - &pins, - &ReplyTables::new(), - ) + let exposed = NodePlaneBaseline::built_in(SUBNET).with_host_exposure(vec![ + crate::net::host_exposures::HostExposure::tcp(3000), + crate::net::host_exposures::HostExposure::tcp(3999), + crate::net::host_exposures::HostExposure::tcp(8080), + crate::net::host_exposures::HostExposure::udp(8080), + ]); + let refused = |result: Result, port: u16| { + result + == Err(GateDrop::HostAlias { + src: LEASE, + dst_port: port, + row: AliasRow::Box, + }) }; - let refused = |result: Result| matches!(result, Err(GateDrop::Infrastructure { dst, .. }) if dst == alias); + for port in [3000, 3999, 8080] { + assert_eq!( + alias_verdict(&table, &exposed, LEASE, false, 6, port), + Ok(GateAdmit::Row), + "an allow-all box reaches the alias at exposed port {port}" + ); + } + assert_eq!( + alias_verdict(&table, &exposed, LEASE, false, 17, 8080), + Ok(GateAdmit::Row) + ); + for port in [2999, 3500, 4000, 22, 53, 443, switch::bep_host::PROXY_PORT] { + assert!( + refused(alias_verdict(&table, &exposed, LEASE, false, 6, port), port), + "the alias at unexposed port {port} is the host-alias drop for a box" + ); + } + assert!( + refused( + alias_verdict(&table, &exposed, LEASE, false, 17, 3000), + 3000 + ), + "a TCP entry does not open its port for UDP" + ); + // Minimal's own listeners are refused ahead of the exposures, the + // hostname proxy at the port the node row holds (NET-162). + for (port, holder) in [ + (7654, "the hostname proxy"), + (crate::net::answerer::DEFAULT_ANSWERER_PORT, "the answerer"), + (switch::bep_host::PROXY_PORT, "the Box Egress Proxy"), + ] { + let at_listener = NodePlaneBaseline::built_in(SUBNET) + .with_host_exposure(vec![crate::net::host_exposures::HostExposure::tcp(port)]); + assert!( + refused( + alias_verdict(&table, &at_listener, LEASE, false, 6, port), + port + ), + "an exposure at {holder}'s port {port} never opens it to a box" + ); + } assert!( - refused(verdict(node, 6, 8080, &built_in)), - "with no configured exposure the alias is closed to the node plane at every port" + refused(alias_verdict(&table, &exposed, LEASE, false, 1, 0), 0), + "ICMP carries no port and opens nothing" ); - - let exposed = built_in - .clone() - .with_host_exposure(vec![(3000, 3999), (8080, 8080)]); assert!( - refused(verdict(node, 6, 2999, &exposed)), - "the port below a range's low edge is the infrastructure drop" + refused( + alias_verdict( + &table, + &NodePlaneBaseline::built_in(SUBNET), + LEASE, + false, + 6, + 8080 + ), + 8080 + ), + "with no exposure the alias is closed to a box at every port" ); - for port in [3000, 3999, 8080] { - assert_eq!( - verdict(node, 6, port, &exposed), - Ok(GateAdmit::Baseline), - "the node plane reaches the host alias at exposed port {port}" + assert!( + matches!( + alias_verdict(&table, &exposed, locked, false, 6, 8080), + Err(GateDrop::Verdict(_)) + ), + "a deny-all box's own rules refuse the exposure" + ); + assert!( + matches!( + alias_verdict( + &table, + &exposed, + laned, + false, + 6, + switch::bep_host::PROXY_PORT + ), + Err(GateDrop::HostAlias { + row: AliasRow::Box, + .. + }) + ), + "a lane opens the proxy's own address, not the alias" + ); + let proxy = SUBNET.box_egress_proxy_address().octets(); + let to_proxy = ipv4_frame(laned, 6, proxy, switch::bep_host::PROXY_PORT); + let resolver = ipv4_frame(LEASE, 17, SUBNET.dns_server().octets(), 53); + let pins = dns_pins::DnsPins::new(SUBNET); + for (frame, what) in [(to_proxy, "the lane's proxy"), (resolver, "the resolver")] { + let summary = sessions::core::egress::summarize(&frame); + assert!( + gate_verdict(&summary, None, &table, &exposed, &pins, &ReplyTables::new()).is_ok(), + "{what} stays open at its own address" ); } - assert!( - refused(verdict(node, 6, 18081, &exposed)), - "the node plane's frame to the host alias outside the exposures is the infrastructure drop" + } + + /// The node-plane identity alone reaches the host alias at the ports + /// the baseline set's own endpoints open there (NET-156). A box row and + /// the host-address cohort, marked frames at the node address, never + /// get that set, and each drop names its row. With no endpoint at the + /// alias the node is closed there too. + #[test] + fn host_alias_admits_baseline_for_node_plane_only() { + let registry = BoxRegistry::new(SUBNET); + registry.register_node_namespace(7654); + registry.register( + BoxRegistration::new("bare", Ipv4Addr::from(LEASE), Ipv4Addr::LOCALHOST) + .with_admitted_ports([8080]), + ); + assert!(registry.admit_host_ip_box( + sessions::SessionId::parse_str(&uuid::Uuid::new_v4().to_string()).unwrap(), + None + )); + let table = registry.table(); + let node = SUBNET.daemon_ip().octets(); + let alias = Ipv4Addr::from(SUBNET.host_alias().octets()); + let built_in = NodePlaneBaseline::built_in(SUBNET); + assert_eq!(built_in.phase(), NodeBaselinePhase::InForce); + let at_alias = built_in.clone().with_cache(vec![format!("{alias}/32")]); + + assert_eq!( + alias_verdict(&table, &built_in, node, false, 6, 443), + Err(GateDrop::HostAlias { + src: node, + dst_port: 443, + row: AliasRow::Node, + }), + "with no endpoint at the alias the node is closed there" ); assert_eq!( - verdict(node, 17, 8080, &exposed), + alias_verdict(&table, &at_alias, node, false, 6, 443), Ok(GateAdmit::Baseline), - "a UDP frame reaches the host alias at an exposed port" + "the node reaches its baseline endpoint at the alias" ); - assert!( - refused(verdict(LEASE, 6, 8080, &exposed)), - "a box row's frame to the host alias is the infrastructure drop at an exposed port" + for (proto, port) in [(17, 443), (6, 80), (1, 0)] { + assert!( + matches!( + alias_verdict(&table, &at_alias, node, false, proto, port), + Err(GateDrop::HostAlias { + row: AliasRow::Node, + .. + }) + ), + "the baseline set opens TCP 443 alone, not protocol {proto} port {port}" + ); + } + assert_eq!( + alias_verdict(&table, &at_alias, node, true, 6, 443), + Err(GateDrop::HostAlias { + src: node, + dst_port: 443, + row: AliasRow::Cohort, + }), + "the host-address cohort never gets the baseline set" ); + assert_eq!( + alias_verdict(&table, &at_alias, LEASE, false, 6, 443), + Err(GateDrop::HostAlias { + src: LEASE, + dst_port: 443, + row: AliasRow::Box, + }), + "a box row never gets the baseline set" + ); + let exposed = at_alias.with_host_exposure(vec![ + crate::net::host_exposures::HostExposure::tcp(8080), + crate::net::host_exposures::HostExposure::tcp(443), + ]); + for marked in [false, true] { + assert!( + alias_verdict(&table, &exposed, node, marked, 6, 8080).is_ok(), + "a host exposure is open to the node and to the cohort" + ); + } + assert_eq!( + alias_verdict(&table, &exposed, node, false, 6, 443), + Ok(GateAdmit::Baseline), + "the node keeps its baseline port" + ); + for (src, marked, row) in [ + (node, true, AliasRow::Cohort), + (LEASE, false, AliasRow::Box), + ] { + assert_eq!( + alias_verdict(&table, &exposed, src, marked, 6, 443), + Err(GateDrop::HostAlias { + src, + dst_port: 443, + row, + }), + "an entry at a baseline port never opens it to the {row:?} row" + ); + } + // A UDP entry at the baseline's TCP port is not a baseline fetch, + // so the table decides it like any other port. + let udp_exposed = + exposed.with_host_exposure(vec![crate::net::host_exposures::HostExposure::udp(443)]); + for marked in [false, true] { + assert!( + alias_verdict(&table, &udp_exposed, node, marked, 17, 443).is_ok(), + "a UDP exposure at a baseline TCP port is open to the node and the cohort" + ); + } + assert_eq!( + GateDrop::HostAlias { + src: node, + dst_port: 0, + row: AliasRow::Node + } + .rule(), + "host-alias" + ); + } - let every_port = built_in.with_host_exposure(vec![(0, u16::MAX)]); + /// An entry added to the `[host-exposures]` table of the client + /// `config.toml` opens its port at the alias for the next flow, with no + /// restart (NET-161). + #[test] + fn host_alias_admits_a_port_the_current_table_opens() { + let dir = tempfile::tempdir().unwrap(); + let config = dir.path().join("config.toml"); + std::fs::write(&config, "[secrets]\n").unwrap(); + let registry = BoxRegistry::new(SUBNET); + registry.register( + BoxRegistration::new("bare", Ipv4Addr::from(LEASE), Ipv4Addr::LOCALHOST) + .with_admitted_ports([8080]), + ); + let table = registry.table(); + let baseline = NodePlaneBaseline::built_in(SUBNET).with_host_exposures( + crate::net::host_exposures::HostExposures::at(config.clone(), Duration::ZERO), + ); assert!( - refused(verdict(node, 1, 0, &every_port)), - "an ICMP frame carries no port and never matches an exposure" + matches!( + alias_verdict(&table, &baseline, LEASE, false, 6, 5173), + Err(GateDrop::HostAlias { .. }) + ), + "no entry, no opening" + ); + std::fs::write( + &config, + "[secrets]\n\n[host-exposures]\nvite = { proto = \"tcp\", port = 5173 }\n", + ) + .unwrap(); + assert_eq!( + alias_verdict(&table, &baseline, LEASE, false, 6, 5173), + Ok(GateAdmit::Row), + "the added entry opens its port" + ); + assert!( + matches!( + alias_verdict(&table, &baseline, LEASE, false, 6, 5174), + Err(GateDrop::HostAlias { .. }) + ), + "the entry opens its own port alone" ); } - /// The in-force arm opens the host alias at the configured host - /// exposures and nowhere else: the registry and the cache are named - /// endpoints now, so no fetch port is held open on the alias, and with - /// no exposure TCP at 443 is the infrastructure drop. With 443 exposed, - /// TCP and UDP pass, and a frame with no transport port — ICMP, or GRE - /// whose bytes read as 443 — never matches the exposure. + /// An entry removed from the `[host-exposures]` table ends each + /// established flow to its port (NET-161). The gate keeps no per-flow + /// state for the alias: it decides every frame of a flow, the frames of + /// a flow already running included, by the table as it is at that + /// frame. So the next frame of a flow that ran through the removed + /// exposure is the `host-alias` drop, and the flow carries nothing more. #[test] - fn in_force_baseline_opens_the_alias_at_the_host_exposures_alone() { + fn host_exposure_removal_ends_established_flows() { + let dir = tempfile::tempdir().unwrap(); + let config = dir.path().join("config.toml"); + std::fs::write( + &config, + "[host-exposures]\ndev = { port = 8080 }\nvite = { proto = \"tcp\", port = 3500 }\n", + ) + .unwrap(); let registry = BoxRegistry::new(SUBNET); - registry.register_node_namespace(7654); + registry.register( + BoxRegistration::new("bare", Ipv4Addr::from(LEASE), Ipv4Addr::LOCALHOST) + .with_admitted_ports([8080]), + ); let table = registry.table(); - let built_in = NodePlaneBaseline::built_in(SUBNET); - let pins = dns_pins::DnsPins::new(SUBNET); - let node = built_in.node_addr(); + let baseline = NodePlaneBaseline::built_in(SUBNET).with_host_exposures( + crate::net::host_exposures::HostExposures::at(config.clone(), Duration::ZERO), + ); + // The flow's frames: one 5-tuple, the box's port 40000 to the alias. + for _ in 0..3 { + assert_eq!( + alias_verdict(&table, &baseline, LEASE, false, 6, 8080), + Ok(GateAdmit::Row), + "the established flow runs while the entry stands" + ); + } + assert_eq!( + alias_verdict(&table, &baseline, LEASE, false, 6, 3500), + Ok(GateAdmit::Row) + ); + std::fs::write( + &config, + "[host-exposures]\nvite = { proto = \"tcp\", port = 3500 }\n", + ) + .unwrap(); + assert_eq!( + alias_verdict(&table, &baseline, LEASE, false, 6, 8080), + Err(GateDrop::HostAlias { + src: LEASE, + dst_port: 8080, + row: AliasRow::Box, + }), + "the next frame of the established flow is refused" + ); + assert_eq!( + alias_verdict(&table, &baseline, LEASE, false, 6, 3500), + Ok(GateAdmit::Row), + "the entry that stays keeps its flows" + ); + std::fs::remove_file(&config).unwrap(); + assert!( + matches!( + alias_verdict(&table, &baseline, LEASE, false, 6, 3500), + Err(GateDrop::HostAlias { .. }) + ), + "a removed file ends the flows of every entry" + ); + } + + /// A box reaches a host exposure only when it names the host: by + /// `host.min.internal` in `egress.allow_dns_hosts`, or by an + /// `allow_subnets` that covers the alias (NET-160). A box that does + /// neither is the `host-alias` drop at the exposure. Naming the host + /// opens the exposures alone, never another port at the alias. + #[test] + fn host_alias_exposure_needs_the_box_to_name_the_host() { let alias = SUBNET.host_alias().octets(); - let decide = |proto: u8, baseline: &NodePlaneBaseline| { - let summary = sessions::core::egress::summarize(&ipv4_frame(node, proto, alias, 443)); - gate_verdict(&summary, None, &table, baseline, &pins, &ReplyTables::new()) + let policy = |subnets: Vec, hosts: Vec<&str>| EgressPolicy { + allow_protocols: None, + allow_subnets: Some(subnets), + allow_dns_hosts: Some(hosts.into_iter().map(str::to_string).collect()), + deny_subnets: None, }; - let refused = |result: Result| matches!(result, Err(GateDrop::Infrastructure { dst, dst_port: 443, .. }) if dst == alias); + let registry = BoxRegistry::new(SUBNET); + let unnamed = LEASE; + let named = [100, 64, 0, 11]; + let covered = [100, 64, 0, 12]; + let dotted = [100, 64, 0, 13]; + for (name, addr, egress) in [ + ( + "unnamed", + unnamed, + policy(vec!["10.0.0.0/8".into()], vec!["example.com"]), + ), + ( + "named", + named, + policy(vec!["10.0.0.0/8".into()], vec!["host.min.internal"]), + ), + ( + "covered", + covered, + policy(vec![format!("{}/32", Ipv4Addr::from(alias))], vec![]), + ), + ("dotted", dotted, policy(vec![], vec!["HOST.min.internal."])), + ] { + registry.register( + BoxRegistration::new(name, Ipv4Addr::from(addr), Ipv4Addr::LOCALHOST) + .with_egress_policy(egress), + ); + } + // A box with no egress section is allow-all: its undeclared + // `allow_subnets` covers the alias, as it covers a sibling box. + let undeclared = [100, 64, 0, 14]; + registry.register(BoxRegistration::new( + "undeclared", + Ipv4Addr::from(undeclared), + Ipv4Addr::LOCALHOST, + )); + let table = registry.table(); + let exposed = NodePlaneBaseline::built_in(SUBNET) + .with_host_exposure(vec![crate::net::host_exposures::HostExposure::tcp(5432)]); + assert_eq!( + alias_verdict(&table, &exposed, unnamed, false, 6, 5432), + Err(GateDrop::HostAlias { + src: unnamed, + dst_port: 5432, + row: AliasRow::Box, + }), + "a box that does not name the host never reaches the exposure" + ); + assert_eq!( + alias_verdict(&table, &exposed, undeclared, false, 6, 5432), + Ok(GateAdmit::Row), + "a box with no egress section reaches the exposure as it reaches a sibling" + ); + for (src, what) in [ + (named, "by name"), + (covered, "by a subnet that covers the alias"), + (dotted, "by the name in any case, with a trailing dot"), + ] { + assert_eq!( + alias_verdict(&table, &exposed, src, false, 6, 5432), + Ok(GateAdmit::Row), + "a box that names the host {what} reaches the exposure" + ); + assert!( + matches!( + alias_verdict(&table, &exposed, src, false, 6, 5433), + Err(GateDrop::HostAlias { + row: AliasRow::Box, + .. + }) + ), + "naming the host {what} opens no other port" + ); + } assert!( - refused(decide(6, &built_in)), - "with no exposure, TCP at 443 on the alias is the infrastructure drop" + matches!( + alias_verdict( + &table, + &NodePlaneBaseline::built_in(SUBNET), + named, + false, + 6, + 5432 + ), + Err(GateDrop::HostAlias { .. }) + ), + "naming the host opens nothing without the table's entry" + ); + } + + /// An entry of the `[host-exposures]` table at a port that one of + /// Minimal's own listeners holds is refused when the table loads, and + /// the daemon log names the entry and the listener (NET-162). The + /// listeners are the hostname proxy at the node row's port, here a + /// drawn one, the box-name answerer, the Box Egress Proxy, and a + /// node-plane baseline endpoint at the alias. + #[test] + fn host_exposure_on_minimal_listener_port_refused() { + let (log, _guard) = capture_log(); + let drawn = 41234; + let answerer = crate::net::answerer::DEFAULT_ANSWERER_PORT; + let proxy = switch::bep_host::PROXY_PORT; + let dir = tempfile::tempdir().unwrap(); + let config = dir.path().join("config.toml"); + std::fs::write( + &config, + format!( + "[host-exposures]\n\ + web = {{ port = 5173 }}\n\ + hp = {{ port = {drawn} }}\n\ + ans = {{ proto = \"udp\", port = {answerer} }}\n\ + bep = {{ port = {proxy} }}\n\ + fetch = {{ port = 443 }}\n" + ), + ) + .unwrap(); + let registry = BoxRegistry::new(SUBNET); + registry.register_node_namespace(drawn); + registry.register(BoxRegistration::new( + "bare", + Ipv4Addr::from(LEASE), + Ipv4Addr::LOCALHOST, + )); + let table = registry.table(); + let alias = Ipv4Addr::from(SUBNET.host_alias().octets()); + let baseline = NodePlaneBaseline::built_in(SUBNET) + .with_cache(vec![format!("{alias}/32")]) + .with_host_exposures(crate::net::host_exposures::HostExposures::at( + config, + Duration::ZERO, + )); + + assert_eq!( + alias_verdict(&table, &baseline, LEASE, false, 6, 5173), + Ok(GateAdmit::Row), + "an entry at a free port opens" ); - let exposed = built_in.clone().with_host_exposure(vec![(443, 443)]); - assert_eq!(decide(6, &exposed), Ok(GateAdmit::Baseline)); - assert_eq!(decide(17, &exposed), Ok(GateAdmit::Baseline)); - for (proto, name) in [(1u8, "ICMP"), (47, "GRE")] { + for (proto, port) in [(6, drawn), (17, answerer), (6, proxy), (6, 443)] { + for _ in 0..2 { + assert_eq!( + alias_verdict(&table, &baseline, LEASE, false, proto, port), + Err(GateDrop::HostAlias { + src: LEASE, + dst_port: port, + row: AliasRow::Box, + }), + "the entry at Minimal's port {port} is refused" + ); + } + } + let logged = log.contents(); + for (entry, holder) in [ + ("hp", "the hostname proxy"), + ("ans", "the box-name answerer"), + ("bep", "the Box Egress Proxy"), + ( + "fetch", + "a node-plane baseline endpoint at host.min.internal", + ), + ] { + let line = format!("entry={entry} "); + let lines: Vec<&str> = logged + .lines() + .filter(|line| line.contains("refused a host exposure")) + .filter(|l| l.contains(&line)) + .collect(); + assert_eq!(lines.len(), 1, "the load names {entry} once: {logged}"); assert!( - refused(decide(proto, &exposed)), - "{name} to alias:443 is the infrastructure drop, in force" + lines[0].contains(&format!("holder=\"{holder}\"")), + "the refusal of {entry} names {holder}: {logged}" + ); + } + assert!( + !logged.contains("entry=web "), + "the open entry is not refused: {logged}" + ); + assert_eq!( + alias_verdict( + &table, + &baseline, + SUBNET.daemon_ip().octets(), + false, + 6, + 443 + ), + Ok(GateAdmit::Baseline), + "the node keeps its own baseline endpoint" + ); + } + + /// A listener of Minimal's own that binds the port of a standing entry + /// later shadows the entry: from the next flow on, a box is refused + /// there, and the daemon log records once which listener shadows which + /// entry (NET-162). Here the hostname proxy binds the entry's port. + #[test] + fn host_exposure_shadowed_by_a_later_minimal_listener() { + let (log, _guard) = capture_log(); + let port = 41001; + let dir = tempfile::tempdir().unwrap(); + let config = dir.path().join("config.toml"); + std::fs::write( + &config, + format!("[host-exposures]\ndev = {{ port = {port} }}\n"), + ) + .unwrap(); + let registry = BoxRegistry::new(SUBNET); + registry.register(BoxRegistration::new( + "bare", + Ipv4Addr::from(LEASE), + Ipv4Addr::LOCALHOST, + )); + let baseline = NodePlaneBaseline::built_in(SUBNET).with_host_exposures( + crate::net::host_exposures::HostExposures::at(config, Duration::ZERO), + ); + assert_eq!( + alias_verdict(®istry.table(), &baseline, LEASE, false, 6, port), + Ok(GateAdmit::Row), + "the entry opens while no listener holds its port" + ); + registry.register_node_namespace(port); + let table = registry.table(); + for _ in 0..3 { + assert_eq!( + alias_verdict(&table, &baseline, LEASE, false, 6, port), + Err(GateDrop::HostAlias { + src: LEASE, + dst_port: port, + row: AliasRow::Box, + }), + "the listener shadows the entry from the next flow on" ); } + let logged = log.contents(); + let shadowed: Vec<&str> = logged + .lines() + .filter(|line| line.contains("now holds the port of a host exposure")) + .collect(); + assert_eq!(shadowed.len(), 1, "the log records it once: {logged}"); + assert!( + shadowed[0].contains("entry=dev ") + && shadowed[0].contains("holder=\"the hostname proxy\""), + "the record names the entry and the listener: {logged}" + ); + assert!( + !logged.contains("refused a host exposure"), + "a shadowed entry is not a load refusal: {logged}" + ); } /// NET-130, un-enrolled, at the gate: the node-plane baseline set comes diff --git a/crates/minvmd/src/net/host_exposures.rs b/crates/minvmd/src/net/host_exposures.rs new file mode 100644 index 0000000000..6a96261a95 --- /dev/null +++ b/crates/minvmd/src/net/host_exposures.rs @@ -0,0 +1,613 @@ +//! The host exposures: the ports at which a box reaches the host through +//! `host.min.internal`, read from the `[host-exposures]` table of the client +//! `config.toml` (design §7.1, NET-159, NET-161, NET-162). +//! +//! The design holds the host alias default-deny except explicitly +//! configured host exposures. The table names each exposure by a key of the +//! user's choice. The value is an inline table with a port and a protocol, +//! and the protocol defaults to `tcp`: +//! +//! ```toml +//! [host-exposures] +//! postgres = { proto = "tcp", port = 5432 } +//! ``` +//! +//! The gate decides each flow by the current table (NET-161). The handle +//! re-reads the file when its stat changes, as `bep serve` does for +//! `[secret-store-rules]`, so an entry added opens its port and an entry +//! removed closes it, with no restart. The handle stats the file at most +//! once per [`RECHECK`], because the gate asks at every frame to the alias. +//! +//! Minimal's own listeners are never exposures (NET-162). The gate hands +//! the handle the listeners that hold a port at each decision +//! ([`Holder`]). An entry at such a port is refused when the table loads, +//! and an entry whose port a listener binds later is shadowed from that +//! decision on. The daemon log names the entry and the listener, once. +//! +//! The reader fails closed. A missing file, a file that does not parse, or a +//! table that is not a table gives no exposure, and the alias stays closed +//! at every port. An entry that is not an inline table with a port from 1 to +//! 65535 and a protocol of `tcp` or `udp` is skipped, and the daemon log +//! names it. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, Instant, SystemTime}; + +use sessions::core::egress::{IPPROTO_TCP, IPPROTO_UDP}; + +/// The table of the client `config.toml` that names the host exposures. +pub(crate) const HOST_EXPOSURES_TABLE: &str = "host-exposures"; + +/// The longest time a changed table waits before the gate decides by it: +/// the handle stats the file at most once in this interval. +pub(crate) const RECHECK: Duration = Duration::from_secs(1); + +/// One port set as inclusive port ranges, `(lo, hi)`: the baseline set's +/// own openings at the alias. +pub(crate) type PortRange = (u16, u16); + +/// One host exposure: the IP protocol and the port a table entry opens. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct HostExposure { + /// The IP protocol number: [`IPPROTO_TCP`] or [`IPPROTO_UDP`]. + pub(crate) protocol: u8, + pub(crate) port: u16, +} + +impl HostExposure { + /// A TCP exposure at `port`. + #[must_use] + pub const fn tcp(port: u16) -> Self { + Self { + protocol: IPPROTO_TCP, + port, + } + } + + /// A UDP exposure at `port`. + #[must_use] + pub const fn udp(port: u16) -> Self { + Self { + protocol: IPPROTO_UDP, + port, + } + } + + /// Whether a frame with this IP protocol and destination port matches. + #[must_use] + pub(crate) fn admits(self, protocol: u8, port: u16) -> bool { + self.protocol == protocol && self.port == port + } +} + +/// One table entry: the name the user gave it and the exposure it opens. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Entry { + pub(crate) name: String, + pub(crate) exposure: HostExposure, +} + +/// One of Minimal's own listeners or sockets on the host, with the ports +/// it holds (NET-162). An entry at a held port is never an exposure: it +/// hands a box the listener in place of a service of the user's. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Holder { + /// The ports the listener holds, inclusive. + pub(crate) ports: PortRange, + /// The IP protocol it holds them for, `None` for both. + pub(crate) protocol: Option, + /// The listener as the daemon log names it. + pub(crate) name: &'static str, +} + +impl Holder { + /// Whether this listener holds the port that `exposure` opens. + #[must_use] + pub(crate) fn holds(&self, exposure: HostExposure) -> bool { + self.holds_port(exposure.protocol, exposure.port) + } + + /// Whether this listener holds `port` for `protocol`. + #[must_use] + pub(crate) fn holds_port(&self, protocol: u8, port: u16) -> bool { + let (lo, hi) = self.ports; + lo <= port && port <= hi && self.protocol.is_none_or(|held| held == protocol) + } +} + +/// The hostname proxy, as the daemon log names it. +pub(crate) const HOSTNAME_PROXY: &str = "the hostname proxy"; +/// The box-name answerer, as the daemon log names it. +pub(crate) const ANSWERER: &str = "the box-name answerer"; +/// The node-local Box Egress Proxy, as the daemon log names it. +pub(crate) const BOX_EGRESS_PROXY: &str = "the Box Egress Proxy"; +/// A node-plane baseline endpoint at the alias, as the daemon log names it. +pub(crate) const BASELINE_ENDPOINT: &str = "a node-plane baseline endpoint at host.min.internal"; + +/// Minimal's own listeners and sockets that hold a port at the host alias, +/// as the gate knows them at one flow (NET-162): +/// +/// - the hostname proxy at each port in `hostname_proxy`: the node row's +/// assigned port, an operator's pin or a drawn fallback; +/// - the box-name answerer at the port each VM-backed node reserves for it; +/// - the node-local Box Egress Proxy at its listener's port; +/// - the node-plane baseline endpoints at `baseline_alias_ports`, over the +/// protocol the baseline fetches with. +#[must_use] +pub(crate) fn minimal_listeners( + hostname_proxy: &[u16], + baseline_alias_ports: &[PortRange], + baseline_protocol: u8, +) -> Vec { + let mut held: Vec = hostname_proxy + .iter() + .map(|&port| Holder { + ports: (port, port), + protocol: None, + name: HOSTNAME_PROXY, + }) + .collect(); + let answerer = crate::net::answerer::DEFAULT_ANSWERER_PORT; + held.push(Holder { + ports: (answerer, answerer), + protocol: None, + name: ANSWERER, + }); + let proxy = switch::bep_host::PROXY_PORT; + held.push(Holder { + ports: (proxy, proxy), + protocol: None, + name: BOX_EGRESS_PROXY, + }); + held.extend(baseline_alias_ports.iter().map(|&ports| Holder { + ports, + protocol: Some(baseline_protocol), + name: BASELINE_ENDPOINT, + })); + held +} + +/// A shared handle on the current host-exposure table. Clones share one +/// table, so the gate and the start-up line read the same ports. +#[derive(Clone)] +pub struct HostExposures { + inner: Arc, +} + +struct Inner { + /// The `config.toml` the table is read from, `None` for a fixed table. + path: Option, + /// The least time between two stats of the file. + recheck: Duration, + state: Mutex, +} + +struct State { + /// The stat of the file version the entries came from. + stamp: Option, + /// When the handle last looked at the file, `None` before the first look. + checked: Option, + /// The entries of the current table, each in the accepted form. + entries: Arc<[Entry]>, + /// The listeners the last decision held the entries against. + held: Vec, + /// The exposures the current table opens: the entries no listener holds. + open: Arc<[HostExposure]>, + /// The entries the log already named as refused or shadowed, each with + /// its holder, so the log names each one once while the holder stays. + reported: Vec<(String, HostExposure, &'static str)>, +} + +/// The stat of one file version: an edit changes at least one field. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Stamp { + modified: Option, + len: u64, + /// The inode change time, the inode and the device: an edit that keeps + /// the length and the modification time still sets the change time, and + /// a file replaced by a rename gets a new inode. + #[cfg(unix)] + unix: (i64, i64, u64, u64), +} + +fn stamp(path: &Path) -> Option { + let meta = std::fs::metadata(path).ok()?; + #[cfg(unix)] + let unix = { + use std::os::unix::fs::MetadataExt as _; + (meta.ctime(), meta.ctime_nsec(), meta.dev(), meta.ino()) + }; + Some(Stamp { + modified: meta.modified().ok(), + len: meta.len(), + #[cfg(unix)] + unix, + }) +} + +impl HostExposures { + /// The table of the user-level client `config.toml` in the default + /// config dir, the file `bep serve` reads its rules from. With no + /// config dir (no absolute `XDG_CONFIG_HOME` and no home directory) + /// there is no table, and the alias stays closed. + #[must_use] + pub fn client_config() -> Self { + let path = client_config_path(); + match &path { + Some(path) => tracing::info!( + config = %path.display(), + table = HOST_EXPOSURES_TABLE, + "reading the host exposures from the client config" + ), + None => tracing::warn!( + table = HOST_EXPOSURES_TABLE, + "found no XDG_CONFIG_HOME and no home directory, so host.min.internal \ + has no host exposure" + ), + } + match path { + Some(path) => Self::at(path, RECHECK), + None => Self::fixed(Vec::new()), + } + } + + /// The table of the `config.toml` at `path`, re-read when the file + /// changes, with at least `recheck` between two stats of it. + #[must_use] + pub fn at(path: PathBuf, recheck: Duration) -> Self { + Self::new(Some(path), recheck, Vec::new()) + } + + /// A table that no file backs: `ports` and nothing else, for the life of + /// the handle. Each entry is named by its `proto/port` term. + #[must_use] + pub fn fixed(ports: Vec) -> Self { + let entries = ports + .into_iter() + .map(|exposure| Entry { + name: render(&[exposure]), + exposure, + }) + .collect(); + Self::new(None, Duration::ZERO, entries) + } + + fn new(path: Option, recheck: Duration, entries: Vec) -> Self { + let open: Arc<[HostExposure]> = entries.iter().map(|entry| entry.exposure).collect(); + Self { + inner: Arc::new(Inner { + path, + recheck, + state: Mutex::new(State { + stamp: None, + checked: None, + entries: entries.into(), + held: Vec::new(), + open, + reported: Vec::new(), + }), + }), + } + } + + /// The exposures the current table opens while the listeners in `held` + /// hold their ports. The handle re-reads the file first when the + /// recheck interval passed and the file's stat changed. + /// + /// An entry at a held port is not opened. When the entry is new in a + /// load, the log says the load refused it. When the entry was open + /// before, the log says the listener shadows it. Each says it once while + /// the listener holds the port, and names the entry and the listener. + #[must_use] + pub(crate) fn current(&self, held: &[Holder]) -> Arc<[HostExposure]> { + let loaded = self.reload(); + let mut state = self.lock(); + if loaded.is_none() && state.held == held { + return Arc::clone(&state.open); + } + let before: Arc<[Entry]> = match loaded { + Some(entries) => std::mem::replace(&mut state.entries, entries), + None => Arc::clone(&state.entries), + }; + let entries = Arc::clone(&state.entries); + let mut open = Vec::with_capacity(entries.len()); + let mut reported = Vec::new(); + for entry in entries.iter() { + let Some(holder) = held.iter().find(|holder| holder.holds(entry.exposure)) else { + open.push(entry.exposure); + continue; + }; + let key = (entry.name.clone(), entry.exposure, holder.name); + if !state.reported.contains(&key) { + let (proto, port) = (proto_name(entry.exposure), entry.exposure.port); + if before.contains(entry) { + tracing::warn!( + table = HOST_EXPOSURES_TABLE, + entry = %entry.name, + proto, + port, + holder = holder.name, + "a Minimal listener now holds the port of a host exposure, so \ + boxes no longer reach it at host.min.internal" + ); + } else { + tracing::warn!( + table = HOST_EXPOSURES_TABLE, + entry = %entry.name, + proto, + port, + holder = holder.name, + "refused a host exposure at a port that a Minimal listener holds" + ); + } + } + reported.push(key); + } + state.reported = reported; + state.held = held.to_vec(); + state.open = open.into(); + Arc::clone(&state.open) + } + + /// Re-reads the file when the recheck interval passed and its stat + /// changed, and returns the entries it read then. The stat and the read + /// run outside the lock, so a slow disk stalls only the frame that found + /// the recheck due, never the other callers. + fn reload(&self) -> Option> { + let path = self.inner.path.as_ref()?; + let (first, known) = { + let mut state = self.lock(); + let now = Instant::now(); + let due = state + .checked + .is_none_or(|checked| now.duration_since(checked) >= self.inner.recheck); + if !due { + return None; + } + // Mark the check as taken before the IO, so the callers that + // come in the meantime keep the current table and do not stat. + let first = state.checked.is_none(); + state.checked = Some(now); + (first, state.stamp.clone()) + }; + let current = stamp(path); + if !first && current == known { + return None; + } + let entries: Arc<[Entry]> = read_table(path).into(); + let mut state = self.lock(); + state.stamp = current; + if first || entries != state.entries { + let ports: Vec = entries.iter().map(|entry| entry.exposure).collect(); + tracing::info!( + config = %path.display(), + host_exposure_ports = %render(&ports), + "read the host exposures for host.min.internal" + ); + } + Some(entries) + } + + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.inner + .state + .lock() + .unwrap_or_else(PoisonError::into_inner) + } + + /// The current table as the daemon log spells it: one `proto/port` term + /// per exposure the last decision opened, and `(none)` for no exposure. + #[must_use] + pub(crate) fn render(&self) -> String { + let held = self.lock().held.clone(); + render(&self.current(&held)) + } +} + +impl std::fmt::Debug for HostExposures { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HostExposures") + .field("path", &self.inner.path) + .finish_non_exhaustive() + } +} + +/// Two handles are equal when they share one table. +impl PartialEq for HostExposures { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.inner, &other.inner) + } +} + +impl Eq for HostExposures {} + +fn proto_name(exposure: HostExposure) -> &'static str { + if exposure.protocol == IPPROTO_UDP { + "udp" + } else { + "tcp" + } +} + +fn render(ports: &[HostExposure]) -> String { + if ports.is_empty() { + return "(none)".to_string(); + } + ports + .iter() + .map(|&exposure| format!("{}/{}", proto_name(exposure), exposure.port)) + .collect::>() + .join(",") +} + +/// The user-level `config.toml` in the default config dir, or `None` when +/// neither an absolute `XDG_CONFIG_HOME` nor a home directory names it. +fn client_config_path() -> Option { + // `paths` takes an absolute `XDG_CONFIG_HOME` first, and stops the + // process when it then finds no home directory. Check the same order. + let xdg = std::env::var_os("XDG_CONFIG_HOME").is_some_and(|dir| Path::new(&dir).is_absolute()); + if !xdg { + std::env::home_dir()?; + } + Some(paths::minimal_config_dir_with_override(None).join("config.toml")) +} + +/// Reads the table of the file at `path`, failing closed: a missing or +/// unparseable file is no exposure. +fn read_table(path: &Path) -> Vec { + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Vec::new(), + Err(error) => { + tracing::warn!( + config = %path.display(), + %error, + "could not read the client config, so host.min.internal has no host exposure" + ); + return Vec::new(); + } + }; + match parse_table(&text) { + Ok(entries) => entries, + Err(error) => { + tracing::warn!( + config = %path.display(), + %error, + "could not parse the client config, so host.min.internal has no host exposure" + ); + Vec::new() + } + } +} + +/// Parses the `[host-exposures]` table of one `config.toml` text. No table +/// is no exposure. An entry that is not an exposure is skipped, and the log +/// names it. The listeners are held against the entries at each decision +/// ([`HostExposures::current`]), not here. +fn parse_table(text: &str) -> Result, toml::de::Error> { + let doc: toml::Table = toml::from_str(text)?; + let Some(table) = doc.get(HOST_EXPOSURES_TABLE) else { + return Ok(Vec::new()); + }; + let Some(table) = table.as_table() else { + tracing::warn!( + table = HOST_EXPOSURES_TABLE, + "the host-exposures key in the client config is not a table, so \ + host.min.internal has no host exposure" + ); + return Ok(Vec::new()); + }; + let mut entries: Vec = table + .iter() + .filter_map(|(name, value)| { + let Some(exposure) = entry_exposure(value) else { + tracing::warn!( + table = HOST_EXPOSURES_TABLE, + entry = %name, + value = %value, + "skipped a host exposure that is not {{ proto = \"tcp\" or \"udp\", \ + port = 1 to 65535 }}" + ); + return None; + }; + Some(Entry { + name: name.clone(), + exposure, + }) + }) + .collect(); + entries.sort_unstable_by(|a, b| (a.exposure, &a.name).cmp(&(b.exposure, &b.name))); + Ok(entries) +} + +/// One entry's value as an exposure: the inline table +/// `{ proto = "tcp", port = N }`, where `proto` is `tcp` or `udp` and +/// defaults to `tcp`. Any other key or form is not an exposure. +fn entry_exposure(value: &toml::Value) -> Option { + let entry = value.as_table()?; + if entry.keys().any(|key| key != "proto" && key != "port") { + return None; + } + let port = entry + .get("port")? + .as_integer() + .and_then(|n| u16::try_from(n).ok()) + .filter(|&port| port != 0)?; + match entry.get("proto").map(toml::Value::as_str) { + None | Some(Some("tcp")) => Some(HostExposure::tcp(port)), + Some(Some("udp")) => Some(HostExposure::udp(port)), + Some(_) => None, + } +} + +#[cfg(test)] +mod tests { + use super::{HostExposure, HostExposures, client_config_path, parse_table}; + use std::time::Duration; + + fn exposures_of(entries: &[super::Entry]) -> Vec { + entries.iter().map(|entry| entry.exposure).collect() + } + + /// NET-159: the host exposures come from the `[host-exposures]` table of + /// the client `config.toml` alone. Each entry names one exposure, its + /// port and its protocol, and `proto` defaults to `tcp`. The handle + /// reads the file at the client config's path and follows it. + #[test] + fn host_exposures_read_from_client_config() { + let entries = parse_table( + "[host-exposures]\n\ + postgres = { proto = \"tcp\", port = 5432 }\n\ + vite = { port = 5173 }\n\ + dns = { proto = \"udp\", port = 5353 }\n\ + bare = 8080\n\ + range = \"3000-3999\"\n\ + text = \"4200\"\n\ + zero = { port = 0 }\n\ + big = { port = 70000 }\n\ + icmp = { proto = \"icmp\", port = 1 }\n\ + extra = { port = 9000, host = \"x\" }\n\ + noport = { proto = \"tcp\" }\n", + ) + .unwrap(); + assert_eq!( + exposures_of(&entries), + vec![ + HostExposure::tcp(5173), + HostExposure::tcp(5432), + HostExposure::udp(5353), + ] + ); + assert_eq!(entries[0].name, "vite", "each entry keeps its name"); + assert!(parse_table("[other]\nx = 1\n").unwrap().is_empty()); + assert!(parse_table("host-exposures = 8080\n").unwrap().is_empty()); + assert!(parse_table("not toml [").is_err()); + + // The file is the client config: `config.toml` in the config dir. + if let Some(path) = client_config_path() { + assert_eq!( + path, + paths::minimal_config_dir_with_override(None).join("config.toml") + ); + } + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + let exposures = HostExposures::at(path.clone(), Duration::ZERO); + assert!(exposures.current(&[]).is_empty(), "no file is no exposure"); + std::fs::write( + &path, + "[secrets]\n\n[host-exposures]\ndev = { port = 8080 }\n", + ) + .unwrap(); + assert_eq!(&*exposures.current(&[]), &[HostExposure::tcp(8080)]); + std::fs::write(&path, "[host-exposures]\n").unwrap(); + assert!(exposures.current(&[]).is_empty(), "a removed entry closes"); + std::fs::write(&path, "broken [").unwrap(); + assert!( + exposures.current(&[]).is_empty(), + "a broken file fails closed" + ); + } +} diff --git a/crates/sessions/src/client/config.rs b/crates/sessions/src/client/config.rs index 39b79edd5e..fec6e4f917 100644 --- a/crates/sessions/src/client/config.rs +++ b/crates/sessions/src/client/config.rs @@ -30,6 +30,33 @@ pub struct Config { /// within stay `snake_case` to match the rest of the config. #[serde(default, rename = "session-keys")] pub session_keys: SessionKeysConfig, + /// `[host-exposures]` section: the host ports `minvmd` opens at + /// `host.min.internal`. `minvmd` reads and checks the entries. The + /// client keeps them as written, so a file that has the table parses. + #[serde( + default, + rename = "host-exposures", + skip_serializing_if = "HostExposuresTable::is_empty" + )] + pub host_exposures: HostExposuresTable, +} + +/// The `[host-exposures]` table as written, one entry per name. `minvmd` +/// owns its form and skips an entry it does not accept. +#[derive(Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(transparent)] +pub struct HostExposuresTable(pub toml::Table); + +// A TOML value compares by value. A NaN float is the one value not equal to +// itself, and no exposure entry holds one. +impl Eq for HostExposuresTable {} + +impl HostExposuresTable { + /// Whether the table has no entry. + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } } /// The `[loadouts]` section. @@ -269,6 +296,26 @@ mod tests { assert_eq!(cfg, Config::default()); } + /// A `[host-exposures]` table parses, so the client still reads a file + /// that opens a host port for `minvmd`. The client keeps an entry it + /// does not accept as well, because `minvmd` decides each entry. + #[test] + fn read_config_file_keeps_host_exposures() { + let dir = TempDir::new().unwrap(); + let path = write_file( + dir.path(), + "config.toml", + indoc::indoc! {r#" + [host-exposures] + postgres = { proto = "tcp", port = 5432 } + legacy = 8080 + "#}, + ); + let cfg = read_config_file(&path).unwrap(); + assert_eq!(cfg.host_exposures.0.len(), 2); + assert!(cfg.host_exposures.0.contains_key("postgres")); + } + /// A missing `[loadouts]` section deserializes to the default /// `LoadoutsConfig` (empty default list) — every subsection is /// optional. diff --git a/docs/reference/cli-minvmd.md b/docs/reference/cli-minvmd.md index ff30364b05..278d345ccf 100644 --- a/docs/reference/cli-minvmd.md +++ b/docs/reference/cli-minvmd.md @@ -55,7 +55,7 @@ An own-IP box that declares no `egress` section gets the deny-all egress default `minvmd` reads the variable only when it starts, and hands it to the guest at boot. Set it in the environment of the process that starts `minvmd`: `minvmd run`, or the first `min` command that starts the VM for you. Setting it for a later `min` command against a running VM changes nothing. To apply a change, stop `minvmd` (`minvmd stop`) and start it again with the new environment. -`minvmd` closes `host.min.internal` at every port unless `MINVMD_HOST_EXPOSURE_PORTS` lists ports. Set the variable to a comma-separated list of ports and ranges, such as `8080,3000-3999`. A box in the default host-address network mode then reaches the host at those ports and no other. The box's egress rules still apply inside an exposure. An own-address box gets a refusal at every port, whatever the list says. Leave the variable unset or empty to keep `host.min.internal` closed. Each port must lie between 1 and 65535, and each range must run from low to high. Any other value fails the start with an error that names the bad term. `minvmd` reads the variable from its own environment each time it boots the VM, so to apply a change, stop `minvmd` and start it again. +`minvmd` closes `host.min.internal` at every port that the `[host-exposures]` table does not open. The table is in the client `config.toml`, `$XDG_CONFIG_HOME/minimal/config.toml` or `~/.config/minimal/config.toml`. Give each exposure a name and an inline table, such as `postgres = { proto = "tcp", port = 5432 }`. `proto` is `tcp` or `udp`, and `tcp` is the default. `port` runs from 1 to 65535. A box in the default host-address network mode then reaches the host at those ports and no other, and the box's egress rules still apply inside an exposure. A deny-all box reaches no exposure. An own-address box gets a refusal at every port, whatever the table says. This is a current limit: the design lets an own-address box reach an exposure when its `egress.allow_dns_hosts` names `host.min.internal` or its `egress.allow_subnets` lists a range that covers that address. [#2990](https://github.com/gominimal/minimal/issues/2990) tracks that change. `minvmd` reads the table again within a second of a change. An added entry opens its port, and a removed entry closes it and refuses the later traffic of its open connections, with no restart. `minvmd` skips an entry in any other form. It also refuses an entry at a port that Minimal itself uses: the hostname proxy's port, the box-name answerer's port, the Box Egress Proxy's port, or a port where the VM's own registry or cache traffic reaches the host. When Minimal starts to use the port of an entry later, `minvmd` closes that entry from then on. The daemon log names each skipped, refused or closed entry, and the Minimal listener that holds its port. `minvmd` no longer reads `MINVMD_HOST_EXPOSURE_PORTS`. When the variable is set, `minvmd` logs a warning and ignores its value. ### `status` diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index 4fa704d8ae..564a0da517 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -359,18 +359,19 @@ case "${1:-}" in esac # The host exposure the VM-backed daemon opens host.min.internal at. minvmd -# closes the alias at every port unless MINVMD_HOST_EXPOSURE_PORTS lists -# ports, and it reads the list from its own environment at each VM boot, so -# the export must precede the first `min` call, whose autospawn inherits it. -# The list is the one port NET-003 reaches the host's loopback at from a -# host-address box (the proxy proof's PROXY_HOST_PORT) and nothing wider: -# the same proof asserts the alias stays refused at an unlisted port, and -# own-address boxes stay refused at every port, listed or not. A native -# daemon has no alias gate and ignores the variable. +# closes the alias at every port unless the [host-exposures] table of the +# client config.toml opens ports. The table is written into the hermetic +# XDG_CONFIG_HOME below, before the first `min` call, whose autospawn hands +# minvmd that config dir. The table is the one port NET-003 reaches the +# host's loopback at from a host-address box (the proxy proof's +# PROXY_HOST_PORT) and nothing wider: the same proof asserts the alias stays +# refused at a port outside the table, and own-address boxes stay refused at +# every port, in the table or not. That own-address refusal is a current +# limit: the design lets an own-address box that declares the name reach an +# exposure, and https://github.com/gominimal/minimal/issues/2990 (T154) +# tracks the guest half. A native daemon has no alias gate and +# ignores the table. E2E_HOST_EXPOSURE_PORT=18081 -if [ -n "$E2E_VM" ]; then - export MINVMD_HOST_EXPOSURE_PORTS="$E2E_HOST_EXPOSURE_PORT" -fi # The non-baseline package the sandbox proof adds and then runs. It must be a # real upstream package that is genuinely ABSENT from a fresh shell-stack @@ -633,6 +634,11 @@ fi # orientation banner (built-in `default` loadout). An operator's own # `default_loadouts`/`default.toml` must not leak into the canonical proof. export XDG_CONFIG_HOME="$WORK/config" +if [ -n "$E2E_VM" ]; then + mkdir -p "$XDG_CONFIG_HOME/minimal" + printf '[host-exposures]\ne2e = { proto = "tcp", port = %s }\n' "$E2E_HOST_EXPOSURE_PORT" \ + >>"$XDG_CONFIG_HOME/minimal/config.toml" +fi # The CLI's tracing layer writes to STDOUT (ot::StdoutWriter, minimal/src/ # main.rs), so at the default level the autospawn INFO lines interleave with @@ -16137,8 +16143,8 @@ proof_min_internal_names_through_proxy() { # ---- the host exposure is the only opening (design §7.1) ---------------- # A VM host's gate opens host.min.internal at the listed exposure alone - # (MINVMD_HOST_EXPOSURE_PORTS, exported at the top), so the same box, the - # same alias, at a port outside the list must not reach the host, even + # (the [host-exposures] table written at the top), so the same box, the + # same alias, at a port outside the table must not reach the host, even # with a host-loopback listener answering there. A native host-address box # shares the host's namespace, with no alias gate to assert. if [ -n "$E2E_VM" ]; then @@ -16178,7 +16184,7 @@ proof_min_internal_names_through_proxy() { case "$proxy_unexposed_rc" in 7 | 28) ;; 0) - echo "::error::the host-address box reached the host through host.min.internal at port $PROXY_UNEXPOSED_PORT, outside MINVMD_HOST_EXPOSURE_PORTS=$MINVMD_HOST_EXPOSURE_PORTS" + echo "::error::the host-address box reached the host through host.min.internal at port $PROXY_UNEXPOSED_PORT, outside the [host-exposures] table, which opens port $E2E_HOST_EXPOSURE_PORT alone" fail ;; *) From ec7524af09c592003eeee115c8b3890a180cde9b Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:46:04 +0000 Subject: [PATCH 17/34] fix(minimald): run a box's package builds under that box's egress verdict (#2720) T104, NET-155. A box's package builds run in the box's classifier leaf, or for an own-address box on a task network of the box's own, so the box's egress verdict decides their connections. That covers build and materialize side operations, launch builds, in-box installs, patched builds and task runs. A box whose launch placed it in a leaf refuses a build, task or check when the leaf is gone, so a build never runs outside the verdict. A build the box's rules refuse names the destination and the box, and each build start logs the box, package and leaf. A box with no leaf and no own address builds in the daemon's cgroup as node-plane traffic, as before, and logs that it does. Backport of #2720. Adapted to the line: - box_actor/setup.rs hunks land in session.rs, box_ops.rs is session_sop.rs, and the tests land in session/tests.rs, with super::setup:: read as super::, a Record literal of the line's fields, and local rewrite_record and read_record helpers. - boxes, sandbox and box_manager read sessions, sandbox2 and sessions, and crate::traced becomes tokio::spawn and tokio::task::spawn_blocking. - mctx keeps the line's build_graph body, without main's dangling-hit replan, and only gains the local_orchestrator call that carries the leaf. Dropped: main's session_id on the env channel, which T104 only carried as context, the dangling-hit test in op/src/patched.rs, and the test for #2234 in op/src/specs.rs, which comes with #2234 below. (cherry picked from commit a26f89bf65cdbd194fd13292e6ed4dc7ba696d09) --- crates/mctx/src/env.rs | 4 + crates/mctx/src/lib.rs | 176 ++++- crates/minimald/src/env.rs | 345 ++++++++- crates/minimald/src/exec.rs | 39 +- crates/minimald/src/session.rs | 251 ++++++- crates/minimald/src/session/tests.rs | 691 ++++++++++++++++++ crates/minimald/src/session_sop.rs | 177 ++++- .../tests/classifier_root_integration.rs | 289 +++++++- crates/mip/src/cmd_pkg_patched_build.rs | 4 + crates/op/src/lib.rs | 2 +- crates/op/src/patched.rs | 21 + crates/op/src/specs.rs | 190 ++++- crates/orchestrator/src/local_backend.rs | 24 + .../18-spec-box-networking.md | 6 + 14 files changed, 2161 insertions(+), 58 deletions(-) diff --git a/crates/mctx/src/env.rs b/crates/mctx/src/env.rs index 0b566b8a9c..eb2bced558 100644 --- a/crates/mctx/src/env.rs +++ b/crates/mctx/src/env.rs @@ -265,6 +265,10 @@ impl EnvChannel<'_> { stdout_writer: Some(Box::new(stdout_writer)), stderr_writer: Some(Box::new(stderr_writer)), remote_cache: remote_cache.as_ref(), + // NET-155: a client-side patched build runs for no box, so it + // stays the caller's own traffic. + classifier_leaf: None, + box_network: None, } .run(&op::Options { cache, diff --git a/crates/mctx/src/lib.rs b/crates/mctx/src/lib.rs index f33c71ace1..3fee2f9ba2 100644 --- a/crates/mctx/src/lib.rs +++ b/crates/mctx/src/lib.rs @@ -267,6 +267,16 @@ pub struct DaemonContext { pub struct Context { pub(crate) daemon: Arc, pub(crate) mfile: mfile::File, + /// The classifier leaf of the box this context builds for, if any + /// (NET-155). Every package build the context starts runs in it, so the + /// box's egress verdict decides the build's connections. `None` for a + /// context that builds as its caller's own traffic. + pub(crate) classifier_leaf: Option, + /// The network of the own-address box this context builds for, if any + /// (NET-155). Every package build the context starts that needs the + /// network attaches to it, so the box's egress verdict decides the + /// build's connections. + pub(crate) box_network: Option, } impl fmt::Display for Context { @@ -422,7 +432,12 @@ impl Context { pub fn new_with_strategy(config: Config, strategy: MFileSearchStrategy) -> Result { let daemon = Arc::new(DaemonContext::init(config)?); let mfile = strategy.find_mfile()?; - Ok(Self { daemon, mfile }) + Ok(Self { + daemon, + mfile, + classifier_leaf: None, + box_network: None, + }) } /// Constructs a new context using the lower-level primitives. @@ -441,7 +456,12 @@ impl Context { vcs, cache, }); - Self { daemon, mfile } + Self { + daemon, + mfile, + classifier_leaf: None, + box_network: None, + } } /// Constructs a `Context` from a shared `DaemonContext` and an @@ -450,7 +470,39 @@ impl Context { /// without repeating the (moderately expensive) /// [`DaemonContext::init`] work per session. pub fn from_daemon(daemon: Arc, mfile: mfile::File) -> Self { - Self { daemon, mfile } + Self { + daemon, + mfile, + classifier_leaf: None, + box_network: None, + } + } + + /// Places every package build this context starts in `leaf`, the + /// classifier leaf of the box the builds run for (NET-155). + #[must_use] + pub fn with_classifier_leaf(mut self, leaf: sandbox2::config::ClassifierLeaf) -> Self { + self.classifier_leaf = Some(leaf); + self + } + + /// The classifier leaf this context's package builds run in, if any. + pub fn classifier_leaf(&self) -> Option<&sandbox2::config::ClassifierLeaf> { + self.classifier_leaf.as_ref() + } + + /// Runs every package build this context starts that needs the network + /// on `network`, the network of the own-address box the builds run for + /// (NET-155). + #[must_use] + pub fn with_box_network(mut self, network: op::BuildNetwork) -> Self { + self.box_network = Some(network); + self + } + + /// The box network this context's package builds attach to, if any. + pub fn box_network(&self) -> Option<&op::BuildNetwork> { + self.box_network.as_ref() } /// Returns a clone of the shared daemon-scoped context. Cheap — @@ -486,7 +538,11 @@ impl Context { MFileSearchStrategy::CurrentDirRecursive }; let mfile = strategy.find_mfile()?; - Ok(Self::from_daemon(Arc::clone(&self.daemon), mfile)) + Ok(Self { + classifier_leaf: self.classifier_leaf.clone(), + box_network: self.box_network.clone(), + ..Self::from_daemon(Arc::clone(&self.daemon), mfile) + }) } /// Builds a [`ProjectSetup`] from this context, for running project-setup @@ -802,6 +858,33 @@ impl Context { .await } + /// The orchestrator this context builds `graph` with. It carries the + /// context's classifier leaf (NET-155), so every spec build it schedules + /// runs in the leaf of the box the context was made for. + async fn local_orchestrator( + &self, + graph: &Graph, + rc: Option>, + log_sink: Option>, + cancel: tokio_util::sync::CancellationToken, + ) -> Result>, Error> + { + Ok(orchestrator::LocalBackend::new_orchestrator( + self.daemon.config.builds_base_dir(), + rc, + self.remote_storage().await?, + self.daemon.config.num_parallel_builds(), + graph.clone(), + self.local_cache(), + self.daemon_id(), + log_sink, + self.daemon.config.ot.clone(), + cancel, + self.classifier_leaf.clone(), + self.box_network.clone(), + )?) + } + pub async fn build_graph_with_cancel( &mut self, graph: &Graph, @@ -817,18 +900,9 @@ impl Context { }; use orchestrator::LocalBackend; - let orchestrator = LocalBackend::new_orchestrator( - self.daemon.config.builds_base_dir(), - rc.clone(), - self.remote_storage().await?, - self.daemon.config.num_parallel_builds(), - graph.clone(), - cache.clone(), - self.daemon_id(), - log_sink, - self.daemon.config.ot.clone(), - cancel, - )?; + let orchestrator = self + .local_orchestrator(graph, rc.clone(), log_sink, cancel) + .await?; let mut bin_provider: Box = match ( self.daemon.config.use_local_cache(), @@ -1451,6 +1525,76 @@ mod tests { )); } + /// NET-155: a context made for a box carries the box's classifier leaf, + /// across a clone and a reload of its mfile, into the orchestrator it + /// builds with. A context made for no box carries no leaf. + #[tokio::test] + async fn box_context_carries_its_classifier_leaf() { + let state = tempdir().unwrap(); + let cache = tempdir().unwrap(); + let config = ConfigBuilder::new() + .with_state_dir(state.path().to_path_buf()) + .with_cache_dir(cache.path().to_path_buf()) + .with_stdlib_dir( + std::path::Path::new(&std::env::var("CARGO_MANIFEST_DIR").unwrap()) + .join("../stdlib/minimal-ncl"), + ) + .build() + .unwrap(); + let daemon = Arc::new(DaemonContext::init(config).unwrap()); + let repo = std::path::Path::new(&std::env::var("CARGO_MANIFEST_DIR").unwrap()) + .join("testdata") + .join("fakerepo"); + let mfile = MFileSearchStrategy::Override(repo) + .find_mfile() + .expect("fakerepo has a minimal.toml"); + let leaf = sandbox2::config::ClassifierLeaf::under( + std::path::Path::new("/sys/fs/cgroup/minimald.slice"), + "box-1", + sandbox2::config::Verdict::Deny, + ); + + let plain = Context::from_daemon(Arc::clone(&daemon), mfile); + assert_eq!(plain.classifier_leaf(), None); + let graph = Graph::new(); + let orchestrator = plain + .local_orchestrator(&graph, None, None, Default::default()) + .await + .unwrap(); + assert_eq!(orchestrator.backend.classifier_leaf(), None); + + let boxed = plain.clone().with_classifier_leaf(leaf.clone()); + assert_eq!(boxed.clone().classifier_leaf(), Some(&leaf)); + assert_eq!( + boxed.cloned_reinit().unwrap().classifier_leaf(), + Some(&leaf) + ); + let orchestrator = boxed + .local_orchestrator(&graph, None, None, Default::default()) + .await + .unwrap(); + assert_eq!(orchestrator.backend.classifier_leaf(), Some(&leaf)); + assert!(orchestrator.backend.box_network().is_none()); + + // An own-address box has no leaf. Its context carries the box's + // network instead, across a clone and a reload, into the + // orchestrator. + assert!(plain.box_network().is_none()); + let on_box_network = plain.with_box_network(op::BuildNetwork::new(|| { + Arc::new(sandbox2::NoNet) as Arc + })); + assert!(on_box_network.clone().box_network().is_some()); + let reloaded = on_box_network.cloned_reinit().unwrap(); + assert!(reloaded.box_network().is_some()); + assert_eq!(reloaded.classifier_leaf(), None); + let orchestrator = reloaded + .local_orchestrator(&graph, None, None, Default::default()) + .await + .unwrap(); + assert!(orchestrator.backend.box_network().is_some()); + assert_eq!(orchestrator.backend.classifier_leaf(), None); + } + #[test] #[ignore] // Do not run in github: does not support nested namespaces fn toplevel_layer_init_and_build() { diff --git a/crates/minimald/src/env.rs b/crates/minimald/src/env.rs index 6f5ea5f4ae..9a4d5d5b6d 100644 --- a/crates/minimald/src/env.rs +++ b/crates/minimald/src/env.rs @@ -447,6 +447,14 @@ impl Env { /// helper), assembles the rootfs, and spawns the command /// channel actor. pub async fn build(mut ctx: Context, mut graph: Graph, args: EnvArgs) -> std::io::Result { + // NET-155: the leaf the launcher created for this box places every + // package build this env starts — the missing packages below and the + // in-box installs later — so the box's egress verdict decides what + // they reach. It replaces any leaf the context carried, because the + // launch's leaf is the one the box runs in now. + if let Some(leaf) = args.classifier_leaf.clone() { + ctx = ctx.with_classifier_leaf(leaf); + } // Resolve the requested package names to top-levels, then ensure the // helper's dependencies (`bash`, `socat`) are present. graph.top_levels = args @@ -470,6 +478,23 @@ impl Env { .all(|bsr| cache.read_dir(&graph.spec_hash(bsr)).is_ok()); if !all_built { tracing::trace!("missing local packages, building session env"); + match ctx.classifier_leaf() { + Some(leaf) => tracing::info!( + box_name = %args.name, + leaf = %leaf.dir().display(), + "starting the box's missing package builds in the box's classifier leaf", + ), + None if ctx.box_network().is_some() => tracing::info!( + box_name = %args.name, + "starting the box's missing package builds on the box's own network, \ + under the box's egress rules", + ), + None => tracing::info!( + box_name = %args.name, + "starting the box's missing package builds in the daemon's own cgroup, \ + with no per-box egress enforcement, because the box has none", + ), + } ctx.build_graph(&graph, false, None) .await .map_err(err_to_io)?; @@ -564,6 +589,7 @@ impl Env { // names the leaf it entered on its launch log line. `None` on a host // that places no box: the config above is the whole story, exactly as // it was before the classifier existed. + let classifier_leaf = args.classifier_leaf.clone(); let config = match args.classifier_leaf { Some(leaf) => config.with_classifier_leaf(leaf), None => config, @@ -598,6 +624,7 @@ impl Env { home: args.home.clone(), has_packages: transitives.keys().copied().collect(), box_id, + classifier_leaf, ot: args.ot.clone(), session: args.session.clone(), ctx, @@ -879,6 +906,10 @@ struct SessionChannel { /// in one, else the session's name — so the daemon's own fetches are /// recorded with the box that asked for them. box_id: String, + /// The box's classifier leaf (NET-155): the in-box patched build runs in + /// it, so the box's egress verdict decides what the build can reach. + /// `None` on a host that places no box. + classifier_leaf: Option, ot: Option, /// Weak handle to the owning session actor, used by session-scoped commands /// (e.g. `min build`) to drive side-ops. @@ -1051,6 +1082,25 @@ impl SessionChannel { // fetch is recorded as the daemon's own node-plane fetch, made for // the box that asked — so the one walk feeds both. let scope = install_scope(&new_graph, pkgs); + // NET-155: `Env::build` placed this context in the box's leaf, so + // the builds this install starts follow the box's egress verdict. + match self.ctx.classifier_leaf() { + Some(leaf) => tracing::info!( + box_name = %self.name, + leaf = %leaf.dir().display(), + "starting an in-box install's package builds in the box's classifier leaf", + ), + None if self.ctx.box_network().is_some() => tracing::info!( + box_name = %self.name, + "starting an in-box install's package builds on the box's own network, \ + under the box's egress rules", + ), + None => tracing::info!( + box_name = %self.name, + "starting an in-box install's package builds in the daemon's own cgroup, \ + with no per-box egress enforcement, because the box has none", + ), + } let progress = crate::sandbox_progress::SandboxProgress::new( self.ctx.op_tracker(), Some(scope.clone()), @@ -1288,6 +1338,10 @@ impl SessionChannel { let mut build_graph = graph.clone(); build_graph.top_levels = vec![bsr]; + let tapped = Arc::new(Mutex::new(std::collections::VecDeque::new())); + // NET-155: an own-address box has no leaf. Its build attaches to the + // box's own network, which the box's context carries. + let box_network = ctx.box_network().cloned(); let result: std::io::Result<()> = async { let remote_storage = ctx.remote_storage().await.map_err(err_to_io)?; let remote_cache = if ctx.use_remote_cache() { @@ -1305,12 +1359,43 @@ impl SessionChannel { let cache = ctx.local_cache(); let (stdout_writer, stderr_writer) = StreamWriter::pair(stream)?; + // NET-155: the build runs in the box's own leaf, or on the own- + // address box's network, so the box's egress verdict decides what + // it can reach. A box with neither has no per-box enforcement, + // and its build runs in the daemon's own cgroup. + match &self.classifier_leaf { + Some(leaf) => tracing::info!( + box_name = %self.name, + package = %pkg_name, + leaf = %leaf.dir().display(), + "starting a package build for a box in the box's classifier leaf", + ), + None if box_network.is_some() => tracing::info!( + box_name = %self.name, + package = %pkg_name, + leaf = "none", + "starting a package build for a box on the box's own network, \ + under the box's egress rules", + ), + None => tracing::info!( + box_name = %self.name, + package = %pkg_name, + leaf = "none", + "starting a package build for a box in the daemon's own cgroup, \ + with no per-box egress enforcement, because the box has none", + ), + } let res = op::PatchedBuild { spec: &bsr, remote_fetcher: &remote_storage, - stdout_writer: Some(Box::new(stdout_writer)), - stderr_writer: Some(Box::new(stderr_writer)), + stdout_writer: Some(Box::new(LineTap::new(stdout_writer, tapped.clone()))), + stderr_writer: Some(Box::new(LineTap::new(stderr_writer, tapped.clone()))), remote_cache: remote_cache.as_ref(), + classifier_leaf: self.classifier_leaf.clone(), + box_network: match &self.classifier_leaf { + Some(_) => None, + None => box_network.clone(), + }, } .run(&op::Options { cache, @@ -1330,6 +1415,20 @@ impl SessionChannel { .await; if let Err(e) = result { + // NET-155: a build the box's rules refused names the destination + // and the box before the build's own error. + let lines = tapped + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(refusal) = patched_build_refusal( + build_denies_all(self.classifier_leaf.as_ref(), box_network.as_ref()), + pkg_name, + &self.name, + &e.to_string(), + lines.iter().map(String::as_str), + ) { + let _ = writeln!(stream, "error: {refusal}"); + } let _ = writeln!(stream, "error: {e}"); return; } @@ -2131,6 +2230,247 @@ impl tokio::io::AsyncWrite for StreamWriter { } } +/// How many output lines of one build [`LineTap`] keeps. +const TAPPED_LINES: usize = 64; + +/// The most bytes of one unfinished line [`LineTap`] holds. Past this bound +/// it keeps only the tail, so output that never ends a line stays bounded. +const TAPPED_LINE_BYTES: usize = 4096; + +/// A writer that passes every byte to `inner` and keeps the last +/// [`TAPPED_LINES`] lines in `lines` (NET-155), so a build that fails can say +/// what it could not reach. A `\n` or a `\r` ends a line, so a progress meter +/// that redraws with `\r` gives lines too. The text after the last line end +/// is kept as a line when the tap shuts down or is dropped. +pub(crate) struct LineTap { + inner: W, + buf: Vec, + lines: Arc>>, +} + +impl LineTap { + pub(crate) fn new(inner: W, lines: Arc>>) -> Self { + Self { + inner, + buf: Vec::new(), + lines, + } + } + + /// How many bytes of an unfinished line the tap holds. + #[cfg(test)] + pub(crate) fn pending(&self) -> usize { + self.buf.len() + } + + fn keep(&self, line: &[u8]) { + if line.is_empty() { + return; + } + let line = String::from_utf8_lossy(line).into_owned(); + let mut lines = self + .lines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if lines.len() == TAPPED_LINES { + lines.pop_front(); + } + lines.push_back(line); + } + + /// Keeps the text after the last line end as a line. + fn flush_line(&mut self) { + let rest = std::mem::take(&mut self.buf); + self.keep(&rest); + } +} + +impl Drop for LineTap { + fn drop(&mut self) { + self.flush_line(); + } +} + +impl tokio::io::AsyncWrite for LineTap { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> Poll> { + let this = &mut *self; + let written = match Pin::new(&mut this.inner).poll_write(cx, buf) { + Poll::Ready(Ok(n)) => n, + other => return other, + }; + this.buf.extend_from_slice(&buf[..written]); + if let Some(last) = this.buf.iter().rposition(|&b| b == b'\n' || b == b'\r') { + let done: Vec = this.buf.drain(..=last).collect(); + for line in done.split(|&b| b == b'\n' || b == b'\r') { + this.keep(line); + } + } + if this.buf.len() > TAPPED_LINE_BYTES { + let cut = this.buf.len() - TAPPED_LINE_BYTES; + this.buf.drain(..cut); + } + Poll::Ready(Ok(written)) + } + + fn poll_flush( + mut self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + Pin::new(&mut self.inner).poll_flush(cx) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + self.flush_line(); + Pin::new(&mut self.inner).poll_shutdown(cx) + } +} + +/// The destination a failed build's output says it could not reach +/// (NET-155), read from the last lines the build printed, newest first, so +/// the failure that ended the build wins over an earlier tolerated one. It +/// knows the failure lines of `curl` and `wget`, which also cover `git` over +/// HTTPS: a host it could not resolve, or a host and port it could not +/// connect to. +/// `None` when no line names a destination. +pub(crate) fn refused_destination<'a>(lines: impl IntoIterator) -> Option { + let word = |rest: &str| -> Option { + let host: String = rest + .chars() + .take_while(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | ':' | '[' | ']')) + .collect(); + let host = host.trim_end_matches(['.', ':']); + (!host.is_empty()).then(|| host.to_string()) + }; + let lines: Vec<&str> = lines.into_iter().collect(); + lines.into_iter().rev().find_map(|line| { + // curl: `Could not resolve host: example.com` + if let Some((_, rest)) = line.split_once("Could not resolve host: ") { + return word(rest); + } + // curl: `Failed to connect to example.com port 443 after 3 ms: ...` + if let Some((_, rest)) = line.split_once("Failed to connect to ") { + let host = word(rest)?; + let port = rest + .split_once(" port ") + .map(|(_, p)| { + p.chars() + .take_while(char::is_ascii_digit) + .collect::() + }) + .filter(|p| !p.is_empty()); + return Some(match port { + Some(port) => format!("{host}:{port}"), + None => host, + }); + } + // wget: `Resolving example.com (example.com)... failed: ...` + if let Some(rest) = line.strip_prefix("Resolving ") + && line.contains("... failed") + { + return word(rest); + } + // wget: `Connecting to example.com (example.com)|1.2.3.4|:443... failed: ...` + if let Some(rest) = line.strip_prefix("Connecting to ") + && line.contains("... failed") + { + let host = word(rest)?; + let port = rest + .rsplit_once("|:") + .map(|(_, p)| { + p.chars() + .take_while(char::is_ascii_digit) + .collect::() + }) + .filter(|p| !p.is_empty()); + return Some(match port { + Some(port) if !host.ends_with(&format!(":{port}")) => format!("{host}:{port}"), + _ => host, + }); + } + None + }) +} + +/// Whether the egress rules a box's package build runs under admit no +/// destination (NET-155): the build runs in the deny subtree of the +/// classifier tree, or on the network of an own-address box whose rules +/// admit nothing. A build with neither runs under no rules of the box. +pub(crate) fn build_denies_all( + leaf: Option<&ClassifierLeaf>, + network: Option<&op::BuildNetwork>, +) -> bool { + match leaf { + Some(leaf) => leaf.is_deny(), + None => network.is_some_and(op::BuildNetwork::denies_all), + } +} + +/// The error line for an in-box patched build that the egress rules of its +/// box refused (NET-155), or `None` when the refusal is not the rules'. +/// `failure` is the build's own error. The error of a patched build names no +/// package, so the refusal holds only when that error itself carries the +/// destination [`refused_destination`] reads from `lines`. A build that +/// tolerated a failed download and then failed for another reason is left +/// with its own error. +pub(crate) fn patched_build_refusal<'a>( + denies_all: bool, + package: &str, + box_name: &str, + failure: &str, + lines: impl IntoIterator, +) -> Option { + let lines: Vec<&str> = lines.into_iter().collect(); + let destination = refused_destination(lines.iter().copied())?; + let host = destination + .rsplit_once(':') + .filter(|(h, p)| !h.is_empty() && p.chars().all(|c| c.is_ascii_digit())) + .map_or(destination.as_str(), |(h, _)| h); + if !failure.contains(host) { + return None; + } + egress_refusal(denies_all, package, box_name, lines) +} + +/// The error line for a package build that the egress rules of its box +/// refused (NET-155), or `None` when the refusal is not the rules'. +/// `denies_all` is [`build_denies_all`]'s answer for the build. A box whose +/// rules admit no destination is refused every one, so any destination +/// outside the box zone that its build could not reach was refused by the +/// box's rules. Any other box's failures are left as the build reported them. +pub(crate) fn egress_refusal<'a>( + denies_all: bool, + package: &str, + box_name: &str, + lines: impl IntoIterator, +) -> Option { + if !denies_all { + return None; + } + let destination = refused_destination(lines)?; + // Box-zone names are not subject to egress rules, so a failure to + // resolve or reach one is not a refusal by the box's rules. + let host = destination + .split(':') + .next() + .unwrap_or_default() + .to_ascii_lowercase(); + let zone_apex = sessions::core::zone_answer::ZONE_APEX; + if host == zone_apex || host.ends_with(&format!(".{zone_apex}")) { + return None; + } + Some(format!( + "the build of {package} could not reach {destination}: \ + the egress rules of box {box_name} do not allow it" + )) +} + #[cfg(test)] mod tests { use super::*; @@ -2511,6 +2851,7 @@ exit $rc .unwrap(), has_packages: HashSet::new(), box_id: "a session in a test".to_string(), + classifier_leaf: None, ot: None, session, runtime_env: RuntimeEnv::default(), diff --git a/crates/minimald/src/exec.rs b/crates/minimald/src/exec.rs index 5a519ef3cf..6fb8130758 100644 --- a/crates/minimald/src/exec.rs +++ b/crates/minimald/src/exec.rs @@ -253,7 +253,7 @@ pub(crate) fn task_network( /// and a host that cannot be asked, read as no row: fail-closed. A box the /// host registered nothing for (a native host, a `host_ip` or `none` box) /// asks nothing. -async fn ensure_task_rows_stand( +pub(crate) async fn ensure_task_rows_stand( record: &sessions::Record, switch: &tokio::sync::Mutex, ) -> io::Result<()> { @@ -339,6 +339,29 @@ fn parse_task_args( .map_err(|e| e.to_string()) } +/// Refuses the package builds of a task (NET-155) when the box's launch +/// placed it in a classifier leaf (`enforcement` is +/// [`sessions::HostIpEnforcement::PerBox`]) and the task's context has no leaf. +/// Such a build would run outside the box's egress verdict. A box that was +/// never placed builds in the daemon's own cgroup, as before. +pub(crate) fn task_build_refusal( + enforcement: Option, + box_name: &str, + has_leaf: bool, +) -> io::Result<()> { + if enforcement == Some(sessions::HostIpEnforcement::PerBox) && !has_leaf { + tracing::warn!( + box_name = %box_name, + "refusing a task's package builds for a placed box whose classifier leaf is missing", + ); + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + crate::session::placed_box_build_refusal(box_name), + )); + } + Ok(()) +} + /// Producer side of [`TaskExec::exec`]. Inlined in one async fn so that /// `env` lives on a single stack frame across every spawn — that frame /// is alive for the whole life of this tokio task, which in turn is @@ -378,6 +401,20 @@ async fn task_producer( return Ok(()); } + // NET-155: a task's package builds run in the box's leaf. A box + // whose launch placed it in a leaf never builds outside it, so a + // placed box whose leaf is gone has its task refused here. The + // context lookup does not refuse, because a relaunch uses it too. + { + let record = session.record().await?; + let box_name = record.name.clone().unwrap_or_else(|| record.id.to_string()); + task_build_refusal( + record.host_ip_enforcement, + &box_name, + ctx.classifier_leaf().is_some(), + )?; + } + // `graph_from_all_packages` is CPU-heavy (nickel evaluation, // graph construction) — run it on the blocking pool so it // doesn't stall the async executor. diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index 02ba2cbc87..1dd32fcac7 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -308,6 +308,135 @@ pub(crate) fn effective_egress_section( } } +/// The classifier leaf a package build for the box `box_id` runs in +/// (NET-155): the leaf the box's launch placed it in, under `root`. The +/// launch places a box in exactly one subtree, so the leaf is the one of the +/// two that exists. +/// +/// `placed` is whether the box's launch recorded a leaf of its own +/// ([`sessions::HostIpEnforcement::PerBox`]). A box that was never placed has no +/// leaf by design (it is not host-address, or the host has no classifier +/// tree), and its builds run in the daemon's own cgroup, as before. A placed +/// box whose leaf is gone fails closed: the build is refused, never run +/// outside the box's egress verdict. The `Err` is the refusal for the person +/// who asked for the build. +pub(crate) fn box_build_leaf( + root: &std::path::Path, + box_id: &str, + box_name: &str, + placed: bool, +) -> Result, String> { + let leaf = [ + sandbox2::config::Verdict::Deny, + sandbox2::config::Verdict::Allow, + ] + .into_iter() + .map(|verdict| sandbox2::config::ClassifierLeaf::under(root, box_id, verdict)) + .find(|leaf| leaf.dir().is_dir()); + match leaf { + Some(leaf) => Ok(Some(leaf)), + None if placed => Err(placed_box_build_refusal(box_name)), + None => Ok(None), + } +} + +/// The network the package builds of the box `record` attach to, when the +/// box has a network namespace of its own and so no classifier leaf: an +/// own-address box (NET-155). Each build gets the network a task of the box +/// gets ([`crate::exec::task_network`]): a task address of the box's own, +/// under the box's egress rules. So the box's egress verdict decides every +/// connection the build makes, and the build is never the daemon's own +/// traffic. `None` for every other box. +pub(crate) fn own_address_build_network( + record: &sessions::Record, + switch: &Arc>, + deny_all_opt_out: bool, +) -> Option { + if record.network != sessions::NetworkMode::OwnIp { + return None; + } + let egress = effective_egress_section( + &record.policy, + record.network, + sessions::EGRESS_DEFAULT_PHASE, + deny_all_opt_out, + ); + let denies_all = + crate::net::classifier::verdict_of(egress.as_ref()) == sandbox2::config::Verdict::Deny; + let record = record.clone(); + let switch = Arc::clone(switch); + Some( + op::BuildNetwork::new(move || { + Arc::new(OwnAddressBuildNetwork { + inner: crate::exec::task_network( + &record, + &switch, + sessions::EGRESS_DEFAULT_PHASE, + deny_all_opt_out, + ), + record: record.clone(), + switch: Arc::clone(&switch), + }) as Arc + }) + .with_denies_all(denies_all), + ) +} + +/// The network of one package build of an own-address box (NET-155). Before +/// each plan it asks, as a task run does, whether the box's host-side row +/// still stands, so the build never attaches under another box's row. +struct OwnAddressBuildNetwork { + record: sessions::Record, + switch: Arc>, + inner: Arc, +} + +impl std::fmt::Debug for OwnAddressBuildNetwork { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OwnAddressBuildNetwork") + .field("box_id", &self.record.id) + .finish_non_exhaustive() + } +} + +impl sandbox2::Network for OwnAddressBuildNetwork { + fn plan(&self) -> sandbox2::PlanFuture<'_> { + Box::pin(async move { + crate::exec::ensure_task_rows_stand(&self.record, &self.switch) + .await + .map_err(sandbox2::NetworkError::new)?; + self.inner.plan().await + }) + } + + fn attach(&self, spawned: sandbox2::Spawned) -> sandbox2::AttachFuture<'_> { + self.inner.attach(spawned) + } + + fn abandon(&self) -> sandbox2::AbandonFuture<'_> { + self.inner.abandon() + } +} + +/// The refusal for a build of the box `box_name` when its launch placed it +/// in a classifier leaf and that leaf is gone (NET-155). Every refusal of +/// this kind uses this text: [`box_build_leaf`] and the task path +/// (`exec::task_build_refusal`). +/// +/// Only an Active box reaches this refusal: `context()` refuses a stopped or +/// exited box first. An Active placed box loses its leaf when its shell ends +/// (the host's teardown removes the leaf). An attach with no running shell +/// mints a new one, and that launch places the box in a leaf again, so the +/// attach is the next step. +pub(crate) fn placed_box_build_refusal(box_name: &str) -> String { + format!( + "the build for box {box_name} did not start: box {box_name} runs under \ + egress rules, and its shell is not running to apply them to the build. \ + Run `min session attach {box_name}` to start its shell, then run the \ + build again." + ) +} + /// The policy the gate enforces for a session: its declared ingress, and its /// egress resolved to the section [`effective_egress_section`] names — the /// deny-all section for an own-address box with no `egress` section once the @@ -3020,6 +3149,39 @@ impl Session { } } + /// Where this box's package builds run (NET-155): the box's own + /// classifier leaf, so its egress verdict decides what the builds can + /// reach. A box whose launch placed it in a leaf never builds outside it, + /// so a missing leaf refuses the build. + async fn build_placement(&self) -> Result { + let record = self.record.record().await?; + let box_id = record.id.to_string(); + let box_name = record.name.clone().unwrap_or_else(|| box_id.clone()); + let placed = record.host_ip_enforcement == Some(sessions::HostIpEnforcement::PerBox); + let leaf = box_build_leaf( + std::path::Path::new(sandbox2::classifier::TREE_ROOT), + &box_id, + &box_name, + placed, + ) + .map_err(|refusal| { + tracing::warn!( + box_name = %box_name, + "refusing a package build for a placed box whose classifier leaf is missing", + ); + std::io::Error::new(std::io::ErrorKind::PermissionDenied, refusal) + })?; + let network = match leaf { + Some(_) => None, + None => own_address_build_network(&record, &self.net_switch, self.deny_all_opt_out), + }; + Ok(crate::session_sop::BuildPlacement { + box_name, + leaf, + network, + }) + } + /// Kicks off a background package build as a side-op and registers it on /// this session, returning the receiver end of its event stream. The build /// runs against a fresh workspace-rooted context (rebuilt per call so it @@ -3032,7 +3194,9 @@ impl Session { pkgs: Vec, ) -> Result, std::io::Error> { let ctx = self.context(false).await.map_err(std::io::Error::other)?; - let (sop, rx) = SideOp::spawn_build(self.weak_self.clone(), rebuild, pkgs, ctx, 64).await?; + let placement = self.build_placement().await?; + let (sop, rx) = + SideOp::spawn_build(self.weak_self.clone(), rebuild, pkgs, ctx, placement, 64).await?; match &mut self.inner { SessionInner::Active { sops, .. } => sops.push(sop), SessionInner::Draft { .. } => { @@ -3055,6 +3219,17 @@ impl Session { opts: CheckOpts, ) -> Result, std::io::Error> { let ctx = self.context(false).await.map_err(std::io::Error::other)?; + // NET-155: a check's package builds follow the box's egress verdict. + // A placed box whose leaf is gone is refused, the same as a task, so + // its builds never run in the daemon's own cgroup. No relaunch asks + // for a check, so the refusal cannot block a launch. + let record = self.record.record().await?; + let box_name = record.name.clone().unwrap_or_else(|| record.id.to_string()); + crate::exec::task_build_refusal( + record.host_ip_enforcement, + &box_name, + ctx.classifier_leaf().is_some(), + )?; let (sop, rx) = SideOp::spawn_check(self.weak_self.clone(), opts, ctx, 64).await?; match &mut self.inner { SessionInner::Active { sops, .. } => sops.push(sop), @@ -3076,7 +3251,39 @@ impl Session { opts: MaterializeOpts, ) -> Result, std::io::Error> { let ctx = self.context(false).await.map_err(std::io::Error::other)?; - let (sop, rx) = SideOp::spawn_materialize(self.weak_self.clone(), opts, ctx, 64).await?; + // NET-155: the package builds a materialize starts run in the box's + // leaf, the same as a build side-op's. + let placement = self.build_placement().await?; + let ctx = match (placement.leaf.clone(), placement.network.clone()) { + (Some(leaf), _) => { + tracing::info!( + box_name = %placement.box_name, + leaf = %leaf.dir().display(), + "starting a materialize for a box in the box's classifier leaf", + ); + ctx.with_classifier_leaf(leaf) + } + (None, Some(network)) => { + tracing::info!( + box_name = %placement.box_name, + leaf = "none", + "starting a materialize for a box on the box's own network, \ + under the box's egress rules", + ); + ctx.with_box_network(network) + } + (None, None) => { + tracing::info!( + box_name = %placement.box_name, + leaf = "none", + "starting a materialize for a box in the daemon's own cgroup, with no \ + per-box egress enforcement, because the box has none", + ); + ctx + } + }; + let (sop, rx) = + SideOp::spawn_materialize(self.weak_self.clone(), opts, ctx, placement, 64).await?; match &mut self.inner { SessionInner::Active { sops, .. } => sops.push(sop), SessionInner::Draft { .. } => { @@ -5968,7 +6175,28 @@ impl Session { } let ctx = self.build_context(scaffold_if_missing).await?; - Ok(ctx) + // NET-155: every context the box hands out places the package builds + // it starts in the box's leaf while the box has one, so a task run, + // a check or a launch's builds follow the box's egress verdict. The + // lookup does not refuse here: a relaunch asks for this context + // before its launch creates the leaf again, and the launch then + // places the env's builds in that leaf itself (`Env::build`). The + // build and materialize side operations refuse a placed box whose + // leaf is gone (`build_placement`). + let box_id = record.id.to_string(); + let box_name = record.name.clone().unwrap_or_else(|| box_id.clone()); + let leaf = box_build_leaf( + std::path::Path::new(sandbox2::classifier::TREE_ROOT), + &box_id, + &box_name, + false, + )?; + // An own-address box has no leaf: `build_context` gave its context + // the box's own network instead. + Ok(match leaf { + Some(leaf) => ctx.with_classifier_leaf(leaf), + None => ctx, + }) } /// The mctx [`Config`] rooted at this session's workspace, shared by the @@ -6070,7 +6298,22 @@ impl Session { _ => scaffold(), }?; } - mctx::Context::new(self.workspace_config(&wsp)?).map_err(|e| e.to_string()) + let ctx = mctx::Context::new(self.workspace_config(&wsp)?).map_err(|e| e.to_string())?; + // NET-155: an own-address box has no classifier leaf. Every package + // build its context starts, the activation launch's among them, + // attaches to a network of the box's own, under the box's egress + // rules. + let record = self + .record + .record() + .await + .map_err(|e| format!("reading session record: {e}"))?; + Ok( + match own_address_build_network(&record, &self.net_switch, self.deny_all_opt_out) { + Some(network) => ctx.with_box_network(network), + None => ctx, + }, + ) } /// Refuse a composition that names a package the session's graph does diff --git a/crates/minimald/src/session/tests.rs b/crates/minimald/src/session/tests.rs index 5814a7bfb8..83d783b597 100644 --- a/crates/minimald/src/session/tests.rs +++ b/crates/minimald/src/session/tests.rs @@ -12996,3 +12996,694 @@ async fn a_hook_launched_shell_of_a_registered_box_is_kept_for_a_terminal() { attach-env; got: {published:?}" ); } +/// The events of one package build named `package`, whose output ends in +/// `lines`, as the build side-op's pump sees them (NET-155). +fn build_events(package: &str, lines: &[&str]) -> Vec { + let start = orchestrator::BuildEvent { + idx: 7, + inner: orchestrator::BuildEventInner::Start { + name: package.to_string(), + full_build: true, + spec_hash: "0".repeat(64), + }, + }; + std::iter::once(start) + .chain(lines.iter().map(|line| orchestrator::BuildEvent { + idx: 7, + inner: orchestrator::BuildEventInner::Log { + is_stderr: true, + line: (*line).to_string(), + }, + })) + .collect() +} + +/// NET-155: a box's package build runs in the box's own classifier leaf, +/// so the box's egress verdict decides it. A deny-all box's build that +/// cannot reach its download is reported as refused by the box's rules. A +/// box allowed the destination is placed in its allow leaf, and its build +/// is never reported as refused. A box never placed in a leaf builds in the +/// daemon's own cgroup, as before. A placed box whose leaf is gone has its +/// build refused, never run outside its egress verdict. +#[test] +fn box_package_build_under_box_egress_verdict() { + use crate::session_sop::{BuildPlacement, BuildTap}; + use sandbox2::config::Verdict; + + let tree = tempfile::tempdir().unwrap(); + let root = tree.path(); + // A box never placed in a leaf builds in the daemon's own cgroup. + assert_eq!( + super::box_build_leaf(root, "sealed", "sealed", false), + Ok(None) + ); + // A placed box whose leaf is gone fails closed: its build is refused. + assert_eq!( + super::box_build_leaf(root, "sealed", "sealed", true), + Err( + "the build for box sealed did not start: box sealed runs under \ + egress rules, and its shell is not running to apply them to the \ + build. Run `min session attach sealed` to start its shell, then \ + run the build again." + .to_string() + ) + ); + + let deny_dir = sandbox2::classifier::box_leaf(root, "sealed", Verdict::Deny); + std::fs::create_dir_all(&deny_dir).unwrap(); + let allow_dir = sandbox2::classifier::box_leaf(root, "open", Verdict::Allow); + std::fs::create_dir_all(&allow_dir).unwrap(); + + let sealed = super::box_build_leaf(root, "sealed", "sealed", true) + .unwrap() + .expect("the deny-all box's leaf"); + assert_eq!(sealed.dir(), deny_dir); + assert!(sealed.is_deny()); + let open = super::box_build_leaf(root, "open", "open", true) + .unwrap() + .expect("the allowed box's leaf"); + assert_eq!(open.dir(), allow_dir); + assert!(!open.is_deny()); + + let download = [ + "curl: (6) Could not resolve host: downloads.example.org", + "build command failed", + ]; + // The error the build returns, as the daemon receives it. + let failed = |package: &str| { + mctx::Error::from(orchestrator::Error::BuildFailed( + package.to_string(), + op::Error::Other(anyhow::anyhow!("build command failed")), + )) + .to_string() + }; + let run = |name: &str, leaf| { + let placement = BuildPlacement { + box_name: name.to_string(), + leaf, + network: None, + }; + let mut tap = BuildTap::default(); + for event in build_events("fetcher", &download) { + tap.observe(&event, &placement); + } + tap.refusal(&placement, &failed("fetcher")) + }; + assert_eq!( + run("sealed", Some(sealed.clone())).as_deref(), + Some( + "the build of fetcher could not reach downloads.example.org: \ + the egress rules of box sealed do not allow it" + ) + ); + assert_eq!(run("open", Some(open)), None); + assert_eq!(run("unplaced", None), None); + + // Only the build that failed is read. A package that printed a failed + // download and still built is not blamed when another package fails. + let placement = BuildPlacement { + box_name: "sealed".to_string(), + leaf: Some(sealed), + network: None, + }; + let mut tap = BuildTap::default(); + let other = build_events("compiler", &["error: cc failed"]) + .into_iter() + .map(|mut event| { + event.idx = 8; + event + }); + for event in build_events("fetcher", &download).into_iter().chain(other) { + tap.observe(&event, &placement); + } + assert_eq!(tap.refusal(&placement, &failed("compiler")), None); + assert!(tap.refusal(&placement, &failed("fetcher")).is_some()); + // An error that names no build reports no refusal. + assert_eq!(tap.refusal(&placement, "the build was cancelled"), None); +} + +/// NET-155: a materialize reports the error of a package build it started +/// the way the build side operation does. A build in a deny leaf whose +/// output names a refused destination fails with the refusal, which names +/// the destination and the box, before the build's own error. +#[test] +fn box_package_build_refusal_names_destination_and_box_for_materialize() { + use crate::session_sop::{BuildPlacement, BuildTap}; + use sandbox2::config::{ClassifierLeaf, Verdict}; + + let root = std::path::Path::new("/sys/fs/cgroup/minimald.slice"); + let placement = BuildPlacement { + box_name: "sealed".to_string(), + leaf: Some(ClassifierLeaf::under(root, "sealed", Verdict::Deny)), + network: None, + }; + let mut tap = BuildTap::default(); + for event in build_events( + "fetcher", + &[ + "curl: (6) Could not resolve host: downloads.example.org", + "build command failed", + ], + ) { + tap.observe(&event, &placement); + } + let failed = mctx::Error::from(orchestrator::Error::BuildFailed( + "fetcher".to_string(), + op::Error::Other(anyhow::anyhow!("build command failed")), + )) + .to_string(); + assert_eq!( + tap.failure(&placement, failed.clone()), + format!( + "the build of fetcher could not reach downloads.example.org: \ + the egress rules of box sealed do not allow it\n{failed}" + ) + ); +} + +/// NET-155: a materialize build in a deny leaf that fails for a reason other +/// than the box's egress rules keeps the build's own error, unchanged. +#[test] +fn box_package_build_refusal_names_destination_and_box_for_materialize_only_when_refused() { + use crate::session_sop::{BuildPlacement, BuildTap}; + use sandbox2::config::{ClassifierLeaf, Verdict}; + + let root = std::path::Path::new("/sys/fs/cgroup/minimald.slice"); + let placement = BuildPlacement { + box_name: "sealed".to_string(), + leaf: Some(ClassifierLeaf::under(root, "sealed", Verdict::Deny)), + network: None, + }; + let mut tap = BuildTap::default(); + for event in build_events("compiler", &["cc -c hello.c", "error: cc failed"]) { + tap.observe(&event, &placement); + } + let failed = mctx::Error::from(orchestrator::Error::BuildFailed( + "compiler".to_string(), + op::Error::Other(anyhow::anyhow!("build command failed")), + )) + .to_string(); + assert_eq!(tap.failure(&placement, failed.clone()), failed); +} + +/// NET-155: the error of a build the box's egress rules refused names the +/// destination the build could not reach and the box whose rules refused +/// it, for the failure lines of `curl`, `wget` and `git`. +#[test] +fn box_package_build_refusal_names_destination_and_box() { + use crate::env::{build_denies_all, egress_refusal, refused_destination}; + use sandbox2::config::{ClassifierLeaf, Verdict}; + + let root = std::path::Path::new("/sys/fs/cgroup/minimald.slice"); + let deny = ClassifierLeaf::under(root, "b1", Verdict::Deny); + let allow = ClassifierLeaf::under(root, "b1", Verdict::Allow); + + let cases: [(&str, &str); 6] = [ + ( + "curl: (7) Failed to connect to example.com port 443 after 2 ms: Couldn't connect to server", + "example.com:443", + ), + ( + "curl: (6) Could not resolve host: example.com", + "example.com", + ), + ( + "fatal: unable to access 'https://github.com/o/r/': Could not resolve host: github.com", + "github.com", + ), + ( + "Resolving example.com (example.com)... failed: Temporary failure in name resolution.", + "example.com", + ), + ( + "Connecting to example.com (example.com)|93.184.215.14|:443... failed: No route to host.", + "example.com:443", + ), + ( + "Connecting to 93.184.215.14:80... failed: No route to host.", + "93.184.215.14:80", + ), + ]; + for (line, destination) in cases { + assert_eq!( + refused_destination(["make: entering", line]).as_deref(), + Some(destination), + "{line}" + ); + assert_eq!( + egress_refusal( + build_denies_all(Some(&deny), None), + "hello", + "my-box", + [line] + ), + Some(format!( + "the build of hello could not reach {destination}: \ + the egress rules of box my-box do not allow it" + )), + ); + // An allowed box's failure is the destination's, not the rules'. + assert_eq!( + egress_refusal( + build_denies_all(Some(&allow), None), + "hello", + "my-box", + [line] + ), + None + ); + assert_eq!( + egress_refusal(build_denies_all(None, None), "hello", "my-box", [line]), + None + ); + } + // An earlier tolerated failure does not hide the one that ended the build. + assert_eq!( + refused_destination([ + "curl: (6) Could not resolve host: optional.example", + "make: entering", + "curl: (7) Failed to connect to example.com port 443 after 2 ms: Couldn't connect to server", + ]) + .as_deref(), + Some("example.com:443"), + ); + // A failure that names no destination is left as the build said it. + assert_eq!( + egress_refusal( + build_denies_all(Some(&deny), None), + "hello", + "my-box", + ["error: cc failed"] + ), + None + ); + // Box-zone names are not subject to egress rules: a deny box that cannot + // resolve or reach one was not refused by its rules. + for line in [ + "curl: (6) Could not resolve host: web.min.internal", + "curl: (7) Failed to connect to host.min.internal port 8080 after 1 ms: Couldn't connect to server", + "curl: (6) Could not resolve host: min.internal", + ] { + assert_eq!( + egress_refusal( + build_denies_all(Some(&deny), None), + "hello", + "my-box", + [line] + ), + None, + "{line}" + ); + } + // A name that only ends in the apex text is not in the zone. + assert!( + egress_refusal( + build_denies_all(Some(&deny), None), + "hello", + "my-box", + ["curl: (6) Could not resolve host: evilmin.internal"] + ) + .is_some() + ); +} + +/// NET-155: the in-box patched build reports a refusal only when the +/// build's own error carries the refused destination. A deny box whose build +/// tolerated a failed download and then failed to compile keeps the compile +/// error. +#[test] +fn box_package_build_refusal_names_destination_and_box_only_for_the_error() { + use crate::env::{build_denies_all, patched_build_refusal}; + use sandbox2::config::{ClassifierLeaf, Verdict}; + + let root = std::path::Path::new("/sys/fs/cgroup/minimald.slice"); + let deny = ClassifierLeaf::under(root, "b1", Verdict::Deny); + let denies_all = build_denies_all(Some(&deny), None); + let download = "curl: (7) Failed to connect to example.com port 443 after 2 ms: Couldn't connect to server"; + + // The download ended the build: its error carries the failed download. + let failure = format!("Invocation 0 failed with exit code 7: exit status\nstderr:\n{download}"); + assert_eq!( + patched_build_refusal( + denies_all, + "hello", + "my-box", + &failure, + ["make: entering", download] + ), + Some( + "the build of hello could not reach example.com:443: \ + the egress rules of box my-box do not allow it" + .to_string() + ) + ); + + // A compile failure after a tolerated download is the build's own error. + let failure = "Invocation 0 failed with exit code 1: exit status\nstderr:\nerror: cc failed"; + assert_eq!( + patched_build_refusal( + denies_all, + "hello", + "my-box", + failure, + [download, "cc -c hello.c", "error: cc failed"] + ), + None + ); +} + +/// NET-155: the build output tap ends a line at `\r` as well as `\n`, holds +/// a bounded part of an unfinished line, and keeps the last unfinished line +/// when it shuts down, so a final failure with no newline is still read. +#[tokio::test] +async fn box_package_build_refusal_names_destination_and_box_from_unfinished_output() { + use crate::env::{LineTap, refused_destination}; + use tokio::io::AsyncWriteExt; + + let lines = std::sync::Arc::new(std::sync::Mutex::new(std::collections::VecDeque::new())); + let mut tap = LineTap::new(tokio::io::sink(), lines.clone()); + for pct in 0..1000 { + tap.write_all(format!(" {pct:>3}% [=====> ] 1.2MB/s\r").as_bytes()) + .await + .unwrap(); + assert_eq!(tap.pending(), 0); + } + // A long run with no line end at all keeps only a bounded tail. + tap.write_all(&vec![b'#'; 64 * 1024]).await.unwrap(); + assert!(tap.pending() <= 4096, "held {} bytes", tap.pending()); + tap.write_all(b"\n").await.unwrap(); + tap.write_all(b"curl: (7) Failed to connect to example.com port 443") + .await + .unwrap(); + tap.shutdown().await.unwrap(); + + let lines = lines.lock().unwrap(); + assert!(lines.len() <= 64); + assert_eq!( + refused_destination(lines.iter().map(String::as_str)).as_deref(), + Some("example.com:443") + ); +} + +/// NET-155: a task of a box whose launch placed it in a classifier leaf is +/// refused when the task's context has no leaf, with the same text the build +/// side operation gives. Its builds never run outside the box's egress verdict. +#[test] +fn box_package_build_under_box_egress_verdict_refuses_task_of_placed_box_without_leaf() { + let refusal = + crate::exec::task_build_refusal(Some(sessions::HostIpEnforcement::PerBox), "sealed", false) + .expect_err("a placed box with no leaf has its task refused"); + assert_eq!(refusal.kind(), std::io::ErrorKind::PermissionDenied); + assert_eq!( + refusal.to_string(), + "the build for box sealed did not start: box sealed runs under \ + egress rules, and its shell is not running to apply them to the \ + build. Run `min session attach sealed` to start its shell, then \ + run the build again." + ); + assert_eq!( + Err(refusal.to_string()), + super::box_build_leaf( + tempfile::tempdir().unwrap().path(), + "sealed", + "sealed", + true + ), + "the task refusal and the build refusal say the same thing" + ); + // A placed box with its leaf builds in that leaf. + crate::exec::task_build_refusal(Some(sessions::HostIpEnforcement::PerBox), "sealed", true) + .expect("a placed box with its leaf builds"); +} + +/// NET-155: a task of a box that was never placed in a leaf still builds, +/// in the daemon's own cgroup, as before. +#[test] +fn box_package_build_under_box_egress_verdict_builds_task_of_box_never_placed() { + for enforcement in [None, Some(sessions::HostIpEnforcement::None)] { + crate::exec::task_build_refusal(enforcement, "plain", false) + .expect("a box that was never placed builds without a leaf"); + } +} + +/// The handle of the box `id` on `server`. +async fn box_handle(server: &TestServer, id: SessionId) -> crate::session::SessionHandle { + server + .state + .sessions_manager() + .await + .get_session(crate::sessions::SessionKeyPredicate::Id(id)) + .await + .unwrap() + .expect("session should resolve") +} + +/// Rewrites the record of `id` on disk. +async fn rewrite_record( + server: &TestServer, + id: SessionId, + change: impl FnOnce(&mut sessions::Record), +) { + let sessions = server + .state + .minimal_state_dir() + .await + .as_utf8_path() + .join("sessions"); + for entry in std::fs::read_dir(&sessions).unwrap() { + let path = entry.unwrap().path().join("record.json"); + let Ok(bytes) = std::fs::read(&path) else { + continue; + }; + let mut record: sessions::Record = serde_json_lenient::from_slice(&bytes).unwrap(); + if record.id == id { + change(&mut record); + std::fs::write(&path, serde_json_lenient::to_vec(&record).unwrap()).unwrap(); + return; + } + } + panic!("no record for {id}"); +} + +/// The record of `id`, as the manager reads it. +async fn read_record(server: &TestServer, id: SessionId) -> sessions::Record { + server + .state + .sessions_manager() + .await + .get_record(crate::sessions::SessionKeyPredicate::Id(id)) + .await + .unwrap() + .expect("the box keeps its record") +} + +/// NET-155: `min check` on a box whose launch placed it in a classifier leaf +/// is refused when that leaf is gone, with the same text a task gives. Its +/// package builds never run in the daemon's own cgroup. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn box_package_build_under_box_egress_verdict_refuses_check_of_placed_box_without_leaf() { + let server = TestServer::new().await; + let mut client = server.connect().await; + let id = create_configured_session(&mut client, "check-sealed", "/uwu").await; + rewrite_record(&server, id, |record| { + record.host_ip_enforcement = Some(sessions::HostIpEnforcement::PerBox); + }) + .await; + + let refusal = box_handle(&server, id) + .await + .start_check(crate::session_sop::CheckOpts::from_args("").unwrap()) + .await + .expect_err("a placed box with no leaf has its check refused"); + assert_eq!(refusal.kind(), std::io::ErrorKind::PermissionDenied); + assert_eq!( + refusal.to_string(), + super::placed_box_build_refusal("check-sealed") + ); +} + +/// NET-155: `min check` on a box that was never placed in a leaf still +/// starts, and its builds run in the daemon's own cgroup, as before. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn box_package_build_under_box_egress_verdict_starts_check_of_box_never_placed() { + let server = TestServer::new().await; + let mut client = server.connect().await; + let id = create_configured_session(&mut client, "check-plain", "/uwu").await; + assert_eq!(read_record(&server, id).await.host_ip_enforcement, None); + + box_handle(&server, id) + .await + .start_check(crate::session_sop::CheckOpts::from_args("").unwrap()) + .await + .expect("a box that was never placed starts a check"); +} + +/// A box record for the NET-155 tests, in `mode` and under `egress`. +fn build_record( + mode: sessions::NetworkMode, + egress: Option, +) -> sessions::Record { + sessions::Record { + id: sessions::SessionId::nil(), + name: Some("sealed".to_string()), + username: None, + project_path: paths::HostAbsPath::try_new("/tmp/project").unwrap(), + network: mode, + policy: sessions::SessionPolicy::new(egress, None), + task_addresses: Vec::new(), + box_addresses: None, + status: sessions::SessionStatus::Active, + hooks_enabled: true, + host_ip_enforcement: None, + box_id: None, + host_row_bound: false, + bound_ports: std::collections::BTreeSet::new(), + attrs: Default::default(), + } +} + +/// A switch on a VM host, which a test never reaches. +fn build_switch() -> std::sync::Arc> { + std::sync::Arc::new(tokio::sync::Mutex::new( + crate::net::SwitchClient::new("/usr/bin/gvproxy", "/run/minimal/gvproxy").with_transport( + crate::net::SwitchTransport::HostShuttle { + cid: crate::net::VSOCK_HOST_CID, + port: crate::net::VSOCK_GVPROXY_SHUTTLE_PORT, + }, + ), + )) +} + +/// NET-155: an own-address box has no classifier leaf. Its package build +/// runs on the box's own network, under the box's egress rules, and never +/// as the daemon's own traffic. A deny-all box's build that cannot reach its +/// download is reported as refused by the box's rules. A box allowed the +/// destination is never reported as refused. Before each build attaches, +/// the network asks whether the box's host-side row stands, as a task run +/// does, so a build never attaches under another box's row. +#[tokio::test] +async fn box_package_build_under_box_egress_verdict_runs_own_address_box_on_its_network() { + use crate::session_sop::{BuildPlacement, BuildTap}; + let switch = build_switch(); + + let sealed = build_record( + sessions::NetworkMode::OwnIp, + Some(sessions::EgressPolicy::deny_all()), + ); + let network = super::own_address_build_network(&sealed, &switch, false) + .expect("an own-address box builds on its own network"); + assert!(network.denies_all()); + assert!(format!("{:?}", network.network()).contains("OwnAddressBuildNetwork")); + // With no `egress` section, the deny-all default applies to the build. + let defaulted = build_record(sessions::NetworkMode::OwnIp, None); + assert!( + super::own_address_build_network(&defaulted, &switch, false) + .unwrap() + .denies_all() + ); + + let open = build_record( + sessions::NetworkMode::OwnIp, + Some(sessions::EgressPolicy { + allow_subnets: Some(Vec::new()), + allow_dns_hosts: Some(vec!["downloads.example.org".to_string()]), + allow_protocols: None, + deny_subnets: None, + }), + ); + let open_network = super::own_address_build_network(&open, &switch, false) + .expect("an allowed own-address box builds on its own network"); + assert!(!open_network.denies_all()); + + let download = [ + "curl: (6) Could not resolve host: downloads.example.org", + "build command failed", + ]; + let failed = mctx::Error::from(orchestrator::Error::BuildFailed( + "fetcher".to_string(), + op::Error::Other(anyhow::anyhow!("build command failed")), + )) + .to_string(); + let run = |network: op::BuildNetwork| { + let placement = BuildPlacement { + box_name: "sealed".to_string(), + leaf: None, + network: Some(network), + }; + let mut tap = BuildTap::default(); + for event in build_events("fetcher", &download) { + tap.observe(&event, &placement); + } + tap.refusal(&placement, &failed) + }; + assert_eq!( + run(network.clone()).as_deref(), + Some( + "the build of fetcher could not reach downloads.example.org: \ + the egress rules of box sealed do not allow it" + ) + ); + assert_eq!(run(open_network), None); + + // A box the VM host registered, whose row the host cannot confirm, has + // its build's network refused before anything attaches. + let mut registered = sealed.clone(); + registered.box_addresses = Some(sessions::BoxAddresses { + switch_address: std::net::Ipv4Addr::new(192, 168, 127, 10), + loopback_address: std::net::Ipv4Addr::new(127, 77, 0, 10), + }); + let network = super::own_address_build_network(®istered, &switch, false).unwrap(); + let refused = network + .network() + .plan() + .await + .expect_err("a build of a box whose row is not confirmed does not attach"); + assert!( + refused.to_string().contains("could not confirm"), + "{refused}" + ); +} + +/// NET-155: a host-address box with no per-box enforcement has no leaf and +/// no network of its own. Its build runs in the daemon's own cgroup, because +/// the box has no enforcement, and no failure is reported as the box's rules. +#[test] +fn box_package_build_under_box_egress_verdict_builds_unenforced_box_without_rules() { + use crate::session_sop::{BuildPlacement, BuildTap}; + let switch = build_switch(); + let mut record = build_record( + sessions::NetworkMode::HostNet, + Some(sessions::EgressPolicy::deny_all()), + ); + record.host_ip_enforcement = Some(sessions::HostIpEnforcement::None); + assert!(super::own_address_build_network(&record, &switch, false).is_none()); + assert_eq!( + super::box_build_leaf( + tempfile::tempdir().unwrap().path(), + "sealed", + "sealed", + false + ), + Ok(None) + ); + let placement = BuildPlacement { + box_name: "sealed".to_string(), + leaf: None, + network: None, + }; + assert!(!placement.denies_all()); + let mut tap = BuildTap::default(); + for event in build_events( + "fetcher", + &["curl: (6) Could not resolve host: downloads.example.org"], + ) { + tap.observe(&event, &placement); + } + assert_eq!( + tap.refusal( + &placement, + "`fetcher` failed to build: build command failed" + ), + None + ); +} diff --git a/crates/minimald/src/session_sop.rs b/crates/minimald/src/session_sop.rs index bdd719a39e..c185e69e1c 100644 --- a/crates/minimald/src/session_sop.rs +++ b/crates/minimald/src/session_sop.rs @@ -392,6 +392,118 @@ enum Op { }, } +/// Where a box's package builds run (NET-155): the box they run for, and +/// its classifier leaf or its own network. A host-address box placed in a +/// leaf builds in that leaf. An own-address box has no leaf, and its builds +/// attach to a network of its own. A box with neither has no per-box egress +/// enforcement, and its builds run in the daemon's own cgroup. +#[derive(Debug, Clone)] +pub(crate) struct BuildPlacement { + /// The box's name, as `min ls` shows it. + pub box_name: String, + /// The box's classifier leaf, if its launch placed it in one. + pub leaf: Option, + /// The own-address box's network, which its builds attach to. + pub network: Option, +} + +impl BuildPlacement { + /// Whether the box's egress rules admit no destination, so a build + /// that could not reach one was refused by the rules. + pub(crate) fn denies_all(&self) -> bool { + crate::env::build_denies_all(self.leaf.as_ref(), self.network.as_ref()) + } +} + +/// How many output lines of one package build [`BuildTap`] keeps. +const TAP_LINES: usize = 64; + +/// Watches the events of a box's build side-op (NET-155). It logs each +/// package build as it starts, and it keeps the last lines of each build's +/// output, so a build the box's egress rules refused can name what it could +/// not reach. +#[derive(Debug, Default)] +pub(crate) struct BuildTap { + builds: std::collections::HashMap)>, +} + +impl BuildTap { + /// Records one build event. + pub(crate) fn observe(&mut self, event: &BuildEvent, placement: &BuildPlacement) { + match &event.inner { + orchestrator::BuildEventInner::Start { name, .. } => { + match &placement.leaf { + Some(leaf) => tracing::info!( + box_name = %placement.box_name, + package = %name, + leaf = %leaf.dir().display(), + "starting a package build for a box in the box's classifier leaf", + ), + None if placement.network.is_some() => tracing::info!( + box_name = %placement.box_name, + package = %name, + leaf = "none", + "starting a package build for a box on the box's own network, \ + under the box's egress rules", + ), + None => tracing::info!( + box_name = %placement.box_name, + package = %name, + leaf = "none", + "starting a package build for a box in the daemon's own cgroup, \ + with no per-box egress enforcement, because the box has none", + ), + } + self.builds + .insert(event.idx, (name.clone(), Default::default())); + } + orchestrator::BuildEventInner::Log { line, .. } => { + if let Some((_, lines)) = self.builds.get_mut(&event.idx) { + if lines.len() == TAP_LINES { + lines.pop_front(); + } + lines.push_back(line.clone()); + } + } + _ => {} + } + } + + /// The refusal to report, if the box's egress rules refused a + /// destination that the failed package build tried to reach. + /// + /// `failure` is the error the build returned. Only a build that the + /// error names as failed is read, so a package that printed a failed + /// download and still built is never blamed for another package's + /// failure. When the error names no build, nothing is reported. + pub(crate) fn refusal(&self, placement: &BuildPlacement, failure: &str) -> Option { + let mut builds: Vec<_> = self + .builds + .iter() + .filter(|(_, (package, _))| failure.contains(&format!("`{package}` failed to build"))) + .collect(); + builds.sort_by_key(|(idx, _)| **idx); + builds.into_iter().find_map(|(_, (package, lines))| { + crate::env::egress_refusal( + placement.denies_all(), + package, + &placement.box_name, + lines.iter().map(String::as_str), + ) + }) + } + + /// The error to report for a failed build. A build the box's egress + /// rules refused names the destination and the box before the build's + /// own error. Any other failure is the build's own error, unchanged. + pub(crate) fn failure(&self, placement: &BuildPlacement, failure: String) -> String { + match self.refusal(placement, &failure) { + Some(refusal) => format!("{refusal}\n{failure}"), + None => failure, + } + } +} + /// Session side-operation. #[derive(Debug)] #[allow(dead_code)] @@ -448,11 +560,19 @@ impl SideOp { rebuild: bool, pkgs: Vec, ctx: Context, + placement: BuildPlacement, buffer: usize, ) -> Result<(Self, mpsc::Receiver), std::io::Error> { let (tx, rx) = mpsc::channel(buffer); - let sop = - Self::new_build(session, rebuild, pkgs, ctx, vec![BuildSink::Structured(tx)]).await?; + let sop = Self::new_build( + session, + rebuild, + pkgs, + ctx, + placement, + vec![BuildSink::Structured(tx)], + ) + .await?; Ok((sop, rx)) } @@ -462,8 +582,15 @@ impl SideOp { rebuild: bool, pkgs: Vec, ctx: Context, + placement: BuildPlacement, sinks: Vec, ) -> Result { + // NET-155: every package build of this op runs in the box's leaf. + let ctx = match (&placement.leaf, &placement.network) { + (Some(leaf), _) => ctx.with_classifier_leaf(leaf.clone()), + (None, Some(network)) => ctx.with_box_network(network.clone()), + (None, None) => ctx, + }; // `graph_from_all_packages` is CPU-heavy, run on blocking pool. let (ctx, graph_result) = tokio::task::spawn_blocking(move || { let mut ctx = ctx; @@ -493,12 +620,15 @@ impl SideOp { // Pump task: drain the log channel until the build drops its // sender (on completion or cancellation). let pump = tokio::task::spawn(async move { + let mut tap = BuildTap::default(); while let Some(event) = log_rx.next().await { + tap.observe(&event, &placement); let inner = pump_inner.lock().await; for sink in &inner.sinks { let _ = sink.try_send(BuildUpdate::Event(event.clone())); } } + (tap, placement) }); // Do the build. Reduce its `!Send` error (`mctx::Error` holds @@ -514,7 +644,7 @@ impl SideOp { Some(e.to_string()) } }; - let _ = pump.await; + let tap = pump.await.ok(); // Classify the terminal outcome. A build that returned `Ok` is // a success even if the token was cancelled afterwards (e.g. a @@ -524,7 +654,12 @@ impl SideOp { let outcome = match build_err { None => BuildOutcome::Success, Some(_) if cancel_flag.is_cancelled() => BuildOutcome::Cancelled, - Some(msg) => BuildOutcome::Failed(msg), + // NET-155: a build the box's rules refused names the + // destination and the box before the build's own error. + Some(msg) => BuildOutcome::Failed(match &tap { + Some((tap, placement)) => tap.failure(placement, msg), + None => msg, + }), }; finish(&build_inner, BuildUpdate::Finished(outcome)).await; @@ -700,11 +835,18 @@ impl SideOp { session: WeakSessionHandle, opts: MaterializeOpts, ctx: Context, + placement: BuildPlacement, buffer: usize, ) -> Result<(Self, mpsc::Receiver), std::io::Error> { let (tx, rx) = mpsc::channel(buffer); - let sop = Self::new_materialize(session, opts, ctx, vec![MaterializeSink::Structured(tx)]) - .await?; + let sop = Self::new_materialize( + session, + opts, + ctx, + placement, + vec![MaterializeSink::Structured(tx)], + ) + .await?; Ok((sop, rx)) } @@ -712,11 +854,13 @@ impl SideOp { /// The output is resolved here, before the op spawns, so a bad name or arch /// errors to the caller rather than through a channel it has yet to be /// handed. The cancel token stops the package build only: assembly is not - /// interruptible. + /// interruptible. `placement` is the box's placement, which the caller + /// already applied to `ctx`: the run reads it to name a refusal. async fn new_materialize( session: WeakSessionHandle, opts: MaterializeOpts, ctx: Context, + placement: BuildPlacement, sinks: Vec, ) -> Result { let output = ctx @@ -763,8 +907,19 @@ impl SideOp { tokio::task::spawn(async move { let mut ctx = ctx; + // NET-155: tap the package builds' output, so a build the + // box's egress rules refused names the destination and the + // box, as the build side-op's does. + let (log_tx, mut log_rx) = futures::channel::mpsc::unbounded::(); + let tap = tokio::task::spawn(async move { + let mut tap = BuildTap::default(); + while let Some(event) = log_rx.next().await { + tap.observe(&event, &placement); + } + (tap, placement) + }); let build_err = match ctx - .build_graph_with_cancel(&graph, false, None, build_cancel) + .build_graph_with_cancel(&graph, false, Some(log_tx), build_cancel) .await { Ok(()) => None, @@ -773,10 +928,14 @@ impl SideOp { Some(e.to_string()) } }; + let tap = tap.await.ok(); if let Some(msg) = build_err { let outcome = match cancel_flag.is_cancelled() { true => MaterializeOutcome::Cancelled, - false => MaterializeOutcome::Failed(msg), + false => MaterializeOutcome::Failed(match &tap { + Some((tap, placement)) => tap.failure(placement, msg), + None => msg, + }), }; finish(&op_inner, MaterializeUpdate::Finished(outcome)).await; return; diff --git a/crates/minimald/tests/classifier_root_integration.rs b/crates/minimald/tests/classifier_root_integration.rs index 3444877614..d5be847bb5 100644 --- a/crates/minimald/tests/classifier_root_integration.rs +++ b/crates/minimald/tests/classifier_root_integration.rs @@ -6,7 +6,7 @@ //! before it is loaded, and a request through the proxy is answered before //! anything refused it. //! -//! Ten proofs, all `#[ignore]`d so the default suite and the surveyed +//! Eleven proofs, all `#[ignore]`d so the default suite and the surveyed //! nextest lines never run them. The native lane's //! `minimald-root-integration` job runs this binary, and its recipe //! (`just test-root-integration`) runs it unprivileged — the same posture @@ -32,7 +32,11 @@ //! own leaf, completes while the same deny-all box's own connect to the //! same server is refused, and that the fetch is recorded as the //! node-plane traffic it is, naming that box -//! (`daemon_fetch_survives_a_live_deny_leaf_over_a_loaded_table`) — and +//! (`daemon_fetch_survives_a_live_deny_leaf_over_a_loaded_table`) — one +//! proves NET-155: a package build placed in a box's deny leaf cannot +//! connect to a local peer, and the same build in an allow leaf connects +//! (`package_build_in_a_box_leaf_follows_its_verdict_over_a_loaded_table`) — +//! and //! four prove the classification itself, live on the kernel that loads //! it: that the installer's whole transaction — the classify chain at //! output's mangle priority, the mark-keyed postrouting — loads where @@ -50,7 +54,7 @@ //! well-known component's range (`install_refuses_a_well_known_mark_range`), //! and the table refuses an allow box's IPv6 connection to a peer //! (`host_ip_box_ipv6_egress_refused`). The identity proof drives an -//! eleventh `#[ignore]`d test as its observer — the half of +//! twelfth `#[ignore]`d test as its observer — the half of //! itself that runs re-exec'd inside the peer's namespace — which passes //! on its own, with nothing to observe. //! @@ -1322,6 +1326,285 @@ fn daemon_fetch_survives_a_live_deny_leaf_over_a_loaded_table() { ); } +/// The connect probe a package build runs (NET-155): a static program that +/// connects to `127.0.0.1:` and reports the result the way `curl` +/// reports a failed connect, so the line reads like a real download's. +const BUILD_CONNECT_PROBE_C: &str = r#" +#include +#include +#include +#include +#include +#include +#include +int main(int argc, char **argv) { + if (argc < 2) return 2; + int port = atoi(argv[1]); + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) { fprintf(stderr, "socket failed: errno %d\n", errno); return 2; } + struct sockaddr_in addr = {0}; + addr.sin_family = AF_INET; + addr.sin_port = htons((unsigned short)port); + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + if (connect(fd, (struct sockaddr *)&addr, sizeof addr) != 0) { + fprintf(stderr, "Failed to connect to 127.0.0.1 port %d: errno %d\n", port, errno); + return 1; + } + fprintf(stderr, "connected to 127.0.0.1 port %d\n", port); + close(fd); + return 0; +} +"#; + +/// Compiles [`BUILD_CONNECT_PROBE_C`] statically and lays it out as a +/// rootfs at `/rootfs-src`, with the probe at `/usr/bin/probe`. Fails +/// the proof when no C compiler is on `PATH`: this lane promised to run it. +fn build_probe_rootfs(base: &Path) -> PathBuf { + let src = base.join("build_connect_probe.c"); + let bin = base.join("build_connect_probe"); + std::fs::write(&src, BUILD_CONNECT_PROBE_C).expect("writing the build probe's source"); + let status = Command::new("gcc") + .args(["-static", "-o"]) + .arg(&bin) + .arg(&src) + .status() + .expect("running gcc to compile the build probe (install build-essential)"); + assert!( + status.success(), + "gcc failed to compile the build probe: {status:?}" + ); + let rootfs = base.join("rootfs-src"); + let usr_bin = rootfs.join("usr").join("bin"); + std::fs::create_dir_all(&usr_bin).expect("the rootfs usr/bin dir"); + std::fs::copy(&bin, usr_bin.join("probe")).expect("copying the probe into the rootfs"); + std::fs::create_dir_all(rootfs.join("usr").join("lib")).expect("the rootfs usr/lib dir"); + std::fs::create_dir_all(rootfs.join("etc")).expect("the rootfs etc dir"); + rootfs +} + +/// An async writer that keeps what a build prints, so the proof can read +/// the build's own failure line. +#[derive(Clone, Default)] +struct BuildOutput(std::sync::Arc>>); + +impl BuildOutput { + fn text(&self) -> String { + String::from_utf8_lossy(&self.0.lock().expect("the build output lock")).into_owned() + } +} + +impl tokio::io::AsyncWrite for BuildOutput { + fn poll_write( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + self.0 + .lock() + .expect("the build output lock") + .extend_from_slice(buf); + std::task::Poll::Ready(Ok(buf.len())) + } + + fn poll_flush( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn poll_shutdown( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } +} + +/// A source fetcher the probe build never calls: its spec has no sources. +#[derive(Debug)] +struct NoSources; + +impl op::SourceFetcher for NoSources { + async fn download_web( + &self, + url: &str, + _sha256: &str, + _op: &ot::OpTracker, + ) -> anyhow::Result { + anyhow::bail!("the probe build has no sources, but it asked for {url}") + } + + async fn download_gcs( + &self, + _bucket_id: String, + file: &str, + _sha256: &str, + _op: &ot::OpTracker, + ) -> anyhow::Result { + anyhow::bail!("the probe build has no sources, but it asked for {file}") + } +} + +/// Runs one package build of the connect probe in `leaf`, against the +/// peer on `port`. Returns whether the build succeeded and what it printed. +fn run_probe_build( + base: &Path, + rootfs: &Path, + leaf: sandbox2::config::ClassifierLeaf, + port: u16, +) -> (bool, String) { + let work = tempfile::tempdir_in(base).expect("a temp dir for the build"); + // The cache opens over a directory that exists: `Cache::at_dir` makes + // its subdirectories, not its base. + let cache_dir = work.path().join("cache"); + std::fs::create_dir_all(&cache_dir).expect("the build's cache dir"); + let cache = lcache::Cache::at_dir(&cache_dir).expect("the build's cache"); + let graph = graph::Graph::new() + .ingest( + decode::Layer::new_for_test(format!( + r#" + let {{BuildSpec, ..}} = import "minimal.ncl" in + {{ + name = "downloads-at-build", + build_deps = [], + cmd = ["/usr/bin/probe", "{port}"], + }} | BuildSpec + "# + )) + .expect("the probe build's spec"), + ) + .expect("the probe build's graph"); + let spec = *graph + .by_name("downloads-at-build") + .expect("the probe build in its graph"); + let output = BuildOutput::default(); + let mut build = op::SpecBuild { + spec: &spec, + remote_fetcher: &NoSources, + // The rootfs is the probe alone. A build with its dependencies + // given shares the host's network, as a build that downloads does. + override_deps: Some(std::iter::once(rootfs.to_path_buf()).collect()), + stdout_writer: Some(Box::new(output.clone())), + stderr_writer: Some(Box::new(output.clone())), + cancel: tokio_util::sync::CancellationToken::new(), + cpu_weight: None, + classifier_leaf: Some(leaf), + box_network: None, + }; + let opts = op::Options { + cache, + graph: &graph, + exec_base: work.path().join("exec"), + ot: None, + daemon_id: None, + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("a runtime for the build"); + let result = runtime.block_on(async { + use op::Runnable as _; + tokio::time::timeout(Duration::from_secs(60), build.run(&opts)).await + }); + let ok = match result { + Ok(Ok(_)) => true, + Ok(Err(e)) => { + eprintln!("the probe build failed: {e}"); + false + } + Err(_) => panic!("the probe build did not finish in 60 s: {}", output.text()), + }; + (ok, output.text()) +} + +/// NET-155, live: a package build placed for a box runs in that box's +/// classifier leaf. In a deny leaf, the build's own connect to a local peer +/// is refused by the loaded table. The same build in an allow leaf connects. +#[test] +#[ignore = "loads the installer's table and runs a real package build in a boxes/deny and a boxes/allow leaf; run by the native lane's minimald-root-integration job (just test-root-integration)"] +fn package_build_in_a_box_leaf_follows_its_verdict_over_a_loaded_table() { + if !nft_present() { + eprintln!( + "skipping package_build_in_a_box_leaf_follows_its_verdict_over_a_loaded_table: \ + no nft on this host — the install's nftables transaction is \ + the artifact under proof (apt install nftables)" + ); + return; + } + let _table = one_table_at_a_time(); + let mountpoint = refuse_unless_the_root_lane_can_run(); + let account = the_delegated_account(); + + let scratch = mountpoint.join(format!("minimald-proof-build-{}", std::process::id())); + let mut install = ScratchInstall { + root: scratch.clone(), + mountpoint: mountpoint.clone(), + leaf: None, + child: None, + }; + install_over_scratch(&scratch, &account); + // In the tree before the build starts: the build's sandbox is born in + // this process's cgroup, and its move into the box leaf is checked at + // the common ancestor of the two. + place_this_process_in_the_tree(&scratch); + let deny_dir = create_box_leaf(&scratch, "builddeny", Verdict::Deny) + .expect("the step's deny subtree takes a box leaf"); + install.leaf = Some(deny_dir.clone()); + let allow_dir = create_box_leaf(&scratch, "buildallow", Verdict::Allow) + .expect("the step's allow subtree takes a box leaf"); + // Dropped before `install`, so the allow leaf goes before the tree. + struct AllowLeaf(PathBuf); + impl Drop for AllowLeaf { + fn drop(&mut self) { + let _ = remove_box_leaf(&self.0); + } + } + let _allow_leaf = AllowLeaf(allow_dir.clone()); + let deny = sandbox2::config::ClassifierLeaf::under(&scratch, "builddeny", Verdict::Deny); + let allow = sandbox2::config::ClassifierLeaf::under(&scratch, "buildallow", Verdict::Allow); + assert_eq!( + deny.dir(), + deny_dir.as_path(), + "the deny leaf the build is given" + ); + assert_eq!( + allow.dir(), + allow_dir.as_path(), + "the allow leaf the build is given" + ); + + // The peer: one loopback listener, the one destination both builds + // reach for, so only the leaf differs between them. + let peer = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).expect("binding the peer's port"); + let port = peer.local_addr().expect("the peer's address").port(); + + let base = tempfile::tempdir().expect("a temp dir for the probe builds"); + let rootfs = build_probe_rootfs(base.path()); + + let (ok, printed) = run_probe_build(base.path(), &rootfs, deny, port); + assert!( + !ok, + "the build in the deny leaf fails, because it cannot reach the peer: {printed}" + ); + let refused = format!( + "Failed to connect to 127.0.0.1 port {port}: errno {}", + libc::EHOSTUNREACH + ); + assert!( + printed.contains(&refused), + "the deny-leaf build's connect is refused with EHOSTUNREACH, the \ + loaded chain's own rejection: {printed}" + ); + + let (ok, printed) = run_probe_build(base.path(), &rootfs, allow, port); + assert!( + ok && printed.contains(&format!("connected to 127.0.0.1 port {port}")), + "the same build in the allow leaf connects to the peer: {printed}" + ); + drop(peer); +} + // ── The classification's own proofs ───────────────────────────────────────── // // The three proofs above read the table's refusal; the four below read its diff --git a/crates/mip/src/cmd_pkg_patched_build.rs b/crates/mip/src/cmd_pkg_patched_build.rs index dad8851eb8..7568f2498b 100644 --- a/crates/mip/src/cmd_pkg_patched_build.rs +++ b/crates/mip/src/cmd_pkg_patched_build.rs @@ -40,6 +40,10 @@ pub async fn cmd_pkg_patched_build( stdout_writer: Some(Box::new(common::TracingWriter::stdout())), stderr_writer: Some(Box::new(common::TracingWriter::stderr())), remote_cache: remote_cache.as_ref(), + // NET-155: a command-line patched build runs for no box, so it + // stays the caller's own traffic. + classifier_leaf: None, + box_network: None, } .run(&op::Options { cache, diff --git a/crates/op/src/lib.rs b/crates/op/src/lib.rs index 5a4dcc405b..904fc019a5 100644 --- a/crates/op/src/lib.rs +++ b/crates/op/src/lib.rs @@ -48,7 +48,7 @@ pub use notices::{NoticeFile, SourceNotices}; mod sources; pub use sources::{SourceFetcher, SourceLoad}; mod specs; -pub use specs::{SpecBuild, SpecBuildResult}; +pub use specs::{BuildNetwork, SpecBuild, SpecBuildResult}; mod oci_image; diff --git a/crates/op/src/patched.rs b/crates/op/src/patched.rs index 73ef0d7a93..82ffdcfe36 100644 --- a/crates/op/src/patched.rs +++ b/crates/op/src/patched.rs @@ -34,6 +34,15 @@ pub struct PatchedBuild<'a, SF: crate::SourceFetcher> { /// The remote cache to consult before reporting a dependency missing. /// When `None`, dependency resolution stays purely local. pub remote_cache: Option<&'a RemoteCache>, + + /// The classifier leaf of the box this build runs for, if any (NET-155). + /// Carried to the spec build, so the box's egress verdict decides the + /// build's connections. `None` keeps the build as the caller's traffic. + pub classifier_leaf: Option, + + /// The network of the own-address box this build runs for, if any + /// (NET-155). Carried to the spec build, the same as the leaf. + pub box_network: Option, } impl<'a, SF: crate::SourceFetcher> Runnable for PatchedBuild<'a, SF> { @@ -146,6 +155,8 @@ impl<'a, SF: crate::SourceFetcher> Runnable for PatchedBuild<'a, SF> { stderr_writer: self.stderr_writer.take(), cancel: tokio_util::sync::CancellationToken::new(), cpu_weight: None, + classifier_leaf: self.classifier_leaf.clone(), + box_network: self.box_network.clone(), } .run(opts) .await?; @@ -226,6 +237,8 @@ mod tests { stdout_writer: None, stderr_writer: None, remote_cache: None, + classifier_leaf: None, + box_network: None, } } @@ -335,6 +348,8 @@ mod tests { stdout_writer: None, stderr_writer: None, remote_cache: Some(&remote_cache), + classifier_leaf: None, + box_network: None, }; let opts = Options { cache: cache.clone(), @@ -393,6 +408,8 @@ mod tests { stdout_writer: None, stderr_writer: None, remote_cache: Some(&remote_cache), + classifier_leaf: None, + box_network: None, }; let opts = Options { cache: cache.clone(), @@ -444,6 +461,8 @@ mod tests { stdout_writer: None, stderr_writer: None, remote_cache: Some(&remote_cache), + classifier_leaf: None, + box_network: None, }; let opts = Options { cache: cache.clone(), @@ -487,6 +506,8 @@ mod tests { stdout_writer: None, stderr_writer: None, remote_cache: Some(&remote_cache), + classifier_leaf: None, + box_network: None, }; let opts = Options { cache: cache.clone(), diff --git a/crates/op/src/specs.rs b/crates/op/src/specs.rs index b3a95634c3..fb11b36598 100644 --- a/crates/op/src/specs.rs +++ b/crates/op/src/specs.rs @@ -48,6 +48,67 @@ pub struct SpecBuild<'a, SF: crate::SourceFetcher> { /// CPU shares, for partitioning CPU when the system is contended. Maps roughly to /// cgroups v2 cpu.weights. pub cpu_weight: Option, + + /// The classifier leaf of the box this build runs for, if any (NET-155). + /// When set, the build's sandbox runs in that leaf, so the box's egress + /// verdict decides every connection the build makes. When `None`, the + /// build runs in the cgroup of its caller, as the caller's own traffic. + pub classifier_leaf: Option, + + /// The network of the box this build runs for, if the box has a network + /// namespace of its own and no classifier leaf (NET-155): an own-address + /// box. A build that needs the network attaches to it, so the box's egress + /// verdict decides every connection the build makes. A build that needs + /// no network stays in its empty namespace. `None` keeps the plan below. + pub box_network: Option, +} + +/// Makes the network a package build attaches to when it runs for a box with +/// a network namespace of its own (NET-155). Each build gets a network of its +/// own, because a network holds the reservation of one sandbox at a time and +/// the dependency builds of one box can run at the same time. +#[derive(Clone)] +pub struct BuildNetwork { + make: std::sync::Arc std::sync::Arc + Send + Sync>, + denies_all: bool, +} + +impl BuildNetwork { + /// A build network that `make` makes again for each build. + pub fn new( + make: impl Fn() -> std::sync::Arc + Send + Sync + 'static, + ) -> Self { + Self { + make: std::sync::Arc::new(make), + denies_all: false, + } + } + + /// Records whether the egress rules of the network admit no destination, + /// so the caller that reads a failed build can name the refusal. + #[must_use] + pub fn with_denies_all(mut self, denies_all: bool) -> Self { + self.denies_all = denies_all; + self + } + + /// Whether the egress rules of the network admit no destination. + pub fn denies_all(&self) -> bool { + self.denies_all + } + + /// A new network for one build. + pub fn network(&self) -> std::sync::Arc { + (self.make)() + } +} + +impl std::fmt::Debug for BuildNetwork { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BuildNetwork") + .field("denies_all", &self.denies_all) + .finish_non_exhaustive() + } } impl<'a, SF: crate::SourceFetcher> SpecBuild<'a, SF> { @@ -276,6 +337,55 @@ impl<'a, SF: crate::SourceFetcher> SpecBuild<'a, SF> { } } +impl<'a, SF: crate::SourceFetcher> SpecBuild<'a, SF> { + /// The sandbox configuration this build runs in. A build placed for a box + /// (NET-155) carries that box's classifier leaf, so its sandbox joins the + /// box's leaf and not the leaf of the process that starts it. + fn sandbox_config( + &self, + build: &BuildSpec, + inputs: Vec, + rootfs: HashSet, + needs_dns: bool, + needs_internet: bool, + daemon_id: Option<&String>, + ) -> sandbox2::config::Config { + let mut config = sandbox2::config::Config::new(&build.name) + .with_isolated_wd(inputs.into_iter()) + .with_rootfs(rootfs.into_iter()) + .with_dns(needs_dns) + // A build that needs neither DNS nor the internet runs in an empty + // network namespace; anything else shares the host's. Said as a plan + // rather than as a mode: a build is not a PTask and has never had + // one, and the sandbox layer acts on plans. + .with_plan(if !needs_dns && !needs_internet { + sandbox2::NetPlan::isolated() + } else { + sandbox2::NetPlan::host() + }); + if let Some(a) = &build.build_args { + config = config.with_build_args(a.iter()); + } + if let Some(w) = self.cpu_weight { + config = config.with_cpu_weight(w); + } + if let Some(id) = daemon_id { + config = config.with_daemon_id(id.clone()); + } + if let Some(leaf) = &self.classifier_leaf { + config = config.with_classifier_leaf(leaf.clone()); + } + // NET-155: a build that needs the network, for an own-address box, + // runs on that box's network and under its egress verdict. + if let Some(network) = &self.box_network + && (needs_dns || needs_internet) + { + config = config.with_network(network.network()); + } + config + } +} + impl<'a, SF: crate::SourceFetcher> Runnable for SpecBuild<'a, SF> { type Result = SpecBuildResult; @@ -310,28 +420,14 @@ impl<'a, SF: crate::SourceFetcher> Runnable for SpecBuild<'a, SF> { let channel = (); - let mut config = sandbox2::config::Config::new(&build.name) - .with_isolated_wd(inputs.into_iter()) - .with_rootfs(rootfs.into_iter()) - .with_dns(needs_dns) - // A build that needs neither DNS nor the internet runs in an empty - // network namespace; anything else shares the host's. Said as a plan - // rather than as a mode: a build is not a PTask and has never had - // one, and the sandbox layer acts on plans. - .with_plan(if !needs_dns && !needs_internet { - sandbox2::NetPlan::isolated() - } else { - sandbox2::NetPlan::host() - }); - if let Some(a) = &build.build_args { - config = config.with_build_args(a.iter()); - } - if let Some(w) = self.cpu_weight { - config = config.with_cpu_weight(w); - } - if let Some(id) = &opts.daemon_id { - config = config.with_daemon_id(id.clone()); - } + let config = self.sandbox_config( + build, + inputs, + rootfs, + needs_dns, + needs_internet, + opts.daemon_id.as_ref(), + ); let mut sandbox = config.build(&opts.exec_base, channel).await?; sandbox.keep_dir(true); @@ -466,6 +562,8 @@ mod tests { stderr_writer: None, cancel: CancellationToken::new(), cpu_weight: None, + classifier_leaf: None, + box_network: None, } } @@ -543,4 +641,52 @@ mod tests { "error should name the unbuildable target, got: {msg}" ); } + + /// NET-155: a spec build placed for a box runs its sandbox in that box's + /// classifier leaf. A build with no placement names no leaf, so its + /// sandbox stays in the cgroup of the process that starts it. + #[test] + fn spec_build_runs_in_the_given_classifier_leaf() { + let dg = graph_from(indoc! {r#" + let {BuildSpec, ..} = import "minimal.ncl" in + { + name = "downloads-at-build", + build_deps = [], + cmd = "curl https://example.invalid/", + } | BuildSpec + "#}); + let bsr = *dg.by_name("downloads-at-build").unwrap(); + let build = dg.get(&bsr).unwrap(); + let fetcher = UnusedFetcher; + let leaf = sandbox2::config::ClassifierLeaf::under( + std::path::Path::new("/sys/fs/cgroup/minimald.slice"), + "box-1", + sandbox2::config::Verdict::Deny, + ); + + let mut placed = spec_build(&bsr, &fetcher); + placed.classifier_leaf = Some(leaf.clone()); + let config = placed.sandbox_config(build, Vec::new(), HashSet::new(), true, true, None); + assert_eq!(config.classifier_leaf, Some(leaf)); + + let unplaced = spec_build(&bsr, &fetcher); + let config = unplaced.sandbox_config(build, Vec::new(), HashSet::new(), true, true, None); + assert_eq!(config.classifier_leaf, None); + assert!(config.network.is_none()); + + // An own-address box has no leaf. Its build runs on the box's own + // network when the build needs the network, and in an empty + // namespace when it does not. + let mut on_box_network = spec_build(&bsr, &fetcher); + on_box_network.box_network = Some(BuildNetwork::new(|| { + std::sync::Arc::new(sandbox2::NoNet) as std::sync::Arc + })); + let config = + on_box_network.sandbox_config(build, Vec::new(), HashSet::new(), true, true, None); + assert_eq!(config.classifier_leaf, None); + assert!(config.network.is_some()); + let config = + on_box_network.sandbox_config(build, Vec::new(), HashSet::new(), false, false, None); + assert!(config.network.is_none()); + } } diff --git a/crates/orchestrator/src/local_backend.rs b/crates/orchestrator/src/local_backend.rs index d673759ec2..3a2e00027a 100644 --- a/crates/orchestrator/src/local_backend.rs +++ b/crates/orchestrator/src/local_backend.rs @@ -137,6 +137,12 @@ pub struct LocalBackend { pub(crate) log_sink: Option>, pub(crate) ot: Option, pub(crate) cancel: tokio_util::sync::CancellationToken, + /// The classifier leaf of the box these builds run for, if any + /// (NET-155). Every spec build this backend schedules runs in it. + pub(crate) classifier_leaf: Option, + /// The network of the own-address box these builds run for, if any + /// (NET-155). Every spec build this backend schedules attaches to it. + pub(crate) box_network: Option, } impl Backend for LocalBackend { @@ -221,6 +227,8 @@ impl Backend for LocalBackend { cpu_weight: Some( ((3 * cost + 60 + depended_on_by * cost.max(1) * 25) as u64).min(10000), ), + classifier_leaf: shared.backend.classifier_leaf.clone(), + box_network: shared.backend.box_network.clone(), }; let res = b @@ -402,6 +410,8 @@ impl LocalBackend { log_sink: Option>, ot: Option, cancel: tokio_util::sync::CancellationToken, + classifier_leaf: Option, + box_network: Option, ) -> Result, Error> { Ok(Orchestrator { backend: LocalBackend:: { @@ -413,6 +423,8 @@ impl LocalBackend { num_concurrent_builds, ot, cancel, + classifier_leaf, + box_network, }, graph, cache, @@ -420,6 +432,18 @@ impl LocalBackend { }) } + /// The classifier leaf every spec build of this backend runs in, if the + /// builds run for a box (NET-155). + pub fn classifier_leaf(&self) -> Option<&sandbox2::config::ClassifierLeaf> { + self.classifier_leaf.as_ref() + } + + /// The network every spec build of this backend attaches to, if the + /// builds run for an own-address box (NET-155). + pub fn box_network(&self) -> Option<&op::BuildNetwork> { + self.box_network.as_ref() + } + /// Executes a local build with the orchestrator. pub async fn run_local_build( orch: Orchestrator, diff --git a/docs/specs/18-spec-box-networking/18-spec-box-networking.md b/docs/specs/18-spec-box-networking/18-spec-box-networking.md index 3a4aad6e57..10ddcb5d9a 100644 --- a/docs/specs/18-spec-box-networking/18-spec-box-networking.md +++ b/docs/specs/18-spec-box-networking/18-spec-box-networking.md @@ -623,6 +623,12 @@ included, with every refusal logged (NET-001 to NET-004). verify: cargo nextest run -p minimald daemon_fetch_survives_cohort_deny +- **NET-155** WHEN `minimald` runs a package build for a box that has a classifier leaf THE SYSTEM SHALL run the build in that leaf. + tier: T0 + verify: cargo nextest run -p minimald box_package_build_under_box_egress_verdict + - IF `minimald` runs a package build for a box that its launch placed in a classifier leaf and that leaf is missing, THEN THE SYSTEM SHALL refuse the build. + + - **NET-081** WHERE the host is VM-backed THE SYSTEM SHALL apply per-box source-addressed egress rules derived from the expanded box specs outside the VM. tier: T0 verify: cargo nextest run -p minvmd host_side_rules_applied_outside_vm From b568f8dae2a87ca051697195eb77e4136b414ea2 Mon Sep 17 00:00:00 2001 From: bryan-minimal Date: Thu, 22 Oct 2026 19:23:37 +0000 Subject: [PATCH 18/34] fix(op): give the sandbox a private copy of non-extracted sources (#2234) A non-extracted Source resolves to the download cache's own file, and it was hardlinked into the build's writable /build, so a build that appended to its input changed the cached source. It is now mapped as FileCopy, as Local files already are. FileCopy no longer truncates an existing destination: copy_input_file creates with create_new, and on a collision it keeps the first file and warns, as the hardlinked File input does. Backport of #2234. Adapted to the line: the sandbox hunk lands in sandbox2, and the new test's SpecBuild names T104's classifier_leaf and box_network fields. (cherry picked from commit 13aaa439e604dc4448fc3c7ebf3098c8627d99f1) --- crates/op/src/specs.rs | 83 +++++++++++++++++++++++++++++++++++++- crates/sandbox2/src/lib.rs | 65 ++++++++++++++++++++++++++++- 2 files changed, 145 insertions(+), 3 deletions(-) diff --git a/crates/op/src/specs.rs b/crates/op/src/specs.rs index fb11b36598..3fbe69c9fa 100644 --- a/crates/op/src/specs.rs +++ b/crates/op/src/specs.rs @@ -141,7 +141,12 @@ impl<'a, SF: crate::SourceFetcher> SpecBuild<'a, SF> { .await?; match resolved_src { Materialized::Given(_) => unreachable!(), - Materialized::File(path) => build_deps.push(SandboxMapped::File(path)), + // A copy, not a hardlink: the path is the download cache's own file, + // and /build is writable, so a hardlink would let a build rewrite the + // cached source under a concurrent build of the same source + // (build-servers#231). The cache re-hashes on every hit, so only a + // reader already past that check could see the change. + Materialized::File(path) => build_deps.push(SandboxMapped::FileCopy(path)), Materialized::TempDir(td) => build_deps.push(SandboxMapped::TempDir(td)), } } @@ -544,6 +549,31 @@ mod tests { } } + /// Serves the same file for every source, as the download cache does on a hit. + #[derive(Debug)] + struct OneFileFetcher(PathBuf); + + impl SourceFetcher for OneFileFetcher { + async fn download_web( + &self, + _url: &str, + _sha256: &str, + _op: &OpTracker, + ) -> anyhow::Result { + Ok(self.0.clone()) + } + + async fn download_gcs( + &self, + _bucket_id: String, + _file: &str, + _sha256: &str, + _op: &OpTracker, + ) -> anyhow::Result { + Ok(self.0.clone()) + } + } + fn graph_from(ncl: &str) -> Graph { Graph::new() .ingest(Layer::new_for_test(ncl.to_string()).unwrap()) @@ -689,4 +719,55 @@ mod tests { on_box_network.sandbox_config(build, Vec::new(), HashSet::new(), false, false, None); assert!(config.network.is_none()); } + + /// A non-extracted source reaches the sandbox's writable /build as a private + /// copy of the cached file. A hardlink there would be the download cache's own + /// inode, so a build appending to its input changed the cached source for any + /// concurrent build of it (build-servers#231, reproduced from inside a real + /// build sandbox). + #[test] + fn a_non_extracted_source_is_mapped_as_a_private_copy() { + let tmp = TempDir::new().unwrap(); + let cache = Cache::at_dir(tmp.path()).unwrap(); + let cached = tmp.path().join("payload.tar.gz"); + std::fs::write(&cached, b"payload").unwrap(); + let dg = graph_from(indoc! {r#" + let {BuildSpec, Source, ..} = import "minimal.ncl" in + { + name = "uses-raw-source", + build_deps = [ + { + url = "https://example.invalid/payload.tar.gz", + sha256 = "0000000000000000000000000000000000000000000000000000000000000000", + extract = false, + } | Source, + ], + cmd = "true", + } | BuildSpec + "#}); + let bsr = *dg.by_name("uses-raw-source").unwrap(); + let fetcher = OneFileFetcher(cached.clone()); + let sb = SpecBuild { + spec: &bsr, + remote_fetcher: &fetcher, + override_deps: None, + stdout_writer: None, + stderr_writer: None, + cancel: CancellationToken::new(), + cpu_weight: None, + classifier_leaf: None, + box_network: None, + }; + let opts = Options { + cache: cache.clone(), + graph: &dg, + exec_base: "/not-exists".into(), + ot: None, + daemon_id: None, + }; + let build = dg.get(&bsr).unwrap(); + let op = OpTracker::new_root(); + let deps = futures::executor::block_on(sb.build_deps_mapped(build, &opts, &op)).unwrap(); + assert_eq!(deps, vec![SandboxMapped::FileCopy(cached)]); + } } diff --git a/crates/sandbox2/src/lib.rs b/crates/sandbox2/src/lib.rs index f5fa2457b7..8ec207453a 100644 --- a/crates/sandbox2/src/lib.rs +++ b/crates/sandbox2/src/lib.rs @@ -316,8 +316,15 @@ impl Sandbox { } config::SandboxMapped::FileCopy(p) => { let dest = b.join(p.file_name().unwrap()); - fs::copy(p, &dest) - .map_err(|e| Error::IO("copying input file", dest, e))?; + if !copy_input_file(p, &dest) + .map_err(|e| Error::IO("copying input file", dest.clone(), e))? + { + tracing::warn!( + "Not copying {} => {}, already exists", + p.display(), + dest.display() + ); + } } config::SandboxMapped::Dir(p) => hardlink_dir_contents(p, &b)?, config::SandboxMapped::TempDir(td) => hardlink_dir_contents(td.path(), &b)?, @@ -2737,6 +2744,28 @@ fn booted_with_systemd() -> bool { .unwrap_or_default() } +/// Copy `src` to `dest` as a new file with `src`'s permission bits. Returns +/// `false`, touching nothing, when `dest` already exists: two inputs with one +/// basename keep the first, as a hardlinked `File` input does. `fs::copy` +/// would truncate an existing `dest`, and a `dest` hardlinked in from a mapped +/// directory shares its inode with that directory's source, so the copy would +/// have rewritten the source in place. +fn copy_input_file(src: &Path, dest: &Path) -> std::io::Result { + let mut out = match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(dest) + { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => return Ok(false), + Err(e) => return Err(e), + }; + let mut input = fs::File::open(src)?; + std::io::copy(&mut input, &mut out)?; + out.set_permissions(input.metadata()?.permissions())?; + Ok(true) +} + fn hardlink_dir_contents(src_dir: &Path, dst_parent_dir: &Path) -> Result<(), Error> { common::hardlink_dir_contents(src_dir, dst_parent_dir).map_err(Error::HardlinkFailed) } @@ -3725,6 +3754,38 @@ fn hosts_entry_present(body: &str, entry: &network::HostEntry) -> bool { #[cfg(test)] mod tests { use super::*; + + /// A copied input is a new file with the source's bytes and mode. + #[test] + fn copy_input_file_copies_bytes_and_mode() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("src.tar.gz"); + fs::write(&src, b"payload").unwrap(); + fs::set_permissions(&src, Permissions::from_mode(0o640)).unwrap(); + let dest = tmp.path().join("dest.tar.gz"); + assert!(copy_input_file(&src, &dest).unwrap()); + assert_eq!(fs::read(&dest).unwrap(), b"payload"); + assert_eq!( + fs::metadata(&dest).unwrap().permissions().mode() & 0o777, + 0o640 + ); + } + + /// An existing destination is left alone, even one that is a hardlink to + /// another file: `fs::copy` would truncate it and so rewrite that file. + #[test] + fn copy_input_file_never_writes_through_an_existing_entry() { + let tmp = tempfile::tempdir().unwrap(); + let shared = tmp.path().join("cached-from-a-dep"); + fs::write(&shared, b"shared bytes").unwrap(); + let dest = tmp.path().join("build-dest"); + fs::hard_link(&shared, &dest).unwrap(); + let src = tmp.path().join("other-source"); + fs::write(&src, b"other").unwrap(); + assert!(!copy_input_file(&src, &dest).unwrap()); + assert_eq!(fs::read(&shared).unwrap(), b"shared bytes"); + assert_eq!(fs::read(&dest).unwrap(), b"shared bytes"); + } use config::{Config, SandboxMapped}; /// The daemon's AppArmor remedy names the install step for the From fd35986de415d3a850a1c2d897bb4c67d9a1b9af Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Sun, 18 Oct 2026 19:25:58 +0000 Subject: [PATCH 19/34] fix(mctx): keep task args from running as shell code (#2296) Task argument values were spliced into command strings unquoted, so in a single exec string a value with spaces shattered into several argv entries, and in a bash script a value could end the quoted region and run as shell code. A single exec string is now split into argv words with each string value standing in as an opaque token, then each value fills its own word, and no shell runs it. A bash script gets ${MIN_ARG_} for each string reference, with the value bound in the invocation's environment. The daemon and mip launch paths pass Invocation::envs to the sandbox command. An enum value keeps its real value, since it is validated against the author's permitted set. Backport of #2296. Adapted to the line: sandbox:: reads sandbox2:: in mctx. (cherry picked from commit cfcff043e9283fd5da9e580ed10aca9af63f762f) --- Cargo.lock | 1 + crates/mctx/Cargo.toml | 1 + crates/mctx/src/env.rs | 606 +++++++++++++++++++++++++++++++++--- crates/mctx/src/lib.rs | 2 +- crates/minimald/src/exec.rs | 2 +- crates/mip/src/cmd_run.rs | 2 +- docs/reference/tasks.md | 55 ++++ 7 files changed, 624 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a93138bc6c..27bae6fc63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3864,6 +3864,7 @@ dependencies = [ "rcache", "sandbox2", "serial_test", + "shlex", "stdlib", "tempfile", "thiserror 2.0.21", diff --git a/crates/mctx/Cargo.toml b/crates/mctx/Cargo.toml index f4de5776b5..2ab01bad65 100644 --- a/crates/mctx/Cargo.toml +++ b/crates/mctx/Cargo.toml @@ -13,6 +13,7 @@ check.workspace = true codespan-reporting.workspace = true google-cloud-auth.workspace = true google-cloud-storage.workspace = true +shlex.workspace = true thiserror.workspace = true toml.workspace = true toml_edit.workspace = true diff --git a/crates/mctx/src/env.rs b/crates/mctx/src/env.rs index eb2bced558..4da441aad6 100644 --- a/crates/mctx/src/env.rs +++ b/crates/mctx/src/env.rs @@ -642,6 +642,27 @@ pub struct EnvArgs<'a> { /// the task's parameters — `task_packages` plus any `parsed_args` — so that /// `%{name}`-style placeholders are resolved. /// +/// Argument values are data, never code, so how a value reaches the +/// program depends on the action: +/// +/// - An `exec` single string is split into argv words first, with every +/// string value standing in as an opaque token, and each value then fills +/// in the one word its reference sits in. The result is a list-form +/// `exec`: no shell runs, and a value never adds, removes or splits a +/// word, whatever it holds and however the reference is quoted. +/// - A `bash` script never has a value spliced into it: a string reference +/// expands to the shell variable `${MIN_ARG_}`, and the value is +/// bound in the action's environment by [`task_arg_env`]. +/// - `echo`, list-form `exec` and `cmdcmd` actions take values verbatim; +/// their runners start no shell. +/// +/// Numbers, booleans and enum values keep their real values in every +/// action: number and boolean text cannot carry shell syntax, and an enum +/// value is one of the author's own permitted values. Only string values +/// stand in as a token or a variable reference while the Nickel expression +/// evaluates, so an expression that compares, measures or parses a string +/// argument does not see its value. +/// /// Shared by the invocation path ([`Env::task_invocations`]) and the /// sandbox-free `echo` short-circuits, so an echoed string is interpolated /// the same way an `exec`/`bash` action would be. @@ -649,7 +670,227 @@ pub fn interpolate_task_strings( task: &mfile::Task, parsed_args: Option<&args::ArgsSet>, ) -> Result { - let base = [( + let args = parsed_args + .map(|args| args.iter().collect::>()) + .unwrap_or_default(); + match &task.action { + mfile::TaskAction::Exec(mfile::StrOrList::Single(template)) => { + if template.contains([SENTINEL_OPEN, SENTINEL_CLOSE]) { + return Err(Error::Other(anyhow::anyhow!( + "exec string contains a reserved character (U+E000 or U+E001): {template}" + ))); + } + let mut values = Vec::new(); + let bound: Vec<_> = args + .into_iter() + .map(|(k, v)| (k.as_str(), sentinel_arg(v, &mut values))) + .collect(); + let var_ctx = task_var_ctx(task, bound); + let interpolated = eval_task_string(&var_ctx, template).map_err(Error::Other)?; + let argv = shlex::Shlex::new(interpolated.trim()) + .map(|word| fill_sentinels(&word, &values)) + .collect(); + Ok(mfile::Task { + action: mfile::TaskAction::Exec(mfile::StrOrList::Multiple(argv)), + ..task.clone() + }) + } + mfile::TaskAction::Bash(_) => { + // Checked here as well as where the environment is built, so a + // colliding pair fails before anything runs. + check_arg_env_names(&args).map_err(Error::Other)?; + let bound: Vec<_> = args + .into_iter() + .map(|(k, v)| (k.as_str(), env_reference_arg(k, v))) + .collect(); + let var_ctx = task_var_ctx(task, bound); + task.map_exec_strings(|s| eval_task_string(&var_ctx, s)) + } + _ => { + let bound: Vec<_> = args + .into_iter() + .map(|(k, v)| (k.as_str(), v.clone())) + .collect(); + let var_ctx = task_var_ctx(task, bound); + task.map_exec_strings(|s| eval_task_string(&var_ctx, s)) + } + } + .map_err(Error::Other) +} + +/// The environment a `bash` action's argument references read: one +/// variable per argument, holding its value as data. +/// +/// A scalar or enum argument `name` binds `MIN_ARG_`, where `` +/// is the argument name uppercased with every character outside `A-Z`/`0-9` +/// replaced by `_`. An array argument binds `MIN_ARG__COUNT` to its +/// length and `MIN_ARG__` to element `i`, counting from 0. Two +/// arguments that would bind the same variable are refused. +pub fn task_arg_env( + parsed_args: Option<&args::ArgsSet>, +) -> Result, Error> { + let args = parsed_args + .map(|args| args.iter().collect::>()) + .unwrap_or_default(); + check_arg_env_names(&args).map_err(Error::Other)?; + let mut env = BTreeMap::new(); + for (name, arg) in args { + let var = arg_env_name(name); + match arg { + args::Arg::Scalar(s) => { + env.insert(var, scalar_text(s)); + } + args::Arg::Enum(s) => { + env.insert(var, s.clone()); + } + args::Arg::Array(items) => { + env.insert(format!("{var}_COUNT"), items.len().to_string()); + for (i, item) in items.iter().enumerate() { + env.insert(format!("{var}_{i}"), scalar_text(item)); + } + } + } + } + Ok(env) +} + +/// The variable a `bash` action binds argument `name` to; see +/// [`task_arg_env`]. +fn arg_env_name(name: &str) -> String { + let upper: String = name + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() { + c.to_ascii_uppercase() + } else { + '_' + } + }) + .collect(); + format!("MIN_ARG_{upper}") +} + +/// Refuses two arguments that would bind the same variable (`a-b` and +/// `a_b`, or array `a` and scalar `a_0`). +fn check_arg_env_names(args: &[(&String, &args::Arg)]) -> Result<(), anyhow::Error> { + let mut owners: BTreeMap = BTreeMap::new(); + for (name, arg) in args { + let var = arg_env_name(name); + let vars = match arg { + args::Arg::Array(items) => std::iter::once(format!("{var}_COUNT")) + .chain((0..items.len()).map(|i| format!("{var}_{i}"))) + .collect(), + _ => vec![var], + }; + for v in vars { + if let Some(other) = owners.insert(v.clone(), name.as_str()) { + anyhow::bail!( + "task arguments `{other}` and `{name}` both bind the environment \ + variable `{v}`; rename one of them" + ); + } + } + } + Ok(()) +} + +/// The text a scalar value takes in an environment variable. +fn scalar_text(s: &args::ScalarArg) -> String { + match s { + args::ScalarArg::String(s) => s.clone(), + args::ScalarArg::Number(n) => n.to_string(), + args::ScalarArg::Boolean(b) => b.to_string(), + } +} + +/// Binds a `bash` argument for Nickel: each string becomes a reference to +/// the variable [`task_arg_env`] holds it in, so the script carries the +/// reference and never the value. An enum value is bound as it is. +fn env_reference_arg(name: &str, arg: &args::Arg) -> args::Arg { + let var = arg_env_name(name); + let reference = |v: String| args::ScalarArg::String(format!("${{{v}}}")); + match arg { + args::Arg::Scalar(args::ScalarArg::String(_)) => args::Arg::Scalar(reference(var)), + // Validated against the author's permitted set, so author text. + args::Arg::Enum(_) => arg.clone(), + args::Arg::Scalar(other) => args::Arg::Scalar(other.clone()), + args::Arg::Array(items) => args::Arg::Array( + items + .iter() + .enumerate() + .map(|(i, item)| match item { + args::ScalarArg::String(_) => reference(format!("{var}_{i}")), + other => other.clone(), + }) + .collect(), + ), + } +} + +/// Open and close the token a string value stands in as while an `exec` +/// string is split into words. Private-use characters: shlex treats them +/// as ordinary word characters, and an author has no reason to type them. +const SENTINEL_OPEN: char = '\u{E000}'; +const SENTINEL_CLOSE: char = '\u{E001}'; + +/// Binds an `exec` argument for Nickel: each string becomes an opaque +/// token naming its slot in `values`, which [`fill_sentinels`] swaps back +/// once the words are split. An enum value is bound as it is. +fn sentinel_arg(arg: &args::Arg, values: &mut Vec) -> args::Arg { + let mut token = |v: &str| { + values.push(v.to_string()); + format!("{SENTINEL_OPEN}{}{SENTINEL_CLOSE}", values.len() - 1) + }; + match arg { + args::Arg::Scalar(args::ScalarArg::String(s)) => { + args::Arg::Scalar(args::ScalarArg::String(token(s))) + } + // Validated against the author's permitted set, so author text. + args::Arg::Enum(_) => arg.clone(), + args::Arg::Scalar(other) => args::Arg::Scalar(other.clone()), + args::Arg::Array(items) => args::Arg::Array( + items + .iter() + .map(|item| match item { + args::ScalarArg::String(s) => args::ScalarArg::String(token(s)), + other => other.clone(), + }) + .collect(), + ), + } +} + +/// Replaces each token [`sentinel_arg`] made in one split word with the +/// value it stands for. Text that is not a whole token stays as it is. +fn fill_sentinels(word: &str, values: &[String]) -> String { + let mut out = String::with_capacity(word.len()); + let mut rest = word; + while let Some(start) = rest.find(SENTINEL_OPEN) { + out.push_str(&rest[..start]); + let after = &rest[start + SENTINEL_OPEN.len_utf8()..]; + let filled = after.find(SENTINEL_CLOSE).and_then(|end| { + let value = values.get(after[..end].parse::().ok()?)?; + Some((value, end)) + }); + match filled { + Some((value, end)) => { + out.push_str(value); + rest = &after[end + SENTINEL_CLOSE.len_utf8()..]; + } + None => { + out.push(SENTINEL_OPEN); + rest = after; + } + } + } + out.push_str(rest); + out +} + +/// The Nickel context a task's strings evaluate in: `task_packages` plus +/// the arguments as `bound`. +fn task_var_ctx(task: &mfile::Task, bound: Vec<(&str, args::Arg)>) -> common::ncl_eval::VarCtx { + let base = ( "task_packages", args::Arg::Array( task.packages @@ -657,31 +898,25 @@ pub fn interpolate_task_strings( .map(|s| args::ScalarArg::String(s.clone())) .collect(), ), - )] - .into_iter(); - let var_ctx = if let Some(args) = parsed_args { - common::ncl_eval::VarCtx::from_iter( - base.chain(args.iter().map(|(k, v)| (k.as_str(), v.clone()))), - ) - } else { - common::ncl_eval::VarCtx::from_iter(base) - }; - task.map_exec_strings(|s| { - var_ctx.eval_string(s).map_err(|_| { - if s.contains("%{") { - anyhow::anyhow!( - "nickel eval failed for string: {s}\n\ - note: command strings are Nickel-evaluated, so `%{{` opens an \ - interpolation even when the task defines no matching arg; to keep a \ - literal `%{{` (as in curl's `-w \"%{{http_code}}\"`), escape it as \ - `%{{\"%\"}}{{http_code}}` — see docs/reference/tasks.md" - ) - } else { - anyhow::anyhow!("nickel eval failed for string: {s}") - } - }) + ); + common::ncl_eval::VarCtx::from_iter(std::iter::once(base).chain(bound)) +} + +/// Evaluates one task string, pointing a failed `%{` at its escape. +fn eval_task_string(var_ctx: &common::ncl_eval::VarCtx, s: &str) -> Result { + var_ctx.eval_string(s).map_err(|_| { + if s.contains("%{") { + anyhow::anyhow!( + "nickel eval failed for string: {s}\n\ + note: command strings are Nickel-evaluated, so `%{{` opens an \ + interpolation even when the task defines no matching arg; to keep a \ + literal `%{{` (as in curl's `-w \"%{{http_code}}\"`), escape it as \ + `%{{\"%\"}}{{http_code}}` — see docs/reference/tasks.md" + ) + } else { + anyhow::anyhow!("nickel eval failed for string: {s}") + } }) - .map_err(Error::Other) } /// Names whoever put a patch path into the environment: the package that @@ -984,16 +1219,22 @@ impl<'a> Env<'a> { ) -> Result<(bool, Vec), Error> { let mapped_task = interpolate_task_strings(task, parsed_args)?; - Ok(( - task.interactive, - op::TaskEnv { - task: &mapped_task, - sandbox: &mut self.sandbox, + let mut invocations = op::TaskEnv { + task: &mapped_task, + sandbox: &mut self.sandbox, + } + .resolve() + .await + .map_err(|e| Error::Other(anyhow::anyhow!("{}", e)))?; + // A `bash` script reads its arguments from these variables; see + // [`interpolate_task_strings`]. + if matches!(task.action, mfile::TaskAction::Bash(_)) { + let arg_env = task_arg_env(parsed_args)?; + for inv in &mut invocations { + inv.envs.extend(arg_env.clone()); } - .resolve() - .await - .map_err(|e| Error::Other(anyhow::anyhow!("{}", e)))?, - )) + } + Ok((task.interactive, invocations)) } /// Step 1 of a launch: the provider's plan, with its release owed. See @@ -1033,6 +1274,24 @@ impl<'a> Env<'a> { .map_err(|e| Error::Other(anyhow::anyhow!("{}", e))) } + /// Builds the command for one invocation: its program, its argv and its + /// own environment variables ([`Invocation::envs`]). + #[cfg(target_os = "linux")] + pub fn invocation_command( + &mut self, + container: &Container, + invocation: &Invocation, + ) -> Result { + self.sandbox + .command( + container, + &invocation.executable, + &invocation.args, + &invocation.envs, + ) + .map_err(|e| Error::Other(anyhow::anyhow!("{}", e))) + } + /// Runs a sequence of invocations inside the sandbox, optionally streaming /// stdout and stderr to the provided async writers. pub async fn run( @@ -1409,11 +1668,18 @@ mod tests { }; let resolved = interpolate_task_strings(&task, None) .expect("the escaped `%{` must evaluate successfully"); - let mfile::TaskAction::Exec(mfile::StrOrList::Single(cmd)) = resolved.action else { - panic!("expected a single exec string"); - }; + let (exec, argv) = resolved.exec_and_args().unwrap(); + assert_eq!(exec, "/bin/curl"); assert_eq!( - cmd, "curl -s -o /dev/null -w \"%{http_code}\" https://example.com", + argv, + [ + "-s", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "https://example.com" + ], "the `%{{\"%\"}}{{...}}` escape should resolve to a literal `%{{http_code}}`" ); } @@ -1435,13 +1701,269 @@ mod tests { .unwrap(); let task = mf.task("useparam").unwrap(); - for (argv, want) in [("", "echo hi"), ("--greeting bye", "echo bye")] { + for (argv, want) in [("", "hi"), ("--greeting bye", "bye")] { let parsed = task.args.parse(argv).unwrap(); let resolved = interpolate_task_strings(&task, Some(&parsed)).unwrap(); - let mfile::TaskAction::Exec(mfile::StrOrList::Single(cmd)) = resolved.action else { - panic!("expected a single exec string"); + assert_eq!( + resolved.exec_and_args(), + Some(("/bin/echo".to_string(), vec![want.to_string()])), + "argv `{argv}`" + ); + } + } + + /// Argument values a hostile caller might pass: words, shell control + /// operators, command substitutions, both quote characters, and nothing. + /// `./victim` and `pwned` are relative, so a value that did run as code + /// would act inside the test's own temporary directory. + const HOSTILE: &[&str] = &[ + "harmless", + "two words", + "x; rm -rf ./victim", + "$(touch pwned)", + "`touch pwned`", + "it's \"quoted\"", + "", + ]; + + /// Parses `--name ` against a one-string-arg schema. + fn name_arg(task: &mfile::Task, value: &str) -> args::ArgsSet { + task.args + .parse_argv_named("task", vec!["--name", value]) + .unwrap() + } + + /// A single `exec` string is split into words before any value is + /// filled in, so each `%{name}` lands in exactly the one argv entry it + /// sits in — unquoted, single-quoted or double-quoted, alone or joined + /// to other text — whatever the value holds. + #[test] + fn interpolate_exec_value_fills_exactly_one_argv_entry() { + let schema: args::ArgsSpec = toml::from_str("name = \"string\"").unwrap(); + for value in HOSTILE { + for (template, want) in [ + ("runner %{name} run", value.to_string()), + ("runner '%{name}' run", value.to_string()), + ("runner \"%{name}\" run", value.to_string()), + ("runner --name=%{name} run", format!("--name={value}")), + ("runner '--name=%{name}' run", format!("--name={value}")), + ("runner \"--name=%{name}\" run", format!("--name={value}")), + ] { + let task = mfile::Task { + action: mfile::TaskAction::exec_from_str(template), + args: schema.clone(), + ..Default::default() + }; + let parsed = name_arg(&task, value); + let resolved = interpolate_task_strings(&task, Some(&parsed)).unwrap(); + assert_eq!( + resolved.exec_and_args(), + Some(("/bin/runner".to_string(), vec![want, "run".to_string()])), + "template {template:?}, value {value:?}" + ); + } + } + } + + /// An array joined into a single `exec` string yields one argv entry + /// per element, each element intact. + #[test] + fn interpolate_exec_array_elements_stay_whole() { + let schema: args::ArgsSpec = toml::from_str("files = \"Array string\"").unwrap(); + let task = mfile::Task { + action: mfile::TaskAction::exec_from_str("runner %{std.string.join \" \" files} done"), + args: schema, + ..Default::default() + }; + let mut argv = Vec::new(); + for v in HOSTILE { + argv.extend(["--files", v]); + } + let parsed = task.args.parse_argv_named("task", argv).unwrap(); + let resolved = interpolate_task_strings(&task, Some(&parsed)).unwrap(); + let mut want: Vec = HOSTILE.iter().map(|v| v.to_string()).collect(); + want.push("done".to_string()); + assert_eq!( + resolved.exec_and_args(), + Some(("/bin/runner".to_string(), want)) + ); + } + + /// Runs `script` the way a `bash` action runs — `bash -c` with the + /// task's argument environment — in a fresh directory holding a + /// `victim` file, and returns its stdout. Fails if the script created + /// `pwned` or removed `victim`, which only a value run as code would do. + fn run_bash_action(task: &mfile::Task, parsed: &args::ArgsSet) -> String { + let resolved = interpolate_task_strings(task, Some(parsed)).unwrap(); + let mfile::TaskAction::Bash(script) = &resolved.action else { + panic!("expected a bash action"); + }; + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("victim"), "").unwrap(); + let out = std::process::Command::new("bash") + .arg("-c") + .arg(script) + .envs(task_arg_env(Some(parsed)).unwrap()) + .current_dir(dir.path()) + .output() + .unwrap(); + assert!(out.status.success(), "script {script:?} failed: {out:?}"); + assert!( + !dir.path().join("pwned").exists(), + "script {script:?} ran a value as code" + ); + assert!( + dir.path().join("victim").exists(), + "script {script:?} ran a value as code" + ); + String::from_utf8(out.stdout).unwrap() + } + + /// A `bash` script never carries a value, only a reference to the + /// variable holding it, so no value runs as code in any quoting + /// context. Double-quoted, the value arrives whole; unquoted, it is + /// word-split like any unquoted variable; single-quoted, the reference + /// itself is printed. + #[test] + fn interpolate_bash_value_never_runs_as_code() { + let schema: args::ArgsSpec = toml::from_str("name = \"string\"").unwrap(); + for value in HOSTILE { + let split: String = match value.split_whitespace().collect::>() { + words if words.is_empty() => "[]\n".to_string(), + words => words.iter().map(|w| format!("[{w}]\n")).collect(), }; - assert_eq!(cmd, want, "argv `{argv}`"); + for (script, want) in [ + ("printf '[%s]\\n' \"%{name}\"", format!("[{value}]\n")), + ( + "printf '[%s]\\n' \"Hello %{name}!\"", + format!("[Hello {value}!]\n"), + ), + ("printf '[%s]\\n' %{name}", split), + ( + "printf '[%s]\\n' '%{name}'", + "[${MIN_ARG_NAME}]\n".to_string(), + ), + ] { + let task = mfile::Task { + action: mfile::TaskAction::Bash(script.to_string()), + args: schema.clone(), + ..Default::default() + }; + let parsed = name_arg(&task, value); + assert_eq!( + run_bash_action(&task, &parsed), + want, + "script {script:?}, value {value:?}" + ); + } + } + } + + /// Array arguments bind one variable per element plus a count, and an + /// element reference expands to its own variable. + #[test] + fn interpolate_bash_array_elements_bind_one_variable_each() { + let schema: args::ArgsSpec = toml::from_str("files = \"Array string\"").unwrap(); + let task = mfile::Task { + action: mfile::TaskAction::Bash( + "printf '[%s]\\n' \"$MIN_ARG_FILES_COUNT\" \"%{std.array.at 2 files}\" \ + \"%{std.array.at 3 files}\"" + .to_string(), + ), + args: schema, + ..Default::default() + }; + let mut argv = Vec::new(); + for v in HOSTILE { + argv.extend(["--files", v]); + } + let parsed = task.args.parse_argv_named("task", argv).unwrap(); + assert_eq!( + run_bash_action(&task, &parsed), + format!("[{}]\n[{}]\n[{}]\n", HOSTILE.len(), HOSTILE[2], HOSTILE[3]) + ); + } + + /// Two argument names that map to the same variable are refused rather + /// than letting one value shadow the other. + #[test] + fn interpolate_bash_refuses_colliding_arg_variables() { + let schema: args::ArgsSpec = toml::from_str("a-b = \"string\"\na_b = \"string\"").unwrap(); + let task = mfile::Task { + action: mfile::TaskAction::Bash("echo \"%{a_b}\"".to_string()), + args: schema, + ..Default::default() + }; + let parsed = task + .args + .parse_argv_named("task", vec!["--a-b", "x", "--a_b", "y"]) + .unwrap(); + let err = interpolate_task_strings(&task, Some(&parsed)).expect_err("names collide"); + assert!( + format!("{err}").contains("MIN_ARG_A_B"), + "error should name the variable, got: {err}" + ); + assert!(task_arg_env(Some(&parsed)).is_err()); + } + + /// An enum value is one of the author's permitted values, so a Nickel + /// expression sees the real value and takes the branch it selects, in + /// `exec` and in `bash`; `bash` still binds `MIN_ARG_` for it. + #[test] + fn interpolate_enum_value_reaches_nickel_expressions() { + let schema: args::ArgsSpec = toml::from_str("profile = \"[debug, release]\"").unwrap(); + let branch = "%{if profile == \"release\" then \"--release\" else \"--debug\"}"; + for (value, want) in [("release", "--release"), ("debug", "--debug")] { + let exec = mfile::Task { + action: mfile::TaskAction::exec_from_str(&format!("runner {branch} done")), + args: schema.clone(), + ..Default::default() + }; + let parsed = exec + .args + .parse_argv_named("task", vec!["--profile", value]) + .unwrap(); + let resolved = interpolate_task_strings(&exec, Some(&parsed)).unwrap(); + assert_eq!( + resolved.exec_and_args(), + Some(( + "/bin/runner".to_string(), + vec![want.to_string(), "done".to_string()] + )), + "exec, profile {value:?}" + ); + + let bash = mfile::Task { + action: mfile::TaskAction::Bash(format!( + "printf '[%s]\\n' {branch} \"$MIN_ARG_PROFILE\"" + )), + args: schema.clone(), + ..Default::default() + }; + assert_eq!( + run_bash_action(&bash, &parsed), + format!("[{want}]\n[{value}]\n"), + "bash, profile {value:?}" + ); + } + } + + /// `echo` runs no shell, so the value is printed verbatim. + #[test] + fn interpolate_echo_value_is_verbatim() { + let schema: args::ArgsSpec = toml::from_str("name = \"string\"").unwrap(); + for value in HOSTILE { + let task = mfile::Task { + action: mfile::TaskAction::Echo("Hello %{name}!".to_string()), + args: schema.clone(), + ..Default::default() + }; + let parsed = name_arg(&task, value); + let resolved = interpolate_task_strings(&task, Some(&parsed)).unwrap(); + assert_eq!( + resolved.action.as_echo(), + Some(format!("Hello {value}!").as_str()) + ); } } diff --git a/crates/mctx/src/lib.rs b/crates/mctx/src/lib.rs index 3fee2f9ba2..1e4717af27 100644 --- a/crates/mctx/src/lib.rs +++ b/crates/mctx/src/lib.rs @@ -34,7 +34,7 @@ pub use mfile_search_strategy::MFileSearchStrategy; mod project_setup; pub use project_setup::ProjectSetup; -pub use env::{Env, PatchHome, WdLayout, interpolate_task_strings}; +pub use env::{Env, PatchHome, WdLayout, interpolate_task_strings, task_arg_env}; use tokio::sync::Semaphore; use toml_edit::{Array, DocumentMut, Item, TableLike, Value}; diff --git a/crates/minimald/src/exec.rs b/crates/minimald/src/exec.rs index 6fb8130758..03a9585c16 100644 --- a/crates/minimald/src/exec.rs +++ b/crates/minimald/src/exec.rs @@ -518,7 +518,7 @@ async fn task_producer( .container(planned.plan()) .map_err(|e| io::Error::other(format!("building container failed: {}", e)))?; let mut cmd = env - .command(&container, &inv.executable, inv.args.iter()) + .invocation_command(&container, &inv) .map_err(|e| io::Error::other(format!("building command failed: {}", e)))?; if let Some(dir) = &start_dir { cmd.current_dir(dir); diff --git a/crates/mip/src/cmd_run.rs b/crates/mip/src/cmd_run.rs index dc36185987..389a359a3f 100644 --- a/crates/mip/src/cmd_run.rs +++ b/crates/mip/src/cmd_run.rs @@ -300,7 +300,7 @@ pub async fn run_task( .container(planned.plan()) .map_err(|e| Error::Other(anyhow!("building container failed: {}", e)))?; let mut cmd = env - .command(&container, &inv.executable, inv.args.iter()) + .invocation_command(&container, inv) .map_err(|e| Error::Other(anyhow!("building command failed: {}", e)))?; let mut child = cmd .spawn() diff --git a/docs/reference/tasks.md b/docs/reference/tasks.md index 8061ac5327..c795d9d948 100644 --- a/docs/reference/tasks.md +++ b/docs/reference/tasks.md @@ -89,6 +89,61 @@ exec = 'curl -s -o /dev/null -w "%{http_code}" https://example.com' exec = 'curl -s -o /dev/null -w "%{"%"}{http_code}" https://example.com' ``` +Argument values are data. What reaches the program depends on the action. + +Minimal splits an `exec` string into words before it fills in any value, and +no shell runs the result. Each string value lands inside the one word that +holds its reference. This holds whatever spaces, quotes, or shell syntax the +value contains. It also holds whether the reference is bare, in single quotes, +or in double quotes: + +```toml +[tasks.deploy] +args.environment = "string" +exec = "runner --environment=%{environment} apply" +``` + +Passing `--environment "stage; rm -rf /"` runs `runner` with the two +arguments `--environment=stage; rm -rf /` and `apply`. + +Minimal never pastes a string value into a `bash` script. Instead, it sets +each argument `name` in the script's environment as `MIN_ARG_`. In that +variable name, the argument name is in upper case, and `_` replaces every +character other than a letter or digit. A reference `%{name}` to a string +argument becomes `${MIN_ARG_NAME}` in the script. The shell expands it as a +variable and never runs the value. Quote it as you quote any shell variable: + +```toml +[tasks.greet] +args.name = "string" +bash = "echo \"Hello %{name}!\"" # same as: echo "Hello $MIN_ARG_NAME!" +``` + +- Inside double quotes, the value arrives whole. +- Outside quotes, the shell splits the value into words and expands + wildcards in it, as it does for any unquoted variable. +- Inside single quotes, the shell prints the reference itself, such as + `${MIN_ARG_NAME}`, not the value. + +An array argument `files` sets `MIN_ARG_FILES_COUNT` to its length and +`MIN_ARG_FILES_0`, `MIN_ARG_FILES_1`, and so on to its elements. Argument +names that map to the same variable, such as `a-b` and `a_b`, are an error. + +Number and boolean arguments keep their real values in `%{...}` expressions, +such as `%{std.string.from_number count}`. Minimal does not replace them with +a variable reference. Enum arguments keep their real values too, because +an enum value is always one of the values the task lists. So +`%{if profile == "release" then "--release" else ""}` takes the branch the +value selects, in `exec` and in `bash`. + +Inside a `%{...}` expression, a string argument in an `exec` string or a +`bash` script is an opaque placeholder. You can place it or join it with other +text. An expression that compares or parses it does not see the value the +caller passed. + +`echo` and list-form `exec` actions take each value as it is. Neither one +runs a shell. + ### `description` - Describe the task From cb85e78e60f3394215ba0a28840a65f0adef56b6 Mon Sep 17 00:00:00 2001 From: Karl Godard Date: Mon, 2 Nov 2026 18:59:20 +0000 Subject: [PATCH 20/34] fix(deps): bump pulldown-cmark to 0.13.6 for RUSTSEC-2026-0433 (#2846) Backport of #2846. Cargo.lock only; the advisory is the one failure in the 0.7.x cargo-deny job. (cherry picked from commit 1aa79abda4d36936e2a66c8a34a549e96a026b45) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 27bae6fc63..e5421aa627 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5615,9 +5615,9 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.13.5" +version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "278e9268c93b9e498a2da46c34d90ba93a3255eb99009646398f34bc73ab36eb" +checksum = "3db2be46835f013dd3f4fa7f5f7402c537787b54ebb6fe2b94bbdd7e2669bb7b" dependencies = [ "bitflags 2.13.1", "memchr", From 82c8fe760046733bc45ee23b17c6747c5a8fa075 Mon Sep 17 00:00:00 2001 From: Norrie Taylor <91171431+norrietaylor@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:30:21 -0700 Subject: [PATCH 21/34] fix(cargo): remove the committed buf registry token (#2204) .cargo/config.toml held one thing: a [registries.buf] entry whose token line was a bearer credential in plain text. No dependency resolves through that registry, no crate names registry = "buf", and no workflow or script reads it, so the whole file goes rather than the one line. The token is still in history and needs rotating at buf.build. Backport of #2204. Applied unchanged; the line's file matched main's. (cherry picked from commit 813e4e590340690833349d582175f65d3e06c07c) --- .cargo/config.toml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index 9096f8dc8d..0000000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,4 +0,0 @@ -[registries.buf] -index = "sparse+https://buf.build/gen/cargo/" -credential-provider = "cargo:token" -token = "Bearer c5ceab0fe4fbdddabe0aefb73a1351aa7f77149a9df3837af3f27e4f4171119f" From 0ce56e4f318832cf131806fb157702c30dc5c720 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:30:51 +0000 Subject: [PATCH 22/34] fix(minimal): show each box's address and egress posture in min ls (#2777) The ListSessions reply gains a map keyed by box id with the host loopback address the box's ports answer at, its effective egress posture, and whether it stands on the 127.0.0.1 fallback. All three are serde-defaulted, so an older daemon reads as unreported. min ls gains an ADDRESS column and --json carries the same facts. The EGRESS column prints the posture in the words min session policy uses, from one shared function, and the activate host line and the BOX NAMES row state how a box name opens through one resolver function. A VM node's host-address box reports no address until the node is granted one, since it is never mirrored onto 127.0.0.1. Backport of #2777. Adapted to the line: the policy.rs and create.rs hunks land in cmd/session.rs, and the minimald-rpc and minimald rpc tests land inline in lib.rs and rpc.rs, with the line's names (SessionStatus, EffectiveSessionPolicy, a built CreateSessionRequest). (cherry picked from commit 08045bfbdeee76d348625385fc43249b0f8c4af7) --- crates/minimal/src/cmd/list.rs | 196 +++++++-- crates/minimal/src/cmd/session.rs | 50 ++- crates/minimal/src/resolver.rs | 377 ++++++++++++------ crates/minimal/src/tests.rs | 355 ++++++++++++++++- crates/minimal/tests/cli.rs | 159 +++++--- crates/minimald-rpc/src/lib.rs | 182 +++++++++ crates/minimald/src/rpc.rs | 241 ++++++++++- .../18-spec-box-networking.md | 28 +- scripts/session-e2e.sh | 52 ++- 9 files changed, 1393 insertions(+), 247 deletions(-) diff --git a/crates/minimal/src/cmd/list.rs b/crates/minimal/src/cmd/list.rs index 8094d47ea9..7aa2fb1a9a 100644 --- a/crates/minimal/src/cmd/list.rs +++ b/crates/minimal/src/cmd/list.rs @@ -786,7 +786,7 @@ pub fn format_ls_on( host: crate::resolver::HostInstall, ) -> Result<(), anyhow::Error> { if args.json { - let json = serde_json_lenient::to_string_pretty(resp) + let json = serde_json_lenient::to_string_pretty(&listing_json(resp)?) .context("Failed to serialize session list")?; writeln!(out, "{json}")?; return Ok(()); @@ -881,7 +881,11 @@ pub fn format_ls_on( writeln!( out, "BOX NAMES: {}{}", - crate::resolver::name_surface_line(surface.clone(), serving_proxy_port), + crate::resolver::name_surface_line_at( + surface.clone(), + serving_proxy_port, + boxes_at_own_address(resp), + ), crate::resolver::install_clause(crate::resolver::install_unfinished( &surface, host )), @@ -912,44 +916,129 @@ pub fn format_ls_on( // Format as a table. The columns mirror the fields `--json` exposes so // the two surfaces present the same session attributes. + let rows: Vec<[String; 8]> = resp + .sessions + .iter() + .map(|entry| session_cells(entry, resp.box_networks.get(&entry.id))) + .collect(); + let egress_width = egress_column_width(rows.iter()); writeln!( out, - "{:<36} {:<20} {:<13} {:<7} {:<20} {:<19} PROJECT PATH", - "SESSION ID", "NAME", "STATUS", "EGRESS", "TITLE", "LAST ACTIVITY" + "{:<36} {:<20} {: bool { + let fallbacks: Vec> = resp + .box_networks + .values() + .map(|facts| facts.at_fallback) + .collect(); + fallbacks.contains(&Some(false)) && !fallbacks.contains(&Some(true)) +} + +/// The width of the ADDRESS column: the longest IPv4 address. +const ADDRESS_COLUMN_WIDTH: usize = 15; + +/// The width of the EGRESS column: its header, or the longest posture any +/// row prints, so a posture with a per-box enforcement note keeps the +/// columns after it aligned. +fn egress_column_width<'a>(rows: impl Iterator) -> usize { + rows.map(|row| row[4].len()) + .fold("EGRESS".len(), usize::max) +} + +/// The `--json` document of one daemon's listing (T110): the reply as it +/// came, except that each box's network facts move from the reply's map +/// into that box's own entry, as `loopback_address` and `egress_posture`. +/// A field the daemon did not report is left out of the entry. +fn listing_json( + resp: &minimald_rpc::ListSessionsResponse, +) -> Result { + let mut document = + serde_json_lenient::to_value(resp).context("Failed to serialize session list")?; + if let Some(object) = document.as_object_mut() { + object.remove("box_networks"); + if let Some(serde_json_lenient::Value::Array(entries)) = object.get_mut("sessions") { + for (entry, session) in entries.iter_mut().zip(&resp.sessions) { + let (Some(entry), Some(facts)) = + (entry.as_object_mut(), resp.box_networks.get(&session.id)) + else { + continue; + }; + if let Some(address) = facts.loopback_address { + entry.insert( + "loopback_address".to_string(), + serde_json_lenient::Value::String(address.to_string()), + ); + } + if let Some(posture) = facts.egress_posture { + entry.insert( + "egress_posture".to_string(), + serde_json_lenient::to_value(posture) + .context("Failed to serialize session list")?, + ); + } + } + } + } + Ok(document) +} + +/// The cells of one session row — id, name, address, status, egress, title, +/// last activity, project path — shared by the single-VM table /// ([`format_ls`]) and the multi-VM one ([`format_ls_across_vms`]), so the /// two surfaces present the same session attributes. /// -/// The egress cell is a host-address box's enforcement (NET-079) in the -/// spelling `--json` carries, `per_box` or `none`; every other box shows -/// `-`, because the daemon reports no enforcement for it. -fn session_cells(entry: &minimald_rpc::ListSessionsEntry) -> [String; 7] { +/// The address cell is the host loopback address the box's ports answer +/// at, and the egress cell is the box's posture in `min session policy`'s +/// words ([`egress_posture_cell`]). +/// Each shows `-` when the daemon did not report it (T110). +fn session_cells( + entry: &minimald_rpc::ListSessionsEntry, + facts: Option<&minimald_rpc::BoxNetworkFacts>, +) -> [String; 8] { let id = entry.id.to_string(); let name = entry.name.as_deref().unwrap_or("-").to_string(); + let address = facts + .and_then(|facts| facts.loopback_address) + .map_or_else(|| "-".to_string(), |address| address.to_string()); let status = status_label(entry.status).to_string(); - let egress = entry - .host_ip_enforcement - .map_or("-", minimald_rpc::HostIpEnforcement::machine_str) - .to_string(); + let egress = egress_posture_cell( + facts.and_then(|facts| facts.egress_posture), + entry.host_ip_enforcement, + ); let project_path = entry .project_path .as_ref() @@ -975,7 +1064,16 @@ fn session_cells(entry: &minimald_rpc::ListSessionsEntry) -> [String; 7] { } None => ("-".to_string(), "-".to_string()), }; - [id, name, status, egress, title, last_activity, project_path] + [ + id, + name, + address, + status, + egress, + title, + last_activity, + project_path, + ] } /// The width of the VM column in the multi-VM table: enough for the default @@ -1060,6 +1158,7 @@ pub fn format_ls_across_vms_on( hostname_proxy_port: None, zone_answerer_port: None, answerer_bound: false, + box_networks: Default::default(), }, None, None, @@ -1080,8 +1179,7 @@ pub fn format_ls_across_vms_on( let mut sessions = Vec::new(); let mut vms = Vec::new(); for listing in listings { - let reply = serde_json_lenient::to_value(&listing.resp) - .context("Failed to serialize session list")?; + let reply = listing_json(&listing.resp)?; let mut object = match reply { serde_json_lenient::Value::Object(map) => map, other => anyhow::bail!("session list did not serialize to an object: {other}"), @@ -1208,7 +1306,11 @@ pub fn format_ls_across_vms_on( writeln!( out, "BOX NAMES: {vm: = listings + .iter() + .flat_map(|listing| { + listing.resp.sessions.iter().map(|entry| { + ( + listing.vm.as_str(), + session_cells(entry, listing.resp.box_networks.get(&entry.id)), + ) + }) + }) + .collect(); + let egress_width = egress_column_width(rows.iter().map(|(_, row)| row)); writeln!( out, - "{: &'static str { + match posture { + minimald_rpc::EgressPosture::DenyAllDefault => "deny-all (default)", + minimald_rpc::EgressPosture::AllowAllDefault => "allow-all (default)", + minimald_rpc::EgressPosture::DenyAll => "deny-all", + minimald_rpc::EgressPosture::DeclaredRules => "declared rules", + minimald_rpc::EgressPosture::NoNetwork => "no network", + } +} + +/// The EGRESS cell `min ls` prints for a box (T110): the posture in +/// [`egress_posture_words`]' words, with `; per-box enforcement none` +/// appended for a host-address box this host cannot decide per box, the +/// fact `min session policy` prints as its `per-box enforcement` row. A +/// posture the daemon did not report prints `-`. +#[must_use] +pub fn egress_posture_cell( + posture: Option, + host_ip_enforcement: Option, +) -> String { + let Some(posture) = posture else { + return "-".to_string(); + }; + let words = egress_posture_words(posture); + match host_ip_enforcement { + Some(sessions::HostIpEnforcement::None) => format!("{words}; per-box enforcement none"), + _ => words.to_string(), + } +} + /// Render a session's effective policy as its rules: the egress the gate /// enforces (NET-074/NET-075) — a declared section's dimensions each /// resolved to its list or its default (`allow-all`; `deny subnets` reads @@ -2605,7 +2648,8 @@ pub fn format_policy( return Ok(()); } writeln!(out, "egress")?; - if let Some(label) = effective.egress.summary_label() { + let posture = minimald_rpc::EgressPosture::of(&effective.egress, network); + if posture != minimald_rpc::EgressPosture::DeclaredRules { // A verdict by name rather than as rows: the defaults marked as // defaults, and the declared deny-all section unmarked — every list // empty would render as blankness, which reads as nothing, the one @@ -2613,8 +2657,8 @@ pub fn format_policy( // nothing"). The mark is the difference between a verdict the box // chose and the one the rollout chose for it, and the JSON document // carries the same distinction as `source`. The label is the - // `sessions` crate's, shared with the TUI's detail pane. - writeln!(out, " {label}")?; + // posture's own words, shared with `min ls` (T110). + writeln!(out, " {}", egress_posture_words(posture))?; } else if let sessions::EffectiveEgress::Declared(egress) = &effective.egress { write_rules(out, "subnets", egress.allow_subnets.as_ref(), "allow-all")?; write_rules( diff --git a/crates/minimal/src/resolver.rs b/crates/minimal/src/resolver.rs index 22b54a60d2..3a3d4a0e0d 100644 --- a/crates/minimal/src/resolver.rs +++ b/crates/minimal/src/resolver.rs @@ -4767,17 +4767,15 @@ pub fn install_clause(unfinished: bool) -> &'static str { } /// The one host line a session start prints (NET-018), naming the box: -/// `names: .min.internal resolves on this machine; open it through -/// the HTTP proxy at 127.0.0.1:` where native DNS is deployed, and -/// `names: .min.internal opens through the HTTP proxy at -/// 127.0.0.1:` where it is not. The native form promises resolution -/// only: the name answers with a reserved local address, which is not a -/// connection a browser is promised, so the line names the proxy — which -/// keeps serving beside native DNS (NET-019) — as the path that opens the -/// box. Independently, the line carries the clause `; finish setup: min -/// finalize-install` while any finalize-install item is not done. One -/// line, never a block, so the activation's own output stays what the -/// operator reads. Pure, so tests assert the words without +/// `names: ` and then how the box name opens on this host, in the words +/// [`box_name_opens`] gives the `min ls` BOX NAMES row too. Where native +/// DNS is deployed that is `.min.internal opens at its name on this +/// machine; the HTTP proxy at 127.0.0.1: opens it too`, and where it +/// is not, `.min.internal opens through the HTTP proxy at +/// 127.0.0.1:`. Independently, the line carries the clause `; finish +/// setup: min finalize-install` while any finalize-install item is not +/// done. One line, never a block, so the activation's own output stays +/// what the operator reads. Pure, so tests assert the words without /// capturing stderr. [`host_line_beside`] is the same line with a /// proxy-down cause (NET-020) in place of the resolution half. #[must_use] @@ -4787,11 +4785,14 @@ pub fn host_line( proxy_port: Option, host: HostInstall, ) -> String { - host_line_beside(box_name, surface, proxy_port, host, None) + host_line_beside(box_name, surface, proxy_port, host, None, true) } /// [`host_line`] beside a proxy-down cause (NET-020), in both of the -/// line's forms: when the hostname proxy failed to bind — the VM host +/// line's forms, for a box whose ports answer at its own address +/// (`at_own_address`) or only at the `127.0.0.1` fallback, as the `min ls` +/// BOX NAMES row reads the same fact from the box's report. When the +/// hostname proxy failed to bind — the VM host /// daemon's sibling cause, or a native daemon's own report — the cause /// replaces the resolution half, `names: .min.internal: the hostname /// proxy is not serving: `, whatever the surface verdict says — the @@ -4807,30 +4808,17 @@ pub fn host_line_beside( proxy_port: Option, host: HostInstall, proxy_cause: Option<&str>, + at_own_address: bool, ) -> String { let name = format!("{box_name}.{ZONE}"); // The not-serving arms keep the box named: the line's contract is one // host line naming the box, and a reader whose proxy is down is the one - // who most needs to know which name the cause is about. - let names = match (proxy_cause, surface) { - (Some(cause), _) => format!("{name}: the hostname proxy is not serving: {cause}"), - (None, LiveSurface::Native) => match proxy_port { - Some(port) => format!( - "{name} resolves on this machine; open it through the HTTP proxy at \ - 127.0.0.1:{port}" - ), - None => format!("{name} resolves on this machine"), - }, - (None, LiveSurface::Proxy) => match proxy_port { - Some(port) => format!("{name} opens through the HTTP proxy at 127.0.0.1:{port}"), - None => format!("{name}: the hostname proxy is not serving"), - }, - (None, LiveSurface::ProxyNotServing { port, cause }) => { - format!( - "{name}: the hostname proxy is not serving: {}", - proxy_down_detail(*port, cause) - ) - } + // who most needs to know which name the cause is about. The caller + // says where the activated box's ports answer, so a box on the + // `127.0.0.1` fallback reads the fallback form, as its `min ls` row does. + let names = match proxy_cause { + Some(cause) => format!("{name}: the hostname proxy is not serving: {cause}"), + None => box_name_opens(&name, surface, proxy_port, at_own_address), }; format!( "names: {names}{}", @@ -4838,6 +4826,58 @@ pub fn host_line_beside( ) } +/// How a box name opens on this host (T110): the one statement the +/// activate host line ([`host_line_beside`]) and the `min ls` BOX NAMES row +/// ([`name_surface_line_at`]) both print, so the two verbs say it in the +/// same words. `name` is the name as the line prints it, such as +/// `web.min.internal` or `.min.internal`. +/// +/// With native DNS live, the words depend on where the box's ports answer. +/// At the box's own address (`at_own_address`) the name opens directly and +/// the proxy opens it too. At the `127.0.0.1` fallback the name only +/// resolves, and the proxy is the path that opens it. Without native DNS +/// the proxy is the only path, and the not-serving arms name the box and +/// then the cause. +#[must_use] +pub fn box_name_opens( + name: &str, + surface: &LiveSurface, + proxy_port: Option, + at_own_address: bool, +) -> String { + match (surface, proxy_port) { + (LiveSurface::Native, Some(port)) if at_own_address => format!( + "{name} opens at its name on this machine; the HTTP proxy at \ + 127.0.0.1:{port} opens it too" + ), + (LiveSurface::Native, None) if at_own_address => { + format!("{name} opens at its name on this machine") + } + (LiveSurface::Native, Some(port)) => format!( + "{name} resolves on this machine; open it through the HTTP proxy at \ + 127.0.0.1:{port}" + ), + (LiveSurface::Native, None) => format!("{name} resolves on this machine"), + (LiveSurface::Proxy, Some(port)) => { + format!("{name} opens through the HTTP proxy at 127.0.0.1:{port}") + } + (LiveSurface::Proxy, None) => format!("{name}: the hostname proxy is not serving"), + (LiveSurface::ProxyNotServing { port, cause }, _) => { + let detail = proxy_down_detail(*port, cause); + match cause { + ProxyDownCause::PortHeld + | ProxyDownCause::RedrawsRanOut + | ProxyDownCause::PortHeldAfterStart { .. } => { + format!("{name}: the hostname proxy is not serving: {detail}") + } + ProxyDownCause::PublishUnconfirmed => { + format!("{name}: the hostname proxy is not confirmed yet: {detail}") + } + } + } + } +} + /// The proxy-down cause a session start's host line names (NET-020), from /// either report the create reply can carry: the VM host daemon's sibling /// cause, in the words [`name_surface_line`]'s folded arm uses, or a native @@ -4894,55 +4934,39 @@ fn proxy_down_detail(port: u16, cause: &ProxyDownCause) -> String { } } -/// NET-018's report: the line `min ls` and `min session activate` print, -/// naming the surface [`live_name_surfaces`] decided is live. `proxy_port` -/// is the port the same reply carries, when the proxy came up: NET-019 -/// keeps it serving beside native DNS, and the line says so, because a -/// client that captured `HTTP(S)_PROXY` at activation keeps routing -/// through it — the export does not go stale when the surface changes. -/// With no port the proxy is down, and the line says that in the daemon's -/// own words rather than claiming it still serves. Pure, so both verbs -/// print the same words and tests assert them without capturing output. +/// NET-018's report: the BOX NAMES row `min ls` prints, naming the surface +/// [`live_name_surfaces`] decided is live, in the words the activate host +/// line uses ([`box_name_opens`]) for `.min.internal`. `proxy_port` is +/// the port the same reply carries, when the proxy came up: NET-019 keeps +/// it serving beside native DNS, and the row says so. With no port the +/// proxy is down, and the row says that rather than claiming it still +/// serves. A native verdict reads the own-address form, because it holds +/// the reserved range on this host's loopback. Pure, so tests assert the +/// words without capturing output. #[must_use] pub fn name_surface_line(surface: LiveSurface, proxy_port: Option) -> String { - let proxy_half = match proxy_port { - Some(port) => format!("; the hostname proxy also listens on 127.0.0.1:{port}"), - None => "; the hostname proxy is not serving".to_string(), - }; - match surface { - LiveSurface::Native => { - format!(".min.internal resolves through this machine's DNS{proxy_half}") - } - // The proxy as the live surface is the host not set up for native - // names (NET-122's opt-in): the install clause `min ls` appends to - // this row names the step that is, and nothing on either verb's - // path prints or runs it. - LiveSurface::Proxy => match proxy_port { - Some(port) => { - format!(".min.internal opens through the HTTP proxy at 127.0.0.1:{port}") - } - None => { - "the hostname proxy is not serving, and box names open only through it".to_string() - } - }, - // The named cause replaces the bare "not serving" (T93), in the - // words [`proxy_down_detail`] spells for both verbs. The port comes - // from the verdict, not `proxy_port`: a reply whose proxy never - // published carries no serving port to name. - LiveSurface::ProxyNotServing { port, cause } => { - let detail = proxy_down_detail(port, &cause); - match cause { - ProxyDownCause::PortHeld - | ProxyDownCause::RedrawsRanOut - | ProxyDownCause::PortHeldAfterStart { .. } => { - format!("the hostname proxy is not serving: {detail}") - } - ProxyDownCause::PublishUnconfirmed => { - format!("the hostname proxy is not confirmed yet: {detail}") - } - } - } - } + name_surface_line_at(surface, proxy_port, true) +} + +/// [`name_surface_line`] for a listing that knows where its boxes' ports +/// answer (T110). The row speaks for every box, so pass `at_own_address` +/// as `true` only when the listing proves a box at its own address and +/// reports none at the `127.0.0.1` fallback. A listing that reports no +/// fallback fact, such as one from a daemon that predates the fact, cannot +/// prove the stronger claim: pass `false`, and the row reads the fallback +/// form. Only [`name_surface_line`] defaults to the own-address form. +#[must_use] +pub fn name_surface_line_at( + surface: LiveSurface, + proxy_port: Option, + at_own_address: bool, +) -> String { + box_name_opens( + &format!(".{ZONE}"), + &surface, + proxy_port, + at_own_address, + ) } /// The reserved range as `network/prefix`, the form the advisory and the @@ -5690,8 +5714,8 @@ mod tests { let names_done = host_line("web", &LiveSurface::Native, Some(7654), others_open); assert_eq!( names_done, - "names: web.min.internal resolves on this machine; open it through the HTTP proxy \ - at 127.0.0.1:7654; finish setup: min finalize-install" + "names: web.min.internal opens at its name on this machine; the HTTP proxy at \ + 127.0.0.1:7654 opens it too; finish setup: min finalize-install" ); let finished = host_line( "web", @@ -5701,8 +5725,8 @@ mod tests { ); assert_eq!( finished, - "names: web.min.internal resolves on this machine; open it through the HTTP proxy \ - at 127.0.0.1:7654" + "names: web.min.internal opens at its name on this machine; the HTTP proxy at \ + 127.0.0.1:7654 opens it too" ); // NET-020: a proxy-down cause replaces the resolution half, and the // rest of the line is unchanged. @@ -5737,6 +5761,7 @@ mod tests { Some(7654), HostInstall::default(), Some(&cause), + true, ); assert_eq!( native_down, @@ -5749,6 +5774,7 @@ mod tests { Some(7654), others_open, Some(&cause), + true, ); assert_eq!( native_down_open, @@ -5768,6 +5794,7 @@ mod tests { Some(7654), HostInstall::default(), Some(&daemon_cause), + true, ); assert_eq!( proxy_down_native_daemon, @@ -5787,7 +5814,14 @@ mod tests { ); assert_eq!(host_line_proxy_cause(None, None), None); assert_eq!( - host_line_beside("web", &LiveSurface::Native, Some(7654), others_open, None), + host_line_beside( + "web", + &LiveSurface::Native, + Some(7654), + others_open, + None, + true + ), names_done, "no cause: the same line host_line prints" ); @@ -5809,11 +5843,12 @@ mod tests { } } - /// NET-142: the resolution clause is keyed on the host-DNS state alone. - /// Native DNS deployed, the box name resolves on this machine and opens - /// through the proxy, whatever else is open — never a promise that any - /// browser connects to the name itself; not deployed, the name routes - /// via the proxy's port, and the native words never print. + /// NET-142, T110: the resolution clause is keyed on the host-DNS state. + /// Native DNS deployed, a box whose ports answer at its own address + /// opens at its name and the proxy opens it too. One that answers only + /// at the `127.0.0.1` fallback resolves, and opens through the proxy. + /// Not deployed, the name routes via the proxy's port, and the native + /// words never print. #[test] fn activate_host_line_names_native_dns_when_deployed() { for host in [ @@ -5823,21 +5858,33 @@ mod tests { }, ] { let native = host_line("api", &LiveSurface::Native, Some(7654), host); - assert!( - native.starts_with( - "names: api.min.internal resolves on this machine; open it through the \ - HTTP proxy at 127.0.0.1:7654" + assert_eq!( + native, + format!( + "names: api.min.internal opens at its name on this machine; the HTTP \ + proxy at 127.0.0.1:7654 opens it too{}", + install_clause(host.other_items_unfinished) ), - "{native}" ); assert!(!native.contains("any browser"), "{native}"); assert_eq!( host_line("api", &LiveSurface::Native, None, host), format!( - "names: api.min.internal resolves on this machine{}", + "names: api.min.internal opens at its name on this machine{}", install_clause(host.other_items_unfinished) ), - "with no proxy port the native line claims resolution only" + "with no proxy port the native line names no proxy" + ); + // The `127.0.0.1` fallback: the name resolves, and the proxy + // is the path that opens it. + assert_eq!( + box_name_opens("api.min.internal", &LiveSurface::Native, Some(7654), false), + "api.min.internal resolves on this machine; open it through the HTTP proxy \ + at 127.0.0.1:7654" + ); + assert_eq!( + box_name_opens("api.min.internal", &LiveSurface::Native, None, false), + "api.min.internal resolves on this machine" ); let proxy = host_line("api", &LiveSurface::Proxy, Some(7654), host); assert!( @@ -6716,35 +6763,120 @@ exit 0 #[test] fn name_surface_line_says_which_surface_and_where_the_proxy_serves() { let native = name_surface_line(LiveSurface::Native, Some(15390)); - assert!( - native.contains(".min.internal resolves through this machine's DNS"), - "the native arm names the surface: {native}" - ); - assert!( - native.contains("the hostname proxy also listens on 127.0.0.1:15390"), - "the native arm says the proxy keeps serving beside it (NET-019): {native}" + assert_eq!( + native, + ".min.internal opens at its name on this machine; the HTTP proxy at \ + 127.0.0.1:15390 opens it too", + "the native arm names the surface and the proxy serving beside it (NET-019)" ); - let not_serving = name_surface_line(LiveSurface::Native, None); + let no_port = name_surface_line(LiveSurface::Native, None); assert!( - not_serving.contains("the hostname proxy is not serving"), - "a daemon with no proxy port is one whose proxy is not serving: {not_serving}" + !no_port.contains("HTTP proxy"), + "with no port to name, the line must not claim the proxy serves: {no_port}" ); - assert!( - !not_serving.contains("also listens"), - "with no port to name, the line must not claim the proxy serves: {not_serving}" + assert_eq!( + name_surface_line_at(LiveSurface::Native, Some(15390), false), + ".min.internal resolves on this machine; open it through the HTTP proxy \ + at 127.0.0.1:15390", + "a listing whose boxes answer only at the fallback reads the resolve form" ); let proxy = name_surface_line(LiveSurface::Proxy, Some(15390)); - assert!( - proxy.contains(".min.internal opens through the HTTP proxy at"), - "the proxy arm names the surface: {proxy}" + assert_eq!( + proxy, + ".min.internal opens through the HTTP proxy at 127.0.0.1:15390" ); - assert!( - proxy.contains("opens through the HTTP proxy at 127.0.0.1:15390"), - "the proxy arm names where it serves: {proxy}" + assert_eq!( + name_surface_line(LiveSurface::Proxy, None), + ".min.internal: the hostname proxy is not serving" ); - assert!( - !proxy.contains("resolves through this machine's DNS"), - "the proxy arm must not say the native words: {proxy}" + } + + /// T110: the `min ls` BOX NAMES row states how a box name opens in the + /// same words the activate host line uses, for every surface: with the + /// box's ports at its own address, at the `127.0.0.1` fallback, through + /// the proxy alone, and with the proxy not serving. The row says + /// `` where the line names the box, and the finish-setup clause + /// is the same on both. + #[test] + fn ls_name_surface_matches_activate_host_line() { + let held = LiveSurface::ProxyNotServing { + port: 7654, + cause: ProxyDownCause::PortHeld, + }; + let unconfirmed = LiveSurface::ProxyNotServing { + port: 7654, + cause: ProxyDownCause::PublishUnconfirmed, + }; + for surface in [ + LiveSurface::Native, + LiveSurface::Proxy, + held.clone(), + unconfirmed, + ] { + for port in [Some(7654), None] { + for host in [ + HostInstall::default(), + HostInstall { + other_items_unfinished: true, + }, + ] { + for at_own_address in [true, false] { + let activate = + host_line_beside("web", &surface, port, host, None, at_own_address); + let row = format!( + "{}{}", + name_surface_line_at(surface.clone(), port, at_own_address), + install_clause(install_unfinished(&surface, host)) + ); + assert_eq!( + activate, + format!("names: {}", row.replace("", "web")), + "{surface:?} {port:?} {host:?} {at_own_address}" + ); + } + assert_eq!( + host_line("web", &surface, port, host), + host_line_beside("web", &surface, port, host, None, true), + ); + } + } + } + assert_eq!( + host_line( + "web", + &LiveSurface::Native, + Some(7654), + HostInstall::default() + ), + "names: web.min.internal opens at its name on this machine; the HTTP proxy at \ + 127.0.0.1:7654 opens it too" + ); + assert_eq!( + name_surface_line(LiveSurface::Native, Some(7654)), + ".min.internal opens at its name on this machine; the HTTP proxy at \ + 127.0.0.1:7654 opens it too" + ); + assert_eq!( + name_surface_line_at(LiveSurface::Native, Some(7654), false), + ".min.internal resolves on this machine; open it through the HTTP proxy \ + at 127.0.0.1:7654" + ); + assert_eq!( + host_line_beside( + "web", + &LiveSurface::Native, + Some(7654), + HostInstall::default(), + None, + false + ), + "names: web.min.internal resolves on this machine; open it through the HTTP \ + proxy at 127.0.0.1:7654" + ); + assert_eq!( + name_surface_line(held, None), + ".min.internal: the hostname proxy is not serving: another program on \ + this machine holds 127.0.0.1:7654" ); } @@ -6898,7 +7030,7 @@ exit 0 Some(19_917), ); assert!( - line.starts_with("the hostname proxy is not confirmed yet"), + line.starts_with(".min.internal: the hostname proxy is not confirmed yet"), "the row says the publish is unconfirmed: {line}" ); assert!( @@ -6934,8 +7066,9 @@ exit 0 ); assert_eq!( line, - "the hostname proxy is not serving: the VM is up without it because \ - pid 4242 (python3) holds 127.0.0.1:19918; free the port, then run \ + ".min.internal: the hostname proxy is not serving: the VM is up \ + without it because pid 4242 (python3) holds 127.0.0.1:19918; free the \ + port, then run \ `min stop` and `min ls` to restart the VM (with the same --vm or \ --provider flags, if you use them)", "the row names the port, the holder, and that the VM is up" diff --git a/crates/minimal/src/tests.rs b/crates/minimal/src/tests.rs index ff17cb8651..4ad01897ef 100644 --- a/crates/minimal/src/tests.rs +++ b/crates/minimal/src/tests.rs @@ -2792,7 +2792,7 @@ async fn ls_shows_vm_per_box() { let api = create_box_on(&default_vm, "api").await; let web = create_box_on(&alpha, "web").await; - let listings = cmd::ls_listings(&vm_globals(state.path(), None)) + let mut listings = cmd::ls_listings(&vm_globals(state.path(), None)) .await .expect("listing a two-VM host succeeds"); assert_eq!(listings.len(), 2, "a two-VM host lists both VMs"); @@ -2813,6 +2813,16 @@ async fn ls_shows_vm_per_box() { // machine's port says so while a sibling registered with another // daemon names that holder instead — one `Holder` and one `Registered` // here, fed the same way. + // The named VM's box reports its own range address, as a published box + // does (T110), so its BOX NAMES row can read the own-address form. + listings[1].resp.box_networks.insert( + web, + minimald_rpc::BoxNetworkFacts { + loopback_address: Some(std::net::Ipv4Addr::new(127, 0, 64, 2)), + egress_posture: None, + at_fallback: Some(false), + }, + ); let surfaces = vec![ Some(crate::resolver::LiveSurface::Proxy), Some(crate::resolver::LiveSurface::Native), @@ -2864,11 +2874,11 @@ async fn ls_shows_vm_per_box() { surface_line_of("default") .contains(".min.internal opens through the HTTP proxy at 127.0.0.1:") || surface_line_of("default") - .contains("the hostname proxy is not serving, and box names open only through it"), + .contains(".min.internal: the hostname proxy is not serving"), "the VM the hook does not route to keeps its proxy line:\n{table}" ); assert!( - surface_line_of("alpha").contains("resolves through this machine's DNS"), + surface_line_of("alpha").contains(".min.internal opens at its name on this machine"), "the VM the hook routes to is named native:\n{table}" ); let answerer_row_of = |vm: &str| { @@ -3739,3 +3749,342 @@ fn an_exact_name_wins_and_an_ambiguous_fold_resolves_to_none() { Some(entries[2].id) ); } + +/// The cell under `column` in the `min ls` row that contains `row_key`, +/// read by the header's positions up to `next`, the column after it. A +/// cell can hold spaces, so the row is not split on whitespace. +fn ls_cell_at(text: &str, row_key: &str, column: &str, next: &str) -> String { + let header = text + .lines() + .find(|l| l.contains("SESSION ID")) + .unwrap_or_else(|| panic!("a header in:\n{text}")); + let start = header.find(column).expect("the column in the header"); + let end = header.find(next).expect("the next column in the header"); + let row = text + .lines() + .find(|l| l.contains(row_key) && !l.contains("SESSION ID")) + .unwrap_or_else(|| panic!("a row for {row_key} in:\n{text}")); + row[start..end].trim().to_string() +} + +/// A listing of one box per entry in `boxes`: its name, its network facts +/// and its per-box enforcement. +fn listing_of( + boxes: &[( + &str, + Option, + Option, + )], +) -> minimald_rpc::ListSessionsResponse { + let id = |n: usize| { + sessions::SessionId::parse_str(&format!("00000000-0000-0000-0000-{n:012}")).unwrap() + }; + minimald_rpc::ListSessionsResponse { + resource_pool: None, + sessions: boxes + .iter() + .enumerate() + .map(|(n, (name, _, enforcement))| { + let mut entry = twin_entry( + &id(n + 1).to_string(), + Some(name), + Some("/work/proj"), + sessions::SessionStatus::Active, + ); + entry.host_ip_enforcement = *enforcement; + entry + }) + .collect(), + daemon_version: None, + hostname_routing_unavailable: None, + hostname_proxy_port: None, + zone_answerer_port: None, + answerer_bound: false, + box_networks: boxes + .iter() + .enumerate() + .filter_map(|(n, (_, facts, _))| facts.clone().map(|facts| (id(n + 1), facts))) + .collect(), + } +} + +/// T110: the BOX NAMES row speaks for every listed box, so it reads the +/// `127.0.0.1` fallback form as soon as one box reports the fallback, even +/// beside a box at its own address. A host-address box at the node address +/// `127.0.0.1` is not at the fallback, so it keeps the own-address form. +#[test] +fn ls_shows_box_loopback_address_names_row_follows_any_fallback_box() { + let at = |address: std::net::Ipv4Addr, at_fallback: bool| { + Some(minimald_rpc::BoxNetworkFacts { + loopback_address: Some(address), + egress_posture: None, + at_fallback: Some(at_fallback), + }) + }; + let own = std::net::Ipv4Addr::new(127, 0, 64, 2); + let localhost = std::net::Ipv4Addr::LOCALHOST; + let row = |boxes: &[( + &str, + Option, + Option, + )]| { + let mut resp = listing_of(boxes); + resp.hostname_proxy_port = Some(7654); + resp.answerer_bound = true; + let mut out = Vec::new(); + format_ls( + &mut out, + &LsArgs { + raw: false, + json: false, + }, + &resp, + Some(crate::resolver::LiveSurface::Native), + None, + ) + .expect("rendering the listing"); + let table = String::from_utf8(out).expect("the listing is UTF-8"); + table + .lines() + .find(|l| l.starts_with("BOX NAMES:")) + .unwrap_or_else(|| panic!("a BOX NAMES row in:\n{table}")) + .to_string() + }; + let opens = ".min.internal opens at its name on this machine; the HTTP proxy at \ + 127.0.0.1:7654 opens it too"; + let resolves = ".min.internal resolves on this machine; open it through the HTTP \ + proxy at 127.0.0.1:7654"; + assert_eq!( + row(&[ + ("own", at(own, false), None), + ("fallback", at(localhost, true), None) + ]), + format!("BOX NAMES: {resolves}"), + "a mixed listing reads the fallback form" + ); + assert_eq!( + row(&[("host", at(localhost, false), None)]), + format!("BOX NAMES: {opens}"), + "a host-address box at 127.0.0.1 is not at the fallback" + ); + assert_eq!( + row(&[("own", at(own, false), None)]), + format!("BOX NAMES: {opens}") + ); + assert_eq!( + row(&[("unreported", None, None)]), + format!("BOX NAMES: {resolves}"), + "a listing that proves no box off the fallback reads the fallback form" + ); + assert_eq!( + row(&[]), + format!("BOX NAMES: {resolves}"), + "a listing with no box reads the fallback form" + ); +} + +/// T110: `min ls` shows each box's address in an ADDRESS column after NAME +/// — its own address, the `127.0.0.1` fallback, or `-` when the daemon +/// did not report one — and `--json` carries it per session as +/// `loopback_address`, with no key for a box that reported none. +#[test] +fn ls_shows_box_loopback_address() { + let at = |address: std::net::Ipv4Addr| { + Some(minimald_rpc::BoxNetworkFacts { + loopback_address: Some(address), + egress_posture: None, + at_fallback: Some(address == std::net::Ipv4Addr::LOCALHOST), + }) + }; + let resp = listing_of(&[ + ("own", at(std::net::Ipv4Addr::new(127, 0, 64, 2)), None), + ("fallback", at(std::net::Ipv4Addr::LOCALHOST), None), + ("unreported", None, None), + ]); + let render = |json: bool, listings: &[cmd::VmListing]| { + let mut out = Vec::new(); + format_ls_across_vms(&mut out, &LsArgs { raw: false, json }, listings, &[], &[]) + .expect("rendering the listing"); + String::from_utf8(out).expect("the listing is UTF-8") + }; + let one = [cmd::VmListing { + vm: "default".to_string(), + resp: resp.clone(), + control_sock: None, + }]; + + let table = render(false, &one); + let header = table + .lines() + .find(|l| l.contains("SESSION ID")) + .expect("a header"); + assert!( + header.find("NAME") < header.find("ADDRESS") + && header.find("ADDRESS") < header.find("STATUS"), + "ADDRESS sits after NAME: {header}" + ); + let address = |name: &str| ls_cell_at(&table, name, "ADDRESS", "STATUS"); + assert_eq!(address("own"), "127.0.64.2", "got:\n{table}"); + assert_eq!(address("fallback"), "127.0.0.1", "got:\n{table}"); + assert_eq!(address("unreported"), "-", "got:\n{table}"); + + let document: serde_json_lenient::Value = + serde_json_lenient::from_str(&render(true, &one)).expect("the listing is JSON"); + let sessions = document["sessions"].as_array().expect("a sessions array"); + assert_eq!(sessions[0]["loopback_address"], "127.0.64.2"); + assert_eq!(sessions[1]["loopback_address"], "127.0.0.1"); + assert!( + sessions[2].get("loopback_address").is_none(), + "an unreported address is no key, never an invented value: {document}" + ); + assert!( + document.get("box_networks").is_none(), + "the facts ride on each session, not beside them: {document}" + ); + + // The multi-VM document carries the same field on each session. + let two = [ + cmd::VmListing { + vm: "default".to_string(), + resp: resp.clone(), + control_sock: None, + }, + cmd::VmListing { + vm: "alpha".to_string(), + resp: listing_of(&[("other", at(std::net::Ipv4Addr::new(127, 0, 64, 9)), None)]), + control_sock: None, + }, + ]; + let document: serde_json_lenient::Value = + serde_json_lenient::from_str(&render(true, &two)).expect("the listing is JSON"); + let other = document["sessions"] + .as_array() + .expect("a sessions array") + .iter() + .find(|s| s["name"] == "other") + .expect("the alpha VM's box"); + assert_eq!(other["loopback_address"], "127.0.64.9", "{document}"); + assert_eq!(other["vm"], "alpha", "{document}"); +} + +/// T110: the EGRESS cell `min ls` prints for a box is the posture `min +/// session policy` prints for the same box, in the same words: the named +/// verdict, `declared rules` for a section listed as rows, and the per-box +/// enforcement note for a host-address box this host cannot decide per +/// box. +#[test] +fn ls_egress_matches_session_policy() { + use sessions::{EffectiveEgress, EgressPolicy, HostIpEnforcement, NetworkMode}; + let rules = EgressPolicy { + allow_subnets: Some(vec!["10.0.0.0/8".to_string()]), + allow_dns_hosts: Some(Vec::new()), + ..EgressPolicy::default() + }; + let cases = [ + ( + "own-default", + EffectiveEgress::DenyAll, + NetworkMode::OwnIp, + None, + "deny-all (default)", + ), + ( + "host-default", + EffectiveEgress::AllowAll, + NetworkMode::HostNet, + Some(HostIpEnforcement::PerBox), + "allow-all (default)", + ), + ( + "host-unenforced", + EffectiveEgress::Declared(EgressPolicy::deny_all()), + NetworkMode::HostNet, + Some(HostIpEnforcement::None), + "deny-all; per-box enforcement none", + ), + ( + "own-rules", + EffectiveEgress::Declared(rules), + NetworkMode::OwnIp, + None, + "declared rules", + ), + ]; + let resp = listing_of( + &cases + .iter() + .map(|(name, egress, network, enforcement, _)| { + ( + *name, + Some(minimald_rpc::BoxNetworkFacts { + loopback_address: None, + egress_posture: Some(minimald_rpc::EgressPosture::of(egress, *network)), + at_fallback: None, + }), + *enforcement, + ) + }) + .collect::>(), + ); + let mut out = Vec::new(); + format_ls( + &mut out, + &LsArgs { + raw: false, + json: false, + }, + &resp, + None, + None, + ) + .expect("rendering the listing"); + let table = String::from_utf8(out).expect("the listing is UTF-8"); + + for (name, egress, network, enforcement, words) in cases { + let cell = ls_cell_at(&table, name, "EGRESS", "TITLE"); + assert_eq!(cell, words, "{name}:\n{table}"); + let mut policy = Vec::new(); + format_policy( + &mut policy, + &sessions::EffectiveSessionPolicy { + egress, + ..Default::default() + }, + network, + enforcement, + &[], + None, + ) + .expect("rendering the policy"); + let policy = String::from_utf8(policy).expect("the policy is UTF-8"); + let (posture, note) = match cell.split_once("; ") { + Some((posture, note)) => (posture, Some(note)), + None => (cell.as_str(), None), + }; + if posture == "declared rules" { + assert!( + policy.contains("egress\n subnets "), + "a declared-rules box lists its rows in the policy view:\n{policy}" + ); + } else { + assert!( + policy.contains(&format!("egress\n {posture}\n")), + "{name}: the policy view names the same posture:\n{policy}" + ); + } + match note { + Some(note) => assert_eq!(note, "per-box enforcement none"), + None => assert!( + !policy.contains("per-box enforcement none"), + "{name}: only an unenforced box carries the note:\n{policy}" + ), + } + if note.is_some() { + assert!( + policy.contains(" per-box enforcement none\n"), + "{name}: the policy view states the same enforcement:\n{policy}" + ); + } + } + assert!(!table.contains("per_box"), "got:\n{table}"); +} diff --git a/crates/minimal/tests/cli.rs b/crates/minimal/tests/cli.rs index 59fade710d..156b28fdca 100644 --- a/crates/minimal/tests/cli.rs +++ b/crates/minimal/tests/cli.rs @@ -162,6 +162,7 @@ async fn finalize_install_show_returns_within_its_deadline_on_a_wedged_daemon() #[test] fn ls_shows_shared_resource_pool() { let resp = ListSessionsResponse { + box_networks: Default::default(), daemon_version: None, hostname_routing_unavailable: None, hostname_proxy_port: None, @@ -205,6 +206,7 @@ fn ls_shows_shared_resource_pool() { #[test] fn ls_table_exposes_project_path_and_status() { let resp = ListSessionsResponse { + box_networks: Default::default(), daemon_version: None, hostname_routing_unavailable: None, hostname_proxy_port: None, @@ -249,14 +251,33 @@ fn ls_table_exposes_project_path_and_status() { ); } -/// NET-079's proof names the listing: a host-address box that runs -/// unenforced shows egress enforcement `none` in the human `min ls`, not -/// only in `--json`; one decided per box shows `per_box`; a box the daemon -/// reports no enforcement for shows `-`. +/// The cell under `column` in the row that contains `row_key`, read by +/// the header's positions: `column` up to `next`, the column after it. A +/// cell can hold spaces, so the row is not split on whitespace. +fn ls_cell(text: &str, row_key: &str, column: &str, next: &str) -> String { + let header = text + .lines() + .find(|l| l.contains("SESSION ID")) + .unwrap_or_else(|| panic!("a header in:\n{text}")); + let start = header.find(column).expect("the column in the header"); + let end = header.find(next).expect("the next column in the header"); + let row = text + .lines() + .find(|l| l.contains(row_key) && !l.contains("SESSION ID")) + .unwrap_or_else(|| panic!("a row for {row_key} in:\n{text}")); + row[start..end].trim().to_string() +} + +/// NET-079's proof names the listing, in `min session policy`'s words +/// (T110): a host-address box that runs unenforced shows its posture with +/// `; per-box enforcement none` in the human `min ls`, one decided per box +/// shows its posture alone, and a box the daemon reports no posture for +/// shows `-`. #[test] fn ls_table_shows_host_address_enforcement() { + let id = |n: u64| SessionId::parse_str(&format!("00000000-0000-0000-0000-{n:012}")).unwrap(); let entry = |name: &str, n: u64, enforcement| minimald_rpc::ListSessionsEntry { - id: SessionId::parse_str(&format!("00000000-0000-0000-0000-{n:012}")).unwrap(), + id: id(n), name: Some(name.to_string()), project_path: Some(paths::HostAbsPath::try_new("/work/proj").unwrap()), status: sessions::SessionStatus::Active, @@ -265,7 +286,18 @@ fn ls_table_shows_host_address_enforcement() { shared_port_collisions: Vec::new(), attrs: None, }; + let posture = |posture| minimald_rpc::BoxNetworkFacts { + loopback_address: Some(std::net::Ipv4Addr::LOCALHOST), + egress_posture: Some(posture), + at_fallback: Some(true), + }; let resp = ListSessionsResponse { + box_networks: [ + (id(1), posture(minimald_rpc::EgressPosture::DenyAll)), + (id(2), posture(minimald_rpc::EgressPosture::AllowAllDefault)), + ] + .into_iter() + .collect(), daemon_version: None, hostname_routing_unavailable: None, hostname_proxy_port: None, @@ -293,18 +325,15 @@ fn ls_table_shows_host_address_enforcement() { .unwrap(); let text = String::from_utf8(out).unwrap(); - let cells_of = |name: &str| -> Vec { - text.lines() - .find(|l| l.contains(name)) - .unwrap_or_else(|| panic!("a row for {name} in:\n{text}")) - .split_whitespace() - .map(str::to_string) - .collect() - }; - assert!(text.contains("EGRESS"), "header should list EGRESS: {text}"); - assert_eq!(cells_of("decided")[3], "per_box", "got:\n{text}"); - assert_eq!(cells_of("unenforced")[3], "none", "got:\n{text}"); - assert_eq!(cells_of("own-address")[3], "-", "got:\n{text}"); + let egress = |name: &str| ls_cell(&text, name, "EGRESS", "TITLE"); + assert_eq!(egress("decided"), "deny-all", "got:\n{text}"); + assert_eq!( + egress("unenforced"), + "allow-all (default); per-box enforcement none", + "got:\n{text}" + ); + assert_eq!(egress("own-address"), "-", "got:\n{text}"); + assert!(!text.contains("per_box"), "got:\n{text}"); } /// NET-129's listing half: a box that yields a declared port at a shared @@ -329,6 +358,7 @@ fn ls_reports_yielded_shared_address_ports() { held_by: "first.min.internal".to_string(), }; let resp = ListSessionsResponse { + box_networks: Default::default(), daemon_version: None, hostname_routing_unavailable: None, hostname_proxy_port: None, @@ -385,12 +415,13 @@ fn ls_reports_yielded_shared_address_ports() { ); } -/// The multi-VM table carries the same EGRESS cell, one column right of the -/// single-VM one because each row leads with its VM. +/// The multi-VM table carries the same EGRESS cell, after the VM column +/// each row leads with. #[test] fn ls_across_vms_table_shows_host_address_enforcement() { + let id = |n: u64| SessionId::parse_str(&format!("00000000-0000-0000-0000-{n:012}")).unwrap(); let entry = |name: &str, n: u64, enforcement| minimald_rpc::ListSessionsEntry { - id: SessionId::parse_str(&format!("00000000-0000-0000-0000-{n:012}")).unwrap(), + id: id(n), name: Some(name.to_string()), project_path: Some(paths::HostAbsPath::try_new("/work/proj").unwrap()), status: sessions::SessionStatus::Active, @@ -399,9 +430,20 @@ fn ls_across_vms_table_shows_host_address_enforcement() { shared_port_collisions: Vec::new(), attrs: None, }; - let listing = |vm: &str, sessions| VmListing { + let posture = |n: u64, posture| { + ( + id(n), + minimald_rpc::BoxNetworkFacts { + loopback_address: None, + egress_posture: Some(posture), + at_fallback: None, + }, + ) + }; + let listing = |vm: &str, sessions, box_networks| VmListing { vm: vm.to_string(), resp: ListSessionsResponse { + box_networks, daemon_version: None, hostname_routing_unavailable: None, hostname_proxy_port: None, @@ -419,6 +461,9 @@ fn ls_across_vms_table_shows_host_address_enforcement() { entry("decided", 1, Some(minimald_rpc::HostIpEnforcement::PerBox)), entry("own-address", 3, None), ], + [posture(1, minimald_rpc::EgressPosture::AllowAllDefault)] + .into_iter() + .collect(), ), listing( "alpha", @@ -427,6 +472,9 @@ fn ls_across_vms_table_shows_host_address_enforcement() { 2, Some(minimald_rpc::HostIpEnforcement::None), )], + [posture(2, minimald_rpc::EgressPosture::DeclaredRules)] + .into_iter() + .collect(), ), ]; let mut out = Vec::new(); @@ -444,18 +492,15 @@ fn ls_across_vms_table_shows_host_address_enforcement() { .unwrap(); let text = String::from_utf8(out).unwrap(); - let cells_of = |name: &str| -> Vec { - text.lines() - .find(|l| l.contains(name)) - .unwrap_or_else(|| panic!("a row for {name} in:\n{text}")) - .split_whitespace() - .map(str::to_string) - .collect() - }; - assert!(text.contains("EGRESS"), "header should list EGRESS: {text}"); - assert_eq!(cells_of("decided")[4], "per_box", "got:\n{text}"); - assert_eq!(cells_of("unenforced")[4], "none", "got:\n{text}"); - assert_eq!(cells_of("own-address")[4], "-", "got:\n{text}"); + let egress = |name: &str| ls_cell(&text, name, "EGRESS", "TITLE"); + assert_eq!(egress("decided"), "allow-all (default)", "got:\n{text}"); + assert_eq!( + egress("unenforced"), + "declared rules; per-box enforcement none", + "got:\n{text}" + ); + assert_eq!(egress("own-address"), "-", "got:\n{text}"); + assert_eq!(ls_cell(&text, "decided", "ADDRESS", "STATUS"), "-"); } #[tokio::test] @@ -3646,7 +3691,7 @@ async fn activate_and_ls_report_native_surface() { // the port the proxy's half of the line names, and the port a hook // would have to route to. let mut client = connect_daemon(&args).await.unwrap(); - let resp = client.oneshot_rpc::(()).await.unwrap(); + let mut resp = client.oneshot_rpc::(()).await.unwrap(); let port = resp .hostname_proxy_port .expect("the proxy must report the port it landed on"); @@ -3678,7 +3723,7 @@ async fn activate_and_ls_report_native_surface() { {ls_stdout}" ); assert!( - !ls_stdout.contains("resolves through this machine's DNS"), + !ls_stdout.contains("on this machine"), "a host with no hook must not be told native DNS is live: {ls_stdout}" ); @@ -3727,7 +3772,7 @@ async fn activate_and_ls_report_native_surface() { "a default start prints exactly one host line, got: {activate_stderr}" ); assert!( - !activate_stderr.contains("resolves on this machine"), + !activate_stderr.contains("on this machine"), "a host with no hook must not be told native DNS is live: {activate_stderr}" ); assert!( @@ -3746,6 +3791,17 @@ async fn activate_and_ls_report_native_surface() { // live surface, the proxy's half stays beside it (NET-019), and neither // the NET-122 advisory nor the pointer a native host no longer needs // rides it. + // A box that reports its own range address (T110): the BOX NAMES row + // reads the own-address form only when a box proves it is off the + // `127.0.0.1` fallback. + resp.box_networks.insert( + sessions::SessionId::parse_str("00000000-0000-0000-0000-000000000001").unwrap(), + minimald_rpc::BoxNetworkFacts { + loopback_address: Some(Ipv4Addr::new(127, 0, 64, 2)), + egress_posture: None, + at_fallback: Some(false), + }, + ); let mut native_out = Vec::new(); format_ls( &mut native_out, @@ -3760,14 +3816,13 @@ async fn activate_and_ls_report_native_surface() { .unwrap(); let native_ls = String::from_utf8(native_out).unwrap(); assert!( - native_ls - .contains("BOX NAMES: .min.internal resolves through this machine's DNS"), + native_ls.contains("BOX NAMES: .min.internal opens at its name on this machine"), "`min ls` must say native DNS is the live surface when the three facts hold, got: \ {native_ls}" ); assert!( native_ls.contains(&format!( - "the hostname proxy also listens on 127.0.0.1:{port}" + "; the HTTP proxy at 127.0.0.1:{port} opens it too" )), "the native arm keeps the proxy's half beside it (NET-019): {native_ls}" ); @@ -3790,8 +3845,8 @@ async fn activate_and_ls_report_native_surface() { assert_eq!( finished, format!( - "names: native-surface.min.internal resolves on this machine; open it through the \ - HTTP proxy at 127.0.0.1:{port}" + "names: native-surface.min.internal opens at its name on this machine; the HTTP \ + proxy at 127.0.0.1:{port} opens it too" ), "a finished host's start line carries no clause" ); @@ -3891,6 +3946,7 @@ async fn ls_keeps_detail_rows_and_name_surface_clause() { #[test] fn ls_names_the_vm_host_daemon_as_the_zone_answerer() { let resp = ListSessionsResponse { + box_networks: Default::default(), daemon_version: None, hostname_routing_unavailable: None, hostname_proxy_port: None, @@ -3927,10 +3983,11 @@ fn ls_names_the_vm_host_daemon_as_the_zone_answerer() { ); // The verdict beside it: the query proved the answerer live and this // host's facts held, so the surface the CLI computed prints through — - // the same `BOX NAMES` line the native host's list carries. + // the same `BOX NAMES` line the native host's list carries. No box is + // listed, so none proves it is off the `127.0.0.1` fallback, and the + // row reads the fallback form (T110). assert!( - holder_ls - .contains("BOX NAMES: .min.internal resolves through this machine's DNS"), + holder_ls.contains("BOX NAMES: .min.internal resolves on this machine"), "the VM-host verdict rides the list like a native one: {holder_ls}" ); assert!( @@ -3968,8 +4025,8 @@ fn ls_names_the_vm_host_daemon_as_the_zone_answerer() { ); assert!( held_ls.contains( - "BOX NAMES: the hostname proxy is not serving, and box names open only \ - through it" + "BOX NAMES: .min.internal: the hostname proxy is not serving; finish \ + setup: min finalize-install" ), "the no-channel verdict reads the proxy: {held_ls}" ); @@ -4031,6 +4088,7 @@ fn ls_names_the_vm_host_daemon_as_the_zone_answerer() { #[test] fn ls_keeps_the_answerer_row_beside_a_down_proxy() { let resp = ListSessionsResponse { + box_networks: Default::default(), daemon_version: None, hostname_routing_unavailable: None, hostname_proxy_port: None, @@ -4147,6 +4205,7 @@ fn ls_keeps_the_answerer_row_beside_a_down_proxy() { // cause row is the one HOSTNAME PROXY row, and neither it nor the NAME // SURFACE row names the reported port as serving. let reported = ListSessionsResponse { + box_networks: Default::default(), hostname_proxy_port: Some(7_654), ..resp.clone() }; @@ -4215,6 +4274,7 @@ fn ls_across_vms_names_one_proxy_verdict_per_vm() { let listing = |vm: &str, proxy_port: u16| VmListing { vm: vm.to_string(), resp: ListSessionsResponse { + box_networks: Default::default(), daemon_version: None, hostname_routing_unavailable: None, hostname_proxy_port: Some(proxy_port), @@ -4300,9 +4360,11 @@ fn ls_across_vms_names_one_proxy_verdict_per_vm() { && !default_surface[0].contains("opens through the HTTP proxy at"), "the down VM's surface row does not name its proxy as serving:\n{text}" ); + // The surface row names no proxy at all (T110): the HOSTNAME PROXY + // row above is where the cause is named. assert!( - default_surface[0].contains("not serving"), - "the down VM's surface row agrees the proxy is not serving:\n{text}" + !default_surface[0].contains("HTTP proxy"), + "the down VM's surface row names no proxy to open it through:\n{text}" ); // The other VM: its own listening row and serving surface, as before. @@ -4425,6 +4487,7 @@ fn retired_surfaces_absent() { #[test] fn session_list_decodes_without_mtls_field() { let resp = ListSessionsResponse { + box_networks: Default::default(), daemon_version: Some("test".to_string()), hostname_routing_unavailable: None, hostname_proxy_port: None, diff --git a/crates/minimald-rpc/src/lib.rs b/crates/minimald-rpc/src/lib.rs index a2d7792cb4..193ddfd6a3 100644 --- a/crates/minimald-rpc/src/lib.rs +++ b/crates/minimald-rpc/src/lib.rs @@ -342,6 +342,80 @@ pub struct ListSessionsResponse { /// native DNS on a host that may not have one. #[serde(default)] pub answerer_bound: bool, + /// Each listed box's network facts, keyed by its id (T110): the host + /// loopback address its ports answer at and its effective egress + /// posture. A box the daemon has no fact for is absent from the map, + /// or carries `None` in the field it cannot fill. Serde-defaulted, so + /// a reply from a daemon that predates the map reads as every box + /// unreported, and omitted when empty, so the common reply from such + /// a daemon is unchanged on the wire. + #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")] + pub box_networks: std::collections::BTreeMap, +} + +/// What a listing reports about one box's network (T110): where its ports +/// answer on the host and what its egress is held to. Both fields are +/// serde-defaulted and omitted when `None`, so a field this daemon could +/// not fill reads as unreported rather than as an invented value. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct BoxNetworkFacts { + /// The host loopback address the box's ports answer at: its own + /// reserved-range address (NET-010), `127.0.0.1` where the range is + /// absent, or the node's address for a host-address box. `None` while + /// the box publishes nowhere, such as before its first launch. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub loopback_address: Option, + /// The egress posture the box is held to, from the same resolution + /// `GetEffectiveSessionPolicy` answers. `None` when the daemon could + /// not read the box's record. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub egress_posture: Option, + /// Whether [`Self::loopback_address`] is the `127.0.0.1` fallback an + /// own-address or `none` box takes where the reserved range is absent. + /// `Some(false)` for a host-address box, whose node address can also be + /// `127.0.0.1`. The client cannot tell those two apart without it. + /// `None` when the box reports no address. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub at_fallback: Option, +} + +/// A box's effective egress, named as one posture (T110): the summary +/// `min session policy` prints for it. The client owns the printed words. +/// This is the machine spelling. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum EgressPosture { + /// No `egress` section, resolved to deny everything (NET-074). + DenyAllDefault, + /// No `egress` section, resolved to allow everything: the opt-out + /// (NET-077), or a box without an address of its own. + AllowAllDefault, + /// A declared section that admits nothing (NET-075). + DenyAll, + /// A declared section with rules of its own. + DeclaredRules, + /// A `none` box: it has no network, so no egress posture applies. + NoNetwork, +} + +impl EgressPosture { + /// The posture of `egress`, the effective egress the gate enforces, + /// for a box in `network` mode. A `none` box has no network, whatever + /// its egress resolves to, as `min session policy` says for it. + #[must_use] + pub fn of(egress: &sessions::EffectiveEgress, network: sessions::NetworkMode) -> Self { + if network == sessions::NetworkMode::NoNet { + return Self::NoNetwork; + } + match egress { + sessions::EffectiveEgress::DenyAll => Self::DenyAllDefault, + sessions::EffectiveEgress::AllowAll => Self::AllowAllDefault, + sessions::EffectiveEgress::Declared(section) if section.admits_nothing() => { + Self::DenyAll + } + sessions::EffectiveEgress::Declared(_) => Self::DeclaredRules, + } + } } impl OneshotSshRpc for ListSessions { @@ -3882,6 +3956,7 @@ mod tests { hostname_proxy_port: None, zone_answerer_port: None, answerer_bound: false, + box_networks: Default::default(), resource_pool: None, sessions: vec![], }; @@ -4071,6 +4146,112 @@ mod tests { assert_eq!(round_trip(&with), with); } + /// T110: the listing carries each box's address and egress posture keyed + /// by its id, both survive the wire, a field the daemon could not fill + /// stays off it, and a reply from a daemon that predates the map decodes + /// with every box unreported. + #[test] + fn list_sessions_reply_carries_box_address_and_posture() { + let own = SessionId::parse_str("00000000-0000-0000-0000-000000000001").unwrap(); + let host = SessionId::parse_str("00000000-0000-0000-0000-000000000002").unwrap(); + let unreported = SessionId::parse_str("00000000-0000-0000-0000-000000000003").unwrap(); + let reply = ListSessionsResponse { + resource_pool: None, + sessions: Vec::new(), + daemon_version: None, + hostname_routing_unavailable: None, + hostname_proxy_port: None, + zone_answerer_port: None, + answerer_bound: false, + box_networks: [ + ( + own, + BoxNetworkFacts { + loopback_address: Some(std::net::Ipv4Addr::new(127, 0, 64, 2)), + egress_posture: Some(EgressPosture::DenyAllDefault), + at_fallback: Some(false), + }, + ), + ( + host, + BoxNetworkFacts { + loopback_address: Some(std::net::Ipv4Addr::LOCALHOST), + egress_posture: Some(EgressPosture::DeclaredRules), + at_fallback: Some(false), + }, + ), + (unreported, BoxNetworkFacts::default()), + ] + .into_iter() + .collect(), + }; + let json = serde_json_lenient::to_value(&reply).expect("serializes"); + assert_eq!( + json["box_networks"][own.to_string()], + serde_json_lenient::json!({ + "loopback_address": "127.0.64.2", + "egress_posture": "deny_all_default", + "at_fallback": false, + }), + "got {json}" + ); + assert_eq!( + json["box_networks"][unreported.to_string()], + serde_json_lenient::json!({}), + "an unreported field stays off the wire: {json}" + ); + let back: ListSessionsResponse = serde_json_lenient::from_value(json).expect("round trips"); + assert_eq!(back.box_networks, reply.box_networks); + + // A daemon that predates the map sends no key: every box unreported. + let older: ListSessionsResponse = + serde_json_lenient::from_str(r#"{"sessions": []}"#).expect("an older reply decodes"); + assert!(older.box_networks.is_empty()); + let quiet = ListSessionsResponse { + box_networks: Default::default(), + ..reply + }; + assert!( + !serde_json_lenient::to_string(&quiet) + .unwrap() + .contains("box_networks"), + "an empty map stays off the wire" + ); + + // The posture is the one `min session policy` names, from the same + // effective egress. + use sessions::{EffectiveEgress, EgressPolicy, NetworkMode}; + assert_eq!( + EgressPosture::of(&EffectiveEgress::DenyAll, NetworkMode::OwnIp), + EgressPosture::DenyAllDefault + ); + assert_eq!( + EgressPosture::of(&EffectiveEgress::AllowAll, NetworkMode::HostNet), + EgressPosture::AllowAllDefault + ); + assert_eq!( + EgressPosture::of( + &EffectiveEgress::Declared(EgressPolicy::deny_all()), + NetworkMode::HostNet + ), + EgressPosture::DenyAll + ); + assert_eq!( + EgressPosture::of( + &EffectiveEgress::Declared(EgressPolicy { + allow_subnets: Some(vec!["10.0.0.0/8".to_string()]), + ..EgressPolicy::default() + }), + NetworkMode::OwnIp + ), + EgressPosture::DeclaredRules + ); + assert_eq!( + EgressPosture::of(&EffectiveEgress::AllowAll, NetworkMode::NoNet), + EgressPosture::NoNetwork + ); + } + /// NET-129's listing half survives version skew both ways without a /// request flag, because neither [`ListSessionsResponse`] nor /// [`ListSessionsEntry`] is `deny_unknown_fields`. A newer daemon's @@ -4099,6 +4280,7 @@ mod tests { hostname_proxy_port: None, zone_answerer_port: None, answerer_bound: false, + box_networks: Default::default(), }; let quiet = serde_json_lenient::to_string(&response(vec![entry(Vec::new())])) diff --git a/crates/minimald/src/rpc.rs b/crates/minimald/src/rpc.rs index d97b33c109..8cc2e8226d 100644 --- a/crates/minimald/src/rpc.rs +++ b/crates/minimald/src/rpc.rs @@ -147,23 +147,54 @@ async fn serve_list_sessions( // listing. let in_microvm = s.in_microvm().await; let fact = crate::session_host::host_ip_enforcement_fact(); - let enforcement = futures::future::join_all(infos.iter().map(|i| async { - let record = mngr - .get_record(SessionKeyPredicate::Id(i.id)) + let opt_out = s.deny_all_opt_out().await; + let records = futures::future::join_all(infos.iter().map(|i| async { + mngr.get_record(SessionKeyPredicate::Id(i.id)) .await .ok() - .flatten(); - record.map(|record| { - crate::session_host::displayed_host_ip_enforcement( - in_microvm, - record.network, - classifier::verdict_of(record.policy.egress.as_ref()), - &fact, - record.host_ip_enforcement, - ) - }) + .flatten() })) .await; + let enforcement: Vec<_> = records + .iter() + .map(|record| { + record.as_ref().map(|record| { + crate::session_host::displayed_host_ip_enforcement( + in_microvm, + record.network, + classifier::verdict_of(record.policy.egress.as_ref()), + &fact, + record.host_ip_enforcement, + ) + }) + }) + .collect(); + // T110: each box's address and egress posture, from its record + // and the hostname registry. A box with no fact for a field + // leaves it `None`, so the listing shows `-` rather than an + // invented value. + let box_networks: std::collections::BTreeMap<_, _> = infos + .iter() + .zip(&records) + .map(|(i, record)| { + ( + i.id, + box_network_facts(&mngr, i.id, record.as_ref(), opt_out, in_microvm), + ) + }) + .collect(); + tracing::info!( + boxes = infos.len(), + with_address = box_networks + .values() + .filter(|f| f.loopback_address.is_some()) + .count(), + with_posture = box_networks + .values() + .filter(|f| f.egress_posture.is_some()) + .count(), + "listed boxes with their address and egress posture" + ); Ok(ListSessionsResponse { daemon_version: Some(OWN_VERSION.to_string()), hostname_routing_unavailable: s.proxy_unavailable().await, @@ -171,6 +202,7 @@ async fn serve_list_sessions( zone_answerer_port, answerer_bound: zone_answerer_port.is_some(), resource_pool, + box_networks, // `git` is left `None`: the daemon cannot probe it — on // macOS it runs in the minvmd guest, where the host's // project paths do not exist and git is not on PATH. The @@ -211,6 +243,82 @@ async fn serve_list_sessions( .await } +/// The address a host-address box reports in the listing (T110): the +/// node's address, except in a microVM whose node address is still +/// `127.0.0.1`, where the box reports none. +fn host_address_loopback(node: std::net::Ipv4Addr, in_microvm: bool) -> Option { + (!in_microvm || node != std::net::Ipv4Addr::LOCALHOST).then_some(node) +} + +/// Whether a listed box stands on the `127.0.0.1` fallback (T110). A +/// host-address box's node address can be `127.0.0.1` too, so only an +/// own-address or `none` box at `127.0.0.1` is at the fallback. A +/// host-address box whose address is withheld (a VM node not yet granted +/// its address) is reached only through the proxy, so it reads as the +/// fallback too (design §7.1). Any other box with no address reports none. +fn box_at_fallback( + host_address: bool, + loopback_address: Option, +) -> Option { + match loopback_address { + None => host_address.then_some(true), + Some(address) => Some(!host_address && address == std::net::Ipv4Addr::LOCALHOST), + } +} + +/// One box's network facts as the listing reports them (T110). The address +/// is the host loopback address its ports answer at: the node's address for +/// a host-address box, and for an own-address or `none` box the address its +/// publish holds (NET-010, or the `127.0.0.1` interim where the range is +/// absent), else the loopback address its registration was handed. The +/// posture is the effective egress `GetEffectiveSessionPolicy` resolves for +/// the same record ([`effective_policy_reply`]). A box whose record cannot +/// be read reports no posture, and one that publishes nowhere reports no +/// address. In a microVM, a host-address box reports no address while the +/// node's address is still `127.0.0.1`: the answerer has not granted the +/// node its address yet, and a VM node's host-address boxes are never +/// mirrored onto `127.0.0.1` (design §7.1). +fn box_network_facts( + mngr: &crate::sessions::ManagerHandle, + id: SessionId, + record: Option<&sessions::Record>, + opt_out: bool, + in_microvm: bool, +) -> minimald_rpc::BoxNetworkFacts { + let host_address = matches!( + record.map(|record| record.network), + Some(sessions::NetworkMode::HostNet) + ); + let loopback_address = { + let registry = mngr.hostnames(); + let routes = registry.read().expect("hostname registry lock poisoned"); + if host_address { + host_address_loopback(routes.node_address(), in_microvm) + } else { + routes.published_own_address(id).or_else(|| { + record + .and_then(|record| record.box_addresses) + .map(|addresses| addresses.loopback_address) + }) + } + }; + let at_fallback = box_at_fallback(host_address, loopback_address); + let egress_posture = record.map(|record| { + let effective = effective_policy_reply( + &record.policy, + record.network, + sessions::EGRESS_DEFAULT_PHASE, + opt_out, + ); + minimald_rpc::EgressPosture::of(&effective.egress, record.network) + }); + minimald_rpc::BoxNetworkFacts { + loopback_address, + egress_posture, + at_fallback, + } +} + /// The ports a box's attach yields because a sibling at the same shared /// loopback address holds them (NET-129, first-come), as the wire names /// them: the one registry read both the runtime facts and the listing answer @@ -7588,4 +7696,111 @@ mod tests { "the live door still answers" ); } + + /// T110: the listing reports each box's address and egress posture, keyed + /// by its id. A host-address box answers at the node's address. An + /// own-address or `none` box answers at the address its publish holds, and + /// reports none before it publishes. The posture is the one the effective + /// policy reply resolves for the same box. + #[tokio::test] + async fn list_sessions_reports_box_address_and_posture() { + let server = TestServer::new().await; + let mut client = server.connect().await; + + let mut host_address = req("host-address", "/uwu"); + host_address.config.network = NetworkMode::HostNet; + let host_id = client + .call::(&host_address) + .await + .unwrap() + .id; + let own_id = own_ip_session( + &mut client, + "own-address", + sessions::SessionPolicy::new(None, None), + ) + .await; + let mut no_net = req("no-network", "/uwu"); + no_net.config.network = NetworkMode::NoNet; + let none_id = client.call::(&no_net).await.unwrap().id; + + // Before either box publishes, neither has an address to report. + let listed = client.call::(&()).await; + assert_eq!( + listed.box_networks[&own_id].loopback_address, None, + "an unpublished box reports no address, never an invented one" + ); + + let mngr = server.state.sessions_manager().await; + let own_address = std::net::Ipv4Addr::new(127, 0, 64, 2); + // The `none` box stands on the `127.0.0.1` fallback, so the reply + // marks it there, while a host-address box at the node address is not. + let none_address = std::net::Ipv4Addr::LOCALHOST; + let node = { + let registry = mngr.hostnames(); + let mut routes = registry.write().unwrap(); + routes.publish_own_address(own_id, "own-address", own_address, Default::default()); + routes.publish_own_address(none_id, "no-network", none_address, Default::default()); + routes.node_address() + }; + + let listed = client.call::(&()).await; + let facts = |id: SessionId| { + listed + .box_networks + .get(&id) + .cloned() + .unwrap_or_else(|| panic!("facts for {id} in {:?}", listed.box_networks)) + }; + assert_eq!( + facts(own_id), + minimald_rpc::BoxNetworkFacts { + loopback_address: Some(own_address), + egress_posture: Some(minimald_rpc::EgressPosture::DenyAllDefault), + at_fallback: Some(false), + } + ); + assert_eq!( + facts(host_id), + minimald_rpc::BoxNetworkFacts { + loopback_address: Some(node), + egress_posture: Some(minimald_rpc::EgressPosture::AllowAllDefault), + at_fallback: Some(false), + } + ); + assert_eq!( + facts(none_id), + minimald_rpc::BoxNetworkFacts { + loopback_address: Some(none_address), + egress_posture: Some(minimald_rpc::EgressPosture::NoNetwork), + at_fallback: Some(true), + } + ); + + // The posture agrees with the reply `min session policy` reads. + let policy = client + .call::(&GetEffectiveSessionPolicyRequest::Id(own_id)) + .await + .unwrap(); + assert_eq!( + minimald_rpc::EgressPosture::of(&policy.egress, NetworkMode::OwnIp), + minimald_rpc::EgressPosture::DenyAllDefault + ); + // In a microVM, a host-address box at the node's `127.0.0.1` is not + // mirrored there, so it reports no address until the node is granted + // one. A native node's `127.0.0.1` is the box's published address. + let localhost = std::net::Ipv4Addr::LOCALHOST; + let granted = std::net::Ipv4Addr::new(127, 0, 64, 1); + assert_eq!(host_address_loopback(localhost, true), None); + assert_eq!(host_address_loopback(granted, true), Some(granted)); + assert_eq!(host_address_loopback(localhost, false), Some(localhost)); + // A withheld host-address box is reached only through the proxy, so it + // reads as the fallback. A box of another mode with no address says + // nothing. + assert_eq!(box_at_fallback(true, None), Some(true)); + assert_eq!(box_at_fallback(true, Some(localhost)), Some(false)); + assert_eq!(box_at_fallback(false, Some(localhost)), Some(true)); + assert_eq!(box_at_fallback(false, Some(granted)), Some(false)); + assert_eq!(box_at_fallback(false, None), None); + } } diff --git a/docs/specs/18-spec-box-networking/18-spec-box-networking.md b/docs/specs/18-spec-box-networking/18-spec-box-networking.md index 10ddcb5d9a..d48e0a45dd 100644 --- a/docs/specs/18-spec-box-networking/18-spec-box-networking.md +++ b/docs/specs/18-spec-box-networking/18-spec-box-networking.md @@ -170,6 +170,11 @@ included, with every refusal logged (NET-001 to NET-004). verify: cargo nextest run -p minvmd released_address_is_not_reused_within_the_answer_ttl +- **NET-147** THE SYSTEM SHALL show each published box's host loopback address in `min ls` and in its machine-readable output. + tier: T0 + verify: cargo nextest run -p minimal ls_shows_box_loopback_address + + - **NET-011** WHEN a session is finalised THE SYSTEM SHALL register `.min.internal` for the box's loopback address. tier: T0 verify: cargo nextest run -p minimald name_registered_at_finalize @@ -247,22 +252,22 @@ included, with every refusal logged (NET-001 to NET-004). verify: cargo nextest run -p minimal activate_and_ls_report_native_surface -- **NET-142** WHEN a box is activated THE SYSTEM SHALL print one host line naming the box, whose resolution clause is `names: .min.internal via 127.0.0.1:`, where `` is the hostname-proxy port. +- **NET-142** WHEN a box is activated THE SYSTEM SHALL print one host line that names the box, tells the user how to open it by name on this host, and gives the hostname-proxy port when the proxy is the path that opens it. tier: T0 verify: cargo nextest run -p minimal activate_prints_one_host_line_per_install_state - - - WHERE host-OS resolution and published addresses are both deployed on the host THE SYSTEM SHALL print the resolution clause as `names: .min.internal resolves on this machine; open it through the HTTP proxy at 127.0.0.1:`, where `` is the hostname-proxy port, or `names: .min.internal resolves on this machine` while the session start has no proxy port. + + - WHERE host-OS resolution and published addresses are both deployed on the host THE SYSTEM SHALL state in the host line that the name resolves on this machine, and whether the box opens at its name directly or through the hostname proxy at its port. tier: T0 verify: cargo nextest run -p minimal activate_host_line_names_native_dns_when_deployed - - - WHILE any `min finalize-install` item is missing THE SYSTEM SHALL end the host line with the install clause `; finish setup: min finalize-install`, whatever its resolution clause. + + - WHILE any `min finalize-install` item is missing THE SYSTEM SHALL end the host line by naming `min finalize-install` as the step that finishes setup. tier: T0 verify: cargo nextest run -p minimal activate_host_line_carries_finish_setup_clause_while_items_missing - - - WHEN `min ls` runs THE SYSTEM SHALL keep its `HOSTNAME PROXY:` and `ZONE ANSWERER:` detail rows, keep the resolution wording of its `NAME SURFACE:` row, and end that row with the same install clause as the activate host line while any item is missing. + + - WHEN `min ls` runs THE SYSTEM SHALL state the same naming surface and the same way to open a box as the activate host line, beside its rows for the hostname-proxy port and the DNS service. tier: T0 - verify: cargo nextest run -p minimal ls_keeps_detail_rows_and_name_surface_clause - + verify: cargo nextest run -p minimal ls_name_surface_matches_activate_host_line + - **NET-019** WHERE host-OS resolution and published addresses are both deployed on the host THE SYSTEM SHALL keep the hostname proxy serving. tier: T0 @@ -447,6 +452,11 @@ included, with every refusal logged (NET-001 to NET-004). verify: cargo nextest run -p minimal policy_shows_effective_egress +- **NET-146** THE SYSTEM SHALL show in `min ls`, for every box, the effective egress posture that `min session policy` shows for that box. + tier: T0 + verify: cargo nextest run -p minimal ls_egress_matches_session_policy + + - **NET-062** WHILE a box runs with an own address, IF it opens a connection to a destination its egress rules do not allow THEN THE SYSTEM SHALL drop the connection without resetting it. tier: T2 verify: cargo nextest run -p minimald disallowed_egress_dropped_not_reset diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index 564a0da517..e5f434b1d6 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -4987,8 +4987,9 @@ echo "::endgroup::" # * `http://.min.internal:8080` returns the in-box server's response, # through the resolver file the same script wrote; # * the box's own range address (the zone answerer's A record for its name) -# returns the same response at port 8080. -# That `min ls` names the address belongs to T110 (#2696), which adds it. +# returns the same response at port 8080; +# * `min ls` shows the same range address in the box's ADDRESS column, +# never `-` or the `127.0.0.1` fallback. # The host is restored on every exit: the range unit, its aliases, the # resolver file and the answerer service go (range_teardown_unit, # answerer_service_teardown — the teardown repeats both if the case dies). @@ -5137,10 +5138,19 @@ fresh_install_range_lands_on_macos() { fi done + # The listing shows the address the zone answers for the box (T110). + mnl ls >"$WORK/fi-range-ls.out" 2>&1 || true + if ! grep -F -- "$frm_name" "$WORK/fi-range-ls.out" | grep -qF -- " $frm_addr "; then + echo "::error::'min ls' does not show $frm_addr on the $frm_name row, the address the zone answers for it" + cat "$WORK/fi-range-ls.out" 2>/dev/null || true + fail + fi + echo "min ls shows $frm_addr on the $frm_name row" + mnl session destroy --force "$frm_sid" >/dev/null 2>&1 || true range_teardown_unit answerer_service_teardown - echo "fresh-install range publish on macOS OK (min finalize-install on a running VM, $frm_name.min.internal and $frm_addr answer at :$frm_port, host restored)" + echo "fresh-install range publish on macOS OK (min finalize-install on a running VM, $frm_name.min.internal and $frm_addr answer at :$frm_port, min ls shows $frm_addr, host restored)" echo "::endgroup::" } @@ -6532,8 +6542,18 @@ __e2e_same_shell=yes' \ fail fi # Keeping at the exit prompt is a detach, and the session must survive it. - if ! mnl ls --raw 2>/dev/null | grep -q -- "$hook_sid"; then + # Capture-then-match, never `mnl ls --raw | grep -q`: grep's early exit + # SIGPIPEs the CLI under pipefail, and a failed listing would read as a + # gone session. On a miss, print what the listing said and its stderr so + # a failed `min ls` is told apart from a session that is truly gone. + hook_ls_status=0 + hook_ls="$(mnl ls --raw 2>"$WORK/hooks-ls.err")" || hook_ls_status=$? + if ! printf '%s\n' "$hook_ls" | grep -Fq -- "$hook_sid"; then echo "::error::session gone after answering the exit prompt with 'keep'" + echo "--- min ls --raw (exit $hook_ls_status) ---"; printf '%s\n' "$hook_ls" + echo "--- min ls --raw stderr ---"; cat "$WORK/hooks-ls.err" 2>/dev/null || true + echo "--- attach transcript ---"; printf '%s\n' "$attach_out" + echo "--- attach stderr ---"; cat "$WORK/hooks-attach.err" 2>/dev/null || true fail fi # `on_detach` is headless (the terminal it would have used is the thing @@ -7706,8 +7726,10 @@ proof_local_range_reserved_by_privileged_step() { echo "--- second activate stderr ---"; cat "$range2_err" 2>/dev/null || true fail fi - if ! grep -q -- 'resolves on this machine' "$range2_err"; then - echo "::error::a session started after the range step does not read native DNS as its live surface (the host probe must read present)" + # The range is present and the box is off the interim, so its ports + # answer at its own address: the host line reads the own-address form. + if ! grep -qF -- "names: $RANGE_NAME-2.min.internal opens at its name on this machine" "$range2_err"; then + echo "::error::a session started after the range step does not say its box opens at its name on this machine (the host probe must read present, and the box must be off the 127.0.0.1 interim)" echo "--- second activate stderr ---"; cat "$range2_err" 2>/dev/null || true fail fi @@ -9203,10 +9225,12 @@ proof_box_name_resolves_natively_without_proxy() { # ---- NET-018: the second box's activate reports the verdict ------------- bn_second_box - bn_api_surface="$(grep -F -- 'resolves on this machine' \ + # The box publishes at its own leased address (asserted below), so its + # host line reads the own-address form, never the 127.0.0.1 fallback form. + bn_api_surface="$(grep -F -- 'opens at its name on this machine' \ "$bn_api_err" 2>/dev/null | head -n1 || true)" if [ -z "$bn_api_surface" ]; then - echo "::error::the second box's activate does not say its name resolves on this machine (NET-018) — the command ran, so the verdict must have moved" + echo "::error::the second box's activate does not say its box opens at its name on this machine (NET-018) — the command ran, so the verdict must have moved" echo "--- activate stderr ---"; cat "$bn_api_err" 2>/dev/null || true fail fi @@ -9215,7 +9239,7 @@ proof_box_name_resolves_natively_without_proxy() { # the hostname proxy still serves beside native DNS (NET-019) is `min # ls`'s to say, on its BOX NAMES row, asserted below. case "$bn_api_surface" in - "names: $BN_API_NAME.min.internal resolves on this machine"*) ;; + "names: $BN_API_NAME.min.internal opens at its name on this machine"*) ;; *) echo "::error::the second box's activate line is not the one host line naming the box (NET-142)" echo "--- surface line ---"; printf '%s\n' "$bn_api_surface" @@ -9298,18 +9322,20 @@ proof_box_name_resolves_natively_without_proxy() { bn_ls="$(mnl ls 2>&1 || true)" echo "--- min ls ---"; printf '%s\n' "$bn_ls" bn_surface_line="$(printf '%s\n' "$bn_ls" | grep -F -- 'BOX NAMES:' | head -n1 || true)" + # Both boxes stand at their own addresses, so no box is on the fallback + # and the row reads the own-address form, as the activate line did. case "$bn_surface_line" in - *"resolves through this machine's DNS"*) ;; + *".min.internal opens at its name on this machine"*) ;; *) - echo "::error::min ls's BOX NAMES line does not say box names resolve through this machine's DNS (NET-018)" + echo "::error::min ls's BOX NAMES line does not say box names open at their names on this machine (NET-018)" echo "--- BOX NAMES line ---"; printf '%s\n' "$bn_surface_line" fail ;; esac case "$bn_surface_line" in - *"the hostname proxy also listens on 127.0.0.1:"*) ;; + *"the HTTP proxy at 127.0.0.1:"*) ;; *) - echo "::error::min ls's surface line does not say the hostname proxy also listens (NET-019)" + echo "::error::min ls's BOX NAMES line does not name the HTTP proxy that still serves (NET-019)" echo "--- BOX NAMES line ---"; printf '%s\n' "$bn_surface_line" fail ;; From 93dd243a34e131bc17002b8dd10171a97bd48a99 Mon Sep 17 00:00:00 2001 From: bryan-minimal Date: Thu, 29 Oct 2026 16:16:00 +0000 Subject: [PATCH 23/34] fix(minimal): resolve a box's VM from its name for exec, task run and net forward (#2626) NET-058 says a box name alone decides which VM a verb looks in, and attach honours it: the selected VM's daemon first, then every VM's socket. min session exec, min session run and min net forward still looked only in the selected VM, so a box activated with --vm was listed by min ls and reachable by attach, and exec answered "No session found matching". The three now go through the resolution attach uses, generalised by the verb that leads its one announcing line, and the hand-off runs over the owning VM's socket. NET-058 names the verbs it now covers. Backport of #2626. Adapted to the line: the cmd/boxes/attach.rs change lands in cmd/session.rs, with the line's session_announce names. (cherry picked from commit f2e6ed6a7399c5466f81f1f4c0b49478e784ff9c) --- crates/minimal/src/cmd/net.rs | 11 +++- crates/minimal/src/cmd/session.rs | 53 +++++++++++++++---- crates/minimal/src/tests.rs | 38 +++++++++++++ .../18-spec-box-networking.md | 2 +- 4 files changed, 93 insertions(+), 11 deletions(-) diff --git a/crates/minimal/src/cmd/net.rs b/crates/minimal/src/cmd/net.rs index c9afd35226..6c76bf3656 100644 --- a/crates/minimal/src/cmd/net.rs +++ b/crates/minimal/src/cmd/net.rs @@ -44,7 +44,16 @@ pub async fn cmd_net_forward( let mut client = client::Client::connect(&sock) .await .context("Failed to connect to minimald")?; - let record = resolve_session_version_gated(&mut client, &args.session).await?; + // The name alone decides where to look (NET-058): a box on another VM is + // forwarded over that VM's socket, with no global flag. + let (record, sock) = crate::cmd::resolve_named_box_version_gated( + global, + &mut client, + sock, + &args.session, + "Forwarding to", + ) + .await?; // Loopback only: the forward publishes a box's service to the laptop // running the command, not to the network the laptop sits on. A local diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index 1739dda7b5..e91eeb9d5c 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -1822,9 +1822,11 @@ pub async fn cmd_attach(global: &GlobalArgs, args: AttachArgs) -> Result<(), any .await } -/// Resolve a named attach target to its record and the socket the hand-off -/// runs over — the target `min session attach` lands in when the name alone -/// decides where to look (NET-058). +/// Resolve a named box to its record and the socket the hand-off runs over: +/// the target `min session attach`, `min session exec`, `min session run` and +/// `min net forward` land in when the name alone decides where to look +/// (NET-058). `verb` leads the one line that says which VM the box was found +/// on ("Attaching to", "Running in", ...). /// /// The selected VM's daemon is asked first (`client`, reached over `sock`): /// the common case — a box on the VM the operator is already on — costs @@ -1836,21 +1838,31 @@ pub async fn cmd_attach(global: &GlobalArgs, args: AttachArgs) -> Result<(), any /// named the box, so a skewed VM cannot be reached through it. Nothing found /// anywhere returns the selected VM's own error — the "no session found" it /// computed, or its skew — as the more useful of the two answers. -pub(crate) async fn resolve_attach_target_version_gated( +pub(crate) async fn resolve_named_box_version_gated( global: &GlobalArgs, client: &mut client::Client, sock: std::path::PathBuf, name: &str, + verb: &str, ) -> Result<(sessions::Record, std::path::PathBuf), anyhow::Error> { match resolve_session_version_gated(client, name).await { Ok(record) => Ok((record, sock)), Err(selected) => match attach::resolve_box_vm(global, name).await? { Some(resolved) => { + // From here the process acts on the owning VM, as `--vm` + // would have made it: everything keyed on the VM's name + // (the VM host control socket the pre-attach re-hold asks, + // the answerer beside it) must be that VM's, or the re-hold + // lands on the selected VM's host daemon as a name with no + // row behind it and the hand-off stalls. The enumeration + // validated the name; a second set (none happens before a + // resolution) is a no-op. + let _ = client::set_vm_name(&resolved.vm); // The operator never chose the VM the way they chose the // session, so tell them which one they're landing in. if should_announce_session(global) { eprintln!( - "Attaching to session {} on VM {}", + "{verb} session {} on VM {}", session_announce_label( &resolved.record.id, resolved.record.name.as_deref() @@ -1865,6 +1877,17 @@ pub(crate) async fn resolve_attach_target_version_gated( } } +/// [`resolve_named_box_version_gated`] for `min session attach`: the attach +/// target, announced as one. +pub(crate) async fn resolve_attach_target_version_gated( + global: &GlobalArgs, + client: &mut client::Client, + sock: std::path::PathBuf, + name: &str, +) -> Result<(sessions::Record, std::path::PathBuf), anyhow::Error> { + resolve_named_box_version_gated(global, client, sock, name, "Attaching to").await +} + /// Executes a command in an existing session. /// /// The session is resolved using the provided predicate, and the connection @@ -1884,8 +1907,12 @@ pub async fn cmd_exec(global: &GlobalArgs, args: ExecArgs) -> Result<(), anyhow: // Gated by hand for the same reason as `cmd_attach`: this hands off into a // live session (the daemon mints the exec channel and runs the attach // hooks around it), which is not something to drive on a skewed pair. The - // gate rides on the lookup's reply — no `GetVersion` ahead of it. - let r = resolve_session_version_gated(&mut client, &args.session).await?; + // gate rides on the lookup's reply — no `GetVersion` ahead of it. The name + // alone decides where to look (NET-058), as it does for attach: a box on + // another VM is reached over that VM's socket, with no global flag. + let (r, sock) = + resolve_named_box_version_gated(global, &mut client, sock, &args.session, "Running in") + .await?; tracing::info!( session_id = %r.id, session_name = ?r.name, @@ -1914,8 +1941,16 @@ pub async fn cmd_session_run( .await .context("Failed to connect to minimald")?; // Gated by hand for the same reason as `cmd_exec`: this hands off into a - // live session, which is not something to drive on a skewed pair. - let r = resolve_session_version_gated(&mut client, &args.session).await?; + // live session, which is not something to drive on a skewed pair. The name + // alone decides where to look (NET-058), as for exec. + let (r, sock) = resolve_named_box_version_gated( + global, + &mut client, + sock, + &args.session, + "Running the task in", + ) + .await?; tracing::info!( session_id = %r.id, session_name = ?r.name, diff --git a/crates/minimal/src/tests.rs b/crates/minimal/src/tests.rs index 4ad01897ef..c09d598e1d 100644 --- a/crates/minimal/src/tests.rs +++ b/crates/minimal/src/tests.rs @@ -289,6 +289,7 @@ fn every_daemon_connection_is_classified() { // what the tests assert rather than a decision they make. "tests.rs::attach_fall_through_hands_off_the_owning_vm = gated", "tests.rs::attach_refuses_a_box_name_two_vms_know = gated", + "tests.rs::exec_fall_through_hands_off_the_owning_vm = gated", ] ); } @@ -3343,6 +3344,43 @@ async fn attach_fall_through_hands_off_the_owning_vm() { ); } +/// NET-058's exec half, and `session run` and `net forward` with it: the +/// three go through the same resolution attach does, so a box activated with +/// `--vm` is reached by name alone, over the owning VM's socket. Before this, +/// `min ls` and attach found such a box and `min session exec` answered "No +/// session found" (lab scenario 720). +#[tokio::test] +async fn exec_fall_through_hands_off_the_owning_vm() { + let (state, default_vm, alpha) = two_vms().await; + create_box_on(&default_vm, "api").await; + let web = create_box_on(&alpha, "web").await; + let global = vm_globals(state.path(), None); + let selected = + client::resolve_socket_path_named(Some(state.path()), true, paths::DEFAULT_VM_NAME) + .expect("the selected VM's socket path"); + let mut client = client::Client::connect(&selected) + .await + .expect("connect to the selected VM's daemon"); + + for verb in ["Running in", "Running the task in", "Forwarding to"] { + let (record, handed_off) = cmd::resolve_named_box_version_gated( + &global, + &mut client, + selected.clone(), + "web", + verb, + ) + .await + .expect("a name alpha owns resolves through the selected VM's miss"); + assert_eq!(record.id, web, "{verb}: the owning VM's record"); + assert_eq!( + handed_off, + state.path().join("providers/local-minvmd0/alpha/ssh.sock"), + "{verb}: the hand-off runs over the owning VM's socket" + ); + } +} + /// The resolution path gets the same treatment as the listing: a stopped /// VM's stale `ssh.sock` is probed once — "not running", not a fault to retry /// for — so a name resolution across every VM costs the VMs that are up, not diff --git a/docs/specs/18-spec-box-networking/18-spec-box-networking.md b/docs/specs/18-spec-box-networking/18-spec-box-networking.md index d48e0a45dd..875a914109 100644 --- a/docs/specs/18-spec-box-networking/18-spec-box-networking.md +++ b/docs/specs/18-spec-box-networking/18-spec-box-networking.md @@ -432,7 +432,7 @@ included, with every refusal logged (NET-001 to NET-004). verify: cargo nextest run -p minimal ls_shows_vm_per_box -- **NET-058** WHILE two VMs run, WHEN attach or `min net expose` names a box THE SYSTEM SHALL resolve the VM from the box name without a global flag. +- **NET-058** WHILE two VMs run, WHEN attach, exec, a task run, `min net forward` or `min net expose` names a box THE SYSTEM SHALL resolve the VM from the box name without a global flag. tier: T0 verify: cargo nextest run -p minimal box_name_resolves_vm_without_flag From 91d492a27fb63832e020b8c352c2429554e13092 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 03:15:29 +0000 Subject: [PATCH 24/34] fix(minimal): find a box on any VM, a stopped VM included (#2811) Policy and destroy resolve a box's VM from its name, as attach, exec, run and net forward do. A box no running VM holds names --vm, a box on a stopped VM names the command that starts it, and a name any two VMs hold, running or stopped, is refused with every VM listed. A relay failure writes the unwind before the attach error prints, so the error stays on screen. Backport of #2811. Adapted to the line: cmd/boxes/attach.rs and policy.rs hunks land in cmd/session.rs. Dropped, because the line has no stopped session status (#2676) and keeps box ids apart from session ids (#2485): min ls rows for a stopped VM's boxes, vm_running in its JSON, and id-prefix matching against a stopped VM's records, which match by name only here. (cherry picked from commit 2e3962f6ff8d6b5750b5e60ee0b7aadebce65e87) --- crates/minimal/src/attach.rs | 206 ++++++++++++++-- crates/minimal/src/cmd/mod.rs | 160 +++++++++++- crates/minimal/src/cmd/net.rs | 2 + crates/minimal/src/cmd/session.rs | 188 +++++++++----- crates/minimal/src/tests.rs | 398 ++++++++++++++++++++++++++++++ crates/minvmd/src/box_registry.rs | 106 ++++++++ 6 files changed, 977 insertions(+), 83 deletions(-) diff --git a/crates/minimal/src/attach.rs b/crates/minimal/src/attach.rs index bc60722018..3631e1c61e 100644 --- a/crates/minimal/src/attach.rs +++ b/crates/minimal/src/attach.rs @@ -153,6 +153,40 @@ impl Drop for TerminalUnwind { } } +/// The interactive attach could not take the terminal, so it never changed +/// it: the marker that tells the attach a failure owes the terminal no +/// unwind ([`crate::cmd::interactive_attach`]). +#[derive(Debug)] +pub(crate) struct TerminalNotTaken; + +impl fmt::Display for TerminalNotTaken { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("the interactive attach needs a terminal, but none could be taken") + } +} + +impl std::error::Error for TerminalNotTaken {} + +/// A relay that failed before ssh's session reached the real terminal: the +/// pty, the spawn or the raw mode failed, and the relay put back whatever +/// it changed. The marker tells the attach that the failure owes the +/// terminal no unwind ([`crate::cmd::interactive_attach`]). It wraps the +/// relay's error and prints exactly as that error does. +#[derive(Debug)] +pub(crate) struct RelayNotStarted(pub(crate) anyhow::Error); + +impl fmt::Display for RelayNotStarted { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +impl std::error::Error for RelayNotStarted { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.0.source() + } +} + /// The current working directory as a host path, respecting the `--repo-dir` /// (`-C`) override. The result is canonicalized so it compares equal to a /// `Record.project_path`, which `cmd_activate` canonicalizes at create time. @@ -265,10 +299,15 @@ pub(crate) struct ResolvedBoxVm { /// original error (the selected VM's — usually the more useful of the two). /// An explicit `--vm` pins where to look, so it resolves nothing: the /// operator chose, and a resolution across the others would override that -/// choice. A VM that is not running cannot own the name — an answer given -/// without a word, since finding a stopped VM is the normal case on a host -/// with several — and a VM whose daemon answers nothing is skipped with a -/// warning, so the name is resolved from the VMs that did answer. Each look +/// choice. A VM that is not running is asked through its persisted box +/// records instead (NET-152): a name only a stopped VM holds is refused with +/// a [`crate::cmd::BoxNotFound`] that names the VM and the command that +/// starts it. Any name that more than one VM holds, running or stopped, is +/// an [`crate::cmd::AmbiguousBoxName`] that lists every VM, each stopped one +/// marked, and names `--vm`: a name that matches on more than one host is an +/// error that lists the candidates, whatever state each box is in. +/// A VM whose daemon answers nothing is skipped with a warning, so the name +/// is resolved from the VMs that did answer. Each look /// is leashed ([`crate::client::PROBE_TIMEOUT`]): the loop spans every VM on /// the host, and a wedged one must not hold an attach for the stacked /// connect, handshake and RPC deadlines (~72 s each). @@ -288,11 +327,19 @@ pub(crate) async fn resolve_box_vm( let vms = crate::client::enumerate_vm_sockets(global.minimal_dir.as_deref(), global.use_minvmd())?; let mut owners: Vec = Vec::new(); + // The stopped VMs whose persisted box records hold the name (NET-152). + // Each with how it holds the name, so an id prefix never competes with a + // name. + let mut stopped_holders: Vec<(String, HeldBy)> = Vec::new(); for vm in vms { - // A VM that is not running cannot own the name. The check is free, so - // it is made first; the probe classifies the cases it cannot see — a - // socket that vanished mid-walk reads the same as one never bound. + // A VM that is not running has no daemon to ask, so its persisted + // records answer for it. The check is free, so it is made first; the + // probe classifies the cases it cannot see — a socket that vanished + // mid-walk reads the same as one never bound. if !vm.sock.exists() { + if let Some(held_by) = stopped_vm_holds(&vm, name) { + stopped_holders.push((vm.vm, held_by)); + } continue; } #[expect( @@ -308,12 +355,17 @@ pub(crate) async fn resolve_box_vm( ) }); match look { - Ok(Ok(Some(record))) => owners.push(ResolvedBoxVm { + Ok(Ok(BoxLook::Found(record))) => owners.push(ResolvedBoxVm { vm: vm.vm, sock: vm.sock, - record, + record: *record, }), - Ok(Ok(None)) => {} + Ok(Ok(BoxLook::Absent)) => {} + Ok(Ok(BoxLook::NotRunning)) => { + if let Some(held_by) = stopped_vm_holds(&vm, name) { + stopped_holders.push((vm.vm, held_by)); + } + } // VMs that could not be looked at are named, not silent: a // resolution that quietly could not look would read as "no such // box". @@ -323,24 +375,63 @@ pub(crate) async fn resolve_box_vm( ), } } - match owners.len() { - 0 => Ok(None), - 1 => { + // A daemon matches an exact name before an id prefix + // (`crate::cmd::match_id_prefix`), and so does the walk across VMs: when + // any VM holds the name as a name, a holder that matched only an id + // prefix is not a candidate. + let any_by_name = owners + .iter() + .any(|owner| held_by(&owner.record, name) == HeldBy::Name) + || stopped_holders.iter().any(|(_, by)| *by == HeldBy::Name); + if any_by_name { + owners.retain(|owner| held_by(&owner.record, name) == HeldBy::Name); + stopped_holders.retain(|(_, by)| *by == HeldBy::Name); + } + let stopped_holders: Vec = stopped_holders.into_iter().map(|(vm, _)| vm).collect(); + match (owners.len(), stopped_holders.as_slice()) { + (0, []) => Ok(None), + // NET-152: the box is there, on a VM that is down. Say which VM and + // the command that starts it: `--vm` selects the VM, and the attach + // autospawns the selected VM's daemon before it looks. + (0, [vm]) => { + tracing::debug!( + box = name, + vm = %vm, + outcome = "on a stopped VM", + "box name resolved to a VM" + ); + Err(crate::cmd::BoxNotFound(crate::cmd::box_on_stopped_vm(name, vm)).into()) + } + (0, several) => Err(crate::cmd::AmbiguousBoxName(format!( + "box name '{name}' exists on more than one stopped VM ({}); \ + name the one you mean with --vm", + several.join(", ") + )) + .into()), + (1, []) => { let resolved = owners.pop().expect("the one owner"); - tracing::debug!(box = name, vm = %resolved.vm, "box name resolved to a VM"); + tracing::debug!( + box = name, + vm = %resolved.vm, + outcome = "on a running VM", + "box name resolved to a VM" + ); Ok(Some(resolved)) } - // Two VMs can both own the name, and picking one silently would send - // an attach into a project the operator may not have meant. The flag - // is the disambiguator, so the error names both and points at it. - _ => Err(anyhow::anyhow!( + // Two VMs can both hold the name, running or stopped, and picking one + // silently would send an attach into a project the operator may not + // have meant. The flag is the disambiguator, so the error names every + // VM, marks each stopped one, and points at it. + (_, stopped) => Err(crate::cmd::AmbiguousBoxName(format!( "box name '{name}' exists on more than one VM ({}); name the one you mean with --vm", owners .iter() - .map(|owner| owner.vm.as_str()) + .map(|owner| owner.vm.clone()) + .chain(stopped.iter().map(|vm| format!("{vm} (stopped)"))) .collect::>() .join(", ") - )), + )) + .into()), } } @@ -356,13 +447,10 @@ pub(crate) async fn resolve_box_vm( /// The owning daemon is gated on the very reply that named it; the probe /// that finds nothing stays ungated, exactly like the dashboard's listing of /// a skewed daemon. -async fn box_record_on( - vm: &crate::client::VmSocket, - name: &str, -) -> Result, anyhow::Error> { +async fn box_record_on(vm: &crate::client::VmSocket, name: &str) -> Result { let mut client = match crate::client::Client::probe(&vm.sock).await { Ok(client) => client, - Err(crate::client::ProbeRefusal::NotRunning) => return Ok(None), + Err(crate::client::ProbeRefusal::NotRunning) => return Ok(BoxLook::NotRunning), Err(crate::client::ProbeRefusal::Unreachable(e)) => { return Err(e.context(format!( "Failed to connect to the daemon at {}", @@ -374,12 +462,76 @@ async fn box_record_on( match resp.record { Some(record) => { crate::client::ensure_version_reported(resp.daemon_version.as_deref())?; - Ok(Some(record)) + Ok(BoxLook::Found(Box::new(record))) } - None => Ok(None), + None => Ok(BoxLook::Absent), + } +} + +/// What one VM's daemon said about a box name ([`box_record_on`]). +enum BoxLook { + /// The daemon owns the name: its record, boxed so the other answers stay + /// small. + Found(Box), + /// The daemon answered and does not know the name. + Absent, + /// No daemon listens: the VM is stopped. + NotRunning, +} + +/// Whether the stopped VM `vm` holds a box named `name` in its persisted box +/// records (NET-152), read from its state dir with no daemon: the name as it +/// was registered, compared ignoring ASCII case as names are made unique. +/// Names only: the records carry a box's box id, which on this line is not +/// the session id an operator types, so an id prefix never matches here. +/// Read-only, so a file this build cannot use holds nothing and stays as it +/// is. +fn stopped_vm_holds(vm: &crate::client::VmSocket, name: &str) -> Option { + stopped_vm_boxes(vm) + .iter() + .any(|h| h.name.eq_ignore_ascii_case(name)) + .then_some(HeldBy::Name) +} + +/// How a VM holds a box name: as the box's name, or only as a prefix of its +/// id. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum HeldBy { + Name, + IdPrefix, +} + +/// How the record a running daemon resolved `name` to holds it. +fn held_by(record: &sessions::Record, name: &str) -> HeldBy { + if record + .name + .as_deref() + .is_some_and(|n| n.eq_ignore_ascii_case(name)) + { + HeldBy::Name + } else { + HeldBy::IdPrefix } } +/// The boxes a stopped VM's persisted records hold (NET-152), read +/// from the state dir beside its socket. One debug line records the count. +pub(crate) fn stopped_vm_boxes( + vm: &crate::client::VmSocket, +) -> Vec { + let held = vm + .sock + .parent() + .map(minvmd::box_registry::persisted_box_names) + .unwrap_or_default(); + tracing::debug!( + vm = %vm.vm, + boxes = held.len(), + "read a stopped VM's persisted box records" + ); + held +} + /// A pickable session. `Display` renders the row the user sees and fuzzy- /// searches against; the carried entry preserves the id/name so the chosen /// one can be attached without a follow-up round-trip. Boxed, as in diff --git a/crates/minimal/src/cmd/mod.rs b/crates/minimal/src/cmd/mod.rs index f6fed53465..7ad4f3eb7e 100644 --- a/crates/minimal/src/cmd/mod.rs +++ b/crates/minimal/src/cmd/mod.rs @@ -389,7 +389,165 @@ pub(crate) fn named_record( record: Option, session: &str, ) -> Result { - record.ok_or_else(|| anyhow::anyhow!("No session found matching '{session}'")) + record.ok_or_else(|| BoxNotFound(format!("No session found matching '{session}'")).into()) +} + +/// A box name that resolved to no box a command can reach; the message says +/// why. Typed so a caller can tell the miss from a failed RPC: the selected +/// VM's own miss ([`named_record`]), no running VM holding the name +/// ([`no_running_vm_has_box`]), or the box on a stopped VM +/// ([`box_on_stopped_vm`]). +#[derive(Debug)] +pub(crate) struct BoxNotFound(pub(crate) String); + +impl std::fmt::Display for BoxNotFound { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for BoxNotFound {} + +/// A box name that more than one VM holds, running or stopped, with no +/// `--vm` to pick one; the message names the VMs. Typed so a caller can tell +/// the ambiguity from a failed RPC, as [`AmbiguousIdPrefix`] is. +#[derive(Debug)] +pub(crate) struct AmbiguousBoxName(pub(crate) String); + +impl std::fmt::Display for AmbiguousBoxName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for AmbiguousBoxName {} + +/// The refusal for a box name no running VM holds, on a host with more than +/// one VM (NET-058). `example` is the command the user ran, after `min` and +/// with the box name in place, such as `session policy beta`. +pub(crate) fn no_running_vm_has_box(name: &str, example: &str) -> String { + format!( + "no running VM has a box named '{name}'. A box on another VM is named with --vm, \ + for example: min --vm {example}" + ) +} + +/// The refusal for a box name only a stopped VM's persisted records hold +/// (NET-152). `min --vm session attach ` starts that VM: +/// `--vm` selects it before any path resolves, and the attach runs +/// `ensure_daemon`, which spawns the selected VM's `minvmd` +/// (`autospawn::ensure_minvmd_running`). +pub(crate) fn box_on_stopped_vm(name: &str, vm: &str) -> String { + format!( + "box '{name}' is on VM '{vm}', which is stopped. \ + Start it with: min --vm {vm} session attach {name}" + ) +} + +/// The box `name` resolved across every VM once the selected VM's daemon +/// missed it with `selected` (NET-058): the VM that owns it, published as +/// this process's VM so everything keyed on the VM's name follows it, as +/// `--vm` would have. +/// +/// No VM owns it: the selected VM's own error, unless that error is the miss +/// itself on a host with more than one VM and no `--vm`. Then the refusal +/// says no running VM holds the name and shows `--vm` on `example` +/// ([`no_running_vm_has_box`]). A box only a stopped VM holds is refused by +/// the resolution itself ([`box_on_stopped_vm`]). +pub(crate) async fn resolve_box_on_another_vm( + global: &GlobalArgs, + name: &str, + example: &str, + selected: anyhow::Error, +) -> Result { + if let Some(resolved) = crate::attach::resolve_box_vm(global, name).await? { + // From here the process acts on the owning VM, as `--vm` would have + // made it: everything keyed on the VM's name (the VM host control + // socket the pre-attach re-hold asks, the answerer beside it) must be + // that VM's. The enumeration validated the name; a second set (none + // happens before a resolution) is a no-op. + let _ = client::set_vm_name(&resolved.vm); + return Ok(resolved); + } + if global.vm.is_none() + && selected.downcast_ref::().is_some() + && several_vms(global) + { + return Err(BoxNotFound(no_running_vm_has_box(name, example)).into()); + } + Err(selected) +} + +/// Refuse a hit on the selected VM when another VM holds the same name +/// (NET-058). A name that matches on more than one VM, running or stopped, is +/// an [`AmbiguousBoxName`] that lists every VM and names `--vm`, so a hit on +/// the selected VM is checked against every VM before it is used. With `--vm`, +/// or on a host with one VM, nothing else can hold the name, and no lookup +/// runs. `selected` is the record the selected VM's daemon returned. +pub(crate) async fn ensure_selected_holds_box_alone( + global: &GlobalArgs, + name: &str, + selected: &sessions::Record, +) -> Result<(), anyhow::Error> { + if global.vm.is_some() || !several_vms(global) { + return Ok(()); + } + match crate::attach::resolve_box_vm(global, name).await? { + // The walk found one holder. It is the selected VM when its record is + // the one the selected daemon returned. + Some(resolved) if resolved.record.id != selected.id => Err(AmbiguousBoxName(format!( + "box name '{name}' exists on more than one VM ({}, {}); \ + name the one you mean with --vm", + client::vm_name(), + resolved.vm + )) + .into()), + // The selected VM is the only holder, or its probe in the walk did + // not answer and no other VM holds the name. + _ => Ok(()), + } +} + +/// Whether this host has more than one VM, running or not: only then can a +/// box be on a VM other than the selected one. +fn several_vms(global: &GlobalArgs) -> bool { + client::enumerate_vm_sockets(global.minimal_dir.as_deref(), global.use_minvmd()) + .is_ok_and(|vms| vms.len() > 1) +} + +/// Resolve `name` to its record and a client connected to the VM that owns +/// it, for the commands that act on a named box over the daemon's RPCs +/// rather than an ssh hand-off: `min session policy` and `min session +/// destroy` (NET-058). The selected VM's `client` is asked first. A hit +/// there is refused when another VM also holds the name +/// ([`ensure_selected_holds_box_alone`]). When its daemon does not know the +/// name, [`resolve_box_on_another_vm`] finds the +/// owning VM, and the client returned is connected to that VM's socket. +/// `example` is the command the user ran, for the not-found refusal. +pub(crate) async fn resolve_box_on_any_vm( + global: &GlobalArgs, + mut client: client::Client, + name: &str, + example: &str, +) -> Result<(client::Client, sessions::Record), anyhow::Error> { + match resolve_session(&mut client, name).await { + Ok(record) => { + ensure_selected_holds_box_alone(global, name, &record).await?; + Ok((client, record)) + } + Err(selected) => { + let resolved = resolve_box_on_another_vm(global, name, example, selected).await?; + let mut client = client::Client::connect(&resolved.sock) + .await + .with_context(|| { + format!("Failed to connect to the daemon of VM {}", resolved.vm) + })?; + // Gated as `connect_daemon` gates the selected VM: a destroy is + // a mutation, and this is the connection it runs over. + ensure_version_match(&mut client).await?; + Ok((client, resolved.record)) + } + } } /// `ListSessions` with the daemon's build asserted off the same reply, for the diff --git a/crates/minimal/src/cmd/net.rs b/crates/minimal/src/cmd/net.rs index 6c76bf3656..84bb916264 100644 --- a/crates/minimal/src/cmd/net.rs +++ b/crates/minimal/src/cmd/net.rs @@ -46,12 +46,14 @@ pub async fn cmd_net_forward( .context("Failed to connect to minimald")?; // The name alone decides where to look (NET-058): a box on another VM is // forwarded over that VM's socket, with no global flag. + let example = format!("net forward {} {}", args.session, args.spec); let (record, sock) = crate::cmd::resolve_named_box_version_gated( global, &mut client, sock, &args.session, "Forwarding to", + &example, ) .await?; diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index e91eeb9d5c..20edbddeb0 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -1828,10 +1828,11 @@ pub async fn cmd_attach(global: &GlobalArgs, args: AttachArgs) -> Result<(), any /// (NET-058). `verb` leads the one line that says which VM the box was found /// on ("Attaching to", "Running in", ...). /// -/// The selected VM's daemon is asked first (`client`, reached over `sock`): -/// the common case — a box on the VM the operator is already on — costs -/// nothing beyond the one lookup the command always made, and its answer is -/// gated on the reply that lookup was already making. When that daemon does +/// The selected VM's daemon is asked first (`client`, reached over `sock`), +/// and its answer is gated on the reply that lookup was already making. On a +/// host with several VMs and no `--vm`, a hit there is checked against every +/// VM, and a name a second VM also holds is an ambiguity +/// ([`ensure_selected_holds_box_alone`]). When that daemon does /// not know the name, [`attach::resolve_box_vm`] resolves the VM that owns it /// across every VM's socket, so no global flag is needed to reach a box on /// another VM — the resolution gates the owning daemon on the very reply that @@ -1844,36 +1845,31 @@ pub(crate) async fn resolve_named_box_version_gated( sock: std::path::PathBuf, name: &str, verb: &str, + example: &str, ) -> Result<(sessions::Record, std::path::PathBuf), anyhow::Error> { match resolve_session_version_gated(client, name).await { - Ok(record) => Ok((record, sock)), - Err(selected) => match attach::resolve_box_vm(global, name).await? { - Some(resolved) => { - // From here the process acts on the owning VM, as `--vm` - // would have made it: everything keyed on the VM's name - // (the VM host control socket the pre-attach re-hold asks, - // the answerer beside it) must be that VM's, or the re-hold - // lands on the selected VM's host daemon as a name with no - // row behind it and the hand-off stalls. The enumeration - // validated the name; a second set (none happens before a - // resolution) is a no-op. - let _ = client::set_vm_name(&resolved.vm); - // The operator never chose the VM the way they chose the - // session, so tell them which one they're landing in. - if should_announce_session(global) { - eprintln!( - "{verb} session {} on VM {}", - session_announce_label( - &resolved.record.id, - resolved.record.name.as_deref() - ), - resolved.vm - ); - } - Ok((resolved.record, resolved.sock)) + Ok(record) => { + // A second VM that holds the name makes it ambiguous. The check + // runs before the attach touches the terminal. + ensure_selected_holds_box_alone(global, name, &record).await?; + Ok((record, sock)) + } + Err(selected) => { + // Every refusal here — the miss, a box on a stopped VM, an + // ambiguous name — is made before the attach touches the + // terminal, so it prints on the terminal as the user left it. + let resolved = resolve_box_on_another_vm(global, name, example, selected).await?; + // The operator never chose the VM the way they chose the + // session, so tell them which one they're landing in. + if should_announce_session(global) { + eprintln!( + "{verb} session {} on VM {}", + session_announce_label(&resolved.record.id, resolved.record.name.as_deref()), + resolved.vm + ); } - None => Err(selected), - }, + Ok((resolved.record, resolved.sock)) + } } } @@ -1885,7 +1881,8 @@ pub(crate) async fn resolve_attach_target_version_gated( sock: std::path::PathBuf, name: &str, ) -> Result<(sessions::Record, std::path::PathBuf), anyhow::Error> { - resolve_named_box_version_gated(global, client, sock, name, "Attaching to").await + let example = format!("session attach {name}"); + resolve_named_box_version_gated(global, client, sock, name, "Attaching to", &example).await } /// Executes a command in an existing session. @@ -1910,9 +1907,16 @@ pub async fn cmd_exec(global: &GlobalArgs, args: ExecArgs) -> Result<(), anyhow: // gate rides on the lookup's reply — no `GetVersion` ahead of it. The name // alone decides where to look (NET-058), as it does for attach: a box on // another VM is reached over that VM's socket, with no global flag. - let (r, sock) = - resolve_named_box_version_gated(global, &mut client, sock, &args.session, "Running in") - .await?; + let example = format!("session exec {} ", args.session); + let (r, sock) = resolve_named_box_version_gated( + global, + &mut client, + sock, + &args.session, + "Running in", + &example, + ) + .await?; tracing::info!( session_id = %r.id, session_name = ?r.name, @@ -1943,12 +1947,14 @@ pub async fn cmd_session_run( // Gated by hand for the same reason as `cmd_exec`: this hands off into a // live session, which is not something to drive on a skewed pair. The name // alone decides where to look (NET-058), as for exec. + let example = format!("session run {} {}", args.session, args.task); let (r, sock) = resolve_named_box_version_gated( global, &mut client, sock, &args.session, "Running the task in", + &example, ) .await?; tracing::info!( @@ -2173,9 +2179,14 @@ pub(crate) async fn session_via_ssh( // runtime's workers. let code = tokio::task::spawn_blocking(move || { interactive_attach(ssh, stdin_is_tty, attach::TerminalUnwind::arm, |ssh| { - minimal_client::attach::run_interactive_attach( - ssh, + // A terminal the relay could not take was never changed, + // so a failure here owes it nothing; the marker tells + // `interactive_attach` so. + let real = minimal_client::tty_relay::RealTty::acquire() + .context(attach::TerminalNotTaken)?; + mark_relay_not_started( subscribe_host_asks(host_asks).map(minimal_client::attach::HostAsks::into_hook), + |hook| minimal_client::attach::run_interactive_attach_on(ssh, real, Some(hook)), ) }) }) @@ -2303,6 +2314,45 @@ async fn kill_ssh(child: &mut tokio::process::Child) { } } +/// Run the relay with a suspend hook that records that the relay started, +/// and mark a failure that came before it with +/// [`attach::RelayNotStarted`]. +/// +/// The relay calls its suspend hook only once `Relay::start` returned: ssh +/// runs on the pty and the real terminal is in raw mode. Every failure of +/// `Relay::start` comes before ssh's output can reach the real terminal, and +/// the relay put back what it changed, so the attach owes the terminal no +/// unwind. `inner` is the caller's own hook, if any, and runs as before. +/// +/// The hook runs on a thread of its own, so a relay that started and then +/// failed at once can return before the hook records it. Then the guard +/// stands down, though the relay restored the termios. The only such +/// failure is the wait on ssh's exit. +pub(crate) fn mark_relay_not_started( + inner: Option, + run: impl FnOnce( + minimal_client::tty_relay::SuspendHook, + ) -> Result, +) -> Result { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + let started = Arc::new(AtomicBool::new(false)); + let hook_started = Arc::clone(&started); + let hook: minimal_client::tty_relay::SuspendHook = Box::new(move |handle| { + hook_started.store(true, Ordering::Release); + if let Some(inner) = inner { + inner(handle); + } + }); + run(hook).map_err(|e| { + if started.load(Ordering::Acquire) { + e + } else { + anyhow::Error::new(attach::RelayNotStarted(e)) + } + }) +} + /// The interactive attach, from the terminal check to ssh's exit code. /// /// `-tt` over a *non-terminal* stdin is a trap: ssh still forces the remote @@ -2321,6 +2371,15 @@ async fn kill_ssh(child: &mut tokio::process::Child) { /// only process left that can still reach the tty. So the guard stays /// armed for the duration and is stood down only once ssh's exit proves /// the daemon was alive and speaking: see `attach::client_must_unwind`. +/// +/// A relay that fails once it holds the terminal ends `min session attach` +/// with an error, which `main` prints after this returns. The relay put the +/// attach-start termios back before it returned, and the guard writes its +/// unwind as it drops here, so the error lands on a restored terminal and +/// stays on screen after the exit. Only a relay that never took the +/// terminal ([`attach::TerminalNotTaken`]) or never started ssh's +/// session on it ([`attach::RelayNotStarted`]) stands the guard +/// down, because nothing of the session reached the terminal. pub(crate) fn interactive_attach( ssh: std::process::Command, stdin_is_tty: bool, @@ -2332,10 +2391,18 @@ pub(crate) fn interactive_attach( let status = match relay(ssh) { Ok(status) => status, Err(e) => { - // ssh never ran (or the relay never took the terminal), so - // nothing of the session reached the terminal and there is - // nothing to put back. - unwind.disarm(); + // The relay never took the terminal, or never started ssh's + // session on it, so nothing of the session reached it and there + // is nothing to put back. Any other failure came after the + // session started, and the session may have left its modes set: + // the guard writes the unwind as it drops, before the error is + // printed. + if e.downcast_ref::().is_some() + || e.downcast_ref::().is_some() + { + unwind.disarm(); + } + drop(unwind); return Err(e).context("failed to run ssh"); } }; @@ -2394,13 +2461,16 @@ pub async fn cmd_session_policy( ensure_daemon(global)?; - let mut client = connect_daemon(global).await?; + let client = connect_daemon(global).await?; // The effective reply carries only the rules, but the ingress block is // suppressed for host-address sessions (see [`format_policy`]), so the // command also resolves the record the policy rides on for its network - // mode. - let record = resolve_session(&mut client, &args.session).await?; + // mode. The name alone decides which VM is asked (NET-058): a box on + // another VM is read from that VM's daemon, with no global flag. + let example = format!("session policy {}", args.session); + let (mut client, record) = + resolve_box_on_any_vm(global, client, &args.session, &example).await?; // The follow-up lookups go by the resolved id, so a session named by an // id prefix reaches the same session they do. let session = record.id.to_string(); @@ -3426,22 +3496,28 @@ async fn session_policy_json_inputs( > { ensure_daemon(global) .map_err(|error| PolicyJsonFailure::DaemonUnreachable(format!("{error:#}")))?; - let mut client = connect_daemon(global) + let client = connect_daemon(global) .await .map_err(|error| PolicyJsonFailure::DaemonUnreachable(format!("{error:#}")))?; // The record the ingress key is gated on, resolved through the same - // lookup the text walk uses, with the failure kinds told apart where - // they can be: an RPC the daemon never answered is no daemon to ask; - // a lookup that answered with nothing is the not-found itself. - let resp = get_session_record(&mut client, session) + // lookup the text walk uses — on the VM that owns the name (NET-058) — + // with the failure kinds told apart where they can be: an RPC the daemon + // never answered is no daemon to ask; a lookup that answered with + // nothing, or a box on a stopped VM, is the not-found itself. + let example = format!("session policy {session} -o json"); + let (mut client, record) = resolve_box_on_any_vm(global, client, session, &example) .await - .map_err(|error| match error.downcast_ref::() { - Some(ambiguous) => PolicyJsonFailure::SessionNotFound(ambiguous.to_string()), - None => PolicyJsonFailure::DaemonUnreachable(format!("{error:#}")), + .map_err(|error| { + if error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + { + PolicyJsonFailure::SessionNotFound(error.to_string()) + } else { + PolicyJsonFailure::DaemonUnreachable(format!("{error:#}")) + } })?; - let record = named_record(resp.record, session) - .map_err(|error| PolicyJsonFailure::SessionNotFound(error.to_string()))?; // By the resolved id, as the text walk does. let session = &record.id.to_string(); @@ -3789,7 +3865,9 @@ pub async fn cmd_destroy(global: &GlobalArgs, args: DestroyArgs) -> Result<(), a .session .as_deref() .context("a session or --all is required")?; - let record = resolve_session(&mut client, session).await?; + // The name alone decides which VM destroys the box (NET-058). + let example = format!("session destroy {session}"); + let (mut client, record) = resolve_box_on_any_vm(global, client, session, &example).await?; // Under --force the at-risk fetch is skipped outright — the gate is // bypassed regardless, so there is nothing to ask the daemon for. diff --git a/crates/minimal/src/tests.rs b/crates/minimal/src/tests.rs index c09d598e1d..70338d773f 100644 --- a/crates/minimal/src/tests.rs +++ b/crates/minimal/src/tests.rs @@ -270,6 +270,10 @@ fn every_daemon_connection_is_classified() { "cmd/list.rs::list_selected_vm = gated", "cmd/mod.rs::arm_activation_interrupt = ungated", "cmd/mod.rs::connect_daemon_unchecked = ungated", + // `min session policy` and `min session destroy` on a box another + // VM owns: the connection to that VM, gated as `connect_daemon` + // gates the selected one. + "cmd/mod.rs::resolve_box_on_any_vm = gated", "cmd/net.rs::cmd_net_forward = gated", "cmd/session.rs::cmd_attach = gated", "cmd/session.rs::cmd_exec = gated", @@ -290,6 +294,9 @@ fn every_daemon_connection_is_classified() { "tests.rs::attach_fall_through_hands_off_the_owning_vm = gated", "tests.rs::attach_refuses_a_box_name_two_vms_know = gated", "tests.rs::exec_fall_through_hands_off_the_owning_vm = gated", + // Test-only: the NET-058 and NET-152 tests' client on the selected + // harness daemon, which is this build. + "tests.rs::selected_client = ungated", ] ); } @@ -3369,6 +3376,7 @@ async fn exec_fall_through_hands_off_the_owning_vm() { selected.clone(), "web", verb, + "session exec web ", ) .await .expect("a name alpha owns resolves through the selected VM's miss"); @@ -4126,3 +4134,393 @@ fn ls_egress_matches_session_policy() { } assert!(!table.contains("per_box"), "got:\n{table}"); } + +/// A fresh box id for a stopped VM's persisted records. +fn stopped_box_id() -> minimald_rpc::BoxId { + static NEXT: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(1); + minimald_rpc::BoxId::from_bytes([NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); 16]) +} + +/// A stopped VM `vm` under `state`: its state dir with the persisted box +/// records `boxes` (name, id), as its VM host daemon wrote them before it +/// stopped, and a stale `ssh.sock` that no daemon listens on. +fn stopped_vm_with_boxes(state: &std::path::Path, vm: &str, boxes: &[(&str, minimald_rpc::BoxId)]) { + let dir = state.join("providers/local-minvmd0").join(vm); + std::fs::create_dir_all(&dir).expect("the stopped VM's dir"); + let records: Vec = boxes + .iter() + .enumerate() + .map(|(index, (name, id))| { + serde_json_lenient::json!({ + "name": name, + "box_id": id.to_string(), + "switch_address": format!("100.64.0.{}", index + 9), + "loopback_address": format!("127.0.64.{}", index + 2), + "ingress_ports": [], + "standing": true, + }) + }) + .collect(); + std::fs::write( + dir.join(minvmd::box_registry::REGISTRY_FILE), + serde_json_lenient::to_vec(&serde_json_lenient::json!({ + "version": 1, + "boxes": records, + })) + .expect("the records serialize"), + ) + .expect("write the stopped VM's box records"); + let stale = std::os::unix::net::UnixListener::bind(dir.join("ssh.sock")) + .expect("the stopped VM's socket path binds"); + drop(stale); +} + +/// The default VM, running, under a fresh state dir: the selected VM of +/// every test below that pairs it with a stopped one. +async fn running_default_vm() -> (tempfile::TempDir, minimald::test_harness::TestServer) { + let state = tempfile::tempdir().expect("a temp minimal state dir"); + std::fs::create_dir_all(state.path().join("providers/local-minvmd0")).expect("the host dir"); + let default_sock = + client::resolve_socket_path_named(Some(state.path()), true, paths::DEFAULT_VM_NAME) + .expect("the default VM's socket path"); + let default_vm = minimald::test_harness::TestServer::new().await; + default_vm.listen_on_uds(&default_sock).await; + (state, default_vm) +} + +/// A client on the selected (default) VM's daemon under `state`. +async fn selected_client(state: &std::path::Path) -> (client::Client, std::path::PathBuf) { + let selected = client::resolve_socket_path_named(Some(state), true, paths::DEFAULT_VM_NAME) + .expect("the selected VM's socket path"); + let client = client::Client::connect(&selected) + .await + .expect("connect to the selected VM's daemon"); + (client, selected) +} + +/// NET-058 for the commands that act on a named box over the daemon's RPCs: +/// `min session policy` and `min session destroy` find a box on another VM +/// by its name alone, and the client they get back is that VM's. +#[tokio::test] +async fn box_name_resolves_vm_without_flag_for_policy_and_destroy() { + let (state, default_vm, alpha) = two_vms().await; + create_box_on(&default_vm, "api").await; + let beta = create_box_on(&alpha, "beta").await; + let global = vm_globals(state.path(), None); + + for example in ["session policy beta", "session destroy beta"] { + let (client, _) = selected_client(state.path()).await; + let (mut owner, record) = cmd::resolve_box_on_any_vm(&global, client, "beta", example) + .await + .expect("a name alpha owns resolves without --vm"); + assert_eq!(record.id, beta, "{example}: alpha's record"); + let again = cmd::resolve_session(&mut owner, "beta") + .await + .expect("the client returned is connected to alpha"); + assert_eq!(again.id, beta, "{example}: the follow-up RPCs reach alpha"); + } +} + +/// NET-058: a box name no running VM holds, on a host with two VMs, is +/// refused with the flag that names a box on another VM, shown on the +/// command the user ran. With `--vm`, the selected VM's own miss stands. +#[tokio::test] +async fn unknown_box_names_vm_flag() { + let (state, _default_vm, _alpha) = two_vms().await; + let global = vm_globals(state.path(), None); + + let (client, _) = selected_client(state.path()).await; + let err = cmd::resolve_box_on_any_vm(&global, client, "beta", "session policy beta") + .await + .map(|(_, record)| record) + .expect_err("no VM holds 'beta'"); + assert_eq!( + format!("error: {err:#}"), + "error: no running VM has a box named 'beta'. A box on another VM is named with --vm, \ + for example: min --vm session policy beta", + "the refusal as main prints it" + ); + + // The hand-off verbs say the same, on their own command. + let (mut client, sock) = selected_client(state.path()).await; + let err = cmd::resolve_named_box_version_gated( + &global, + &mut client, + sock, + "beta", + "Forwarding to", + "net forward beta 8080:3000", + ) + .await + .expect_err("no VM holds 'beta'"); + assert_eq!( + err.to_string(), + "no running VM has a box named 'beta'. A box on another VM is named with --vm, \ + for example: min --vm net forward beta 8080:3000" + ); + + // An explicit --vm is the operator's choice: the selected VM's miss. + let pinned = vm_globals(state.path(), Some(paths::DEFAULT_VM_NAME.to_string())); + let (client, _) = selected_client(state.path()).await; + let err = cmd::resolve_box_on_any_vm(&pinned, client, "beta", "session policy beta") + .await + .map(|(_, record)| record) + .expect_err("no VM holds 'beta'"); + assert_eq!(err.to_string(), "No session found matching 'beta'"); +} + +/// NET-152: a box only a stopped VM holds is named with that VM and the +/// command that starts it, on every verb that resolves a box by name. A +/// running VM's box of the same name and the stopped VM's record are an +/// ambiguity that names both VMs. +#[tokio::test] +async fn box_on_stopped_vm_names_start_command() { + let (state, default_vm) = running_default_vm().await; + create_box_on(&default_vm, "api").await; + let beta = stopped_box_id(); + stopped_vm_with_boxes(state.path(), "alpha", &[("beta", beta)]); + let global = vm_globals(state.path(), None); + let expected = "box 'beta' is on VM 'alpha', which is stopped. \ + Start it with: min --vm alpha session attach beta"; + + let err = attach::resolve_box_vm(&global, "beta") + .await + .expect_err("a box on a stopped VM is refused"); + assert_eq!(format!("error: {err:#}"), format!("error: {expected}")); + + // The attach path itself: the selected VM's miss falls through to the + // stopped VM's records. + let (mut client, sock) = selected_client(state.path()).await; + let err = cmd::resolve_attach_target_version_gated(&global, &mut client, sock, "beta") + .await + .expect_err("attach refuses a box on a stopped VM"); + assert_eq!(err.to_string(), expected); + + // And the RPC verbs, with the policy's machine reading still a miss. + let (client, _) = selected_client(state.path()).await; + let err = cmd::resolve_box_on_any_vm(&global, client, "beta", "session policy beta") + .await + .map(|(_, record)| record) + .expect_err("policy refuses a box on a stopped VM"); + assert_eq!(err.to_string(), expected); + assert!(err.downcast_ref::().is_some()); + + // A name two stopped VMs hold is ambiguous, and typed so the policy's + // machine reading reports a miss, not an unreachable daemon. + stopped_vm_with_boxes(state.path(), "delta", &[("dup", stopped_box_id())]); + stopped_vm_with_boxes(state.path(), "epsilon", &[("dup", stopped_box_id())]); + let (client, _) = selected_client(state.path()).await; + let err = cmd::resolve_box_on_any_vm(&global, client, "dup", "session policy dup") + .await + .map(|(_, record)| record) + .expect_err("a name two stopped VMs hold is ambiguous"); + assert_eq!( + err.to_string(), + "box name 'dup' exists on more than one stopped VM (delta, epsilon); \ + name the one you mean with --vm" + ); + assert!(err.downcast_ref::().is_some()); + + // A name a running VM owns and a stopped VM's records hold is ambiguous, + // though the running box is active: the error lists both VMs, marks the + // stopped one, and names --vm. + stopped_vm_with_boxes(state.path(), "gamma", &[("api", stopped_box_id())]); + let err = attach::resolve_box_vm(&global, "api") + .await + .expect_err("a running owner and a stopped holder are ambiguous"); + assert_eq!( + err.to_string(), + "box name 'api' exists on more than one VM (default, gamma (stopped)); \ + name the one you mean with --vm" + ); + assert!(err.downcast_ref::().is_some()); + + // The selected VM is the only holder: it resolves there. + create_box_on(&default_vm, "solo").await; + let (client, _) = selected_client(state.path()).await; + let (_, record) = cmd::resolve_box_on_any_vm(&global, client, "solo", "session policy solo") + .await + .expect("the selected VM's own box resolves"); + assert_eq!(record.name.as_deref(), Some("solo")); + + // A hit on the selected VM is ambiguous when a stopped VM also holds the + // name. + let (client, _) = selected_client(state.path()).await; + let err = cmd::resolve_box_on_any_vm(&global, client, "api", "session policy api") + .await + .map(|(_, record)| record) + .expect_err("the selected VM and a stopped VM both hold 'api'"); + assert_eq!( + err.to_string(), + "box name 'api' exists on more than one VM (default, gamma (stopped)); \ + name the one you mean with --vm" + ); + + // A stopped VM's records carry box ids, not the session ids an operator + // types, so an id prefix never matches there: a running box named 'cafe' + // resolves though a stopped VM holds a box whose box id starts with + // 'cafe', and that box id's prefix alone finds nothing. + create_box_on(&default_vm, "cafe").await; + let mut cafe_id = [0x11; 16]; + cafe_id[0] = 0xca; + cafe_id[1] = 0xfe; + stopped_vm_with_boxes( + state.path(), + "eta", + &[("other", minimald_rpc::BoxId::from_bytes(cafe_id))], + ); + let resolved = attach::resolve_box_vm(&global, "cafe") + .await + .expect("a name match is not ambiguous with a box id") + .expect("the running VM owns 'cafe'"); + assert_eq!(resolved.vm, "default"); + assert_eq!(resolved.record.name.as_deref(), Some("cafe")); + assert!( + attach::resolve_box_vm(&global, "cafe11") + .await + .expect("a stopped VM's box id is not a candidate") + .is_none() + ); +} + +/// NET-058: a name the selected VM holds is ambiguous when a second running +/// VM holds it too. `min session destroy` and `min session attach` refuse it, +/// name both VMs and `--vm`, and do not act on the selected VM's box. With +/// `--vm`, the selected VM's box resolves. +#[tokio::test] +async fn box_name_resolves_vm_without_flag_refuses_a_name_two_vms_hold() { + let (state, default_vm, alpha) = two_vms().await; + let shared = create_box_on(&default_vm, "shared").await; + create_box_on(&alpha, "shared").await; + let global = vm_globals(state.path(), None); + let expected = "box name 'shared' exists on more than one VM (default, alpha); \ + name the one you mean with --vm"; + + let (client, _) = selected_client(state.path()).await; + let err = cmd::resolve_box_on_any_vm(&global, client, "shared", "session destroy shared") + .await + .map(|(_, record)| record) + .expect_err("destroy refuses a name two VMs hold"); + assert_eq!(err.to_string(), expected); + assert!(err.downcast_ref::().is_some()); + + let (mut client, sock) = selected_client(state.path()).await; + let err = cmd::resolve_attach_target_version_gated(&global, &mut client, sock, "shared") + .await + .expect_err("attach refuses a name two VMs hold"); + assert_eq!(err.to_string(), expected); + assert!(err.downcast_ref::().is_some()); + + // The selected VM's box is still there: nothing acted on it. + let (mut client, _) = selected_client(state.path()).await; + let record = cmd::resolve_session(&mut client, "shared") + .await + .expect("the selected VM still holds 'shared'"); + assert_eq!(record.id, shared); + + // With --vm, the operator chose: the selected VM's box resolves. + let pinned = vm_globals(state.path(), Some(paths::DEFAULT_VM_NAME.to_string())); + let (client, _) = selected_client(state.path()).await; + let (_, record) = + cmd::resolve_box_on_any_vm(&pinned, client, "shared", "session destroy shared") + .await + .expect("--vm picks the selected VM's box"); + assert_eq!(record.id, shared); +} + +/// NET-152: an attach that fails once the relay holds the terminal ends +/// `min session attach` with an error that `main` prints after the attach +/// returns. By then the relay put the attach-start termios back and the +/// unwind codes are on the terminal, so the error prints after them and +/// stays on screen. A relay that never took the terminal writes nothing. +#[test] +fn attach_error_printed_after_terminal_restore() { + use std::cell::RefCell; + use std::rc::Rc; + + /// The user's terminal: everything written to it, in order. + #[derive(Clone, Default)] + struct Screen(Rc>>); + impl std::io::Write for Screen { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let screen = Screen::default(); + let err = interactive_attach( + std::process::Command::new("ssh"), + true, + || attach::TerminalUnwind::arm_on(screen.clone(), true), + |_| Err(anyhow::anyhow!("the relay lost the terminal mid-session")), + ) + .expect_err("a failed relay ends the attach with an error"); + assert!( + screen.0.borrow().ends_with(b"\x1b[?1004l"), + "the unwind is on the terminal before the attach returns" + ); + // What main prints for the error, on the same terminal. + let printed = format!("error: {err:#}\n"); + std::io::Write::write_all(&mut screen.clone(), printed.as_bytes()).expect("print"); + let seen = screen.0.borrow().clone(); + let at = seen + .windows(printed.len()) + .position(|window| window == printed.as_bytes()) + .expect("the error is on the terminal"); + assert!( + seen[..at].ends_with(b"\x1b[?1004l"), + "the error prints after the restore sequence: {:?}", + String::from_utf8_lossy(&seen) + ); + assert_eq!(at + printed.len(), seen.len(), "nothing writes after it"); + + // A relay that never took the terminal never changed it. + let untouched = Screen::default(); + interactive_attach( + std::process::Command::new("ssh"), + true, + || attach::TerminalUnwind::arm_on(untouched.clone(), true), + |_| { + Err(anyhow::Error::new(std::io::Error::other("no tty")) + .context(attach::TerminalNotTaken)) + }, + ) + .expect_err("an attach with no terminal fails"); + assert!( + untouched.0.borrow().is_empty(), + "no unwind for a terminal the attach never took" + ); + + // A relay that took the terminal and then failed to start ssh's session + // on it (ssh does not spawn) never let the session reach the terminal. + // The relay drops its suspend hook unrun, so the failure is marked, and + // the guard writes nothing. The error prints as the relay wrote it. + let not_started = Screen::default(); + let err = interactive_attach( + std::process::Command::new("ssh"), + true, + || attach::TerminalUnwind::arm_on(not_started.clone(), true), + |_| { + mark_relay_not_started(None, |_hook| { + Err( + anyhow::Error::new(std::io::Error::other("No such file or directory")) + .context("spawning the ssh attach"), + ) + }) + }, + ) + .expect_err("an attach whose ssh does not spawn fails"); + assert!( + not_started.0.borrow().is_empty(), + "no unwind for a relay that never started the session: {:?}", + String::from_utf8_lossy(¬_started.0.borrow()) + ); + assert_eq!( + format!("{err:#}"), + "failed to run ssh: spawning the ssh attach: No such file or directory" + ); +} diff --git a/crates/minvmd/src/box_registry.rs b/crates/minvmd/src/box_registry.rs index 81d97cc5f4..e06047b2d7 100644 --- a/crates/minvmd/src/box_registry.rs +++ b/crates/minvmd/src/box_registry.rs @@ -1155,6 +1155,53 @@ fn read_registry_file(path: &std::path::Path) -> Option> { } } +/// One box a VM's persisted registry ([`REGISTRY_FILE`]) holds: its name +/// and its box id, as [`persisted_box_names`] reads them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PersistedBox { + /// The box's name, as its creator registered it. + pub name: String, + /// The box's id. + pub id: minimald_rpc::BoxId, +} + +/// Reads the boxes a VM's state dir `state_dir` holds in its persisted +/// registry ([`REGISTRY_FILE`]), each one's name and id, with no daemon +/// running: what the CLI reads for a stopped VM (NET-148, NET-152). +/// +/// Read-only, unlike [`read_registry_file`]: a file that is missing, that +/// cannot be read, that does not parse, or that is of another version than +/// [`REGISTRY_FILE_VERSION`] reads as no boxes, and it is never written, +/// renamed or set aside. The daemon that owns the file decides what to do +/// with it on its next start. Each box id is listed once, in file order. +#[must_use] +pub fn persisted_box_names(state_dir: &std::path::Path) -> Vec { + #[derive(serde::Deserialize)] + struct Version { + version: u32, + } + let path = state_dir.join(REGISTRY_FILE); + let Ok(bytes) = std::fs::read(&path) else { + return Vec::new(); + }; + match serde_json_lenient::from_slice::(&bytes) { + Ok(Version { version }) if version == REGISTRY_FILE_VERSION => {} + _ => return Vec::new(), + } + let Ok(file) = serde_json_lenient::from_slice::(&bytes) else { + return Vec::new(); + }; + let mut seen = std::collections::HashSet::new(); + file.boxes + .into_iter() + .filter(|creation| seen.insert(creation.box_id)) + .map(|creation| PersistedBox { + name: creation.name, + id: creation.box_id, + }) + .collect() +} + /// Renames the unusable registry file at `path` aside, to the same name /// with an `.unusable-` suffix, and says so as an error line /// naming `why`. Returns no creations when it is set aside, and `None` — @@ -7039,6 +7086,65 @@ mod tests { BoxRegistry::new(SUBNET).persisting_to(dir.path().join(REGISTRY_FILE)) } + /// NET-148, NET-152: a VM's persisted box records read without a daemon + /// — each box's name and id — and a file of another version, or one that + /// does not parse, reads as no boxes and is left exactly as it was. + #[test] + fn persisted_box_names_read_without_a_daemon() { + let dir = tempfile::tempdir().expect("a temp dir"); + assert!( + persisted_box_names(dir.path()).is_empty(), + "a VM with no file holds no boxes" + ); + let registry = persisted_registry(&dir); + let web = registry + .register_client_box(client_spec("web")) + .expect("the plan has an address for the box"); + let api = registry + .register_client_box(client_spec("api")) + .expect("the plan has an address for another box"); + drop(registry); + + let mut read = persisted_box_names(dir.path()); + read.sort_by(|a, b| a.name.cmp(&b.name)); + assert_eq!( + read, + vec![ + PersistedBox { + name: "api".to_string(), + id: minimald_rpc::BoxId::from_bytes(api.box_id()), + }, + PersistedBox { + name: "web".to_string(), + id: minimald_rpc::BoxId::from_bytes(web.box_id()), + }, + ], + "each persisted box, by name and id" + ); + + let path = dir.path().join(REGISTRY_FILE); + for unusable in [ + r#"{"version": 99, "boxes": []}"#.as_bytes(), + b"not json at all".as_slice(), + ] { + std::fs::write(&path, unusable).expect("write the unusable file"); + assert!( + persisted_box_names(dir.path()).is_empty(), + "an unusable file reads as no boxes" + ); + assert_eq!( + std::fs::read(&path).expect("the file is still there"), + unusable, + "and is never written or set aside" + ); + let set_aside = std::fs::read_dir(dir.path()) + .expect("list the state dir") + .map(|entry| entry.expect("an entry").file_name()) + .any(|name| name.to_string_lossy().contains("unusable")); + assert!(!set_aside, "a read never sets the file aside"); + } + } + /// NET-138: the client boxes' registrations persist host-side, and a /// registry started over the file reinstates each standing row — its /// name, both addresses, its box id and its compiled rules — detached From 14c0e8a8d0e04a273e00d803c300962e29324370 Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Sun, 18 Oct 2026 07:12:32 +0000 Subject: [PATCH 25/34] fix(minimald): bound wedged transport writes with an inactivity deadline (#2303) russh bounds every session-loop transport write only by the inactivity timer, and minimald set it to None, so a write that never completed parked the session loop forever. Keepalives are dispatched from that same loop, so they stopped firing too: an attach whose transport wedged went dark in both directions and survived every keepalive budget. The timer is now armed one interval past the keepalive budget, so a healthy idle attach is never reaped and a wedged one is torn down. Backport of #2303. Adapted to the line: the test module gains the small accept-all client handler main's connection tests already had. (cherry picked from commit 02f3af98369388a47937a2f26d0478dde752808a) --- crates/minimald/src/connection.rs | 189 ++++++++++++++++++++++++++++++ crates/minimald/src/diag.rs | 10 +- crates/minimald/src/server.rs | 40 ++++++- 3 files changed, 231 insertions(+), 8 deletions(-) diff --git a/crates/minimald/src/connection.rs b/crates/minimald/src/connection.rs index 34005d6fc7..f4fd7d8eaf 100644 --- a/crates/minimald/src/connection.rs +++ b/crates/minimald/src/connection.rs @@ -1324,4 +1324,193 @@ kill "$peer" 2>/dev/null .expect("read on the channel after teardown"); assert_eq!(n, 0, "the forward's channel must see EOF, not more bytes"); } + + /// A passthrough transport whose write side can be wedged at will: + /// reads always delegate (the client's replies keep flowing until the + /// wedge), while writes, flushes and shutdowns park forever once + /// `stuck` is set — a transport that accepted the connection and then + /// stopped draining. + struct Wedgeable { + inner: T, + stuck: Arc, + } + + impl Wedgeable { + fn is_stuck(&self) -> bool { + self.stuck.load(std::sync::atomic::Ordering::Relaxed) + } + } + + impl AsyncRead for Wedgeable { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.inner).poll_read(cx, buf) + } + } + + impl AsyncWrite for Wedgeable { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + if self.is_stuck() { + return Poll::Pending; + } + Pin::new(&mut self.inner).poll_write(cx, buf) + } + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.is_stuck() { + return Poll::Pending; + } + Pin::new(&mut self.inner).poll_flush(cx) + } + fn poll_shutdown( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + if self.is_stuck() { + return Poll::Pending; + } + Pin::new(&mut self.inner).poll_shutdown(cx) + } + } + + /// The tight keepalive/inactivity pair both transport-deadline tests + /// run under. The keepalive fires well inside the inactivity window, so + /// a healthy peer answers several before it could lapse; the window is + /// wide enough that a slow runner's gaps during KEX and auth (which run + /// under it too) do not fire it early. + const TEST_KEEPALIVE: Duration = Duration::from_millis(100); + const TEST_INACTIVITY: Duration = Duration::from_secs(1); + + /// Accepts any host key: the test server's is ephemeral. + struct ForwardHandler; + + impl russh::client::Handler for ForwardHandler { + type Error = russh::Error; + async fn check_server_key( + &mut self, + _key: &russh::keys::PublicKeyOrCertificate, + ) -> Result { + Ok(true) + } + } + + /// A real russh client handshaken and authenticated against the server + /// through a [`Wedgeable`] transport under [`TEST_KEEPALIVE`] / + /// [`TEST_INACTIVITY`]. Returns the server's session future, the + /// client (which must be held: dropping it EOFs the server's read side) + /// and the wedge switch. + async fn wedgeable_session( + server: &crate::test_harness::TestServer, + ) -> ( + RunningSession, + russh::client::Handle, + Arc, + ) { + let host_key = server.state.host_key().await.unwrap(); + let (server_side, client_side) = tokio::net::UnixStream::pair().unwrap(); + let stuck = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let transport = Wedgeable { + inner: server_side, + stuck: stuck.clone(), + }; + let server_setup = { + let state = server.state.clone(); + async move { + Connection::from_stream( + transport, + Arc::new(russh::server::Config { + keys: vec![host_key], + auth_rejection_time_initial: Some(Duration::ZERO), + nodelay: true, + keepalive_interval: Some(TEST_KEEPALIVE), + // Far above the inactivity budget: a wedged write + // must be resolved by the inactivity deadline, not + // by exhausting keepalives. + keepalive_max: 100, + inactivity_timeout: Some(TEST_INACTIVITY), + ..Default::default() + }), + state, + true, + ) + .await + .expect("handshake over the in-memory pair") + } + }; + let client_setup = russh::client::connect_stream( + Arc::new(russh::client::Config::default()), + client_side, + ForwardHandler, + ); + let ((_conn, session_fut), client) = tokio::join!(server_setup, client_setup); + let mut client = client.expect("client handshake"); + let auth = client + .authenticate_none(&SessionId::nil().to_string()) + .await + .unwrap(); + assert!(auth.success(), "auth_none should succeed on local UDS"); + (session_fut, client, stuck) + } + + /// A write that never completes must not park the session loop forever. + /// russh bounds every transport write only by the inactivity timer, so + /// with `inactivity_timeout: None` a wedged write parks the loop for + /// good — and keepalives are dispatched from that same loop, so they + /// stop firing too: the connection outlives every keepalive budget + /// while both directions sit dark. Arming the timer bounds the park: + /// the session future must resolve with an inactivity error instead of + /// hanging. + #[tokio::test] + async fn a_wedged_transport_write_ends_the_session_on_the_inactivity_deadline() { + let server = crate::test_harness::TestServer::new().await; + // Hold the client open: dropping it would EOF the server's read + // side and end the session for the wrong reason. The wedge is the + // write side only. + let (session_fut, _client, stuck) = wedgeable_session(&server).await; + stuck.store(true, std::sync::atomic::Ordering::Relaxed); + + // The next keepalive queues a transport write that can never + // complete. Under the old `inactivity_timeout: None` this future + // never resolves and the timeout fires instead. + let outcome = tokio::time::timeout(Duration::from_secs(10), session_fut).await; + let err = outcome + .expect("a wedged write must not park the session loop forever") + .expect_err("the wedged transport ends the session with an error"); + assert!( + matches!( + err, + ConnectionError::Protocol(russh::Error::InactivityTimeout) + ), + "the session must end on the inactivity deadline, got: {err}" + ); + } + + /// The other side of arming the inactivity timer: a healthy peer that + /// sends nothing of its own but answers every keepalive must outlive the + /// deadline. russh resets the inactivity clock on every loop pass that + /// is not a bare keepalive send, so each keepalive answer pushes the + /// deadline out; a russh change that stopped counting the answer would + /// reap every quiet attach once the deadline elapsed. + #[tokio::test] + async fn a_healthy_idle_peer_outlives_the_inactivity_deadline() { + let server = crate::test_harness::TestServer::new().await; + let (session_fut, client, _stuck) = wedgeable_session(&server).await; + tokio::pin!(session_fut); + + // Idle for several inactivity windows: only keepalives and their + // answers cross the transport. + let outcome = tokio::time::timeout(TEST_INACTIVITY * 3, session_fut.as_mut()).await; + assert!( + outcome.is_err(), + "a peer answering keepalives must not be reaped, session ended: {:?}", + outcome.map(|r| r.map_err(|e| e.to_string())) + ); + drop(client); + } } diff --git a/crates/minimald/src/diag.rs b/crates/minimald/src/diag.rs index 9b6cc7ba1a..fbff90093f 100644 --- a/crates/minimald/src/diag.rs +++ b/crates/minimald/src/diag.rs @@ -42,8 +42,8 @@ const MAX_REQUEST_BYTES: u64 = 8 * 1024; /// Bounding the *size* of the request is not the same as bounding the *wait* /// for it: a client that writes half an object and then holds the channel open /// sends nothing more and never signals EOF, so the read below would park -/// forever. Nothing else reclaims it — the daemon deliberately runs with -/// `inactivity_timeout: None` (a live attach must not be reaped), and the SSH +/// forever. Nothing else reclaims it — the daemon arms its inactivity timeout +/// at the full keepalive budget (a live attach must not be reaped), and the SSH /// keepalives it uses instead only detect a peer that has *stopped answering*, /// not one that answers and stays silent. The body is a two-field object /// written immediately after the subsystem opens, so this is a generous @@ -1378,9 +1378,9 @@ mod tests { /// R6.3, the other dimension: bounding the request's *size* does not bound /// the *wait* for it. A client that writes half an object and then holds /// the channel open never sends EOF, and nothing else reclaims the task — - /// the daemon runs `inactivity_timeout: None` by design, and its keepalives - /// only notice a peer that has stopped answering, not one that answers and - /// stays silent. + /// the daemon arms its inactivity timeout at the full keepalive budget, + /// and its keepalives only notice a peer that has stopped answering, not + /// one that answers and stays silent. #[tokio::test] async fn a_request_that_never_ends_is_abandoned_on_the_deadline() { // Kept alive for the whole call: dropping it would deliver the EOF diff --git a/crates/minimald/src/server.rs b/crates/minimald/src/server.rs index 17bbc14adf..320dff848e 100644 --- a/crates/minimald/src/server.rs +++ b/crates/minimald/src/server.rs @@ -1384,6 +1384,14 @@ const KEEPALIVE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(3 /// Unanswered keepalives tolerated before the connection is torn down. const KEEPALIVE_MAX: usize = 3; +/// Wall-clock budget for tearing down a connection whose transport has +/// stopped making progress — a full keepalive cycle plus one interval, so +/// an otherwise-quiet but healthy attach (which answers each keepalive, +/// resetting the clock) is never reaped: a healthy idle attach sees a +/// keepalive every [`KEEPALIVE_INTERVAL`], long before this fires. +const INACTIVITY_TIMEOUT: std::time::Duration = + KEEPALIVE_INTERVAL.saturating_mul(KEEPALIVE_MAX as u32 + 1); + /// Builds the shared russh server config from the server state. async fn build_russh_config( state: &ServerStateHandle, @@ -1392,9 +1400,14 @@ async fn build_russh_config( keys: vec![state.host_key().await?], auth_rejection_time_initial: Some(std::time::Duration::ZERO), nodelay: true, - // Keepalives own deadness detection; the default 600s inactivity - // reaper would kill healthy idle attaches, so it is explicitly off. - inactivity_timeout: None, + // Keepalives own deadness detection, so this must never reap an + // idle-but-alive attach (the default 600s would). But russh bounds + // every transport write by the inactivity timer, and with `None` a + // wedged write parks the session loop forever — keepalives are + // dispatched from that same loop, so they stop firing too and the + // connection outlives every budget. Arming it at the keepalive + // budget bounds a stuck transport without touching a healthy one. + inactivity_timeout: Some(INACTIVITY_TIMEOUT), keepalive_interval: Some(KEEPALIVE_INTERVAL), keepalive_max: KEEPALIVE_MAX, ..Default::default() @@ -3027,6 +3040,27 @@ mod tests { super::test_config(dir.path()) } + /// The inactivity deadline is the whole keepalive budget, derived from the + /// keepalive settings rather than restated: a healthy idle attach answers a + /// keepalive every interval (resetting the clock), so the deadline must + /// outlast every keepalive round the budget allows. + #[test] + fn inactivity_deadline_spans_the_full_keepalive_budget() { + assert_eq!(INACTIVITY_TIMEOUT, Duration::from_secs(2 * 60 * 60)); + assert!(INACTIVITY_TIMEOUT > KEEPALIVE_INTERVAL * KEEPALIVE_MAX as u32); + } + + /// The production config arms the inactivity deadline: with `None` a + /// wedged transport write parks the session loop forever. + #[tokio::test] + async fn the_production_config_arms_the_inactivity_deadline() { + let server = crate::test_harness::TestServer::new().await; + let config = build_russh_config(&server.state).await.unwrap(); + assert_eq!(config.inactivity_timeout, Some(INACTIVITY_TIMEOUT)); + assert_eq!(config.keepalive_interval, Some(KEEPALIVE_INTERVAL)); + assert_eq!(config.keepalive_max, KEEPALIVE_MAX); + } + /// The node address's interim opening follows the port the proxy actually /// bound, not the configured one the switch was built with: a daemon whose /// default was busy binds an OS-chosen port, and `record_bound` re-points From 3d4e6ee183c2e0520e897edb78054743ef3b0fbe Mon Sep 17 00:00:00 2001 From: Norrie Taylor <91171431+norrietaylor@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:47:46 +0000 Subject: [PATCH 26/34] fix(common): write file cache downloads atomically via temp file and rename (#2242) FileCache::write_through created the content-addressed file in place and streamed into it. A stream error left partial bytes under the pinned hash, and a concurrent fetcher could read the truncated file while the download was in flight. It now streams into a uniquely named temp file in the same sha directory, fsyncs it, verifies the SHA-256, and only then renames it onto the final path. The temp file goes on every error path, and a write error returns an I/O error instead of panicking. A failed fsync of the directory warns and does not fail the download. Backport of #2242. Applied without adaptation. (cherry picked from commit 40420c960d6c0d93ca23ed6061ab838f36fead37) --- crates/common/src/file_cache.rs | 226 ++++++++++++++++++++++++++++++-- 1 file changed, 217 insertions(+), 9 deletions(-) diff --git a/crates/common/src/file_cache.rs b/crates/common/src/file_cache.rs index a8d6ee9517..7b78271528 100644 --- a/crates/common/src/file_cache.rs +++ b/crates/common/src/file_cache.rs @@ -158,6 +158,23 @@ impl FileCache { /// Drives the given response to read bytes and write them into the cache with the given filename /// and hash. + /// + /// The bytes go to a uniquely named temp file in the same directory, which is renamed onto the + /// content-addressed path only after its SHA-256 matches. A failed or mismatched download + /// therefore never appears under the hash, and concurrent readers see either nothing or the + /// whole verified file. + /// + /// Returns the cached path, replacing any existing entry on success. `sha256` must match the + /// lowercase hexadecimal digest of the response bytes. Updates `op` with bytes received, using + /// the response's content length, when available, as the progress total only. + /// + /// # Errors + /// + /// Returns `Either::Right` for response stream errors. Returns `Either::Left` for + /// [FileCacheError::BadSha] if `sha256` contains `/`, [FileCacheError::HashMismatch] if the + /// digest differs, or [FileCacheError::IO] if accessing or creating the cache directory, + /// creating the temporary file, writing or syncing it, or persisting it fails. Failure to sync + /// the directory after publication does not prevent a successful return. pub async fn write_through( &self, filename: &str, @@ -168,12 +185,24 @@ impl FileCache { where R: fetchers::FetchResponse, { - let cached_path = self.sha_dir(sha256).map_err(Either::Left)?.join(filename); + use std::os::unix::fs::PermissionsExt; + + let dir = self.sha_dir(sha256).map_err(Either::Left)?; + let cached_path = dir.join(filename); + + // Dropping `tmp` on any early return removes the temp file. The prefix is fixed rather than + // derived from `filename`, so a filename near NAME_MAX still fits. + let mut tmp = tempfile::Builder::new() + .prefix(".dl.") + .suffix(".part") + .permissions(fs::Permissions::from_mode(0o666)) + .tempfile_in(&dir) + .map_err(|e| Either::Left(e.into()))?; let mut hasher = super::HashWriter(Sha256::new()); { - let mut f = fs::File::create(&cached_path).map_err(|e| Either::Left(e.into()))?; - let mut w = Tee::new(&mut f, &mut hasher); + let f = tmp.as_file_mut(); + let mut w = Tee::new(&mut *f, &mut hasher); if let Some(size) = response.content_length() { op.set_length(size); @@ -182,7 +211,7 @@ impl FileCache { match response.chunk().await { Ok(Some(chunk)) => { op.increment(chunk.len() as u64); - w.write_all(&chunk).unwrap(); + w.write_all(&chunk).map_err(|e| Either::Left(e.into()))?; } Ok(None) => { break; @@ -200,14 +229,22 @@ impl FileCache { let computed_hex = hex::encode(computed_hash); if computed_hex != sha256 { - fs::remove_file(&cached_path).ok(); // ignore error, best effort - Err(Either::Left(FileCacheError::HashMismatch { + return Err(Either::Left(FileCacheError::HashMismatch { want: sha256.to_string(), got: computed_hex, - })) - } else { - Ok(cached_path) + })); } + + // Renaming over an existing entry is fine: it holds the same verified bytes. + tmp.persist(&cached_path) + .map_err(|e| Either::Left(e.error.into()))?; + // Fsync the directory so the rename itself survives a crash. This buys durability, not + // integrity (`read_cached` re-hashes every read), so a failure only warns: some + // filesystems refuse a directory fsync. + if let Err(e) = fs::File::open(&dir).and_then(|d| d.sync_all()) { + tracing::warn!("could not fsync cache directory {}: {e}", dir.display()); + } + Ok(cached_path) } } @@ -350,6 +387,177 @@ mod tests { ); } + fn sha256_hex(data: &[u8]) -> String { + let mut h = sha2::Sha256::new(); + sha2::Digest::update(&mut h, data); + hex::encode(h.finalize()) + } + + /// A [FetchResponse] that yields one chunk, waits for `resume` to be + /// notified, then yields `tail` (or fails if `tail` is `Err`). + #[derive(Debug)] + struct PausingResponse { + head: Option, + tail: Option>, + paused: std::sync::Arc, + resume: std::sync::Arc, + } + + impl fetchers::FetchResponse for PausingResponse { + type Error = String; + + fn error_for_status(self) -> Result { + Ok(self) + } + fn is_success(&self) -> bool { + true + } + fn status_code(&self) -> usize { + 200 + } + fn content_length(&self) -> Option { + None + } + async fn bytes(self) -> Result { + unimplemented!("write_through streams via chunk()") + } + async fn chunk(&mut self) -> Result, Self::Error> { + if let Some(head) = self.head.take() { + return Ok(Some(head)); + } + match self.tail.take() { + Some(tail) => { + self.paused.notify_one(); + self.resume.notified().await; + tail.map(Some) + } + None => Ok(None), + } + } + } + + /// Lists the entries of a cache sha directory. + fn dir_entries(dir: &std::path::Path) -> Vec { + let mut v: Vec = fs::read_dir(dir) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + v.sort(); + v + } + + /// A stream that fails part-way through must leave nothing at the + /// content-addressed path, and no temp file behind either. + #[tokio::test] + async fn write_through_stream_error_leaves_no_file() { + let tmp = tempfile::tempdir().unwrap(); + let cache = FileCache::new(tmp.path().to_path_buf()).unwrap(); + let op = OpTracker::new_with_root(&None); + let sha = sha256_hex(b"partial-and-the-rest"); + + let resume = std::sync::Arc::new(tokio::sync::Notify::new()); + resume.notify_one(); // do not actually pause + let mut resp = PausingResponse { + head: Some(Bytes::from_static(b"partial-")), + tail: Some(Err("connection reset".into())), + paused: std::sync::Arc::new(tokio::sync::Notify::new()), + resume, + }; + + let err = cache + .write_through("src.tar.gz", &sha, &mut resp, &op) + .await + .unwrap_err(); + assert_eq!(err.unwrap_right(), "connection reset"); + + let dir = cache.sha_dir(&sha).unwrap(); + assert!( + !dir.join("src.tar.gz").exists(), + "a failed stream must not leave a file under the pinned hash" + ); + assert_eq!(dir_entries(&dir), Vec::::new(), "temp file leaked"); + } + + /// While a download is in flight, a concurrent reader must never see a + /// partial file at the content-addressed path; once it completes, the + /// whole verified file appears. + #[tokio::test] + async fn write_through_in_flight_is_invisible_to_readers() { + let tmp = tempfile::tempdir().unwrap(); + let cache = FileCache::new(tmp.path().to_path_buf()).unwrap(); + let op = OpTracker::new_with_root(&None); + let content = b"first-half|second-half"; + let sha = sha256_hex(content); + + let paused = std::sync::Arc::new(tokio::sync::Notify::new()); + let resume = std::sync::Arc::new(tokio::sync::Notify::new()); + let mut resp = PausingResponse { + head: Some(Bytes::copy_from_slice(&content[..11])), + tail: Some(Ok(Bytes::copy_from_slice(&content[11..]))), + paused: paused.clone(), + resume: resume.clone(), + }; + + let writer = { + let cache = cache.clone(); + let sha = sha.clone(); + tokio::spawn(async move { + cache + .write_through("src.tar.gz", &sha, &mut resp, &op) + .await + .map_err(|e| format!("{e:?}")) + }) + }; + + // The writer has written the first chunk and is blocked mid-stream. + paused.notified().await; + let final_path = cache.sha_dir(&sha).unwrap().join("src.tar.gz"); + assert!( + !final_path.exists(), + "a partial download must not be visible under the pinned hash" + ); + assert_eq!(cache.read_cached("src.tar.gz", &sha).unwrap(), None); + + resume.notify_one(); + let path = writer.await.unwrap().expect("download should succeed"); + assert_eq!(path, final_path); + assert_eq!(fs::read(&path).unwrap(), content); + assert_eq!( + dir_entries(path.parent().unwrap()), + vec!["src.tar.gz".to_string()], + "only the final file should remain" + ); + } + + /// A filename close to NAME_MAX (255) must still cache: the temp file + /// name must not grow with the filename. + #[tokio::test] + async fn write_through_accepts_filename_near_name_max() { + let tmp = tempfile::tempdir().unwrap(); + let cache = FileCache::new(tmp.path().to_path_buf()).unwrap(); + let op = OpTracker::new_with_root(&None); + let content = b"long-name"; + let sha = sha256_hex(content); + let filename = format!("{}.tar.gz", "a".repeat(247)); + assert_eq!(filename.len(), 254); + + let resume = std::sync::Arc::new(tokio::sync::Notify::new()); + resume.notify_one(); + let mut resp = PausingResponse { + head: Some(Bytes::from_static(content)), + tail: None, + paused: std::sync::Arc::new(tokio::sync::Notify::new()), + resume, + }; + + let path = cache + .write_through(&filename, &sha, &mut resp, &op) + .await + .map_err(|e| format!("{e:?}")) + .expect("a 254-byte filename should cache"); + assert_eq!(fs::read(&path).unwrap(), content); + } + /// A fake URL whose filename is whatever the test sets. #[derive(Debug, Clone)] struct FakeUrl { From 169704442fed45a99d391ae45827e4c58e65e967 Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Sun, 18 Oct 2026 19:34:07 +0000 Subject: [PATCH 27/34] fix(minimald): leave HEAD alone when a push targets another branch (#2319) The post-receive hook checked out every pushed branch, so pushing a branch other than the workspace's current one moved HEAD and swapped the worktree to it. A push to a non-checked-out branch now only moves the ref. A first push into a fresh workspace checks out the first branch and leaves the rest as refs, and a push onto a detached HEAD says so on a remote: line with the git switch that re-attaches it. The cli-min push paragraph describes all three. Backport of #2319. Applied without adaptation. (cherry picked from commit 0a297d0d1c39fe0756490a58efbcaa77f2821bab) --- crates/minimal/tests/git_remote.rs | 317 +++++++++++++++++++++++++++++ crates/minimald/src/exec.rs | 40 +++- docs/reference/cli-min.md | 10 + 3 files changed, 365 insertions(+), 2 deletions(-) diff --git a/crates/minimal/tests/git_remote.rs b/crates/minimal/tests/git_remote.rs index 01bf93049d..af3899c75c 100644 --- a/crates/minimal/tests/git_remote.rs +++ b/crates/minimal/tests/git_remote.rs @@ -86,3 +86,320 @@ async fn git_push_through_min_binary_lands_commit_in_workspace() { let contents = sftp.read("/workbench/hello.txt").await.unwrap(); assert_eq!(contents, b"hi there\n"); } + +/// `git push ...` from `src`, with the `git-remote-min` helper +/// in `helpers` first on PATH and the daemon socket under `state_home`. +/// Returns the push's stderr, which carries the hook's `remote:` lines. +async fn push_refs( + src: &std::path::Path, + helpers: &std::path::Path, + state_home: &std::path::Path, + url: &str, + refs: &[&str], +) -> String { + let inherited_path = std::env::var("PATH").unwrap_or_default(); + let out = tokio::process::Command::new("git") + .current_dir(src) + .env("PATH", format!("{}:{inherited_path}", helpers.display())) + .env("XDG_STATE_HOME", state_home) + .arg("push") + .arg(url) + .args(refs) + .output() + .await + .expect("git push should run"); + assert!( + out.status.success(), + "git push {refs:?} failed: stdout={} stderr={}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr), + ); + String::from_utf8_lossy(&out.stderr).into_owned() +} + +/// The commit `rev` names in the repo at `cwd`. +async fn rev_parse(cwd: &std::path::Path, rev: &str) -> String { + let out = tokio::process::Command::new("git") + .current_dir(cwd) + .args(["rev-parse", rev]) + .output() + .await + .expect("git rev-parse should run"); + assert!(out.status.success(), "git rev-parse {rev} failed"); + String::from_utf8(out.stdout).unwrap().trim().to_string() +} + +#[tokio::test] +async fn git_push_of_other_branch_leaves_workspace_on_current_branch() { + // Same harness shape as the push test above: TestServer on the default + // socket path, helper symlinked on PATH. + let state_home = tempfile::TempDir::new().unwrap(); + let sock_dir = state_home.path().join("minimal/providers/local-minimald0"); + std::fs::create_dir_all(&sock_dir).unwrap(); + + let server = TestServer::new().await; + server.listen_on_uds(&sock_dir.join("ssh.sock")).await; + + let mut client = server.connect().await; + let session_id = create_configured_session(&mut client, "git-push-other-branch", "/tmp").await; + + let helpers = tempfile::TempDir::new().unwrap(); + std::os::unix::fs::symlink( + env!("CARGO_BIN_EXE_min"), + helpers.path().join("git-remote-min"), + ) + .unwrap(); + + // Source repo: `main` with one commit, then a `feature` branch with a + // second commit on top. + let src_dir = tempfile::TempDir::new().unwrap(); + let src = src_dir.path(); + run_git(src, &["init", "-q", "-b", "main"]).await; + run_git(src, &["config", "user.email", "t@example.com"]).await; + run_git(src, &["config", "user.name", "Test"]).await; + tokio::fs::write(src.join("hello.txt"), b"hi there\n") + .await + .unwrap(); + run_git(src, &["add", "hello.txt"]).await; + run_git(src, &["commit", "-q", "-m", "first commit"]).await; + run_git(src, &["checkout", "-q", "-b", "feature"]).await; + tokio::fs::write(src.join("feature.txt"), b"feature\n") + .await + .unwrap(); + run_git(src, &["add", "feature.txt"]).await; + run_git(src, &["commit", "-q", "-m", "second commit"]).await; + let feature_tip = rev_parse(src, "feature").await; + + let session_url = format!("min://{session_id}"); + for remote_ref in ["main", "feature"] { + push_refs( + src, + helpers.path(), + state_home.path(), + &session_url, + &[remote_ref], + ) + .await; + } + + // The pushes ran in order: the first populated the workspace with + // `main` checked out; the second pushed `feature`, a branch that is + // NOT the checked-out one. HEAD must stay on main ... + + let sftp = client.open_sftp(session_id).await; + let head = sftp.read("/workbench/.git/HEAD").await.unwrap(); + assert_eq!(head, b"ref: refs/heads/main\n"); + // ... the worktree still shows main's files and not feature's ... + let hello = sftp.read("/workbench/hello.txt").await.unwrap(); + assert_eq!(hello, b"hi there\n"); + assert!(sftp.read("/workbench/feature.txt").await.is_err()); + // ... and the feature ref landed at the pushed tip. + let feature_ref = sftp + .read("/workbench/.git/refs/heads/feature") + .await + .unwrap(); + assert_eq!(String::from_utf8_lossy(&feature_ref).trim(), feature_tip); +} + +#[tokio::test] +async fn git_push_of_several_refs_syncs_only_the_current_branch() { + // Same harness shape as the tests above. + let state_home = tempfile::TempDir::new().unwrap(); + let sock_dir = state_home.path().join("minimal/providers/local-minimald0"); + std::fs::create_dir_all(&sock_dir).unwrap(); + + let server = TestServer::new().await; + server.listen_on_uds(&sock_dir.join("ssh.sock")).await; + + let mut client = server.connect().await; + let session_id = create_configured_session(&mut client, "git-push-several-refs", "/tmp").await; + + let helpers = tempfile::TempDir::new().unwrap(); + std::os::unix::fs::symlink( + env!("CARGO_BIN_EXE_min"), + helpers.path().join("git-remote-min"), + ) + .unwrap(); + let url = format!("min://{session_id}"); + + // Populate the workspace with `main` checked out. + let src_dir = tempfile::TempDir::new().unwrap(); + let src = src_dir.path(); + run_git(src, &["init", "-q", "-b", "main"]).await; + run_git(src, &["config", "user.email", "t@example.com"]).await; + run_git(src, &["config", "user.name", "Test"]).await; + tokio::fs::write(src.join("hello.txt"), b"hi there\n") + .await + .unwrap(); + run_git(src, &["add", "hello.txt"]).await; + run_git(src, &["commit", "-q", "-m", "first commit"]).await; + push_refs(src, helpers.path(), state_home.path(), &url, &["main"]).await; + + // Advance `main`, and grow a new `feature` branch off the first commit. + tokio::fs::write(src.join("main2.txt"), b"main two\n") + .await + .unwrap(); + run_git(src, &["add", "main2.txt"]).await; + run_git(src, &["commit", "-q", "-m", "main second commit"]).await; + run_git(src, &["checkout", "-q", "-b", "feature", "HEAD~1"]).await; + tokio::fs::write(src.join("feature.txt"), b"feature\n") + .await + .unwrap(); + run_git(src, &["add", "feature.txt"]).await; + run_git(src, &["commit", "-q", "-m", "feature commit"]).await; + let feature_tip = rev_parse(src, "feature").await; + + // One push carrying both refs: the checked-out `main` syncs into the + // worktree, while `feature` only moves its ref. + push_refs( + src, + helpers.path(), + state_home.path(), + &url, + &["main", "feature"], + ) + .await; + + let sftp = client.open_sftp(session_id).await; + let head = sftp.read("/workbench/.git/HEAD").await.unwrap(); + assert_eq!(head, b"ref: refs/heads/main\n"); + let main2 = sftp.read("/workbench/main2.txt").await.unwrap(); + assert_eq!(main2, b"main two\n"); + assert!(sftp.read("/workbench/feature.txt").await.is_err()); + let feature_ref = sftp + .read("/workbench/.git/refs/heads/feature") + .await + .unwrap(); + assert_eq!(String::from_utf8_lossy(&feature_ref).trim(), feature_tip); +} + +#[tokio::test] +async fn git_first_push_of_several_refs_checks_out_only_the_first() { + // Same harness shape as the tests above. + let state_home = tempfile::TempDir::new().unwrap(); + let sock_dir = state_home.path().join("minimal/providers/local-minimald0"); + std::fs::create_dir_all(&sock_dir).unwrap(); + + let server = TestServer::new().await; + server.listen_on_uds(&sock_dir.join("ssh.sock")).await; + + let mut client = server.connect().await; + let session_id = create_configured_session(&mut client, "git-first-push-several", "/tmp").await; + + let helpers = tempfile::TempDir::new().unwrap(); + std::os::unix::fs::symlink( + env!("CARGO_BIN_EXE_min"), + helpers.path().join("git-remote-min"), + ) + .unwrap(); + + // Source repo: `main` with one commit, then `feature` one commit ahead. + let src_dir = tempfile::TempDir::new().unwrap(); + let src = src_dir.path(); + run_git(src, &["init", "-q", "-b", "main"]).await; + run_git(src, &["config", "user.email", "t@example.com"]).await; + run_git(src, &["config", "user.name", "Test"]).await; + tokio::fs::write(src.join("hello.txt"), b"hi there\n") + .await + .unwrap(); + run_git(src, &["add", "hello.txt"]).await; + run_git(src, &["commit", "-q", "-m", "first commit"]).await; + run_git(src, &["checkout", "-q", "-b", "feature"]).await; + tokio::fs::write(src.join("feature.txt"), b"feature\n") + .await + .unwrap(); + run_git(src, &["add", "feature.txt"]).await; + run_git(src, &["commit", "-q", "-m", "feature commit"]).await; + let feature_tip = rev_parse(src, "feature").await; + + // The very first push into the fresh workspace carries both branches, + // `main` first. The hook checks out `main` to populate the worktree; + // `feature`, arriving second, only moves its ref. + let url = format!("min://{session_id}"); + push_refs( + src, + helpers.path(), + state_home.path(), + &url, + &["main", "feature"], + ) + .await; + + let sftp = client.open_sftp(session_id).await; + let head = sftp.read("/workbench/.git/HEAD").await.unwrap(); + assert_eq!(head, b"ref: refs/heads/main\n"); + let hello = sftp.read("/workbench/hello.txt").await.unwrap(); + assert_eq!(hello, b"hi there\n"); + assert!(sftp.read("/workbench/feature.txt").await.is_err()); + let feature_ref = sftp + .read("/workbench/.git/refs/heads/feature") + .await + .unwrap(); + assert_eq!(String::from_utf8_lossy(&feature_ref).trim(), feature_tip); +} + +#[tokio::test] +async fn git_push_into_detached_head_moves_ref_and_says_so() { + // Same harness shape as the tests above. + let state_home = tempfile::TempDir::new().unwrap(); + let sock_dir = state_home.path().join("minimal/providers/local-minimald0"); + std::fs::create_dir_all(&sock_dir).unwrap(); + + let server = TestServer::new().await; + server.listen_on_uds(&sock_dir.join("ssh.sock")).await; + + let mut client = server.connect().await; + let session_id = create_configured_session(&mut client, "git-push-detached", "/tmp").await; + + let helpers = tempfile::TempDir::new().unwrap(); + std::os::unix::fs::symlink( + env!("CARGO_BIN_EXE_min"), + helpers.path().join("git-remote-min"), + ) + .unwrap(); + let url = format!("min://{session_id}"); + + // Populate the workspace with `main` checked out. + let src_dir = tempfile::TempDir::new().unwrap(); + let src = src_dir.path(); + run_git(src, &["init", "-q", "-b", "main"]).await; + run_git(src, &["config", "user.email", "t@example.com"]).await; + run_git(src, &["config", "user.name", "Test"]).await; + tokio::fs::write(src.join("hello.txt"), b"hi there\n") + .await + .unwrap(); + run_git(src, &["add", "hello.txt"]).await; + run_git(src, &["commit", "-q", "-m", "first commit"]).await; + let first = rev_parse(src, "main").await; + push_refs(src, helpers.path(), state_home.path(), &url, &["main"]).await; + + // Detach the workspace HEAD at the first commit. The detached form is + // longer than `ref: refs/heads/main\n`, so a plain write replaces it. + let sftp = client.open_sftp(session_id).await; + sftp.write("/workbench/.git/HEAD", format!("{first}\n").as_bytes()) + .await + .unwrap(); + + // Advance `main` and push it. + tokio::fs::write(src.join("main2.txt"), b"main two\n") + .await + .unwrap(); + run_git(src, &["add", "main2.txt"]).await; + run_git(src, &["commit", "-q", "-m", "main second commit"]).await; + let main_tip = rev_parse(src, "main").await; + let stderr = push_refs(src, helpers.path(), state_home.path(), &url, &["main"]).await; + + // The push says HEAD is detached ... + assert!( + stderr.contains("remote worktree HEAD is detached; moved refs without checking out"), + "push stderr should note the detached HEAD: {stderr}", + ); + // ... HEAD stays detached at the first commit and the worktree is + // unchanged ... + let head = sftp.read("/workbench/.git/HEAD").await.unwrap(); + assert_eq!(String::from_utf8_lossy(&head).trim(), first); + assert!(sftp.read("/workbench/main2.txt").await.is_err()); + // ... and the main ref landed at the pushed tip. + let main_ref = sftp.read("/workbench/.git/refs/heads/main").await.unwrap(); + assert_eq!(String::from_utf8_lossy(&main_ref).trim(), main_tip); +} diff --git a/crates/minimald/src/exec.rs b/crates/minimald/src/exec.rs index 03a9585c16..5dcee1c182 100644 --- a/crates/minimald/src/exec.rs +++ b/crates/minimald/src/exec.rs @@ -2308,6 +2308,19 @@ async fn handle_git_receive( WORK_TREE=$(dirname "$GIT_DIR_ABS") unset GIT_DIR GIT_WORK_TREE GIT_QUARANTINE_PATH g() { git --git-dir="$GIT_DIR_ABS" --work-tree="$WORK_TREE" "$@"; } + # The branch currently checked out in the worktree, and + # whether HEAD resolves to a commit (`born`). The first push + # into a fresh workspace has no commit on HEAD, so the first + # pushed branch that checks out cleanly populates the + # worktree and becomes HEAD. After that, a push to any branch + # other than the checked-out one must not move HEAD or touch + # the worktree. post-receive runs after the refs moved, so a + # first push that includes HEAD's own branch already reads as + # born, and that branch is the one checked out. + cur=$(g symbolic-ref -q HEAD) + born= + detached_moved= + g rev-parse -q --verify HEAD >/dev/null && born=yes zero=0000000000000000000000000000000000000000 while read -r old new ref; do @@ -2318,6 +2331,18 @@ async fn handle_git_receive( [ "$new" = "$zero" ] && continue branch=${ref#refs/heads/} + # A push to a branch that is not the one checked out + # leaves HEAD and the worktree alone: the ref itself has + # already moved in receive-pack, so there is nothing more + # to do. Only on the still-unborn first push does the + # hook check out the first arriving branch. + # A detached HEAD (`cur` empty) matches no branch, so + # every push lands here; note it for the message below. + if [ "$ref" != "$cur" ] && [ -n "$born" ]; then + [ -z "$cur" ] && detached_moved=yes + continue + fi + if [ "$old" = "$zero" ]; then # Previously-unborn branch: nothing tracked, so tracked # dirtiness cannot exist — but uploaded/untracked files @@ -2326,7 +2351,12 @@ async fn handle_git_receive( # pushed tip and checks it out; WITHOUT -f, git refuses # to overwrite untracked files (its stderr names them), # and we surface the skip like the dirty case below. - if ! g checkout -q -B "$branch" "$new"; then + if g checkout -q -B "$branch" "$new"; then + # HEAD is born now: later refs in this same push + # are other branches and must not move it. + cur=$ref + born=yes + else echo "remote worktree has local files the push would overwrite; skipping checkout of $branch" >&2 fi continue @@ -2341,7 +2371,13 @@ async fn handle_git_receive( continue fi g checkout -qf "$branch" - done"# + done + + # A detached HEAD stays detached, as with plain git; say so + # once, so the pusher knows the worktree did not change. + if [ -n "$detached_moved" ]; then + echo "remote worktree HEAD is detached; moved refs without checking out (run git switch in the session)" >&2 + fi"# .as_bytes(), ) .await diff --git a/docs/reference/cli-min.md b/docs/reference/cli-min.md index 8206a00834..4eb29e0ee4 100644 --- a/docs/reference/cli-min.md +++ b/docs/reference/cli-min.md @@ -888,6 +888,16 @@ git remote add session min:// git push session ``` +When the workspace's worktree is clean, a push to its checked-out branch also +checks out the new tip. That rewrites files under any process running in the +session. When the worktree holds uncommitted changes, the branch ref still +moves, and the push warns that it skipped the checkout. A push to any other +branch moves only that branch's ref. A push into a worktree with a detached +HEAD moves only the refs and warns that it skipped the checkout. HEAD stays +detached. The first push into an empty workspace checks out the branch that +the fresh repository's HEAD already names, if the push includes it. +Otherwise it checks out the first pushed branch. + Only the `connect` capability is implemented: on `connect`, the helper opens an exec channel to `minimald` over its Unix domain socket (the same transport the rest of the CLI uses) and bridges git's pack-protocol From 94eb0ebe61a392635c3d61b3cc2458f1c809daf3 Mon Sep 17 00:00:00 2001 From: bryan-minimal Date: Sun, 25 Oct 2026 22:09:29 +0000 Subject: [PATCH 28/34] fix(minimald): fork a task run's container on the sandbox fork thread (#2572) hakoniwa arms PR_SET_PDEATHSIG(SIGKILL) in every container it forks, and Linux delivers that signal when the forking thread exits. The task-run path in exec.rs forked its container on whichever tokio worker ran the exec. Any block_in_place on that worker displaces it, tokio retires it after its 10 s keep-alive, and every idle task session forked on it dies with exit 125. The spawn now goes through on_fork_thread, as the session launch and run_with_cancel do. Backport of #2572. Adapted to the line: sandbox:: reads sandbox2::. (cherry picked from commit cafa6743216b61006dc439843b48eb137695ff33) --- crates/minimald/src/exec.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/minimald/src/exec.rs b/crates/minimald/src/exec.rs index 5dcee1c182..d2d8f58b64 100644 --- a/crates/minimald/src/exec.rs +++ b/crates/minimald/src/exec.rs @@ -526,8 +526,15 @@ async fn task_producer( cmd.stdin(hakoniwa::Stdio::piped()) .stdout(hakoniwa::Stdio::piped()) .stderr(hakoniwa::Stdio::piped()); - let mut child = cmd - .spawn() + // On the fork thread, never this one: hakoniwa arms + // `PR_SET_PDEATHSIG(SIGKILL)` in the container, so it dies + // with the thread that forked it, and a runtime worker is + // retired ~10 s after any `block_in_place` displaces it (see + // `sandbox2::forker`). #1748 moved the session launch and + // `Sandbox::run_with_cancel` there; this spawn, a task run's, + // stayed on the worker, and idle task sessions died with it + // (inbox#710). + let mut child = sandbox2::forker::on_fork_thread(move || cmd.spawn()) .map_err(|e| io::Error::other(format!("command launch failed: {}", e)))?; let spawned = sandbox2::Spawned::from_child(&mut child); let guard = attach_or_reap(planned, spawned, &mut child).await?; From e34798455ca28185778171a3a4fabdccbcc1888b Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 08:26:53 +0000 Subject: [PATCH 29/34] fix(installer,dash): stop the daemon across uninstall and reinstall (#2342) An uninstall deleted the binaries but left a running daemon in place, so a reinstall autospawned nothing and every daemon-backed command then hit the old daemon across a version skew. The uninstall walk now stops the daemon first, while the min that matches it is still on disk, with the same escalation as an upgrade. An install that starts with no min on disk compares the installed min's version with the answering daemon's and force-stops a mismatched one. The dashboard's discovery connections are now gated as min ls is, so it refuses a skewed daemon before the TUI takes the terminal, and a provider that reappears skewed joins only under the override. The skew message's recovery line names min stop --force. Backport of #2342. Adapted to the line: dash/ is the minimal-tui crate, so its inventory test carries the gated rows, and the rows in minimal's connection inventory drop out. (cherry picked from commit 2e67f8ae99bea693d4a4d344192bc53646d657a4) --- crates/minimal-tui/src/app.rs | 5 +- crates/minimal-tui/src/rpc.rs | 76 ++++++++------ crates/minimal/src/cmd/list.rs | 7 +- crates/minimald-rpc/src/lib.rs | 3 +- scripts/install.sh | 181 ++++++++++++++++++++------------- scripts/install_test.sh | 155 ++++++++++++++++++++++++++++ 6 files changed, 319 insertions(+), 108 deletions(-) diff --git a/crates/minimal-tui/src/app.rs b/crates/minimal-tui/src/app.rs index b18f6bc93f..69cfb258ae 100644 --- a/crates/minimal-tui/src/app.rs +++ b/crates/minimal-tui/src/app.rs @@ -913,7 +913,10 @@ pub async fn run(opts: DashOptions) -> Result<(), anyhow::Error> { anyhow::bail!("min dash requires an interactive terminal on stdin and stdout"); } - let mut providers = rpc::discover(opts.minimal_dir.as_deref()).await; + // Gated before the terminal is taken over, so a skewed daemon refuses the + // dashboard the same way it refuses `min ls` — plain message, terminal + // untouched. + let mut providers = rpc::discover(opts.minimal_dir.as_deref()).await?; let contribution = opts.contribution.clone(); let mut last_rediscovery = std::time::Instant::now(); let saved = state::load(opts.minimal_dir.as_deref()); diff --git a/crates/minimal-tui/src/rpc.rs b/crates/minimal-tui/src/rpc.rs index 9f25d59b7d..40c73cb4f8 100644 --- a/crates/minimal-tui/src/rpc.rs +++ b/crates/minimal-tui/src/rpc.rs @@ -141,34 +141,41 @@ fn push_candidate(candidates: &mut Vec<(String, PathBuf)>, label: String, sock: /// Probe both known socket paths and connect to each reachable daemon. /// Sockets that don't exist are skipped without a connect attempt, so /// discovery never pays the connect-retry delay for a down provider. -pub async fn discover(minimal_dir: Option<&Path>) -> Vec { +/// +/// Each connection is version-gated like `min ls`'s: the dashboard's create +/// path (`activate`) is a multi-step mutation, and opening on a skewed pair +/// is refused before the TUI takes over the terminal. The override var +/// downgrades each refusal to a warning, as everywhere else. +pub async fn discover(minimal_dir: Option<&Path>) -> Result, anyhow::Error> { let mut providers = Vec::new(); for (label, sock) in probe_candidates(minimal_dir) { if !sock.exists() { continue; } - // Deliberately not version-gated: the dashboard is read-only until the - // user acts, it renders each provider's version in the sidebar, and a - // skewed daemon is precisely when an operator needs to see (and be able - // to destroy) the sessions on it. The one multi-step mutation the TUI - // drives, `activate`, gates on its own connection instead. - match Client::connect(&sock).await { - Ok(client) => providers.push(Provider { + let mut client = match Client::connect(&sock).await { + Ok(client) => client, + Err(e) => { + tracing::warn!(provider = label, error = %e, "socket present but daemon unreachable"); + continue; + } + }; + match minimal_client::ensure_version_match(&mut client).await { + Ok(()) => providers.push(Provider { label, sock, client, }), - Err(e) => { - tracing::warn!(provider = label, error = %e, "socket present but daemon unreachable"); - } + Err(e) => return Err(e.context(format!("provider {label}"))), } } - providers + Ok(providers) } /// Connect to any canonical provider not already served, so a daemon that /// (re)appears mid-run joins the dashboard. Each attempt is bounded by -/// [`CONNECT_TIMEOUT`] so a wedged acceptor can't stall the UI loop. +/// [`CONNECT_TIMEOUT`] so a wedged acceptor can't stall the UI loop. A +/// provider whose build is skewed is skipped with a warning rather than +/// joined, but never fails the loop for the providers already listed. pub async fn connect_missing(providers: &mut Vec, minimal_dir: Option<&Path>) { for (label, sock) in probe_candidates(minimal_dir) { if providers.iter().any(|p| p.label == label || p.sock == sock) { @@ -177,24 +184,31 @@ pub async fn connect_missing(providers: &mut Vec, minimal_dir: Option< if !sock.exists() { continue; } - // Deliberately not version-gated, for the same reason as `discover`: - // a provider that reappears mid-run must still be listable. - match tokio::time::timeout(CONNECT_TIMEOUT, Client::connect(&sock)).await { - Ok(Ok(client)) => { - tracing::info!(provider = label, "provider connected"); - providers.push(Provider { - label, - sock, - client, - }); - } + let mut client = match tokio::time::timeout(CONNECT_TIMEOUT, Client::connect(&sock)).await { + Ok(Ok(client)) => client, Ok(Err(e)) => { tracing::debug!(provider = label, error = %e, "provider reconnect failed"); + continue; } Err(_) => { tracing::debug!(provider = label, "provider reconnect timed out"); + continue; } + }; + // Version-gated like `discover`, but a skewed provider that appears + // mid-run must not kill the dashboard the other providers are still + // serving, so the refusal skips it — the warning it logged is the + // operator's cue to `min stop --force` and reopen. + if let Err(e) = minimal_client::ensure_version_match(&mut client).await { + tracing::warn!(provider = label, error = %e, "provider build is skewed"); + continue; } + tracing::info!(provider = label, "provider connected"); + providers.push(Provider { + label, + sock, + client, + }); } } @@ -1538,21 +1552,21 @@ mod tests { #[cfg(test)] mod version_gate_tests { /// The dashboard opens three daemon connections, and #1251's gate reached - /// none of them. Only `activate` runs a multi-step mutation, so only it is - /// gated; the two discovery connections stay open on purpose and must say - /// so, since a reader who finds an ungated connect has no other way to tell - /// an audited decision from an oversight. + /// none of them. `activate` and the two discovery connections are all + /// gated now — the dashboard opens on the same terms `min ls` does, and a + /// provider that reappears mid-run with a skewed build joins only under + /// the override. /// /// Asserted as an inventory rather than a count, so adding, removing, or /// reclassifying a connection fails here and forces the same judgement. #[test] - fn only_the_activation_path_is_version_gated() { + fn every_dashboard_connection_is_version_gated() { assert_eq!( connect_site_inventory(), [ "activate = gated", - "connect_missing = ungated (explained)", - "discover = ungated (explained)", + "connect_missing = gated", + "discover = gated", ] ); } diff --git a/crates/minimal/src/cmd/list.rs b/crates/minimal/src/cmd/list.rs index 7aa2fb1a9a..c403d9c60e 100644 --- a/crates/minimal/src/cmd/list.rs +++ b/crates/minimal/src/cmd/list.rs @@ -374,10 +374,9 @@ async fn listings_across_vms( let mut listings = Vec::new(); for vm in vms { // The selected VM gets the gate `min ls` has always applied - // (`connect_daemon`'s); the others are listable ungated, for the same - // reason the dashboard lists them ungated — a listing is read-only, - // and a skewed VM is precisely one whose boxes an operator still - // needs to see. + // (`connect_daemon`'s); the others are listable ungated — a listing is + // read-only, and a skewed VM is precisely one whose boxes an + // operator still needs to see. let gate = vm.vm == selected; // A VM that is not running hosts no boxes: an absent socket is "no // boxes here", not a fault. The selected VM is exempt even so — it is diff --git a/crates/minimald-rpc/src/lib.rs b/crates/minimald-rpc/src/lib.rs index 193ddfd6a3..b5701b898e 100644 --- a/crates/minimald-rpc/src/lib.rs +++ b/crates/minimald-rpc/src/lib.rs @@ -134,7 +134,8 @@ pub fn version_skew_message(cli: &str, daemon: &str) -> Option { "This CLI is minimal {cli}, but the running minimald is {daemon}. \ The two speak the same RPCs only when built together, so continuing \ would fail partway through and tear down whatever it had created. \ - Restart the daemon on the new build: run `min stop`, then re-run this \ + Restart the daemon on the new build: run `min stop --force` (the \ + plain stop is refused while sessions are live), then re-run this \ command (the daemon is started again automatically). \ Set {SKEW_OVERRIDE_VAR}=1 to proceed anyway." ) diff --git a/scripts/install.sh b/scripts/install.sh index 94e879e4fc..48ea408eb0 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -31,9 +31,12 @@ # offer. # # Uninstall walks the local install record (no network) and removes each file -# whose on-disk bytes still match what the installer recorded writing; it accepts -# --force (remove modified files too), --purge (also delete the minimal data/ -# state/cache trees), and --dry-run. +# whose on-disk bytes still match what the installer recorded writing. It +# stops the running daemon first — asking on the terminal when that daemon +# has active sessions, exactly as an upgrade does — and accepts --force +# (remove modified files too), --force-stop (stop live sessions without +# asking), --purge (also delete the minimal data/state/cache trees), and +# --dry-run. # # After the files are placed, the installer wires up shell integration: # it generates per-shell init files that prepend the bin dir to PATH, installs @@ -243,8 +246,9 @@ if [ "${1-}" = "--uninstall" ]; then case "$1" in --force) uninstall_force=1 ;; --purge) uninstall_purge=1 ;; + --force-stop) force_stop=1 ;; -n|--dry-run) dry_run=1 ;; - *) die "unknown uninstall option '$1' (allowed: --force, --purge, --dry-run)" ;; + *) die "unknown uninstall option '$1' (allowed: --force, --force-stop, --purge, --dry-run)" ;; esac shift done @@ -622,6 +626,19 @@ do_uninstall() { say "" [ "$dry_run" -eq 1 ] && say "uninstall: dry run — nothing will be removed" + # The walk below deletes `min` itself, so the daemon it runs must be + # stopped first, while the `min` that matches it is still on disk — a + # daemon left running through an uninstall+reinstall keeps serving the old + # build, and every daemon-backed command on the fresh install then hits a + # version skew. Same escalation as the upgrade swap: graceful stop, consent + # on the terminal if sessions are live, --force after. Uninstall and the + # upgrade share the once-guard, so a `min` removed here stays removed. + if [ "$dry_run" -eq 1 ]; then + say " would stop the running daemon (it is stopped on the real run)" + else + stop_running_daemon || die "aborted: the daemon is still running; nothing was removed" + fi + # Before the walk (which deletes `min` and the shipped loader), offer to # tear down what was installed with root outside the record too. maybe_remove_finalize_install @@ -806,6 +823,75 @@ do_uninstall() { return 0 } +# --- Daemon stop (shared by the upgrade swap and the uninstall walk) --------- + +# Swapping binaries under a running daemon wedges it: the daemon keeps +# serving from the old image while the new `min` talks to it. Stop it first, +# using the `min` ALREADY on disk — that one matches the daemon it started, and +# it is about to be overwritten. Deliberately best-effort and silent: nothing +# installed yet, no daemon running, or a `min` too old to have `stop` all mean +# "nothing to stop", and none of them should fail an install whose binaries are +# otherwise fine. `min stop` only connects (it never autospawns), so with no +# daemon up this is a failed connect and nothing more. +# +# The graceful `min stop` goes first because it already refuses while sessions +# are live, printing exactly this message — the one signal that an upgrade is +# about to destroy someone's work, and worth a question. Matched literally +# rather than on exit status: a bare non-zero also means no daemon, a failed +# connect, or a transport drop on an otherwise successful stop, and none of +# those may turn every upgrade into a prompt. +# +# Every `min` below reads from /dev/null: a child that ever read stdin would +# eat the applicable-manifest rows still to be installed (the upgrade calls +# this inside the component loop). +sessions_live_msg='daemon has active sessions' + +# Ask, on the CONTROLLING TERMINAL, whether to end the live sessions. Under +# `curl … | sh` stdin is the script pipe, so reading the answer from it would +# consume the script; /dev/tty is the only sound source (overridable so the +# test harness can drive both answers). No terminal to ask on — CI, a +# non-interactive shell — means nobody can consent: say so and let the caller +# abort, rather than hang or silently destroy work. Returns non-zero when the +# upgrade must not proceed. +confirm_force_stop() { + _tty="${MINIMAL_OVERRIDE_TTY:-/dev/tty}" + say "" + say "warning: the running daemon has active sessions:" + "$bindir/min" ls >&2 /dev/null; then + say "there is no terminal to confirm on; rerun with --force-stop" + say "(or MINIMAL_INSTALL_FORCE_STOP=1) to stop those sessions anyway." + return 1 + fi + printf 'Stopping the daemon ends them. Continue? [y/N] ' >&2 + _ans= + read -r _ans <"$_tty" || _ans= + case "$_ans" in + [Yy]*) return 0 ;; + *) return 1 ;; + esac +} + +daemon_stop_tried=0 +stop_running_daemon() { + [ "$daemon_stop_tried" -eq 0 ] || return 0 + daemon_stop_tried=1 + [ -x "$bindir/min" ] || return 0 + + if [ "$force_stop" -eq 0 ]; then + # Stopped gracefully (or there was nothing to stop): done, no --force. + if _stop_out="$("$bindir/min" stop 2>&1 /dev/null 2>&1 "$records" installed=0 skipped=0 -# Swapping binaries under a running daemon wedges it: the daemon keeps -# serving from the old image while the new `min` talks to it. Stop it first, -# using the `min` ALREADY on disk — that one matches the daemon it started, and -# it is about to be overwritten. Deliberately best-effort and silent: nothing -# installed yet, no daemon running, or a `min` too old to have `stop` all mean -# "nothing to stop", and none of them should fail an install whose binaries are -# otherwise fine. `min stop` only connects (it never autospawns), so with no -# daemon up this is a failed connect and nothing more. -# -# The graceful `min stop` goes first because it already refuses while sessions -# are live, printing exactly this message — the one signal that an upgrade is -# about to destroy someone's work, and worth a question. Matched literally -# rather than on exit status: a bare non-zero also means no daemon, a failed -# connect, or a transport drop on an otherwise successful stop, and none of -# those may turn every upgrade into a prompt. -# -# Every `min` below reads from /dev/null: this runs inside the component loop, -# whose stdin is the applicable-manifest file, and a child that ever read stdin -# would eat the rows still to be installed. -sessions_live_msg='daemon has active sessions' - -# Ask, on the CONTROLLING TERMINAL, whether to end the live sessions. Under -# `curl … | sh` stdin is the script pipe, so reading the answer from it would -# consume the script; /dev/tty is the only sound source (overridable so the -# test harness can drive both answers). No terminal to ask on — CI, a -# non-interactive shell — means nobody can consent: say so and let the caller -# abort, rather than hang or silently destroy work. Returns non-zero when the -# upgrade must not proceed. -confirm_force_stop() { - _tty="${MINIMAL_OVERRIDE_TTY:-/dev/tty}" - say "" - say "warning: the running daemon has active sessions:" - "$bindir/min" ls >&2 /dev/null; then - say "there is no terminal to confirm on; rerun with --force-stop" - say "(or MINIMAL_INSTALL_FORCE_STOP=1) to stop those sessions anyway." - return 1 - fi - printf 'Stopping the daemon ends them. Continue? [y/N] ' >&2 - _ans= - read -r _ans <"$_tty" || _ans= - case "$_ans" in - [Yy]*) return 0 ;; - *) return 1 ;; - esac -} - -daemon_stop_tried=0 -stop_running_daemon() { - [ "$daemon_stop_tried" -eq 0 ] || return 0 - daemon_stop_tried=1 - [ -x "$bindir/min" ] || return 0 - - if [ "$force_stop" -eq 0 ]; then - # Stopped gracefully (or there was nothing to stop): done, no --force. - if _stop_out="$("$bindir/min" stop 2>&1 /dev/null 2>&1 >"$records" done <"$applicable" +# The pre-swap stop used the OLD `min` — the one that matched the daemon. +# But an install can also START with no `min` on disk (a fresh prefix over a +# leftover socket, or an uninstall whose walk kept a modified `min` that a +# later --force removed), and then nothing above stopped anything: the daemon +# from the previous build is still up, and every daemon-backed command on the +# fresh install now speaks to it across a version skew. Probe with the NEW +# `min` and stop what it finds mismatched — deliberately silent when the +# versions match or no daemon answers, so an ordinary install or an +# up-to-date rerun stops nothing. `min version` never autospawns; it reports +# the client and, when a daemon answers, its version too. +if [ "$installed" -gt 0 ] && [ -x "$bindir/min" ]; then + _vout="$("$bindir/min" version 2>&1 /dev/null 2>&1 >"$HOME/stop.calls" @@ -2021,6 +2036,146 @@ case_uninstall() { want_ok "dump kept when no zsh completions were installed (R9.4)" \ grep -q '# untouched user cache' "$HUD/.zcompdump" cp "$root/good-components" "$mock/versions/v1/components" # restore + + # --- Uninstall vs a running daemon -------------------------------------- + # The uninstall walk deletes `min` itself, so the daemon must be stopped + # before it, while the `min` that matches the daemon is still on disk. A + # daemon left running through an uninstall+reinstall keeps serving the old + # build and every daemon-backed command on the fresh install then hits a + # version skew. Same escalation as the upgrade swap: graceful stop, a + # consent question when sessions are live, --force-stop after. + + # Live sessions: the installer lists them and asks; answering yes stops the + # daemon and the uninstall completes. + HU10="$root/hu10"; mkdir -p "$HU10" + run u10_install "$HU10" + check 0 "$rc" "daemon-uninstall: seed install exits 0" + : >"$HU10/sessions.live" + rm -f "$HU10/stop.calls" + printf 'y\n' >"$root/tty-yes10" + TTY_FILE="$root/tty-yes10" + run u10_live "$HU10" --uninstall + TTY_FILE= + check 0 "$rc" "uninstall with a live daemon exits 0" + want_ok "the running sessions are listed" grep -q "session-alpha" "$OUT" + want_ok "the user is asked before sessions die" grep -q "Continue?" "$OUT" + want_ok "the graceful stop is tried first" grep -qx "stop" "$HU10/stop.calls" + want_ok "confirmation escalates to --force" grep -qx "stop --force" "$HU10/stop.calls" + want_err "uninstall removed minimald" test -e "$HU10/bin/minimald" + want_err "uninstall removed the record" test -e "$HU10/xdg-state/minimal/installed" + + # Declining leaves everything exactly as it was: nothing removed, daemon + # still up, record intact. + HU11="$root/hu11"; mkdir -p "$HU11" + run u11_install "$HU11" + : >"$HU11/sessions.live" + rm -f "$HU11/stop.calls" + printf 'n\n' >"$root/tty-no11" + TTY_FILE="$root/tty-no11" + run u11_decline "$HU11" --uninstall + TTY_FILE= + check 1 "$rc" "declining aborts the uninstall" + want_ok "the abort reports nothing was removed" grep -q "nothing was removed" "$OUT" + want_err "declining never force-stops" grep -qx "stop --force" "$HU11/stop.calls" + want_ok "the record survives the abort" test -f "$HU11/xdg-state/minimal/installed" + want_ok "the binaries survive the abort" test -f "$HU11/bin/minimald" + + # No terminal to ask on: abort naming the escape hatch, remove nothing. + HU12="$root/hu12"; mkdir -p "$HU12" + run u12_install "$HU12" + : >"$HU12/sessions.live" + rm -f "$HU12/stop.calls" + run u12_notty "$HU12" --uninstall + check 1 "$rc" "an unconfirmable uninstall aborts" + want_ok "the abort names the escape hatch" grep -q -- "--force-stop" "$OUT" + want_err "nothing is asked without a terminal" grep -q "Continue?" "$OUT" + want_ok "the record survives the abort" test -f "$HU12/xdg-state/minimal/installed" + + # --force-stop skips the question and the graceful stop entirely. + HU13="$root/hu13"; mkdir -p "$HU13" + run u13_install "$HU13" + : >"$HU13/sessions.live" + rm -f "$HU13/stop.calls" + run u13_force "$HU13" --uninstall --force-stop + check 0 "$rc" "uninstall --force-stop exits 0" + want_err "--force-stop asks nothing" grep -q "Continue?" "$OUT" + want_err "--force-stop skips the graceful stop" grep -qx "stop" "$HU13/stop.calls" + want_ok "--force-stop force-stops" grep -qx "stop --force" "$HU13/stop.calls" + want_err "uninstall removed the binaries" test -e "$HU13/bin/minimald" + + # --dry-run announces the stop and stops nothing. + HU14="$root/hu14"; mkdir -p "$HU14" + run u14_install "$HU14" + : >"$HU14/sessions.live" + rm -f "$HU14/stop.calls" + run u14_dry "$HU14" --uninstall --dry-run + check 0 "$rc" "uninstall --dry-run with a live daemon exits 0" + want_ok "dry-run announces the stop" grep -q "would stop the running daemon" "$OUT" + want_err "dry-run stops nothing" test -e "$HU14/stop.calls" + want_ok "dry-run keeps minimald" test -f "$HU14/bin/minimald" + + # A daemon with no sessions stops gracefully on a plain uninstall: no + # question, no --force. + HU15="$root/hu15"; mkdir -p "$HU15" + run u15_install "$HU15" + rm -f "$HU15/stop.calls" + run u15_plain "$HU15" --uninstall + check 0 "$rc" "plain uninstall with an idle daemon exits 0" + want_ok "an idle daemon is stopped gracefully" grep -qx "stop" "$HU15/stop.calls" + want_err "an idle daemon is never asked about" grep -q "Continue?" "$OUT" + want_err "an idle daemon is never force-stopped" grep -qx "stop --force" "$HU15/stop.calls" + want_err "uninstall removed minimald" test -e "$HU15/bin/minimald" + + # --- Uninstall, then reinstall: the post-swap stale-daemon probe ---------- + # The uninstall stops the daemon it finds, but nothing stops the user from + # having an old-build daemon running again by the time they reinstall + # (started by hand, autospawned from a stale PATH). That reinstall has no + # `min` on disk at its first swap, so the pre-swap stop sees nothing; the + # post-swap probe — the installed `min` comparing its version against the + # answering daemon's — is what must catch it. minver.server seeds the + # daemon side of the mock `min version`; no file means no daemon answers. + HU16="$root/hu16"; mkdir -p "$HU16" + run u16_install "$HU16" + check 0 "$rc" "stale-daemon: seed install exits 0" + run u16_uninstall "$HU16" --uninstall + check 0 "$rc" "stale-daemon: uninstall exits 0" + printf 'v9\n' >"$HU16/minver.server" # an old-build daemon answers + rm -f "$HU16/stop.calls" + run u16_reinstall "$HU16" + check 0 "$rc" "reinstall over a stale daemon exits 0" + want_ok "the stale daemon is reported" \ + grep -q "stopping a daemon still running from build v9" "$OUT" + want_ok "the stale daemon is force-stopped" grep -qx "stop --force" "$HU16/stop.calls" + + # Matching versions: the probe runs (a component was re-placed) but stops + # nothing and says nothing — the pre-swap stop already handled the daemon. + HU17="$root/hu17"; mkdir -p "$HU17" + run u17_install "$HU17" + check 0 "$rc" "matching-daemon: seed install exits 0" + printf 'stale\n' >"$HU17/bin/minimald" # force a re-placement, so the probe runs + printf 'v1\n' >"$HU17/minver.server" # a daemon of this build answers + rm -f "$HU17/stop.calls" + run u17_upgrade "$HU17" + check 0 "$rc" "upgrade with a matching daemon exits 0" + want_ok "the pre-swap stop still ran" grep -qx "stop" "$HU17/stop.calls" + want_err "a matching daemon is never force-stopped" \ + grep -qx "stop --force" "$HU17/stop.calls" + want_err "a matching daemon is never reported" \ + grep -q "stopping a daemon still running" "$OUT" + + # An unreachable daemon (no minver.server) answers nothing: the probe stays + # silent, the pre-swap stop is all that ran. + HU18="$root/hu18"; mkdir -p "$HU18" + run u18_install "$HU18" + check 0 "$rc" "unreachable-daemon: seed install exits 0" + printf 'stale\n' >"$HU18/bin/minimald" # force a component re-placement + rm -f "$HU18/stop.calls" + run u18_upgrade "$HU18" + check 0 "$rc" "upgrade over an unreachable daemon exits 0" + want_ok "the pre-swap stop still ran" grep -qx "stop" "$HU18/stop.calls" + want_err "an unreachable daemon is not asked about" grep -q "Continue?" "$OUT" + want_err "no version probe fires without a server line" \ + grep -q "stopping a daemon still running" "$OUT" } case_gvproxy_rename_migration() { From aeaf53cf93f19032937184a64459061a0e5e77f9 Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:15:08 +0000 Subject: [PATCH 30/34] fix(installer): stop a daemon left on an older build after the swaps (#2338) A client attached during an upgrade autospawns a replacement daemon the moment the pre-swap stop ends, and that daemon serves the old version from the binaries the run is about to replace. The installer now stops once more after every swap, only a daemon whose version differs from the installed min, and --force only with consent. A pre-swap yes covers only the sessions it listed: sessions opened after it are asked about again, and with no terminal the daemon stays running and the installer names min stop --force. An EXIT trap runs the same stop when the run dies after the first executable swap. Spec 07 R5.5 describes it. Backport of #2338. Applied without adaptation, after #2342 as on main. (cherry picked from commit e6cbd4616f4a01767886a076a1afaf3e8ae970d0) --- .../07-spec-installer/07-spec-installer.md | 58 ++++- scripts/install.sh | 138 +++++++++-- scripts/install_test.sh | 224 +++++++++++++++++- 3 files changed, 388 insertions(+), 32 deletions(-) diff --git a/docs/specs/07-spec-installer/07-spec-installer.md b/docs/specs/07-spec-installer/07-spec-installer.md index 7e579b7e91..16528acd27 100644 --- a/docs/specs/07-spec-installer/07-spec-installer.md +++ b/docs/specs/07-spec-installer/07-spec-installer.md @@ -316,9 +316,43 @@ them may raise the question — treating a bare failure as "sessions are live" would make every upgrade ask one. They fall through to the force stop, which remains unconditional for them. -It is attempted **at most once per run**, and only from the path that actually -replaces a file: a rerun where every component is already up to date, or a run -that dies fetching or checksum-verifying, must leave a healthy daemon running. +This pre-swap stop is attempted **at most once per run**, and only from the +path that actually replaces a file: a rerun where every component is already up +to date, or a run that dies fetching or checksum-verifying before the first +executable is replaced, must leave a healthy daemon running. + +After the swaps, the installer stops a daemon once more if that daemon is on an +older build. An attached client can autospawn a new daemon as soon as the +pre-swap stop ends. That daemon runs from the binaries that the installer is +replacing. An install can also start with no `min` on disk, so no pre-swap stop +runs. The installer runs `/min version` with the newly installed `min`. It +stops the daemon only when the daemon reports a version and that version is not +the client's. When the versions match, or no daemon answers, it stops nothing +and prints nothing. + +This **post-swap stop** never ends a session without consent. A session can +open in a respawned daemon while the run continues. The stop is graceful unless +the run already has consent: `--force-stop`, `MINIMAL_INSTALL_FORCE_STOP`, or a +yes to the pre-swap question — and that yes covers only the sessions it listed +at the time. Any session opened after it (or a failed listing of the live +sessions) makes the stop graceful again for the sessions now live, so the +installer asks again. With consent it is `/min stop --force`. When the +graceful stop refuses because sessions are live, the installer asks on the +terminal, as the pre-swap stop does, and a yes runs `stop --force`. On a no, or +with no terminal, it leaves the daemon running. It tells the user to run +`min stop` after those sessions end, or `min stop --force` to end them now — +never to rerun the installer, which cannot do this stop, because a rerun +replaces nothing. The upgrade still succeeds. Any other failure of the graceful +stop falls through to `stop --force`, as the pre-swap stop does. When a +`stop --force` of the post-swap stop fails, the installer says the old daemon +can still be running and names `min stop --force`. + +The post-swap stop also runs when the run dies after the pre-swap stop, for +example on a failed download or checksum of a later component. The installer +runs it from its exit handler, so a daemon respawned from half-replaced +binaries does not outlive the failed run. The exit handler runs it once the run +replaces any executable. An up-to-date rerun and a data-only replacement +replace no executable, so they do not run it from the exit handler. **R5.6**, A `kind = symlink` component places a symbolic link at `dest` pointing at `src` (the link target, see R3.1). The target is validated with @@ -347,7 +381,23 @@ both hash columns in place of digests. up-to-date rerun stops nothing (nothing replaced); an upgrade whose components are stale runs the on-disk `min` with exactly `stop`, once, however many components it replaces, never escalating to `--force` and never asking - anything when the graceful stop succeeds. + anything when the graceful stop succeeds and no daemon on an older build + answers after the swap. +- **Test** (R5.5): a client that respawns the daemon from the binaries on disk + after each stop leaves a daemon on the old build. After a yes to the pre-swap + question, the run ends with exactly `stop`, `stop --force`, `stop --force`. + The daemon it leaves runs from the newly installed `minimald`. +- **Test** (R5.5): the respawned daemon holds a session, and the run has no + consent and no terminal. The post-swap stop is then graceful only. The session + survives, the output names `min stop --force`, and the upgrade succeeds. With + a yes on the terminal, the run asks and ends with `stop`, `stop`, + `stop --force`. +- **Test** (R5.5): a checksum failure on a component after the first + executable swap exits non-zero and still runs the post-swap stop. The daemon + it leaves runs from the newly installed `minimald`. +- **Test** (R5.5): a reinstall with no `min` on disk over a daemon on an older + build reports that daemon and stops it gracefully. A daemon of the same build + is never stopped after the swap. - **Test** (R5.5): an installed `min` whose `stop` exits non-zero and writes to both stdout and stderr still yields exit 0, leaks neither stream into the installer's output, asks nothing, and completes the upgrade. diff --git a/scripts/install.sh b/scripts/install.sh index 48ea408eb0..295490c740 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -852,7 +852,10 @@ sessions_live_msg='daemon has active sessions' # test harness can drive both answers). No terminal to ask on — CI, a # non-interactive shell — means nobody can consent: say so and let the caller # abort, rather than hang or silently destroy work. Returns non-zero when the -# upgrade must not proceed. +# upgrade must not proceed. The argument names the calling stop: `pre` (the +# default) may tell the user to rerun with --force-stop, which works before +# the swaps; `post` runs after them, where a rerun replaces nothing and could +# never reach this stop, so it must not suggest one. confirm_force_stop() { _tty="${MINIMAL_OVERRIDE_TTY:-/dev/tty}" say "" @@ -860,8 +863,12 @@ confirm_force_stop() { "$bindir/min" ls >&2 /dev/null; then - say "there is no terminal to confirm on; rerun with --force-stop" - say "(or MINIMAL_INSTALL_FORCE_STOP=1) to stop those sessions anyway." + if [ "${1:-pre}" = post ]; then + say "there is no terminal to confirm on." + else + say "there is no terminal to confirm on; rerun with --force-stop" + say "(or MINIMAL_INSTALL_FORCE_STOP=1) to stop those sessions anyway." + fi return 1 fi printf 'Stopping the daemon ends them. Continue? [y/N] ' >&2 @@ -873,10 +880,19 @@ confirm_force_stop() { esac } +# daemon_stop_consent records that this run may end live sessions: the user +# passed --force-stop, or answered yes on the terminal. The post-swap stop +# reads it, so it never forces a stop the user did not agree to. A yes covers +# only the sessions the question listed, so their ids are kept in +# pre_swap_sessions (`min ls --raw`, one per line). A listing that fails leaves +# it empty, and then every session the post-swap stop finds counts as new. daemon_stop_tried=0 +daemon_stop_consent=0 +pre_swap_sessions= stop_running_daemon() { [ "$daemon_stop_tried" -eq 0 ] || return 0 daemon_stop_tried=1 + [ "$force_stop" -eq 0 ] || daemon_stop_consent=1 [ -x "$bindir/min" ] || return 0 if [ "$force_stop" -eq 0 ]; then @@ -885,7 +901,12 @@ stop_running_daemon() { return 0 fi case "$_stop_out" in - *"$sessions_live_msg"*) confirm_force_stop || return 1 ;; + *"$sessions_live_msg"*) + pre_swap_sessions="$("$bindir/min" ls --raw 2>/dev/null "$records" installed=0 skipped=0 +# The pre-swap stop used the OLD `min`, the one that matched the daemon. Two +# ways leave a daemon of the old build running after the swap anyway: +# +# - An install can START with no `min` on disk (a fresh prefix over a leftover +# socket, or an uninstall whose walk kept a modified `min` that a later +# --force removed). Then nothing above stopped anything. +# - An attached client autospawns a replacement daemon the moment the pre-swap +# stop ends, and that window spans every swap after the first. The daemon it +# spawns serves the old version from the binaries the run is replacing. +# +# Either way every daemon-backed command on the new install then speaks to it +# across a version skew. So once the swaps are done, probe with the NEW `min` +# and stop only a daemon whose version differs from it. `min version` never +# autospawns. It reports the client and, when a daemon answers, its version +# too. Matching versions or no answer stop nothing and say nothing, so an +# ordinary install, an up-to-date rerun, and a daemon respawned from the new +# binaries are all left alone. +# +# A session can open in a respawned daemon while the run is still going (a +# `min new`, an unattended job), and R5.5 never ends a session unasked. So the +# stop is graceful unless this run already holds consent: --force-stop, or a +# yes to the pre-swap question. The yes covers only the sessions that question +# listed. So before it forces, the stop lists the daemon's live sessions, and +# any session the yes did not see (or a listing that fails) makes the stop +# graceful again. A graceful stop refused for live sessions asks +# on the terminal, as the pre-swap stop does. Declined, or with no terminal to +# ask on, it leaves the daemon running and says how to stop it: the swaps are +# done, so the install itself still succeeds. +stale_stop_tried=0 +stop_stale_daemon() { + [ "$stale_stop_tried" -eq 0 ] || return 0 + stale_stop_tried=1 + [ -x "$bindir/min" ] || return 0 + _vout="$("$bindir/min" version 2>&1 &1 /dev/null /dev/null 2>&1 >"$records" done <"$applicable" -# The pre-swap stop used the OLD `min` — the one that matched the daemon. -# But an install can also START with no `min` on disk (a fresh prefix over a -# leftover socket, or an uninstall whose walk kept a modified `min` that a -# later --force removed), and then nothing above stopped anything: the daemon -# from the previous build is still up, and every daemon-backed command on the -# fresh install now speaks to it across a version skew. Probe with the NEW -# `min` and stop what it finds mismatched — deliberately silent when the -# versions match or no daemon answers, so an ordinary install or an -# up-to-date rerun stops nothing. `min version` never autospawns; it reports -# the client and, when a daemon answers, its version too. -if [ "$installed" -gt 0 ] && [ -x "$bindir/min" ]; then - _vout="$("$bindir/min" version 2>&1 /dev/null 2>&1 >"$HOME/stop.calls" if [ -f "$HOME/sessions.live" ] && [ "${2:-}" != "--force" ]; then echo "daemon has active sessions; pass --force to shut down anyway" >&2 exit 1 fi + # $HOME/stopforce.fail: a forced stop fails and the daemon keeps running. + if [ -f "$HOME/stopforce.fail" ] && [ "${2:-}" = "--force" ]; then + echo "error: stop failed" >&2 + exit 1 + fi + # A daemon that stops is gone; --force takes its live sessions with + # it. $HOME/daemon.running carries the daemon binary the running + # daemon serves from, and a $HOME/daemon.reconnect client autospawns + # a replacement daemon from whatever is on disk the moment the + # previous one stops — a respawn holds no live sessions of its own, + # and serves from the binaries present at the moment it spawns. + rm -f "$HOME/daemon.running" + [ "${2:-}" = "--force" ] && rm -f "$HOME/sessions.live" "$HOME/sessions.ids" + if [ -f "$HOME/daemon.reconnect" ] && [ -f "$HOME/bin/minimald" ]; then + cat "$HOME/bin/minimald" >"$HOME/daemon.running" + # $HOME/respawn.sessions: the client opens a session in the + # respawned daemon, as a `min new` during the install would. A + # non-empty file holds the ids of the sessions it opens. + if [ -f "$HOME/respawn.sessions" ]; then + : >"$HOME/sessions.live" + [ -s "$HOME/respawn.sessions" ] && cp "$HOME/respawn.sessions" "$HOME/sessions.ids" + fi + # $HOME/respawn.notty names a terminal file that goes away at the + # respawn: the user answered once, then left. + if [ -s "$HOME/respawn.notty" ]; then rm -f "$(cat "$HOME/respawn.notty")"; fi + fi ;; finalize-install) # Every invocation is recorded, arguments and all, in $HOME/finalize.calls. @@ -1434,10 +1480,12 @@ case_daemon_stop() { check 0 "$rc" "up-to-date rerun exits 0" want_err "up-to-date rerun stops no daemon (R5.5)" test -e "$H8/stop.calls" - # An upgrade (two components stale) stops the daemon exactly once, before the - # swap, via the min that is on disk now — an older build than the manifest's, - # still runnable, still able to reach the daemon it started. With no sessions - # running the graceful stop succeeds, so nothing is forced and nothing is asked. + # An upgrade (two components stale) stops the daemon before the swap via the + # min that is on disk now — an older build than the manifest's, still + # runnable, still able to reach the daemon it started. With no sessions + # running the graceful stop succeeds, so nothing is forced and nothing is + # asked. No daemon answers the post-swap version probe here, so the + # graceful stop is the only one. printf 'stale\n' >"$H8/bin/minimald" write_min_stub "$H8/bin/min" "previous release" run daemonupgrade "$H8" @@ -1568,6 +1616,168 @@ EOF check 0 "$rc" "target plus --force-stop exits 0 (R5.5/R2.1)" want_ok "the target survives the option filter (R2.1)" grep -q "target 'unstable'" "$OUT" want_ok "the trailing flag still force-stops (R5.5)" grep -qx "stop --force" "$HL6/stop.calls" + + # A daemon respawned mid-run serves the old version from the replaced + # binaries: the stop above happened before the first swap, and an attached + # client autospawns a replacement daemon within that window. The installer + # must stop again once every swap is done, so the daemon the run leaves — + # if any — is the one it just installed. The respawned daemon holds no + # live sessions of its own, so the closing stop is a --force one, already + # consented to by the pre-swap answer. A client that reconnects after the + # last stop respawns the daemon from the NEW binaries, which is the + # version the run just installed — the acceptable end state. + HL7="$root/hl7"; mkdir -p "$HL7" + stage_live_upgrade "$HL7" livereconnect + printf 'stale\n' >"$HL7/daemon.running" + : >"$HL7/daemon.reconnect" + printf 'y\n' >"$root/tty-yes7" + TTY_FILE="$root/tty-yes7" + run livereconnect "$HL7" + TTY_FILE= + check 0 "$rc" "an upgrade with a respawning client exits 0 (R5.5)" + check "stop +stop --force +stop --force" "$(cat "$HL7/stop.calls")" \ + "graceful stop, consented force stop, then the closing force stop (R5.5)" + check "$h_minimald" "$(hash_file "$HL7/daemon.running")" \ + "the daemon the run leaves is the one just installed (R5.5)" + check "$h_minimald" "$(hash_file "$HL7/bin/minimald")" "the upgrade still completes (R5.5)" + + # A session opened in the respawned daemon while the run is still going + # (a `min new`, an unattended job) is never ended unasked. Without + # --force-stop or a yes on the terminal, the closing stop is graceful. + # Refused for live sessions, it asks; with no terminal to ask on, it + # leaves the old-build daemon running and tells the user how to stop it, + # and the upgrade itself still succeeds. + HL8="$root/hl8"; mkdir -p "$HL8" + stage_live_upgrade "$HL8" liverespawnsession + rm -f "$HL8/sessions.live" + printf 'stale\n' >"$HL8/daemon.running" + : >"$HL8/daemon.reconnect" + : >"$HL8/respawn.sessions" + run liverespawnsession "$HL8" + check 0 "$rc" "an upgrade with a session opened mid-run exits 0 (R5.5)" + check "stop +stop" "$(cat "$HL8/stop.calls")" \ + "graceful stop before the swap, graceful closing stop, no --force (R5.5)" + want_ok "the session opened mid-run survives (R5.5)" test -f "$HL8/sessions.live" + want_ok "the user is told how to stop the old daemon (R5.5)" \ + grep -q "min stop --force" "$OUT" + want_ok "with no terminal, the closing stop says it cannot ask (R5.5)" \ + grep -q "there is no terminal to confirm on" "$OUT" + want_err "the no-terminal closing stop does not suggest a rerun (R5.5)" \ + grep -q "rerun with --force-stop" "$OUT" + check "$h_minimald" "$(hash_file "$HL8/bin/minimald")" "the upgrade still completes (R5.5)" + + # The same mid-run session with a yes on the terminal: the closing stop + # asks, and only then forces the stop. + HL8Y="$root/hl8y"; mkdir -p "$HL8Y" + stage_live_upgrade "$HL8Y" liverespawnyes + rm -f "$HL8Y/sessions.live" + printf 'stale\n' >"$HL8Y/daemon.running" + : >"$HL8Y/daemon.reconnect" + : >"$HL8Y/respawn.sessions" + printf 'y\n' >"$root/tty-yes8" + TTY_FILE="$root/tty-yes8" + run liverespawnyes "$HL8Y" + TTY_FILE= + check 0 "$rc" "an upgrade that ends a mid-run session on a yes exits 0 (R5.5)" + want_ok "the closing stop asks before forcing (R5.5)" grep -q "Continue?" "$OUT" + check "stop +stop +stop --force" "$(cat "$HL8Y/stop.calls")" \ + "graceful stop, graceful closing stop, then the consented force stop (R5.5)" + + # The same yes, but the forced closing stop fails: the run does not leave + # the "stopping a daemon" line looking like success. It says the old + # daemon may still be running and names the command to stop it. + HL8F="$root/hl8f"; mkdir -p "$HL8F" + stage_live_upgrade "$HL8F" liverespawnforcefail + rm -f "$HL8F/sessions.live" + printf 'stale\n' >"$HL8F/daemon.running" + : >"$HL8F/daemon.reconnect" + : >"$HL8F/respawn.sessions" + : >"$HL8F/stopforce.fail" + TTY_FILE="$root/tty-yes8" + run liverespawnforcefail "$HL8F" + TTY_FILE= + check 0 "$rc" "an upgrade whose closing force stop fails still exits 0 (R5.5)" + check "stop +stop +stop --force" "$(cat "$HL8F/stop.calls")" \ + "the closing stop asks, then tries the force stop once (R5.5)" + want_ok "a failed closing force stop is reported (R5.5)" \ + grep -q "the daemon on build old may still be running; run 'min stop --force'" "$OUT" + + # A yes to the pre-swap question covers only the sessions it listed. A + # session the client opens in the respawned daemon after that yes is one + # the user never saw, so the closing stop does not force it on the old + # answer: it stops gracefully, and when that is refused it asks again. + HL10="$root/hl10"; mkdir -p "$HL10" + stage_live_upgrade "$HL10" livenewsessionyes + printf 'stale\n' >"$HL10/daemon.running" + : >"$HL10/daemon.reconnect" + printf 'session-beta\n' >"$HL10/respawn.sessions" + printf 'y\n' >"$root/tty-yes10" + TTY_FILE="$root/tty-yes10" + run livenewsessionyes "$HL10" + TTY_FILE= + check 0 "$rc" "an upgrade with a session opened after the pre-swap yes exits 0 (R5.5)" + check 2 "$(grep -c "Continue?" "$OUT")" \ + "a session opened after the pre-swap yes is asked about again (R5.5)" + check "stop +stop --force +stop +stop --force" "$(cat "$HL10/stop.calls")" \ + "the closing stop is graceful first, then forced on the second yes (R5.5)" + check "$h_minimald" "$(hash_file "$HL10/daemon.running")" \ + "the daemon the run leaves is the one just installed (R5.5)" + + # The same session, but the terminal is gone after the pre-swap yes: the + # closing stop never forces it. The session survives, and the user is + # told how to stop the old-build daemon. + HL11="$root/hl11"; mkdir -p "$HL11" + stage_live_upgrade "$HL11" livenewsessionnotty + printf 'stale\n' >"$HL11/daemon.running" + : >"$HL11/daemon.reconnect" + printf 'session-beta\n' >"$HL11/respawn.sessions" + printf 'y\n' >"$root/tty-yes11" + printf '%s\n' "$root/tty-yes11" >"$HL11/respawn.notty" + TTY_FILE="$root/tty-yes11" + run livenewsessionnotty "$HL11" + TTY_FILE= + check 0 "$rc" "an upgrade that cannot ask about a new session exits 0 (R5.5)" + check "stop +stop --force +stop" "$(cat "$HL11/stop.calls")" \ + "the pre-swap yes does not force the closing stop over a new session (R5.5)" + want_ok "the session opened after the yes survives (R5.5)" test -f "$HL11/sessions.live" + want_ok "the user is told how to stop the old daemon (R5.5)" \ + grep -q "min stop --force" "$OUT" + check "$h_minimald" "$(hash_file "$HL11/bin/minimald")" "the upgrade still completes (R5.5)" + + # A download that fails after the first swap still runs the closing stop, + # from the EXIT trap, so the daemon a client respawned from the + # half-replaced binaries does not outlive the failed run. + # minimald is swapped first (the pre-swap stop runs there and the client + # respawns from the old minimald), then minvmd fails its checksum. + HL9="$root/hl9"; mkdir -p "$HL9" + stage_live_upgrade "$HL9" livediemidrun + rm -f "$HL9/sessions.live" + printf 'stale\n' >"$HL9/bin/minvmd" + printf 'stale\n' >"$HL9/daemon.running" + : >"$HL9/daemon.reconnect" + awk '$1=="minvmd" && $2=="linux" {$5="deadbeef"} {print}' "$root/good-components" \ + >"$mock/versions/v1/components" + run livediemidrun "$HL9" + cp "$root/good-components" "$mock/versions/v1/components" # restore + check 1 "$rc" "a checksum failure after the first swap exits non-zero (R5.3)" + want_ok "the failure is named" grep -q "checksum mismatch for minvmd" "$OUT" + check "stop +stop" "$(cat "$HL9/stop.calls")" \ + "the failed run still takes the closing stop (R5.5)" + check "$h_minimald" "$(hash_file "$HL9/daemon.running")" \ + "the daemon the failed run leaves serves the minimald it installed (R5.5)" } case_shell_integration() { @@ -2145,7 +2355,9 @@ case_uninstall() { check 0 "$rc" "reinstall over a stale daemon exits 0" want_ok "the stale daemon is reported" \ grep -q "stopping a daemon still running from build v9" "$OUT" - want_ok "the stale daemon is force-stopped" grep -qx "stop --force" "$HU16/stop.calls" + want_ok "the stale daemon is stopped gracefully" grep -qx "stop" "$HU16/stop.calls" + want_err "nothing consented, so the stale daemon is never force-stopped" \ + grep -qx "stop --force" "$HU16/stop.calls" # Matching versions: the probe runs (a component was re-placed) but stops # nothing and says nothing — the pre-swap stop already handled the daemon. From 2b8ead991151dc19026ac42a57559e4198dad854 Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Sun, 18 Oct 2026 19:26:30 +0000 Subject: [PATCH 31/34] fix(net): keep restored own-ip boxes' rows and ingress across a VM restart (#2298) A minvmd load reinstated each standing row detached, arming the 45 s detach grace. A VM's cold boot outlasts that, so the row, and with it the box's published name and declared ingress, was gone before the guest daemon started. Reloaded rows now wait under the resume bound. The guest resume also started a restored box's actor but never its host, so its shuttle never attached and ingress stayed unbound until a client's first exec or attach. Resume now launches an own-address box's host detached from the resume, and the attach announces the box's lease, so an idle box carries its row. Backport of #2298. Adapted to the line: box_actor and box_manager hunks land in session/tests.rs and sessions.rs, crate::traced::spawn_detached becomes an instrumented tokio::spawn, and line names replace BoxGate, BoxKeyPredicate and boxes::. #2333 below moves the announcement. (cherry picked from commit 10d34a52275b9eb02ae208e97791e25c8e8c8582) --- crates/minimald/src/net/switch.rs | 131 ++++++++++++++- crates/minimald/src/session/tests.rs | 62 +++++++ crates/minimald/src/sessions.rs | 43 +++++ crates/minvmd/src/box_registry.rs | 152 +++++++++++++----- crates/minvmd/src/net/egress_gate.rs | 50 ++++++ .../18-spec-box-networking.md | 5 +- 6 files changed, 397 insertions(+), 46 deletions(-) diff --git a/crates/minimald/src/net/switch.rs b/crates/minimald/src/net/switch.rs index cac728690d..69815f6503 100644 --- a/crates/minimald/src/net/switch.rs +++ b/crates/minimald/src/net/switch.rs @@ -415,6 +415,9 @@ pub async fn attach_to_switch( ) -> io::Result { let mut sock = UnixStream::connect(api_sock).await?; sock.write_all(CONNECT_REQUEST).await?; + if gate.is_some() { + announce_lease(&mut sock, lease).await?; + } let (sock_rx, sock_tx) = sock.into_split(); spawn_relay(tap_fd, sock_rx, sock_tx, gate, lease, subnet) } @@ -442,11 +445,62 @@ pub async fn attach_to_switch_vsock( lease: Ipv4Addr, subnet: SwitchSubnet, ) -> io::Result { - let sock = connect_switch_vsock(cid, port).await?; + let mut sock = connect_switch_vsock(cid, port).await?; + if gate.is_some() { + announce_lease(&mut sock, lease).await?; + } let (sock_rx, sock_tx) = tokio::io::split(sock); spawn_relay(tap_fd, sock_rx, sock_tx, gate, lease, subnet) } +/// A gratuitous ARP announcing `lease` from its own MAC +/// ([`super::MacAddr::for_switch_ip`]): a broadcast request whose sender and +/// target protocol addresses are both the lease, padded to Ethernet's +/// 60-byte minimum. It is the frame a box's kernel would send at link-up +/// with `arp_notify` set, and it passes the relay's lease check (NET-084) +/// by construction. +fn gratuitous_arp(lease: Ipv4Addr) -> Vec { + const ETHERTYPE_ARP: u16 = 0x0806; + let mac = super::MacAddr::for_switch_ip(lease).0; + let ip = lease.octets(); + let mut frame = Vec::with_capacity(60); + frame.extend_from_slice(&[0xff; 6]); // destination MAC: broadcast + frame.extend_from_slice(&mac); // source MAC + frame.extend_from_slice(ÐERTYPE_ARP.to_be_bytes()); + frame.extend_from_slice(&1u16.to_be_bytes()); // htype: Ethernet + frame.extend_from_slice(ÐERTYPE_IPV4.to_be_bytes()); // ptype: IPv4 + frame.extend_from_slice(&[6, 4]); // hlen, plen + frame.extend_from_slice(&1u16.to_be_bytes()); // oper: request + frame.extend_from_slice(&mac); // sender MAC + frame.extend_from_slice(&ip); // sender protocol address + frame.extend_from_slice(&[0; 6]); // target MAC + frame.extend_from_slice(&ip); // target protocol address + frame.resize(60, 0); + frame +} + +/// Writes one gratuitous ARP for `lease` onto a box relay's switch +/// connection, framed as every relayed frame is, before the relay starts. +/// +/// The host-side gate counts a box's row carried only once a frame from the +/// box's address crosses it (minvmd's `BoxTable::carry`), and the tap's +/// static configuration sends none: no DHCP, and no gratuitous ARP at +/// link-up. A box whose host runs idle — a resumed box's host, launched +/// with no client — would otherwise leave a reloaded row awaiting its first +/// frame until its resume bound withdrew it, its name and ingress with it +/// (NET-138). The announcement is that first frame, sent at every box +/// attach. +async fn announce_lease(sock: &mut W, lease: Ipv4Addr) -> io::Result<()> +where + W: AsyncWriteExt + Unpin, +{ + let frame = gratuitous_arp(lease); + let mut framed = Vec::with_capacity(2 + frame.len()); + framed.extend_from_slice(&(frame.len() as u16).to_le_bytes()); + framed.extend_from_slice(&frame); + sock.write_all(&framed).await +} + /// Attaches the guest's **root** namespace tap — the daemon's own, which /// every host-address box on the guest shares — to the host gvproxy switch /// over AF_VSOCK, the way [`attach_to_switch_vsock`] attaches a box's, with @@ -5956,6 +6010,81 @@ pub(crate) mod tests { assert_eq!(err.kind(), io::ErrorKind::InvalidInput); } + /// NET-138: a box's attach announces its lease before any frame of the + /// box's own, so an idle box — a resumed box's host, launched with no + /// client — still reaches the host-side gate with a frame from its + /// address, the one thing that carries its row. The announcement is a + /// gratuitous ARP from the lease at the lease's own MAC, admitted by + /// the lease check; the daemon's own, ungated attach announces nothing. + #[tokio::test] + async fn a_box_attach_announces_its_lease_before_any_box_frame() { + let lease = Ipv4Addr::new(100, 64, 0, 9); + let dir = tempfile::TempDir::new().expect("a temp dir"); + let api_sock = dir.path().join("gvproxy.sock"); + let listener = tokio::net::UnixListener::bind(&api_sock).expect("binding the stand-in"); + + for gated in [true, false] { + let mut fds = [0 as libc::c_int; 2]; + // SAFETY: `socketpair` with a valid domain/type either returns -1 + // (checked) or fills `fds` with two fresh descriptors. + let rc = + unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_DGRAM, 0, fds.as_mut_ptr()) }; + assert_eq!(rc, 0, "socketpair: {}", io::Error::last_os_error()); + // SAFETY: each fd in `fds` is a fresh, valid, owned descriptor + // just returned by socketpair. + let tap_fd = unsafe { OwnedFd::from_raw_fd(fds[0]) }; + // SAFETY: as above; held so the tap side stays open and idle. + let _box_end = unsafe { std::fs::File::from_raw_fd(fds[1]) }; + let gate = gated.then(|| { + Arc::new(SessionGate::for_session( + lease.to_string(), + lease, + &sessions::SessionPolicy::default(), + SwitchSubnet::default(), + None, + )) + }); + let (relay, accepted) = tokio::join!( + attach_to_switch(tap_fd, &api_sock, gate, lease, SwitchSubnet::default()), + listener.accept(), + ); + let _relay = relay.expect("the attach completes against the stand-in"); + let (mut switch, _) = accepted.expect("the attach dials the stand-in"); + + let mut head = vec![0u8; CONNECT_REQUEST.len()]; + switch.read_exact(&mut head).await.expect("the upgrade"); + assert_eq!(head, CONNECT_REQUEST, "the upgrade comes first"); + let mut len_buf = [0u8; 2]; + let read = + tokio::time::timeout(Duration::from_millis(300), switch.read_exact(&mut len_buf)) + .await; + if !gated { + assert!(read.is_err(), "the daemon's own attach announces nothing"); + continue; + } + read.expect("a box's attach speaks without any box frame") + .expect("the announcement's length"); + let mut frame = vec![0u8; u16::from_le_bytes(len_buf) as usize]; + switch + .read_exact(&mut frame) + .await + .expect("the announcement"); + let summary = egress::summarize(&frame); + assert_eq!(summary.family(), egress::FrameFamily::Arp, "an ARP frame"); + assert_eq!(summary.source(), Some(lease.octets()), "from the lease"); + assert!( + egress::foreign_source(&summary, lease.octets()).is_none(), + "the lease check admits it" + ); + assert_eq!( + &frame[6..12], + &super::super::MacAddr::for_switch_ip(lease).0[..], + "sent at the lease's own MAC" + ); + assert_eq!(frame.len(), 60, "padded to Ethernet's minimum"); + } + } + #[test] fn tap_netns_commands_move_then_configure_by_pid() { use std::net::Ipv4Addr; diff --git a/crates/minimald/src/session/tests.rs b/crates/minimald/src/session/tests.rs index 83d783b597..c018e38df1 100644 --- a/crates/minimald/src/session/tests.rs +++ b/crates/minimald/src/session/tests.rs @@ -5435,6 +5435,68 @@ async fn a_restarted_daemon_routes_restored_sessions_names_without_an_rpc() { ); } +/// A restarted daemon's resume launches the host of a restored own-address +/// box — no RPC, no exec, no attach — so the box's declared ingress is +/// bound and its name keeps answering without a client. A `HostNet` box +/// gets no such launch: it owns no declared ingress and publishes no +/// address, so its restored actor alone is the whole box, as before. +#[cfg(target_os = "linux")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_restarted_daemon_launches_a_restored_own_ip_boxs_host_on_resume() { + let handed = std::net::Ipv4Addr::new(127, 0, 64, 10); + let server = TestServer::new().await; + let mut client = server.connect().await; + let web = finalize_handed_own_ip_session( + &mut client, + "restored-own-ip", + std::net::Ipv4Addr::new(100, 64, 128, 10), + handed, + ) + .await; + create_configured_session(&mut client, "restored-hostnet", "/uwu").await; + + let server = restart(server, client).await; + let manager = server.state.sessions_manager().await; + assert_eq!( + manager.resume_active_sessions().await.unwrap(), + 2, + "both active sessions are resumed" + ); + + let web = manager + .get_session(crate::sessions::SessionKeyPredicate::Id(web)) + .await + .unwrap() + .expect("the own-address box's session resolves"); + for _ in 0..100 { + if web.get_attrs().await.is_some() { + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + web.get_attrs().await.is_some(), + "the resumed own-address box holds a host no client asked for" + ); + + let hostnet = manager + .get_session(crate::sessions::SessionKeyPredicate::Name( + "restored-hostnet".to_owned(), + )) + .await + .unwrap() + .expect("the host-network box's session resolves"); + // Held for a settle rather than read once: a launch wrongly spawned + // for it would be in flight alongside the own-address box's. + for _ in 0..20 { + assert!( + hostnet.get_attrs().await.is_none(), + "a resumed host-network box launches no host" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + /// One restored session whose actor cannot start costs only its own name: /// the resume logs it and goes on, and the other session still routes. #[cfg(target_os = "linux")] diff --git a/crates/minimald/src/sessions.rs b/crates/minimald/src/sessions.rs index 2f3ff1e2b4..a13f942fc3 100644 --- a/crates/minimald/src/sessions.rs +++ b/crates/minimald/src/sessions.rs @@ -1316,6 +1316,18 @@ impl Manager { /// session, so until then the name of a session `min ls` lists as active /// answers NXDOMAIN and the proxy has no route for it. /// + /// A resumed box with an own address its VM handed it + /// (`record.box_addresses`) also gets its host launched, detached from + /// this resume and unattached to any client: the host's launch is what + /// attaches the box's shuttle, whose lease announcement — a gratuitous + /// ARP from the box's address, the one frame an idle box sends — marks + /// the box's host-side row carried (minvmd's `BoxTable::carry`), and + /// the attach binds its declared ingress forwards — so the box's + /// published name keeps answering and its ingress binds without waiting + /// for a client's first exec or attach. + /// The row a reloaded host registry reinstated awaits this first frame + /// under its resume bound, which the VM's cold boot must fit inside. + /// /// Resuming through [`Session::run`] keeps one registration path, with /// its collision rules and its rule never to wait on the range verdict. /// It is idempotent: a session already in `running` is skipped, and a @@ -1391,6 +1403,37 @@ impl Manager { } match Session::run(self.session_config(handle)).await { Ok(session) => { + // An own-address box's host is what carries its shuttle, + // and with the shuttle the row's first frame — the lease + // announcement its attach sends (`net::switch`), the frame + // that stands the reloaded row and binds the declared + // ingress. Launched detached from this resume and never + // attached to a client, so the next attach or exec finds + // it and reuses it, as an exec-minted host is reused. + if matches!(record.network, sessions::NetworkMode::OwnIp) + && record.box_addresses.is_some() + { + let session = session.clone(); + let username = record.username.clone().unwrap_or_else(|| "user".into()); + let span = tracing::info_span!( + parent: None, + "resume_own_ip_host", + session_id = %session_id + ); + tokio::spawn(tracing::Instrument::instrument( + async move { + if let Err(error) = session.ensure_host(username).await { + tracing::warn!( + %error, + "resume: could not launch the own-address \ + box's host; its ingress binds at its \ + first exec or attach", + ); + } + }, + span, + )); + } self.running.insert(session_id, session); resumed += 1; } diff --git a/crates/minvmd/src/box_registry.rs b/crates/minvmd/src/box_registry.rs index e06047b2d7..105067ddca 100644 --- a/crates/minvmd/src/box_registry.rs +++ b/crates/minvmd/src/box_registry.rs @@ -51,7 +51,8 @@ //! again, so a box whose shuttle reconnects keeps its row. The client boxes' //! registrations — the creator's declarations, never anything the guest said //! — are persisted in this daemon's state dir ([`BoxRegistry::persisting_to`]) -//! and reloaded detached at start, and a creator whose box outlived its row +//! and reloaded under the resume bound at start — a VM's cold boot +//! outlasts the detach grace — and a creator whose box outlived its row //! asks for the row back from that persisted record //! ([`BoxRegistry::resume_client_box`]). @@ -99,9 +100,9 @@ const ADMIT_RATE_WINDOW: std::time::Duration = std::time::Duration::from_secs(1) /// answers it. const ALLOW_ALL_SUBNET: &str = "0.0.0.0/0"; -/// How long a detached row — one whose last carrying relay ended, or one -/// reloaded from the persisted registry at start — stands before it is -/// withdrawn, unless a relay carries its box's frames again first. +/// How long a detached row — one whose last carrying relay ended — stands +/// before it is withdrawn, unless a relay carries its box's frames again +/// first. /// /// NET-138 bounds a row's withdrawal at 60 seconds from the end of the /// box's attachment. The grace runs from the drainer's arrival of the @@ -128,10 +129,13 @@ pub const DETACH_GRACE: Duration = Duration::from_secs(45); /// row it was resumed for, while a resume nobody follows with an attach /// holds its addresses no longer than this. NET-138's 60 s runs from the /// end of an attachment, and a reinstated row has had none since it was -/// published. The bound is armed only when a resume reinstates a withdrawn -/// row: a resume of a row that stands changes nothing about its liveness, -/// so it never stretches a detached row's grace past -/// [`NET_138_WITHDRAWAL_BOUND`] nor puts a bound on a row a relay carries. +/// published. The bound is armed when a resume reinstates a withdrawn row +/// and when a load reinstates a standing creation +/// ([`BoxRegistry::persisting_to`]) — the same wait for the box's first +/// frame, over the same cold boot: a resume of a row that stands changes +/// nothing about its liveness, so it never stretches a detached row's +/// grace past [`NET_138_WITHDRAWAL_BOUND`] nor puts a bound on a row a +/// relay carries. pub const RESUME_ATTACH_BOUND: Duration = Duration::from_secs(300); /// NET-138's bound on a detached row: withdrawn no later than 60 seconds @@ -455,14 +459,14 @@ pub struct BoxRecord { /// - **awaiting** — no relay has carried it yet, and no grace runs: a /// fresh registration stands until its box's first frame or its /// creator's withdrawal; one its creator reinstated -/// ([`BoxRegistry::resume_client_box`]) stands until a relay carries its -/// box or one of its task rows, or until [`RESUME_ATTACH_BOUND`] passes -/// and it is withdrawn as at a grace's end; +/// ([`BoxRegistry::resume_client_box`]) or the load reloaded from the +/// persisted registry ([`BoxRegistry::persisting_to`]) stands until a +/// relay carries its box or one of its task rows, or until +/// [`RESUME_ATTACH_BOUND`] passes and it is withdrawn as at a grace's +/// end; /// - **attached** — at least one relay carries it; -/// - **detached** — its last relay ended, or it was reloaded from the -/// persisted registry at start ([`BoxRegistry::persisting_to`]); it is -/// withdrawn once [`DETACH_GRACE`] passes unless a relay carries it -/// again first. +/// - **detached** — its last relay ended; it is withdrawn once +/// [`DETACH_GRACE`] passes unless a relay carries it again first. /// /// A detached row decides frames exactly as an attached one does: the /// gate's verdict reads the row's rules, never its liveness, and the row's @@ -996,11 +1000,12 @@ pub struct ClientBoxSpec { /// a row stands for it. A creation lives from its registration until its /// creator withdraws the box ([`BoxRegistry::withdraw_client_box`]) or a /// new registration takes its name; a row withdrawn at the end of its -/// detach grace leaves the creation **dormant** (`standing: false`) for -/// its creator to resume ([`BoxRegistry::resume_client_box`]). A creation, -/// standing or dormant, keeps its addresses reserved: its switch and task -/// addresses stay out of the hand-out book — taken again from a fresh book -/// at reload — and its published loopback address stays the answerer's +/// detach grace or its resume bound leaves the creation **dormant** +/// (`standing: false`) for its creator to resume +/// ([`BoxRegistry::resume_client_box`]). A creation, standing or dormant, +/// keeps its addresses reserved: its switch and task addresses stay out of +/// the hand-out book — taken again from a fresh book at reload — and its +/// published loopback address stays the answerer's /// link for its name, so a resume finds every address its box was created /// with. Its creator's withdrawal releases them all. There is at most one /// creation per box name. @@ -3171,12 +3176,14 @@ impl BoxRegistry { /// /// Every creation whose row stood at the last write is reinstated at /// its own addresses with its own box id, compiled from its creator's - /// declaration as a registration is, and marked **detached** from this - /// instant ([`RowLiveness`]): the VM the row's box ran in ended with - /// the previous run, so the row is withdrawn once [`DETACH_GRACE`] - /// passes unless the box's shuttle carries a frame from its address - /// again. A resume does not extend that grace; once the row has gone, - /// its creator's resume reinstates it ([`Self::resume_client_box`]). A + /// declaration as a registration is, and awaits its box's first frame + /// under [`RESUME_ATTACH_BOUND`] from this instant ([`RowLiveness`]): + /// the VM the row's box ran in ended with the previous run, so the + /// box's host is not launched in the new VM until the guest daemon + /// starts, and the row is withdrawn once the bound passes unless the + /// box's shuttle carries a frame from its address again. A resume + /// does not extend that bound; once the row has gone, its creator's + /// resume reinstates it ([`Self::resume_client_box`]). A /// dormant creation stays dormant. The hand-out book takes every /// creation's switch and task addresses, dormant ones' included, so a /// fresh book hands none of them to a new box; their quarantine is not @@ -3260,13 +3267,14 @@ impl BoxRegistry { ); match self.try_register_since(registration, None, None) { Ok(record) => { - record.liveness().detached_since = Some(now); + record.liveness().resumed_since = Some(now); tracing::info!( box = %record.name(), switch_addr = %record.switch_addr(), - grace = ?DETACH_GRACE, - "reloaded a persisted box's row detached; it is withdrawn after the \ - grace unless its box attaches again or its creator resumes it" + bound = ?RESUME_ATTACH_BOUND, + "reloaded a persisted box's row; it awaits its box's first frame \ + under the resume bound and is withdrawn once that bound passes \ + unless its box attaches again or its creator resumes it" ); true } @@ -7147,10 +7155,11 @@ mod tests { /// NET-138: the client boxes' registrations persist host-side, and a /// registry started over the file reinstates each standing row — its - /// name, both addresses, its box id and its compiled rules — detached - /// from the load, with its switch address out of the hand-out book. + /// name, both addresses, its box id and its compiled rules — awaiting + /// its box's first frame under the resume bound, with its switch + /// address out of the hand-out book. #[test] - fn registry_persists_and_reloads_rows_as_detached() { + fn registry_persists_and_reloads_rows_awaiting_their_boxes() { let dir = tempfile::tempdir().expect("a temp dir"); let before = persisted_registry(&dir); let mut spec = client_spec("web"); @@ -7176,7 +7185,10 @@ mod tests { web.egress(), "compiled from its declaration" ); - assert!(reloaded.is_detached(), "reloaded detached"); + assert!( + !reloaded.is_detached(), + "reloaded awaiting its box's first frame" + ); assert_eq!(after.live_switch_addrs(), 1, "its switch address is out"); let api = after .register_client_box(client_spec("api")) @@ -7188,11 +7200,11 @@ mod tests { ); } - /// NET-138: a reloaded row no relay carries again is withdrawn after its - /// grace, as any detached row is, and stays withdrawn across the next - /// reload: its creation is kept dormant, its switch address reserved - /// for it across the reload too, so the reloaded book hands it to no - /// other box. + /// NET-138: a reloaded row no relay carries again is withdrawn once its + /// resume bound passes, as any row awaiting its box is, and stays + /// withdrawn across the next reload: its creation is kept dormant, its + /// switch address reserved for it across the reload too, so the + /// reloaded book hands it to no other box. #[test] fn a_reloaded_row_not_reattributed_is_withdrawn() { let dir = tempfile::tempdir().expect("a temp dir"); @@ -7205,10 +7217,10 @@ mod tests { after.spawn_withdrawal_drainer(move |name| { let _ = released.send(name.to_string()); }); - after.advance_clock(DETACH_GRACE); + after.advance_clock(RESUME_ATTACH_BOUND); wait_until( || after.row_by_name("web").is_none(), - "the reloaded row is withdrawn once its grace passes", + "the reloaded row is withdrawn once its bound passes", ); assert!( releases.recv_timeout(Duration::from_millis(200)).is_err(), @@ -7243,6 +7255,60 @@ mod tests { assert_eq!(again.live_switch_addrs(), 1, "only the new box's is out"); } + /// NET-138 (finding: a reload armed the detach grace): a reloaded row + /// awaits its box's first frame under the resume bound, not the detach + /// grace — the VM's cold boot outlasts the grace — so it stands past + /// [`DETACH_GRACE`] and is withdrawn only once + /// [`RESUME_ATTACH_BOUND`] passes; a frame carried within the bound + /// stands it back up the way any attributed row stands. + #[test] + fn a_reloaded_row_awaits_its_box_under_the_resume_bound() { + let dir = tempfile::tempdir().expect("a temp dir"); + persisted_registry(&dir) + .register_client_box(client_spec("web")) + .expect("the plan has an address for the box"); + + let registry = persisted_registry(&dir); + registry.spawn_withdrawal_drainer(|_| {}); + registry.advance_clock(DETACH_GRACE); + let_the_drainer_sweep(); + assert!( + registry.row_by_name("web").is_some(), + "the reloaded row stands past the detach grace: its VM's cold boot \ + outlasts the grace" + ); + + registry.advance_clock(RESUME_ATTACH_BOUND.saturating_sub(DETACH_GRACE)); + wait_until( + || registry.row_by_name("web").is_none(), + "the reloaded row is withdrawn at its bound", + ); + assert_eq!( + registry.live_switch_addrs(), + 1, + "the dormant creation keeps its switch address reserved" + ); + + // A reload alone, no resume: the reloaded row's first frame clears + // the bound the load armed. + let dir = tempfile::tempdir().expect("a temp dir"); + let web = persisted_registry(&dir) + .register_client_box(client_spec("web")) + .expect("the plan has an address for the box"); + let reloaded = persisted_registry(&dir); + reloaded.spawn_withdrawal_drainer(|_| {}); + reloaded + .table() + .mark_attributed(web.switch_addr().octets()) + .expect("the reloaded row stands at its switch address"); + reloaded.advance_clock(RESUME_ATTACH_BOUND); + let_the_drainer_sweep(); + assert!( + reloaded.row_by_name("web").is_some(), + "a reloaded row its box's first frame reached stands past the bound" + ); + } + /// NET-138: the persisted registry is its owner's alone — mode 0600, /// whatever a stale temporary file left behind held — written whole by /// a rename, versioned, and a file of another version reloads nothing. @@ -8299,10 +8365,10 @@ mod tests { assert_eq!(registry.live_switch_addrs(), 3); registry.spawn_withdrawal_drainer(|_| {}); - registry.advance_clock(DETACH_GRACE); + registry.advance_clock(RESUME_ATTACH_BOUND); wait_until( || registry.table().is_empty(), - "the reloaded box goes at its grace's end, its task rows with it", + "the reloaded box goes at its bound, its task rows with it", ); let claim = registry.begin_registration("web"); let resumed = registry diff --git a/crates/minvmd/src/net/egress_gate.rs b/crates/minvmd/src/net/egress_gate.rs index 771d558a1a..35a2c3ac6f 100644 --- a/crates/minvmd/src/net/egress_gate.rs +++ b/crates/minvmd/src/net/egress_gate.rs @@ -19034,6 +19034,56 @@ mod tests { } } + /// NET-138, at the gate: a row reloaded from the persisted registry + /// awaits its box's first frame under the resume bound, and the + /// gratuitous ARP the guest's relay sends at every box attach — the one + /// frame an idle box, a resumed box's host with no client, ever sends — + /// is that frame: admitted, carried, and the row stands past + /// [`RESUME_ATTACH_BOUND`](crate::box_registry::RESUME_ATTACH_BOUND). + #[tokio::test] + async fn an_idle_reloaded_box_s_attach_announce_stands_its_row_past_the_bound() { + use crate::box_registry::{ + ClientBoxSpec, DETACH_SWEEP_INTERVAL, REGISTRY_FILE, RESUME_ATTACH_BOUND, + }; + let dir = TempDir::new().expect("a temp dir"); + let path = dir.path().join(REGISTRY_FILE); + let web = BoxRegistry::new(SUBNET) + .persisting_to(path.clone()) + .register_client_box(ClientBoxSpec { + name: "web".to_string(), + ingress_ports: vec![8080], + egress: None, + credentialed_upstream: None, + dynamic_ingress: None, + dynamic_allowed_range: None, + }) + .expect("the plan has an address for the box"); + let addr = web.switch_addr().octets(); + + // The VM restarted: the registry reloads the row, awaiting its box. + let registry = BoxRegistry::new(SUBNET).persisting_to(path); + registry.spawn_withdrawal_drainer(|_| {}); + let clock = registry.clone(); + let mut h = gate_over(registry).await; + + // The box's host launches with no client: its relay's announcement + // is the only frame it sends. + let announce = arp_frame(addr); + send_frame(&mut h.guest, &announce).await; + assert_eq!( + expect_frame(&mut h.switch).await, + announce, + "the announcement reaches the switch" + ); + + clock.advance_clock(RESUME_ATTACH_BOUND); + tokio::time::sleep(DETACH_SWEEP_INTERVAL * 2 + Duration::from_millis(200)).await; + assert!( + h.table.by_source(addr).is_some(), + "the announced row stands past its resume bound with no client traffic" + ); + } + /// The row is withdrawn on the ingress-wins interleaving too: the switch /// closes its side while the guest is still on the connection, so the /// ingress leg ends first and the egress leg is dropped un-drained. The diff --git a/docs/specs/18-spec-box-networking/18-spec-box-networking.md b/docs/specs/18-spec-box-networking/18-spec-box-networking.md index 875a914109..2a6caa0c3a 100644 --- a/docs/specs/18-spec-box-networking/18-spec-box-networking.md +++ b/docs/specs/18-spec-box-networking/18-spec-box-networking.md @@ -701,10 +701,11 @@ included, with every refusal logged (NET-001 to NET-004). tier: T0 verify: cargo nextest run -p minvmd admit_report_refused_past_row_cap_or_rate - - WHEN a box's attachment to the switch ends or its creator destroys it THE SYSTEM SHALL withdraw its row within 60 seconds. + - WHEN a box's attachment ends during the VM host daemon's run or its creator destroys it THE SYSTEM SHALL withdraw its row within 60 seconds. tier: T0 verify: cargo nextest run -p minvmd host_table_row_withdrawn_within_60s_of_box_end - + verify: cargo nextest run -p minvmd a_reloaded_row_awaits_its_box_under_the_resume_bound + - WHILE a host-address box is published THE SYSTEM SHALL admit the node's mirrored binds for it at the node's address at any port. tier: T0 verify: cargo nextest run -p minvmd host_address_mirrored_binds_admitted_unfiltered From 89a676d3efd8875103b7fd5d676e6123a5613c82 Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Sun, 18 Oct 2026 23:10:35 -0700 Subject: [PATCH 32/34] fix: send the lease ARP as the relay's first frame (#2333) A gated attach wrote its framed gratuitous ARP into the /connect handshake, before the relay owned the stream. gvproxy's hijack discards bytes already buffered, so the ARP's tail was read as a bogus length prefix and box ingress died. The handshake now speaks only the upgrade, and the relay sends the announcement as its first frame, one short delay after it starts. Only the vsock attach announces: on the local-socket attach the tap wears a kernel-chosen MAC, so the announcement would point the gateway at a MAC the box does not wear, and there is no host-side row for it to carry. Backport of #2333. Required follow-up to #2298. Adapted to the line: crate::traced::spawn reads tokio::spawn, and line names replace BoxGate and boxes::. (cherry picked from commit 1bec1171bc0c92a18e788c3d2256762bb690b409) --- crates/minimald/src/net/switch.rs | 176 ++++++++++++++++++++++++------ 1 file changed, 145 insertions(+), 31 deletions(-) diff --git a/crates/minimald/src/net/switch.rs b/crates/minimald/src/net/switch.rs index 69815f6503..8d442bbb3c 100644 --- a/crates/minimald/src/net/switch.rs +++ b/crates/minimald/src/net/switch.rs @@ -94,6 +94,15 @@ const CONNECT_REQUEST: &[u8] = b"POST /connect HTTP/1.0\r\nHost: localhost\r\n\r /// guest-egress / session bring-up indefinitely. const VSOCK_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +/// How long a gated vsock attach waits after the relay starts before its first +/// frame — the lease announcement — leaves toward the switch. gvproxy's +/// `/connect` handler hijacks the stream after reading the upgrade request, +/// and a hijack discards whatever bytes already sit in its read buffer, so +/// the announcement must not be written into the stream before the hijack +/// can take it; this delay puts it safely past the hijack on the relay's +/// clock, short enough to stay within the row's resume bound. +const ATTACH_ANNOUNCE_DELAY: Duration = Duration::from_millis(100); + /// `struct ifreq` reduced to the two fields `TUNSETIFF` reads, padded to the /// kernel's `sizeof(struct ifreq)` (40 bytes on every LP64 Linux target) so the /// kernel's `copy_from_user` never reads past the allocation. @@ -398,8 +407,19 @@ impl Drop for SwitchRelay { /// deprecated host-alias literal it notices (NET-004); the daemon relay ignores /// it. /// -/// Spawns two background tasks — tap→switch and switch→tap — and returns a -/// [`SwitchRelay`] handle whose lifetime keeps the attachment alive. +/// Spawns the relay's two background tasks — tap→switch and switch→tap — and +/// returns a [`SwitchRelay`] handle whose lifetime keeps the attachment alive. +/// +/// This attach never announces the lease ([`announce_lease`]), gated or not. +/// The local control socket is DM2's, whose box tap the sandbox builds in its +/// own namespace with a kernel-chosen MAC (`provider::tap_mechanism`), not the +/// lease's derived one. gvproxy's NIC spoofs, so it treats every ARP target as +/// local and learns the sender of a gratuitous ARP: an announcement at the +/// derived MAC would point the gateway's neighbour entry for the lease at a +/// MAC the box does not wear, and every reply the gateway sends the box (its +/// DNS answers among them) would be dropped by the box's kernel until the +/// entry re-resolved. DM2 has no host-side row for the announcement to carry +/// either (NET-138 is minvmd's). /// /// # Errors /// @@ -415,11 +435,8 @@ pub async fn attach_to_switch( ) -> io::Result { let mut sock = UnixStream::connect(api_sock).await?; sock.write_all(CONNECT_REQUEST).await?; - if gate.is_some() { - announce_lease(&mut sock, lease).await?; - } let (sock_rx, sock_tx) = sock.into_split(); - spawn_relay(tap_fd, sock_rx, sock_tx, gate, lease, subnet) + spawn_relay(tap_fd, sock_rx, sock_tx, gate, false, lease, subnet) } /// Attaches `tap_fd` to the **host** gvproxy switch over AF_VSOCK (DM1/3/4) and @@ -432,6 +449,11 @@ pub async fn attach_to_switch( /// raw-L2 relay as [`attach_to_switch`] — the shuttle is *not* a second TCP/IP /// stack — so exactly one gVisor stack (the host gvproxy) sits in the path. /// +/// A gated attach here announces the lease as its relay's first frame +/// ([`announce_lease`], NET-138): this transport's box tap is the one the +/// daemon configures itself ([`tap_netns_commands`]), which wears the lease's +/// derived MAC, so the announcement tells the switch the truth. +/// /// # Errors /// /// Returns an error if the vsock connection cannot be established, the connect @@ -445,12 +467,10 @@ pub async fn attach_to_switch_vsock( lease: Ipv4Addr, subnet: SwitchSubnet, ) -> io::Result { - let mut sock = connect_switch_vsock(cid, port).await?; - if gate.is_some() { - announce_lease(&mut sock, lease).await?; - } + let sock = connect_switch_vsock(cid, port).await?; + let announce = gate.is_some(); let (sock_rx, sock_tx) = tokio::io::split(sock); - spawn_relay(tap_fd, sock_rx, sock_tx, gate, lease, subnet) + spawn_relay(tap_fd, sock_rx, sock_tx, gate, announce, lease, subnet) } /// A gratuitous ARP announcing `lease` from its own MAC @@ -479,8 +499,9 @@ fn gratuitous_arp(lease: Ipv4Addr) -> Vec { frame } -/// Writes one gratuitous ARP for `lease` onto a box relay's switch -/// connection, framed as every relayed frame is, before the relay starts. +/// Sends one gratuitous ARP for `lease` as a gated relay's first frame — +/// [`ATTACH_ANNOUNCE_DELAY`] after the relay starts — onto the relay's +/// shared switch write half. /// /// The host-side gate counts a box's row carried only once a frame from the /// box's address crosses it (minvmd's `BoxTable::carry`), and the tap's @@ -488,17 +509,27 @@ fn gratuitous_arp(lease: Ipv4Addr) -> Vec { /// link-up. A box whose host runs idle — a resumed box's host, launched /// with no client — would otherwise leave a reloaded row awaiting its first /// frame until its resume bound withdrew it, its name and ingress with it -/// (NET-138). The announcement is that first frame, sent at every box +/// (NET-138). The announcement is that first frame, sent at every gated vsock /// attach. -async fn announce_lease(sock: &mut W, lease: Ipv4Addr) -> io::Result<()> +/// +/// The delay is the point: gvproxy hijacks the `/connect` stream after the +/// upgrade and discards whatever bytes already sit in its read buffer, so +/// writing the announcement into the handshake would lose it and desync the +/// framing. Sent here instead — after the relay owns the stream — it is the +/// relay's first frame, past the hijack on the relay's clock. +/// +/// Runs as its own task: it holds the lock only for its one write, so the +/// relay's egress leg never waits on it. +async fn announce_lease(sock: Arc>, lease: Ipv4Addr) -> io::Result<()> where - W: AsyncWriteExt + Unpin, + W: AsyncWriteExt + Unpin + Send + 'static, { + tokio::time::sleep(ATTACH_ANNOUNCE_DELAY).await; let frame = gratuitous_arp(lease); let mut framed = Vec::with_capacity(2 + frame.len()); framed.extend_from_slice(&(frame.len() as u16).to_le_bytes()); framed.extend_from_slice(&frame); - sock.write_all(&framed).await + sock.lock().await.write_all(&framed).await } /// Attaches the guest's **root** namespace tap — the daemon's own, which @@ -599,6 +630,7 @@ where sock_rx, sock_tx, None, + false, Some(node_dns), lease, subnet, @@ -617,11 +649,16 @@ where /// the DM1/3/4 vsock path ([`attach_to_switch_vsock`]); the relay loops are /// transport-agnostic (`AsyncRead`/`AsyncWrite`), so only the connect step /// differs. +/// +/// `announce` says whether this relay's first frame is the lease announcement +/// ([`announce_lease`]): true for a gated vsock attach, false for the daemon's +/// own relay and anything else. fn spawn_relay( tap_fd: OwnedFd, sock_rx: R, sock_tx: W, gate: Option>, + announce: bool, lease: Ipv4Addr, subnet: SwitchSubnet, ) -> io::Result @@ -629,17 +666,22 @@ where R: AsyncReadExt + Unpin + Send + 'static, W: AsyncWriteExt + Unpin + Send + 'static, { - spawn_relay_with(tap_fd, sock_rx, sock_tx, gate, None, lease, subnet) + spawn_relay_with( + tap_fd, sock_rx, sock_tx, gate, announce, None, lease, subnet, + ) } /// [`spawn_relay`], with the node's DNS layer the daemon's own relay carries /// ([`spawn_node_relay`]); `None` for every other relay, whose DNS gate, if -/// any, is its session gate's own. +/// any, is its session gate's own. `announce` is [`spawn_relay`]'s — whether +/// the relay's first frame is the lease announcement. +#[allow(clippy::too_many_arguments)] fn spawn_relay_with( tap_fd: OwnedFd, sock_rx: R, sock_tx: W, gate: Option>, + announce: bool, node_dns: Option, lease: Ipv4Addr, subnet: SwitchSubnet, @@ -716,6 +758,13 @@ where let box_reset_writer = box_resets_rx .map(|box_resets_rx| tokio::spawn(write_box_resets(box_resets_rx, Arc::clone(&tap)))); let switch_to_tap = tokio::spawn(relay_switch_to_tap(sock_rx, tap, gate.clone())); + // A gated vsock attach announces its lease as its relay's first frame — the + // delay carries it past gvproxy's `/connect` hijack, and the row it + // carries is NET-138's. Discarded like every fire-and-forget writer the + // relay owns: it ends with its one write. + if announce { + tokio::spawn(announce_lease(Arc::clone(&sock_tx), lease)); + } Ok(SwitchRelay { tap_to_switch, switch_to_tap, @@ -3925,6 +3974,7 @@ pub(crate) mod tests { sock_rx, sock_tx, Some(gate), + false, lease, SwitchSubnet::default(), ), @@ -6010,14 +6060,16 @@ pub(crate) mod tests { assert_eq!(err.kind(), io::ErrorKind::InvalidInput); } - /// NET-138: a box's attach announces its lease before any frame of the - /// box's own, so an idle box — a resumed box's host, launched with no - /// client — still reaches the host-side gate with a frame from its - /// address, the one thing that carries its row. The announcement is a - /// gratuitous ARP from the lease at the lease's own MAC, admitted by - /// the lease check; the daemon's own, ungated attach announces nothing. + /// The local-socket attach — DM2's, whose box tap wears a kernel-chosen + /// MAC — speaks only the `/connect` upgrade and never announces the + /// lease, gated or not: gvproxy hijacks the stream and discards whatever + /// already sits in its read buffer, and an announcement at the lease's + /// derived MAC would teach gvproxy's spoofing gateway a MAC the box does + /// not wear, so the gateway's replies — a deny-all box's switch-resolver + /// answers among them (NET-074) — would never reach the box. The read + /// runs past the announcement delay, so a late announcement fails it. #[tokio::test] - async fn a_box_attach_announces_its_lease_before_any_box_frame() { + async fn a_local_socket_attach_speaks_only_the_upgrade_and_never_announces() { let lease = Ipv4Addr::new(100, 64, 0, 9); let dir = tempfile::TempDir::new().expect("a temp dir"); let api_sock = dir.path().join("gvproxy.sock"); @@ -6054,15 +6106,77 @@ pub(crate) mod tests { let mut head = vec![0u8; CONNECT_REQUEST.len()]; switch.read_exact(&mut head).await.expect("the upgrade"); assert_eq!(head, CONNECT_REQUEST, "the upgrade comes first"); + let mut byte = [0u8; 1]; + let after = tokio::time::timeout( + ATTACH_ANNOUNCE_DELAY + Duration::from_millis(300), + switch.read_exact(&mut byte), + ) + .await; + assert!( + after.is_err(), + "nothing follows the upgrade on a local-socket attach (gated: {gated})" + ); + } + } + + /// NET-138: a relay told to announce — a gated vsock attach, whose tap + /// wears the lease's derived MAC — sends nothing inside its start delay, + /// which carries it past gvproxy's `/connect` hijack, and its first frame + /// is then the lease announcement: a gratuitous ARP from the lease at the + /// lease's own MAC, admitted by the lease check, so an idle box — a + /// resumed box's host, launched with no client — still reaches the + /// host-side gate with a frame from its address, the one thing that + /// carries its row. A relay not told to announce sends nothing. + #[tokio::test] + async fn an_announcing_relay_s_first_frame_is_the_lease_announcement() { + let lease = Ipv4Addr::new(100, 64, 0, 9); + for announce in [true, false] { + let mut fds = [0 as libc::c_int; 2]; + // SAFETY: `socketpair` with a valid domain/type either returns -1 + // (checked) or fills `fds` with two fresh descriptors. + let rc = + unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_DGRAM, 0, fds.as_mut_ptr()) }; + assert_eq!(rc, 0, "socketpair: {}", io::Error::last_os_error()); + // SAFETY: each fd in `fds` is a fresh, valid, owned descriptor + // just returned by socketpair. + let tap_fd = unsafe { OwnedFd::from_raw_fd(fds[0]) }; + // SAFETY: as above; held so the tap side stays open and idle. + let _box_end = unsafe { std::fs::File::from_raw_fd(fds[1]) }; + let (relay_end, mut switch) = UnixStream::pair().expect("a stream pair"); + let (sock_rx, sock_tx) = relay_end.into_split(); + let gate = Arc::new(SessionGate::for_session( + lease.to_string(), + lease, + &sessions::SessionPolicy::default(), + SwitchSubnet::default(), + None, + )); + let _relay = spawn_relay( + tap_fd, + sock_rx, + sock_tx, + Some(gate), + announce, + lease, + SwitchSubnet::default(), + ) + .expect("the relay starts"); + let mut len_buf = [0u8; 2]; - let read = - tokio::time::timeout(Duration::from_millis(300), switch.read_exact(&mut len_buf)) + let early = + tokio::time::timeout(Duration::from_millis(50), switch.read_exact(&mut len_buf)) .await; - if !gated { - assert!(read.is_err(), "the daemon's own attach announces nothing"); + assert!(early.is_err(), "nothing leaves inside the start delay"); + let read = tokio::time::timeout( + ATTACH_ANNOUNCE_DELAY + Duration::from_millis(900), + switch.read_exact(&mut len_buf), + ) + .await; + if !announce { + assert!(read.is_err(), "a relay not told to announce sends nothing"); continue; } - read.expect("a box's attach speaks without any box frame") + read.expect("the relay's first frame") .expect("the announcement's length"); let mut frame = vec![0u8; u16::from_le_bytes(len_buf) as usize]; switch From 8c4bcd6d26f0a6382a412e70a5de90a8d18dd018 Mon Sep 17 00:00:00 2001 From: Norrie Taylor Date: Wed, 4 Nov 2026 16:42:08 -0700 Subject: [PATCH 33/34] build: declare 0.7.3-rc1 as the first release candidate The release run stages the version that package.version declares, and a staged version is permanent. A candidate number keeps 0.7.3 free for the final build. --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e5421aa627..6a1d4779ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3986,7 +3986,7 @@ dependencies = [ [[package]] name = "minimal" -version = "0.7.3" +version = "0.7.3-rc1" dependencies = [ "anyhow", "async-compression", @@ -4031,7 +4031,7 @@ dependencies = [ [[package]] name = "minimal-client" -version = "0.7.3" +version = "0.7.3-rc1" dependencies = [ "anyhow", "async-compression", @@ -4062,7 +4062,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "minimal-tui" -version = "0.7.3" +version = "0.7.3-rc1" dependencies = [ "anyhow", "camino", @@ -4089,7 +4089,7 @@ dependencies = [ [[package]] name = "minimald" -version = "0.7.3" +version = "0.7.3-rc1" dependencies = [ "anyhow", "args", @@ -4192,7 +4192,7 @@ dependencies = [ [[package]] name = "minvmd" -version = "0.7.3" +version = "0.7.3-rc1" dependencies = [ "anyhow", "camino", @@ -4226,7 +4226,7 @@ dependencies = [ [[package]] name = "minzoned" -version = "0.7.3" +version = "0.7.3-rc1" dependencies = [ "anyhow", "clap", @@ -4250,7 +4250,7 @@ dependencies = [ [[package]] name = "mip" -version = "0.7.3" +version = "0.7.3-rc1" dependencies = [ "anyhow", "args", @@ -8162,7 +8162,7 @@ checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "version" -version = "0.7.3" +version = "0.7.3-rc1" [[package]] name = "version_check" diff --git a/Cargo.toml b/Cargo.toml index 47d63dad62..98e8ed9b34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,7 +58,7 @@ package.rust-version = "1.95" # (scripts/next-version.sh). `scripts/next-version.sh --check` (the # workspace `package_version` test, `just check-version`) fails the PR that # leaves it stale or not strictly greater than the newest v* tag. -package.version = "0.7.3" +package.version = "0.7.3-rc1" # Workspace lint policy. A crate opts in with `[lints] workspace = true`. # From 38c080199d75a92ba4543bf6d43a4f565161ea13 Mon Sep 17 00:00:00 2001 From: Norrie Taylor Date: Wed, 4 Nov 2026 18:05:07 -0700 Subject: [PATCH 34/34] test(e2e): read the macOS DNS SERVICE port without the word listening A macOS VM host prints "this VM's host daemon answers its box names on 127.0.0.1: (UDP)", so the parser in fresh_install_range_lands_on_macos found no port and failed the lane. This takes the one line from #2741 (de32bd3d4). The rest of that commit stays off the 0.7 line. The case now matches main. --- scripts/session-e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index e5f434b1d6..e1c6f5f0f4 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -5093,7 +5093,7 @@ fresh_install_range_lands_on_macos() { # The box's own address: the zone answerer's A record for its name. for _ in $(seq 1 40); do frm_ans_port="$(mnl ls 2>/dev/null \ - | sed -n 's/^DNS SERVICE:.*listening on 127\.0\.0\.1:\([0-9][0-9]*\) (UDP).*/\1/p' \ + | sed -En 's/^DNS SERVICE:.* on 127\.0\.0\.1:([0-9]+) \(UDP\).*/\1/p' \ | head -n1)" [ -n "$frm_ans_port" ] && break sleep 0.25