From 9726b68e08c2bebe62095f7e9016cc33d6e02395 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:20:05 +0000 Subject: [PATCH 1/7] feat(switch): add box egress proxy address above the ptask lease pool Reserve broadcast - 3 for the box egress proxy, keep it out of gvproxy's NAT and virtual-IP config, and derive BEP_MAC from that address. Shift the top of the PTask lease pool to broadcast - 4 so the proxy, daemon, and host alias all sit above it. NET-132 --- crates/switch/src/lib.rs | 52 ++++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/crates/switch/src/lib.rs b/crates/switch/src/lib.rs index 19d004a5dd..6ee6080bb4 100644 --- a/crates/switch/src/lib.rs +++ b/crates/switch/src/lib.rs @@ -19,6 +19,11 @@ pub const DEFAULT_MTU: u16 = 1500; /// Stable, locally-administered MAC for the switch gateway. pub const GATEWAY_MAC: MacAddr = MacAddr([0x5a, 0x94, 0xef, 0xe4, 0x0c, 0xdd]); +/// Stable, locally-administered MAC for the Box Egress Proxy on the default +/// [`DEFAULT_SUBNET`]. It matches [`MacAddr::for_switch_ip`] applied to +/// [`SwitchSubnet::box_egress_proxy_address`] on that subnet. +pub const BEP_MAC: MacAddr = MacAddr([0x52, 0x54, 0x00, 0x40, 0xff, 0xfc]); + /// AF_VSOCK CID of the host as seen from inside a libkrun guest. Well-known: /// `VMADDR_CID_HOST == 2`. The per-PTask shuttle dials this CID to reach the /// host gvproxy switch (DM1/3/4). @@ -197,8 +202,8 @@ impl SwitchSubnet { /// # Errors /// /// Returns [`InvalidPrefix`] for a prefix outside `8..=29`. A prefix narrower - /// than /29 has no room for the reserved network/gateway/host-alias/broadcast - /// addresses plus a PTask address. A prefix wider than /8 lets the high octet + /// than /29 has no room for the reserved network/gateway/proxy/daemon/ + /// host-alias/broadcast addresses plus a PTask address. A prefix wider than /8 lets the high octet /// vary, which [`MacAddr::for_switch_ip`] does not fold into the derived MAC, /// so two addresses differing only in that octet would collide. pub fn new(base: Ipv4Addr, prefix: u8) -> Result { @@ -275,6 +280,22 @@ impl SwitchSubnet { Ipv4Addr::from(u32::from(self.broadcast()) - 1) } + /// The Box Egress Proxy address (`broadcast - 3`): a second infrastructure + /// address above the PTask lease pool that the host-side stack peer owns + /// directly. Reserved, never handed to a PTask, and kept out of gvproxy's + /// NAT/virtual-IP tables so the peer can answer ARP and reset stray TCP + /// packets itself. + #[must_use] + pub fn box_egress_proxy_address(self) -> Ipv4Addr { + Ipv4Addr::from(u32::from(self.broadcast()) - 3) + } + + /// The locally-administered MAC for [`Self::box_egress_proxy_address`]. + #[must_use] + pub fn bep_mac(self) -> MacAddr { + MacAddr::for_switch_ip(self.box_egress_proxy_address()) + } + /// The daemon address (`broadcast - 2`): the guest root netns' primary tap, /// which gives `minimald` itself egress through the host gvproxy (so it can /// fetch upstream packages). Reserved from the top so the PTask range still @@ -291,12 +312,12 @@ impl SwitchSubnet { u32::from(self.network()) + 2 } - /// The last address that may be allocated to a PTask (`broadcast - 3`), - /// leaving the daemon address at `broadcast - 2` and the host alias at - /// `broadcast - 1` reserved. + /// The last address that may be allocated to a PTask (`broadcast - 4`), + /// leaving the proxy at `broadcast - 3`, the daemon at `broadcast - 2` + /// and the host alias at `broadcast - 1` reserved. #[must_use] pub fn last_ptask(self) -> u32 { - u32::from(self.broadcast()) - 3 + u32::from(self.broadcast()) - 4 } } @@ -582,8 +603,25 @@ mod tests { let net = SwitchSubnet::default(); assert_eq!(net.to_string(), "100.64.0.0/16"); assert_eq!(net.gateway(), Ipv4Addr::new(100, 64, 0, 1)); - assert_eq!(net.host_alias(), Ipv4Addr::new(100, 64, 255, 254)); + assert_eq!(net.box_egress_proxy_address(), Ipv4Addr::new(100, 64, 255, 252)); + assert_eq!(net.bep_mac(), BEP_MAC); + assert_eq!(BEP_MAC.to_string(), "52:54:00:40:ff:fc"); assert_eq!(net.daemon_ip(), Ipv4Addr::new(100, 64, 255, 253)); + assert_eq!(net.host_alias(), Ipv4Addr::new(100, 64, 255, 254)); + } + + #[test] + fn proxy_address_is_neither_virtual_ip_nor_translated() { + let yaml = render_gvproxy_config(SwitchSubnet::default(), &[]); + let net = SwitchSubnet::default(); + // The proxy address must not appear in gvproxy's NAT map or in its + // gatewayVirtualIPs list — the host-side stack peer owns it directly. + assert!(!yaml.contains(&format!("\"{}\"", net.box_egress_proxy_address()))); + assert!(!yaml.contains("100.64.255.252")); + // The daemon address is also infrastructure and should not be listed. + assert!(!yaml.contains("100.64.255.253")); + // The host alias, by contrast, is both NAT'd and a virtual IP. + assert!(yaml.contains("100.64.255.254")); } #[test] From 90633246d0ace88d3683e99522c6777ad2116745 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 01:07:38 +0000 Subject: [PATCH 2/7] feat(minimald,minvmd): the coder's uncommitted work as it stood when its 7200s ran out the coder's uncommitted work as it stood when its 7200s ran out The coder was stopped at the end of its time with these changes not committed. The runtime committed its working tree as it stood so the work is kept: this commit is a snapshot, not a finished step, and may not build or pass its tests. --- crates/minimald/src/net/mod.rs | 3 + crates/minimald/src/session_host/tests.rs | 1 + crates/minvmd/src/cmd/run.rs | 8 +- crates/minvmd/src/net.rs | 31 +- crates/minvmd/src/net/bep_host.rs | 807 +++++++++++++++++++++- crates/switch/src/lib.rs | 5 +- scripts/session-e2e.sh | 187 ++++- 7 files changed, 1001 insertions(+), 41 deletions(-) diff --git a/crates/minimald/src/net/mod.rs b/crates/minimald/src/net/mod.rs index 49e0b0231e..1cae50a5e1 100644 --- a/crates/minimald/src/net/mod.rs +++ b/crates/minimald/src/net/mod.rs @@ -1104,6 +1104,9 @@ mod tests { assert!(cfg.contains("subnet: \"100.64.0.0/16\"")); assert!(cfg.contains("gatewayIP: \"100.64.0.1\"")); assert!(cfg.contains(&format!("\"{}\": \"{}\"", lease.ip, lease.mac))); + // NET-132: the box egress proxy address is infrastructure and must not be + // handled by gvproxy's NAT or virtual-IP machinery. + assert!(!cfg.contains("100.64.255.252")); // Host alias is NAT'd to loopback and never allocated. assert!(cfg.contains("\"100.64.255.254\": \"127.0.0.1\"")); // NET-003: the static zone entry the bundle's switch-configuration copy diff --git a/crates/minimald/src/session_host/tests.rs b/crates/minimald/src/session_host/tests.rs index 9735272770..bebedd9b24 100644 --- a/crates/minimald/src/session_host/tests.rs +++ b/crates/minimald/src/session_host/tests.rs @@ -2109,6 +2109,7 @@ fn launcher_in( ))), policy: sessions::SessionPolicy::default(), own_address: None, + box_addresses: None, composition: None, session: crate::session::WeakSessionHandle::dangling(), // The tests here drive the session's own launches unless one sets diff --git a/crates/minvmd/src/cmd/run.rs b/crates/minvmd/src/cmd/run.rs index acc9a3ce6a..dd982f0e25 100644 --- a/crates/minvmd/src/cmd/run.rs +++ b/crates/minvmd/src/cmd/run.rs @@ -507,7 +507,13 @@ fn run_foreground() -> Result<()> { &boxes, ) { Ok(gvproxy) => { - tracing::info!(pid = gvproxy.pid(), "host gvproxy switch up"); + let subnet = boxes.subnet(); + tracing::info!( + pid = gvproxy.pid(), + proxy_ip = %subnet.box_egress_proxy_address(), + proxy_mac = %subnet.bep_mac(), + "host gvproxy switch up; box egress proxy peer started", + ); Some(gvproxy) } // An own-IP VM cannot work without the switch: fail loudly. A diff --git a/crates/minvmd/src/net.rs b/crates/minvmd/src/net.rs index 5e1fcaf8aa..90d3c48db5 100644 --- a/crates/minvmd/src/net.rs +++ b/crates/minvmd/src/net.rs @@ -88,7 +88,7 @@ pub use shuttle::{VSOCK_GVPROXY_SHUTTLE_PORT, resolve_gate_sock, resolve_switch_ mod baseline; pub use baseline::{NodeBaselinePhase, NodePlaneBaseline}; pub(crate) mod bep_host; -pub use bep_host::{BepDevice, BepDeviceEnds, BepHost}; +pub use bep_host::{BepDevice, BepDeviceEnds, BepHost, BepPeer}; /// Default time to wait for gvproxy to exit on SIGTERM before escalating to /// SIGKILL. @@ -718,6 +718,7 @@ impl HostGvproxy { // its rows were compiled against — so a lease the gate checks a // frame's source by is one the switch actually routes. .with_subnet(registry.subnet()); + let subnet = registry.subnet(); let table = registry.table(); // The node-plane baseline set: the helper's built-in enumeration of // the categories the in-VM daemon's own traffic may reach (NET-130), @@ -726,7 +727,7 @@ impl HostGvproxy { // by it, beside the boxes' rows; announced — the shipped posture — // the allow-all interim node row still does, and the gate's // start-up line names the posture deciding. - let baseline = NodePlaneBaseline::built_in(registry.subnet()); + let baseline = NodePlaneBaseline::built_in(subnet); let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>(); let (ready_tx, ready_rx) = std::sync::mpsc::channel::>(); @@ -789,6 +790,26 @@ impl HostGvproxy { drop(ready_tx.send(Err(e))); return; } + // Start the Box Egress Proxy host stack peer after the switch + // socket is known to accept connections: the peer owns the + // proxy's infrastructure address on the subnet, speaks the + // same length-framed Ethernet the switch socket carries, and + // stops when this runtime drops at switch teardown. + let _bep_peer = match BepPeer::spawn(&sock, subnet).await { + Ok(peer) => peer, + Err(e) => { + tracing::error!(error = %e, "failed to start box egress proxy peer"); + switch.stop().await; + drop(ready_tx.send(Err(e))); + return; + } + }; + tracing::info!( + proxy_ip = %subnet.box_egress_proxy_address(), + proxy_mac = %subnet.bep_mac(), + "box egress proxy peer started", + ); + if ready_tx.send(Ok(pid)).is_err() { // Caller went away before learning the PID; tear down. switch.stop().await; @@ -799,6 +820,8 @@ impl HostGvproxy { tokio::select! { _ = stop_rx => { switch.stop().await; + drop(_bep_peer); + drop(_gate); } status = exit.recv() => { tracing::error!( @@ -806,7 +829,9 @@ impl HostGvproxy { code = status.and_then(|s| s.code()), "host gvproxy switch exited unexpectedly", ); - // gvproxy is already gone; drop the handle (no signal). + // gvproxy is already gone; drop the handles (no signal). + drop(_bep_peer); + drop(_gate); drop(switch); } } diff --git a/crates/minvmd/src/net/bep_host.rs b/crates/minvmd/src/net/bep_host.rs index 4c2cd2f19e..8dd68d05ec 100644 --- a/crates/minvmd/src/net/bep_host.rs +++ b/crates/minvmd/src/net/bep_host.rs @@ -8,8 +8,8 @@ //! see [`egress_gate`]'s relay). The host kernel knows nothing of the switch //! subnet, so the leg terminates those frames itself: one smoltcp //! [`Interface`] over a channel-backed [`Device`], answering ARP for the leg's -//! address (the proof the first test pins) and carrying the proxy's sockets -//! once the proxy's own work lands. +//! address, resetting TCP to unlistened proxy ports, and sending ICMP +//! port-unreachable for UDP at the address. //! //! [`BepDevice`] is that [`Device`]: a pair of unbounded tokio channels, one //! raw Ethernet frame per message on each. [`BepDevice::pair`] hands the @@ -19,16 +19,42 @@ //! [`BepHost`] builds the interface over it: the leg's address on a //! [`SwitchSubnet`], its MAC derived the switch's way //! ([`MacAddr::for_switch_ip`]), stepped with [`BepHost::poll`]. +//! +//! [`BepPeer`] dials the switch `-listen` socket once, upgrades the connection +//! with the HyperKit `/connect` request, and runs the stack in a dedicated +//! local task that is woken by inbound frames, by the outbound pump, and by a +//! periodic `poll_delay`. +use std::collections::VecDeque; use std::fmt; +use std::io; use std::net::Ipv4Addr; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; use smoltcp::iface::{Config as InterfaceConfig, Interface, SocketSet}; -use smoltcp::phy::{Device, DeviceCapabilities, Medium, RxToken, TxToken}; +use smoltcp::phy::{Checksum, ChecksumCapabilities, Device, DeviceCapabilities, Medium}; use smoltcp::time::Instant; -use smoltcp::wire::{EthernetAddress, HardwareAddress, IpAddress, IpCidr, Ipv4Address}; +use smoltcp::wire::{ + ArpOperation, ArpPacket, ArpRepr, EthernetAddress, EthernetFrame, EthernetProtocol, + HardwareAddress, IPV4_HEADER_LEN, IpAddress, IpCidr, IpProtocol, Ipv4Address, Ipv4Packet, + Ipv4Repr, TcpControl, TcpPacket, TcpRepr, TcpSeqNumber, UdpPacket, +}; use switch::{DEFAULT_MTU, MacAddr, SwitchSubnet}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::UnixStream; +use tokio::sync::Notify; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; +use tokio::task::JoinSet; + +/// The HTTP upgrade request that turns a gvproxy control socket connection +/// into a raw Ethernet frame stream (the same head the guest shuttle uses). +const CONNECT_REQUEST: &[u8] = b"POST /connect HTTP/1.0\r\nHost: localhost\r\n\r\n"; + +/// Maximum interval between stack polls even when no traffic arrives, so smoltcp +/// TCP timers still advance. +const POLL_DELAY: Duration = Duration::from_millis(100); /// A channel-backed smoltcp [`Device`] over the switch's Ethernet lane. /// @@ -40,6 +66,8 @@ use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; pub struct BepDevice { rx: UnboundedReceiver>, tx: UnboundedSender>, + /// Frames the host loop has admitted and wants the interface to process. + pending: VecDeque>, } /// The peer ends of a [`BepDevice`]'s channels: what feeds the stack and what @@ -66,6 +94,7 @@ impl BepDevice { Self { rx: inbound_rx, tx: outbound_tx, + pending: VecDeque::new(), }, BepDeviceEnds { inbound: inbound_tx, @@ -73,6 +102,13 @@ impl BepDevice { }, ) } + + fn send_out(&self, frame: Vec) { + // A failed send means the leg's peer end is gone — the lane is closed + // and the frame has nowhere to go. Drop it rather than fail the poll; + // the lane's teardown is the leg's wiring's to log. + let _ = self.tx.send(frame); + } } impl Device for BepDevice { @@ -86,8 +122,7 @@ impl Device for BepDevice { Self: 'a; fn receive(&mut self, _timestamp: Instant) -> Option<(Self::RxToken<'_>, Self::TxToken<'_>)> { - // No buffered frame: nothing for the interface to process this poll. - let frame = self.rx.try_recv().ok()?; + let frame = self.pending.pop_front()?; Some(( BepRxToken { frame }, BepTxToken { @@ -111,6 +146,12 @@ impl Device for BepDevice { let mut caps = DeviceCapabilities::default(); caps.medium = Medium::Ethernet; caps.max_transmission_unit = usize::from(DEFAULT_MTU); + let mut checksum_caps = ChecksumCapabilities::default(); + checksum_caps.ipv4 = Checksum::Both; + checksum_caps.tcp = Checksum::Both; + checksum_caps.udp = Checksum::Both; + checksum_caps.icmpv4 = Checksum::Both; + caps.checksum = checksum_caps; caps } } @@ -120,7 +161,7 @@ pub struct BepRxToken { frame: Vec, } -impl RxToken for BepRxToken { +impl smoltcp::phy::RxToken for BepRxToken { fn consume(self, f: F) -> R where F: FnOnce(&[u8]) -> R, @@ -135,17 +176,14 @@ pub struct BepTxToken { tx: UnboundedSender>, } -impl TxToken for BepTxToken { +impl smoltcp::phy::TxToken for BepTxToken { fn consume(self, len: usize, f: F) -> R where F: FnOnce(&mut [u8]) -> R, { let mut frame = vec![0u8; len]; let result = f(&mut frame); - // A failed send means the leg's peer end is gone — the lane is closed - // and the frame has nowhere to go. Drop it rather than fail the poll; - // the lane's teardown is the leg's wiring's to log. - drop(self.tx.send(frame)); + let _ = self.tx.send(frame); result } } @@ -194,6 +232,8 @@ impl BepHost { .push(IpCidr::new(IpAddress::Ipv4(ip), subnet.prefix())) .expect("the leg holds one address and the interface's address capacity is four"); }); + // any_ip stays off: the interface owns this address and nothing else. + debug_assert!(!iface.any_ip()); Self { device, iface, @@ -215,18 +255,441 @@ impl BepHost { self.mac } + /// Send a gratuitous ARP announcement for the leg's address. + /// + /// The peer originates no frame unless addressed, but at attach it tells + /// the switch and any neighbors its MAC for the proxy address so inbound + /// frames can be addressed to it. + pub fn announce(&self) { + let repr = ArpRepr::EthernetIpv4 { + operation: ArpOperation::Request, + source_hardware_addr: self.mac, + source_protocol_addr: self.ip, + target_hardware_addr: EthernetAddress::BROADCAST, + target_protocol_addr: self.ip, + }; + self.send_arp(repr, EthernetAddress::BROADCAST); + } + /// Step the stack at `now`: process every buffered inbound frame, then /// emit whatever the interface has to send. Both drain to quiescence, so /// one call is one full turn of the stack. pub fn poll(&mut self, now: Instant) { + while let Ok(frame) = self.device.rx.try_recv() { + self.handle_frame(&frame); + } self.iface.poll(now, &mut self.device, &mut self.sockets); } + + fn handle_frame(&mut self, frame: &[u8]) { + let eth = match EthernetFrame::new_checked(frame) { + Ok(eth) => eth, + Err(_) => return, + }; + let dst = eth.dst_addr(); + if dst != self.mac && dst != EthernetAddress::BROADCAST { + return; + } + + match eth.ethertype() { + EthernetProtocol::Arp => { + if let Ok(arp) = ArpPacket::new_checked(eth.payload()) + && let Ok(repr) = ArpRepr::parse(&arp) + { + self.handle_arp(repr); + } + } + EthernetProtocol::Ipv4 => { + if let Ok(ip) = Ipv4Packet::new_checked(eth.payload()) + && let Ok(repr) = Ipv4Repr::parse(&ip, &ChecksumCapabilities::ignored()) + { + if repr.dst_addr != self.ip { + // any_ip is off: traffic not for the proxy address + // is dropped without a reply. + return; + } + let src_mac = eth.src_addr(); + match repr.next_header { + IpProtocol::Tcp => self.handle_tcp(src_mac, repr, ip.payload()), + IpProtocol::Udp => self.handle_udp(src_mac, repr, ip.payload()), + _ => {} + } + } + } + _ => {} + } + } + + fn handle_arp(&self, repr: ArpRepr) { + let (operation, target_protocol_addr, source_hardware_addr, source_protocol_addr) = + match repr { + ArpRepr::EthernetIpv4 { + operation, + target_protocol_addr, + source_hardware_addr, + source_protocol_addr, + .. + } => ( + operation, + target_protocol_addr, + source_hardware_addr, + source_protocol_addr, + ), + _ => return, + }; + if operation != ArpOperation::Request || target_protocol_addr != self.ip { + return; + } + let reply = ArpRepr::EthernetIpv4 { + operation: ArpOperation::Reply, + source_hardware_addr: self.mac, + source_protocol_addr: self.ip, + target_hardware_addr: source_hardware_addr, + target_protocol_addr: source_protocol_addr, + }; + self.send_arp(reply, source_hardware_addr); + } + + fn handle_tcp(&self, src_mac: EthernetAddress, ip_repr: Ipv4Repr, tcp_payload: &[u8]) { + let tcp = match TcpPacket::new_checked(tcp_payload) { + Ok(tcp) => tcp, + Err(_) => return, + }; + let seq = tcp.seq_number(); + let ack = tcp.ack_number(); + let (rst_seq, rst_ack, rst_ack_set) = if tcp.ack() { + (ack, TcpSeqNumber(0), false) + } else { + let len = tcp.payload().len() + if tcp.syn() || tcp.fin() { 1 } else { 0 }; + ( + TcpSeqNumber(0), + TcpSeqNumber(seq.0.wrapping_add(len as i32)), + true, + ) + }; + let rst = TcpRepr { + src_port: tcp.dst_port(), + dst_port: tcp.src_port(), + control: TcpControl::Rst, + seq_number: rst_seq, + ack_number: if rst_ack_set { Some(rst_ack) } else { None }, + window_len: 0, + window_scale: None, + max_seg_size: None, + sack_permitted: false, + sack_ranges: [None; 3], + timestamp: None, + payload: &[], + }; + self.send_ip_packet( + src_mac, + ip_repr.src_addr, + IpProtocol::Tcp, + rst.header_len(), + |tcp_buf| { + let mut tcp_packet = TcpPacket::new_unchecked(tcp_buf); + rst.emit( + &mut tcp_packet, + &IpAddress::Ipv4(self.ip), + &IpAddress::Ipv4(ip_repr.src_addr), + &self.device.capabilities().checksum, + ); + }, + ); + tracing::debug!( + src = %Ipv4Addr::from(u32::from_be_bytes(ip_repr.src_addr.octets())), + dst_port = tcp.dst_port(), + "sent TCP reset for unlistened proxy port" + ); + } + + fn handle_udp(&self, src_mac: EthernetAddress, ip_repr: Ipv4Repr, udp_payload: &[u8]) { + let udp = match UdpPacket::new_checked(udp_payload) { + Ok(udp) => udp, + Err(_) => return, + }; + let src_port = udp.src_port(); + let dst_port = udp.dst_port(); + let original_ip_header_len = 20_usize; + let original_total_len = (original_ip_header_len + udp.len() as usize) as u16; + let original_src = ip_repr.src_addr; + let original_dst = self.ip; + + // ICMP port-unreachable payload: original IPv4 header + first 8 bytes + // of the original datagram (the UDP header). + let returned_len = original_ip_header_len + 8; + self.send_ip_packet( + src_mac, + original_src, + IpProtocol::Icmp, + 8 + returned_len, + |icmp_buf| { + // Type 3, code 3, checksum, unused = 4 zero bytes. + icmp_buf[0] = 3; // Destination Unreachable + icmp_buf[1] = 3; // Port Unreachable + icmp_buf[2] = 0; + icmp_buf[3] = 0; + icmp_buf[4..8].fill(0); + + // Copy original IP header. + let mut orig_header = [0u8; 20]; + { + let mut orig_ip = Ipv4Packet::new_unchecked(&mut orig_header[..]); + orig_ip.set_version(4); + orig_ip.set_header_len(20); + orig_ip.set_dscp(0); + orig_ip.set_ecn(0); + orig_ip.set_total_len(original_total_len); + orig_ip.set_ident(0); + orig_ip.clear_flags(); + orig_ip.set_more_frags(false); + orig_ip.set_dont_frag(true); + orig_ip.set_frag_offset(0); + orig_ip.set_hop_limit(64); + orig_ip.set_next_header(IpProtocol::Udp); + orig_ip.set_src_addr(original_src); + orig_ip.set_dst_addr(original_dst); + orig_ip.fill_checksum(); + } + icmp_buf[8..28].copy_from_slice(&orig_header); + // Copy first 8 bytes of original UDP datagram (the UDP header). + let udp_header_len = core::cmp::min(8, udp_payload.len()); + icmp_buf[28..28 + udp_header_len].copy_from_slice(&udp_payload[..udp_header_len]); + + // Compute ICMP checksum over the ICMP message. + let checksum = smoltcp::wire::checksum::data(icmp_buf); + let checksum = !checksum; + icmp_buf[2..4].copy_from_slice(&checksum.to_be_bytes()); + }, + ); + tracing::debug!( + src = %Ipv4Addr::from(u32::from_be_bytes(original_src.octets())), + src_port, + dst_port, + "sent ICMP port-unreachable for UDP to proxy address" + ); + } + + fn send_arp(&self, repr: ArpRepr, dst_mac: EthernetAddress) { + let len = EthernetFrame::<&[u8]>::buffer_len(repr.buffer_len()); + let mut buf = vec![0u8; len]; + let mut frame = EthernetFrame::new_unchecked(&mut buf); + frame.set_dst_addr(dst_mac); + frame.set_src_addr(self.mac); + frame.set_ethertype(EthernetProtocol::Arp); + let mut packet = ArpPacket::new_unchecked(frame.payload_mut()); + repr.emit(&mut packet); + self.device.send_out(buf); + } + + fn send_ip_packet( + &self, + dst_mac: EthernetAddress, + dst_ip: Ipv4Address, + protocol: IpProtocol, + payload_len: usize, + mut build_payload: F, + ) where + F: FnMut(&mut [u8]), + { + // Largest buffer we'll need for the payloads this peer emits. + let mut buf = vec![0u8; 14 + IPV4_HEADER_LEN + payload_len]; + let mut frame = EthernetFrame::new_unchecked(&mut buf); + frame.set_dst_addr(dst_mac); + frame.set_src_addr(self.mac); + frame.set_ethertype(EthernetProtocol::Ipv4); + + let ip_repr = Ipv4Repr { + src_addr: self.ip, + dst_addr: dst_ip, + next_header: protocol, + payload_len, + hop_limit: 64, + }; + { + let mut ip_packet = Ipv4Packet::new_unchecked(frame.payload_mut()); + ip_repr.emit(&mut ip_packet, &self.device.capabilities().checksum); + } + build_payload(&mut frame.payload_mut()[IPV4_HEADER_LEN..]); + self.device.send_out(buf); + } +} + +/// A running Box Egress Proxy host peer. +/// +/// It owns the upstream switch socket connection (the `POST /connect` +/// upgrade), the channel-backed [`BepDevice`] that talks to the smoltcp +/// [`BepHost`], the two flow pumps that move framed Ethernet between the socket +/// and the device channels, and the dedicated local task that polls the stack. +/// Dropping the handle aborts those tasks and closes the socket. +#[derive(Debug)] +#[must_use = "dropping BepPeer stops the host-side proxy stack"] +pub struct BepPeer { + /// The stack poll task; aborting it stops the interface. + poll_task: tokio::task::JoinHandle<()>, + /// Pump tasks moving frames between socket and device channels. + pumps: JoinSet<()>, +} + +impl Drop for BepPeer { + fn drop(&mut self) { + self.pumps.abort_all(); + self.poll_task.abort(); + } +} + +impl BepPeer { + /// Start a peer for `subnet.box_egress_proxy_address()` on the switch + /// socket at `switch_sock`. + /// + /// The peer dials the switch, upgrades the connection with the HyperKit + /// `/connect` request, sends one gratuitous ARP, and then runs three + /// cooperating tasks: + /// + /// - a socket reader that reads length-framed Ethernet frames from the + /// switch and feeds them into the stack's inbound channel; + /// - a socket writer that reads from the stack's outbound channel and + /// writes length-framed Ethernet frames onto the switch; + /// - a dedicated local task that owns the smoltcp [`BepHost`] and polls it + /// whenever inbound frames arrive, whenever the outbound pump has + /// drained, and at least every [`POLL_DELAY`]. + /// + /// Must be called inside a tokio runtime. + /// + /// # Errors + /// + /// Returns an I/O error if the switch socket cannot be connected to. + pub async fn spawn(switch_sock: &Path, subnet: SwitchSubnet) -> io::Result { + let proxy_ip = subnet.box_egress_proxy_address(); + let proxy_mac = MacAddr::for_switch_ip(proxy_ip); + + let mut stream = UnixStream::connect(switch_sock).await?; + stream.write_all(CONNECT_REQUEST).await?; + stream.flush().await?; + + tracing::info!( + switch_socket = %switch_sock.display(), + proxy_ip = %proxy_ip, + proxy_mac = %proxy_mac, + "box egress proxy peer attached" + ); + + // The socket is split so the reader and writer can run independently; + // the stack poll task cannot own the `UnixStream` because the smoltcp + // `Interface` is not `Send`. + let (mut read_half, write_half) = stream.into_split(); + + // Channel-backed device: raw Ethernet frames in both directions. The + // reader/reader tasks are `Send`; the `BepHost` built around the device + // lives in the local poll task. + let (device, ends) = BepDevice::pair(); + let inbound_tx = ends.inbound; + let mut outbound_rx = ends.outbound; + + // Shared wake source: the reader calls `notify_one` after every frame + // it enqueues, and the writer calls `notify_one` whenever it drains a + // frame so the stack can make progress while the outbound channel has + // capacity. Because the channels are unbounded, the writer waking the + // poll task is the back-pressure substitute the spec asks for. + let notify = Arc::new(Notify::new()); + + let mut pumps = JoinSet::new(); + + // Pump 1: switch socket -> stack. Reads a 2-byte little-endian length, + // then that many frame bytes, and feeds the device. + let reader_notify = Arc::clone(¬ify); + pumps.spawn(async move { + let mut len_buf = [0u8; 2]; + loop { + match read_half.read(&mut len_buf).await { + Ok(0) => return, + Ok(2) => {} + Ok(_) => { + // A short length read means the socket is closing. + return; + } + Err(_) => return, + } + let len = u16::from_le_bytes(len_buf) as usize; + if len == 0 || len > usize::from(DEFAULT_MTU) + 14 + 4 { + // Malformed length; drop the connection. + return; + } + let mut frame = vec![0u8; len]; + if read_half.read_exact(&mut frame).await.is_err() { + return; + } + if inbound_tx.send(frame).is_err() { + // The poll task is gone. + return; + } + reader_notify.notify_one(); + } + }); + + // Pump 2: stack -> switch socket. Reads outbound frames, prefixes them + // with a 2-byte little-endian length, and writes them. + let writer_notify = Arc::clone(¬ify); + pumps.spawn(async move { + let mut stream = write_half; + while let Some(frame) = outbound_rx.recv().await { + let len = frame.len() as u16; + if len == 0 { + continue; + } + let mut buf = Vec::with_capacity(2 + frame.len()); + buf.extend_from_slice(&len.to_le_bytes()); + buf.extend_from_slice(&frame); + if stream.write_all(&buf).await.is_err() { + return; + } + if stream.flush().await.is_err() { + return; + } + writer_notify.notify_one(); + } + }); + + // Poll task: the only task that owns the smoltcp `Interface`. It runs + // on the current-thread runtime so `BepHost` (which is not `Send`) can + // be held across `.await` points. We block_in_place in an async task + // that is itself `Send`, and pass frames through channels. + let poll_task = tokio::task::spawn_blocking(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime for BepHost"); + rt.block_on(async move { + let mut host = BepHost::new(device, subnet, proxy_ip); + host.announce(); + host.poll(Instant::from_millis(0)); + + let start = tokio::time::Instant::now(); + let mut interval = tokio::time::interval(POLL_DELAY); + loop { + tokio::select! { + _ = notify.notified() => {} + _ = interval.tick() => {} + } + while let Ok(frame) = host.device.rx.try_recv() { + host.device.pending.push_back(frame); + } + let elapsed = start.elapsed().as_millis() as i64; + host.poll(Instant::from_millis(elapsed)); + } + }); + }); + + Ok(Self { poll_task, pumps }) + } } #[cfg(test)] mod tests { use super::*; - use smoltcp::wire::{ArpOperation, ArpPacket, ArpRepr, EthernetFrame, EthernetProtocol}; + use smoltcp::wire::{ + ArpOperation, ArpPacket, EthernetFrame, EthernetProtocol, Icmpv4Packet, UdpRepr, + }; /// One ARP request frame, as a box on the plan would send it: broadcast, /// from the asker's switch-derived MAC, asking who has `target_ip`. @@ -252,10 +715,135 @@ mod tests { buf } - /// The skeleton's first proof: build an interface over an in-process - /// device and read its ARP reply. A box on the switch's plan asks who has - /// the leg's address; the stack answers for the address it was given, - /// from the switch-derived MAC, pointed back at the asker. + /// Build a minimal IPv4 frame carrying the given ethertype payload. + fn ip_frame( + src_mac: EthernetAddress, + dst_mac: EthernetAddress, + src_ip: Ipv4Address, + dst_ip: Ipv4Address, + protocol: IpProtocol, + payload: &[u8], + ) -> Vec { + let ip_repr = Ipv4Repr { + src_addr: src_ip, + dst_addr: dst_ip, + next_header: protocol, + payload_len: payload.len(), + hop_limit: 64, + }; + let mut buf = + vec![0u8; EthernetFrame::<&[u8]>::buffer_len(ip_repr.buffer_len() + payload.len())]; + let mut frame = EthernetFrame::new_unchecked(&mut buf); + frame.set_dst_addr(dst_mac); + frame.set_src_addr(src_mac); + frame.set_ethertype(EthernetProtocol::Ipv4); + let mut ip_packet = Ipv4Packet::new_unchecked(frame.payload_mut()); + ip_repr.emit(&mut ip_packet, &ChecksumCapabilities::ignored()); + // Fill IP header checksum explicitly since ignored caps skipped it. + ip_packet.fill_checksum(); + ip_packet.payload_mut().copy_from_slice(payload); + buf + } + + /// Build a TCP SYN from `src:src_port` to `dst:dst_port`. + fn tcp_syn(src_ip: Ipv4Addr, dst_ip: Ipv4Addr, src_port: u16, dst_port: u16) -> Vec { + let src_mac = EthernetAddress(MacAddr::for_switch_ip(src_ip).0); + let dst_mac = EthernetAddress(MacAddr::for_switch_ip(dst_ip).0); + let repr = TcpRepr { + src_port, + dst_port, + control: TcpControl::Syn, + seq_number: TcpSeqNumber(1_000_000), + ack_number: None, + window_len: 1024, + window_scale: None, + max_seg_size: None, + sack_permitted: false, + sack_ranges: [None; 3], + timestamp: None, + payload: &[], + }; + let mut tcp_buf = vec![0u8; repr.buffer_len()]; + let mut tcp = TcpPacket::new_unchecked(&mut tcp_buf); + repr.emit( + &mut tcp, + &IpAddress::Ipv4(src_ip), + &IpAddress::Ipv4(dst_ip), + &ChecksumCapabilities::ignored(), + ); + ip_frame(src_mac, dst_mac, src_ip, dst_ip, IpProtocol::Tcp, &tcp_buf) + } + + /// Build a UDP datagram from `src:src_port` to `dst:dst_port`. + fn udp_datagram( + src_ip: Ipv4Addr, + dst_ip: Ipv4Addr, + src_port: u16, + dst_port: u16, + payload: &[u8], + ) -> Vec { + let src_mac = EthernetAddress(MacAddr::for_switch_ip(src_ip).0); + let dst_mac = EthernetAddress(MacAddr::for_switch_ip(dst_ip).0); + let repr = UdpRepr { src_port, dst_port }; + let mut udp_buf = vec![0u8; 8 + payload.len()]; + let mut udp = UdpPacket::new_unchecked(&mut udp_buf); + repr.emit( + &mut udp, + &IpAddress::Ipv4(src_ip), + &IpAddress::Ipv4(dst_ip), + payload.len(), + |buf| buf.copy_from_slice(payload), + &ChecksumCapabilities::ignored(), + ); + ip_frame(src_mac, dst_mac, src_ip, dst_ip, IpProtocol::Udp, &udp_buf) + } + + fn expect_one_arp_reply( + ends: &mut BepDeviceEnds, + host_mac: EthernetAddress, + host_ip: Ipv4Addr, + peer_mac: EthernetAddress, + peer_ip: Ipv4Addr, + ) { + let reply = ends.outbound.try_recv().expect("expected an ARP reply"); + let frame = EthernetFrame::new_checked(&reply).expect("reply is ethernet"); + assert_eq!(frame.dst_addr(), peer_mac); + assert_eq!(frame.src_addr(), host_mac); + assert_eq!(frame.ethertype(), EthernetProtocol::Arp); + let arp = ArpPacket::new_checked(frame.payload()).unwrap(); + assert_eq!( + ArpRepr::parse(&arp).unwrap(), + ArpRepr::EthernetIpv4 { + operation: ArpOperation::Reply, + source_hardware_addr: host_mac, + source_protocol_addr: host_ip, + target_hardware_addr: peer_mac, + target_protocol_addr: peer_ip, + } + ); + } + + #[test] + fn proxy_address_answers_arp_from_the_host_stack() { + let subnet = SwitchSubnet::default(); + let ip = subnet.box_egress_proxy_address(); + let (device, mut ends) = BepDevice::pair(); + let mut host = BepHost::new(device, subnet, ip); + + let peer_ip = Ipv4Addr::from(subnet.first_ptask()); + let peer_mac = EthernetAddress(MacAddr::for_switch_ip(peer_ip).0); + ends.inbound + .send(arp_request(peer_mac, peer_ip, ip)) + .expect("send the ARP request into the device"); + host.poll(Instant::from_millis(0)); + + expect_one_arp_reply(&mut ends, host.mac(), ip, peer_mac, peer_ip); + assert!(matches!( + ends.outbound.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + )); + } + #[test] fn host_stack_answers_arp_for_its_address() { let subnet = SwitchSubnet::default(); @@ -270,26 +858,175 @@ mod tests { .expect("send the ARP request into the device"); host.poll(Instant::from_millis(0)); - let reply = ends.outbound.try_recv().expect("the stack answered"); - let frame = EthernetFrame::new_checked(&reply).expect("the reply is an Ethernet frame"); - assert_eq!(frame.ethertype(), EthernetProtocol::Arp); + expect_one_arp_reply(&mut ends, host.mac(), ip, peer_mac, peer_ip); + } + + #[test] + fn tcp_to_unlistened_proxy_port_is_reset() { + let subnet = SwitchSubnet::default(); + let proxy_ip = subnet.box_egress_proxy_address(); + let (device, mut ends) = BepDevice::pair(); + let mut host = BepHost::new(device, subnet, proxy_ip); + + let peer_ip = Ipv4Addr::from(subnet.first_ptask()); + ends.inbound + .send(tcp_syn(peer_ip, proxy_ip, 1234, 443)) + .expect("send TCP SYN"); + host.poll(Instant::from_millis(0)); + + let reply = ends.outbound.try_recv().expect("expected a TCP reset"); + let frame = EthernetFrame::new_checked(&reply).unwrap(); assert_eq!(frame.src_addr(), host.mac()); - assert_eq!(frame.dst_addr(), peer_mac); - let arp = ArpPacket::new_checked(frame.payload()).expect("the reply carries ARP"); + let ip = Ipv4Packet::new_checked(frame.payload()).unwrap(); + assert_eq!(ip.src_addr(), proxy_ip); + assert_eq!(ip.dst_addr(), peer_ip); + let tcp = TcpPacket::new_checked(ip.payload()).unwrap(); + assert!(tcp.rst()); + assert_eq!(tcp.dst_port(), 1234); + assert_eq!(tcp.src_port(), 443); + } + + #[test] + fn udp_to_proxy_address_gets_port_unreachable() { + let subnet = SwitchSubnet::default(); + let proxy_ip = subnet.box_egress_proxy_address(); + let (device, mut ends) = BepDevice::pair(); + let mut host = BepHost::new(device, subnet, proxy_ip); + + let peer_ip = Ipv4Addr::from(subnet.first_ptask()); + ends.inbound + .send(udp_datagram(peer_ip, proxy_ip, 1234, 53, b"query")) + .expect("send UDP datagram"); + host.poll(Instant::from_millis(0)); + + let reply = ends.outbound.try_recv().expect("expected an ICMP reply"); + let frame = EthernetFrame::new_checked(&reply).unwrap(); + assert_eq!(frame.src_addr(), host.mac()); + let ip = Ipv4Packet::new_checked(frame.payload()).unwrap(); + assert_eq!(ip.src_addr(), proxy_ip); + assert_eq!(ip.dst_addr(), peer_ip); + let icmp = Icmpv4Packet::new_checked(ip.payload()).unwrap(); assert_eq!( - ArpRepr::parse(&arp).expect("the reply parses as ARP"), - ArpRepr::EthernetIpv4 { - operation: ArpOperation::Reply, - source_hardware_addr: host.mac(), - source_protocol_addr: ip, - target_hardware_addr: peer_mac, - target_protocol_addr: peer_ip, - } + icmp.msg_type(), + smoltcp::wire::Icmpv4Message::DstUnreachable ); - // One request, one reply: the stack emitted nothing else. - assert!(matches!( - ends.outbound.try_recv(), - Err(tokio::sync::mpsc::error::TryRecvError::Empty) - )); + assert_eq!(icmp.msg_code(), 3); // port unreachable + assert!(icmp.verify_checksum()); + // The returned data must carry the original IP header + UDP header. + let returned = icmp.data(); + assert!(returned.len() >= 28); + let expected_dst = u32::from(proxy_ip).to_be_bytes(); + assert_eq!(&returned[16..20], &expected_dst); + let expected_src_port = 1234u16.to_be_bytes(); + let expected_dst_port = 53u16.to_be_bytes(); + assert_eq!(&returned[20..22], &expected_src_port); + assert_eq!(&returned[22..24], &expected_dst_port); + } + + #[test] + fn stack_peer_originates_frames_only_to_the_box_that_addressed_it() { + let subnet = SwitchSubnet::default(); + let proxy_ip = subnet.box_egress_proxy_address(); + let (device, mut ends) = BepDevice::pair(); + let mut host = BepHost::new(device, subnet, proxy_ip); + + let peer_a_ip = Ipv4Addr::from(subnet.first_ptask()); + let peer_a_mac = EthernetAddress(MacAddr::for_switch_ip(peer_a_ip).0); + let peer_b_ip = Ipv4Addr::from(subnet.first_ptask() + 1); + let peer_b_mac = EthernetAddress(MacAddr::for_switch_ip(peer_b_ip).0); + + // A asks for proxy ARP. + ends.inbound + .send(arp_request(peer_a_mac, peer_a_ip, proxy_ip)) + .unwrap(); + host.poll(Instant::from_millis(0)); + let reply = ends.outbound.try_recv().unwrap(); + let frame = EthernetFrame::new_checked(&reply).unwrap(); + assert_eq!(frame.dst_addr(), peer_a_mac); + assert!(frame.ethertype() == EthernetProtocol::Arp); + + // B asks for proxy ARP. + ends.inbound + .send(arp_request(peer_b_mac, peer_b_ip, proxy_ip)) + .unwrap(); + host.poll(Instant::from_millis(0)); + let reply = ends.outbound.try_recv().unwrap(); + let frame = EthernetFrame::new_checked(&reply).unwrap(); + assert_eq!(frame.dst_addr(), peer_b_mac); + assert!(frame.ethertype() == EthernetProtocol::Arp); + + // A sends a TCP SYN; the reply goes back to A, not B. + ends.inbound + .send(tcp_syn(peer_a_ip, proxy_ip, 1000, 80)) + .unwrap(); + host.poll(Instant::from_millis(0)); + let reply = ends.outbound.try_recv().unwrap(); + let frame = EthernetFrame::new_checked(&reply).unwrap(); + assert_eq!(frame.dst_addr(), peer_a_mac); + assert!(frame.ethertype() == EthernetProtocol::Ipv4); + + // A UDP from B to the proxy address goes back to B. + ends.inbound + .send(udp_datagram(peer_b_ip, proxy_ip, 1000, 53, b"x")) + .unwrap(); + host.poll(Instant::from_millis(0)); + let reply = ends.outbound.try_recv().unwrap(); + let frame = EthernetFrame::new_checked(&reply).unwrap(); + assert_eq!(frame.dst_addr(), peer_b_mac); + assert!( + matches!( + ends.outbound.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + ), + "no extra broadcast frames should be emitted" + ); + } + + /// NET-132: the host-side stack peer starts when the switch socket is ready + /// and its drop aborts the socket connection and the poll task. We stand in + /// for gvproxy with a bound listener that accepts the connect upgrade; the + /// peer dials it, sends the upgrade head, and then runs until dropped. + #[tokio::test] + async fn stack_peer_stops_with_the_switch() { + let dir = tempfile::tempdir().expect("tempdir"); + let sock = dir.path().join("switch.sock"); + let listener = tokio::net::UnixListener::bind(&sock).expect("bind stand-in switch socket"); + + // Accept the peer's connection on a separate task so `spawn` completes. + let accept_fut = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("peer connected"); + // The peer must send the HyperKit upgrade head verbatim. + let mut head = vec![0u8; super::CONNECT_REQUEST.len()]; + stream + .read_exact(&mut head) + .await + .expect("read upgrade head"); + assert_eq!(&head, super::CONNECT_REQUEST); + stream + }); + + let subnet = SwitchSubnet::default(); + let peer = BepPeer::spawn(&sock, subnet) + .await + .expect("peer spawns against a listening switch socket"); + + // Wait for the acceptor to confirm the upgrade head was received; once it + // has, the connection is live from the peer's side. Dropping the peer + // must then close that connection, which the acceptor observes as EOF. + let mut accepted = accept_fut.await.expect("acceptor task completed"); + + drop(peer); + + // Give the peer tasks a moment to be aborted and the socket closed. + tokio::time::sleep(Duration::from_millis(100)).await; + + // A read on the still-open accepted socket must now return EOF because + // the peer's side has gone away. + let mut buf = [0u8; 1]; + let n = accepted + .read(&mut buf) + .await + .expect("read after peer drop should not error"); + assert_eq!(n, 0, "peer socket must close when the peer is dropped"); } } diff --git a/crates/switch/src/lib.rs b/crates/switch/src/lib.rs index 6ee6080bb4..e2eaafffa9 100644 --- a/crates/switch/src/lib.rs +++ b/crates/switch/src/lib.rs @@ -603,7 +603,10 @@ mod tests { let net = SwitchSubnet::default(); assert_eq!(net.to_string(), "100.64.0.0/16"); assert_eq!(net.gateway(), Ipv4Addr::new(100, 64, 0, 1)); - assert_eq!(net.box_egress_proxy_address(), Ipv4Addr::new(100, 64, 255, 252)); + assert_eq!( + net.box_egress_proxy_address(), + Ipv4Addr::new(100, 64, 255, 252) + ); assert_eq!(net.bep_mac(), BEP_MAC); assert_eq!(BEP_MAC.to_string(), "52:54:00:40:ff:fc"); assert_eq!(net.daemon_ip(), Ipv4Addr::new(100, 64, 255, 253)); diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index ad15523bb4..2e82f294b3 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -4616,6 +4616,184 @@ while True: # needs no listener, and says what it did not run. # CI's native lane has neither conflict and runs every assertion. # +# --------------------------------------------------------------------------- +# NET-132: the host-side stack peer owns the proxy's infrastructure address. +# These two cases are gated like the own-IP proof: +# * a switch must exist (`MINVMD_GVPROXY_BIN` on the lane), and +# * inside the VM the peer's MAC and ARP behaviour are observable only where +# /dev/net/tun and userns let an own-address box attach. +# Where the prereqs are absent the case prints what it cannot assert and +# returns 0, so a lane that cannot run them reports honestly rather than green. +# +# What is proved: +# * switch_steers_proxy_mac_frames_to_the_host_stack: a frame addressed to the +# proxy's MAC (52:54:00:40:ff:fc) leaves the guest, reaches the host-side +# peer, and is answered (or reset/ICMP'd) back to the originating box, never +# to any other box. +# * switch_answers_no_arp_for_the_proxy_address: the gvproxy switch itself does +# not answer ARP for 100.64.255.252; the host stack peer does. +proof_switch_steers_proxy_mac_frames_to_the_host_stack() { + echo "::group::switch steers proxy-MAC frames to the host stack peer (NET-132)" + if [ -z "${MINVMD_GVPROXY_BIN:-}" ]; then + echo "switch_steers_proxy_mac_frames_to_the_host_stack SKIPPED (no MINVMD_GVPROXY_BIN: this target has no switch)" + echo "::endgroup::" + return 0 + fi + if [ ! -c /dev/net/tun ] || [ ! -r /dev/net/tun ]; then + echo "switch_steers_proxy_mac_frames_to_the_host_stack SKIPPED (/dev/net/tun not available; own-address boxes cannot attach on this host)" + echo "::endgroup::" + return 0 + fi + if [ ! -e /proc/sys/kernel/unprivileged_userns_clone ] && [ "$(id -u)" -ne 0 ]; then + echo "switch_steers_proxy_mac_frames_to_the_host_stack SKIPPED (unprivileged user namespaces unavailable and not root; own-address boxes cannot attach)" + echo "::endgroup::" + return 0 + fi + + # When we can run, the test is identical in shape to the own-IP proof: boot an + # own-address box and, from inside it, send an Ethernet frame addressed to the + # proxy MAC. Because the box has no raw socket privileges (NET-083), we instead + # route to the proxy's address: the in-box stack emits the frame with the right + # destination MAC and source MAC derived from the box's lease. A TCP SYN to the + # proxy address on an unlistened port proves the frame reached the peer (it + # returns a TCP RST) and that the answer comes back to this box only (no other + # guest sees it). The RST is what the BepHost unit test already pins; here we + # prove the end-to-end path through the switch handles the proxy MAC correctly. + local bep_sid bep_ip proxy_ip proxy_mac + proxy_ip="100.64.255.252" + proxy_mac="52:54:00:40:ff:fc" + + BEP_SEED_DIR="$(hook_mktemp /tmp/mnlbep.XXXXXX)" + hook_seed_preamble > "$BEP_SEED_DIR/minimal.toml" + mkdir "$BEP_SEED_DIR/.git" + + bep_sid="$(cd "$BEP_SEED_DIR" && mnl session activate . --no-prompt \ + --name e2e-bep-mac --network own_ip 2>"$WORK/bep-mac.err")" || { + echo "::error::'min session activate --network own_ip' failed for BEP MAC test" + cat "$WORK/bep-mac.err" 2>/dev/null || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + } + bep_sid="$(printf '%s\n' "$bep_sid" | tail -n1 | tr -d '\r')" + + if ! mnl session exec "$bep_sid" sh -c 'cat /proc/net/dev' >"$WORK/bep-dev.out" 2>"$WORK/bep-dev.err"; then + echo "::error::could not read /proc/net/dev from the own-IP box" + cat "$WORK/bep-dev.err" 2>/dev/null || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + fi + if [ "$(grep -c ':' "$WORK/bep-dev.out")" -lt 2 ]; then + echo "::error::own-IP box has no tap interface; the switch did not attach" + cat "$WORK/bep-dev.out" + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + fi + + bep_ip="$(mnl session exec "$bep_sid" sh -c 'ip -4 -o addr show | awk \!/127\.0\.0\.1/{print $4}' 2>/dev/null | head -n1 | cut -d/ -f1)" + if [ -z "$bep_ip" ]; then + echo "::error::could not determine the own-IP box's switch address" + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + fi + + # RST proves the peer received the frame and answered; a non-RST fast refusal + # would mean the switch dropped or mis-routed it. + rst="$(mnl session exec "$bep_sid" \ + "python3 -c \"import socket; s=socket.socket(); s.settimeout(5); s.connect(('$proxy_ip', 443)); s.close()\"" \ + 2>"$WORK/bep-mac-probe.err" || true)" + # The python connect will raise ConnectionRefusedError when the RST arrives, + # which on the exec path is a non-zero exit but no stdout body. + if grep -q "ConnectionRefusedError\|Connection refused" "$WORK/bep-mac-probe.err" || \ + grep -q "Connection refused" "$WORK/bep-mac-probe.err"; then + echo "BEP MAC test OK: TCP SYN to $proxy_ip reached the host stack peer and returned RST" + else + echo "::error::TCP SYN to $proxy_ip did not produce a RST; proxy-MAC frame may not have reached the peer" + cat "$WORK/bep-mac-probe.err" 2>/dev/null || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + fi + + mnl session destroy --force "$bep_sid" >/dev/null 2>&1 || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + echo "switch steers proxy-MAC frames to the host stack peer OK" + echo "::endgroup::" +} + +proof_switch_answers_no_arp_for_the_proxy_address() { + echo "::group::switch answers no ARP for the proxy address (NET-132)" + if [ -z "${MINVMD_GVPROXY_BIN:-}" ]; then + echo "switch_answers_no_arp_for_the_proxy_address SKIPPED (no MINVMD_GVPROXY_BIN: this target has no switch)" + echo "::endgroup::" + return 0 + fi + if [ ! -c /dev/net/tun ] || [ ! -r /dev/net/tun ]; then + echo "switch_answers_no_arp_for_the_proxy_address SKIPPED (/dev/net/tun not available; own-address boxes cannot attach on this host)" + echo "::endgroup::" + return 0 + fi + if [ ! -e /proc/sys/kernel/unprivileged_userns_clone ] && [ "$(id -u)" -ne 0 ]; then + echo "switch_answers_no_arp_for_the_proxy_address SKIPPED (unprivileged user namespaces unavailable and not root; own-address boxes cannot attach)" + echo "::endgroup::" + return 0 + fi + + # The BepHost unit test pins this for the peer's stack; the e2e case checks + # that gvproxy itself does not answer. We boot an own-address box and arping + # the proxy address: the reply's MAC must be the peer's, not the gateway's. + local bep_sid proxy_ip proxy_mac + proxy_ip="100.64.255.252" + proxy_mac="52:54:00:40:ff:fc" + + BEP_SEED_DIR="$(hook_mktemp /tmp/mnlbep.XXXXXX)" + hook_seed_preamble > "$BEP_SEED_DIR/minimal.toml" + mkdir "$BEP_SEED_DIR/.git" + + bep_sid="$(cd "$BEP_SEED_DIR" && mnl session activate . --no-prompt \ + --name e2e-bep-arp --network own_ip 2>"$WORK/bep-arp.err")" || { + echo "::error::'min session activate --network own_ip' failed for BEP ARP test" + cat "$WORK/bep-arp.err" 2>/dev/null || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + } + bep_sid="$(printf '%s\n' "$bep_sid" | tail -n1 | tr -d '\r')" + + if ! mnl session exec "$bep_sid" sh -c 'test -x /usr/sbin/arping || test -x /sbin/arping' >/dev/null 2>&1; then + echo "::error::own-IP box has no arping utility; cannot verify ARP reply source" + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + fi + + # arping -c 1 -I ; success means a reply arrived. We then + # inspect the neighbor table to confirm the resolved MAC is the peer's MAC. + mnl session exec "$bep_sid" "sh -c 'iface=\$(ip -o link show | awk -F: \'!/lo/{print \$2}\' | head -n1 | tr -d \" \"); arping -c 1 -I \"\$iface\" $proxy_ip >/dev/null 2>&1; ip neigh show $proxy_ip'" \ + >"$WORK/bep-arp.out" 2>"$WORK/bep-arp-run.err" || true + + if grep -qF "$proxy_mac" "$WORK/bep-arp.out"; then + echo "BEP ARP test OK: proxy address $proxy_ip resolves to peer MAC $proxy_mac, not the gateway" + else + echo "::error::proxy address $proxy_ip did not resolve to the peer MAC $proxy_mac; the switch may be answering ARP itself" + echo "--- arp/neighbor output ---"; cat "$WORK/bep-arp.out" 2>/dev/null || true + echo "--- stderr ---"; cat "$WORK/bep-arp-run.err" 2>/dev/null || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + fi + + mnl session destroy --force "$bep_sid" >/dev/null 2>&1 || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + echo "switch answers no ARP for the proxy address OK" + echo "::endgroup::" +} + # Ordered LAST in the whole-lane run on purpose: it restarts the daemon (see # the RUST_LOG note inside) and nothing after it depends on the one before. proof_min_internal_names_through_proxy() { @@ -6725,6 +6903,8 @@ case "${1:-}" in proof_min_internal_names_through_proxy proof_proxy_refuses_like_direct proof_retired_surfaces_gone + proof_switch_steers_proxy_mac_frames_to_the_host_stack + proof_switch_answers_no_arp_for_the_proxy_address ;; lifecycle | session_exec | session_outbound_request | own_ip | own_ip_egress_declared_and_enforced | task_run | hooks \ | skip_scaffold | sandbox | restart | fresh_install_own_ip_ingress_publishes_loopback \ @@ -6732,7 +6912,11 @@ case "${1:-}" in | hostnames_recover_and_two_daemons_route \ | min_internal_names_through_proxy | proxy_refuses_like_direct | retired_surfaces_gone \ | fresh_linux_kvm_activate_local_minvmd | fresh_arm64_kvm_activate_local_minvmd \ - | linux_stock_install_runs_vm_boxes) + | linux_stock_install_runs_vm_boxes \ + | switch_steers_proxy_mac_frames_to_the_host_stack | switch_answers_no_arp_for_the_proxy_address) + "proof_$1" + ;; + switch_steers_proxy_mac_frames_to_the_host_stack | switch_answers_no_arp_for_the_proxy_address) "proof_$1" ;; *) @@ -6745,6 +6929,7 @@ case "${1:-}" in echo " linux_stock_install_runs_vm_boxes" echo " hostnames_recover_and_two_daemons_route" echo " min_internal_names_through_proxy proxy_refuses_like_direct retired_surfaces_gone" + echo " switch_steers_proxy_mac_frames_to_the_host_stack switch_answers_no_arp_for_the_proxy_address" exit 2 ;; esac From 1a5c7e0f0bf4e9d89860510abb28ac8487fa7f58 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:24:52 -0700 Subject: [PATCH 3/7] fix(minvmd): let the proxy peer's poll loop exit when the peer drops The smoltcp poll loop runs on a blocking thread with a runtime of its own, and aborting a blocking task does not stop it, so dropping the peer left the loop running and the switch runtime waited forever for it at shutdown. Every test that stops the runtime hung, and minvmd stop would have hung the same way. The peer now holds a oneshot the loop selects on and drops it on Drop; the loop also returns once its reader pump is gone and no frame can arrive. --- crates/minvmd/src/net/bep_host.rs | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/crates/minvmd/src/net/bep_host.rs b/crates/minvmd/src/net/bep_host.rs index 8dd68d05ec..b21d47464f 100644 --- a/crates/minvmd/src/net/bep_host.rs +++ b/crates/minvmd/src/net/bep_host.rs @@ -525,8 +525,13 @@ impl BepHost { #[derive(Debug)] #[must_use = "dropping BepPeer stops the host-side proxy stack"] pub struct BepPeer { - /// The stack poll task; aborting it stops the interface. + /// The stack poll task. It runs on a blocking thread, which `abort` cannot + /// stop, so the loop watches `stop` and returns when the sender drops. poll_task: tokio::task::JoinHandle<()>, + /// Dropped by `Drop`: the poll loop exits when this closes. Without it the + /// runtime that spawned the peer waits forever for the blocking task at + /// shutdown, which is how the switch runtime hung at stop. + stop: Option>, /// Pump tasks moving frames between socket and device channels. pumps: JoinSet<()>, } @@ -534,6 +539,7 @@ pub struct BepPeer { impl Drop for BepPeer { fn drop(&mut self) { self.pumps.abort_all(); + drop(self.stop.take()); self.poll_task.abort(); } } @@ -654,6 +660,7 @@ impl BepPeer { // on the current-thread runtime so `BepHost` (which is not `Send`) can // be held across `.await` points. We block_in_place in an async task // that is itself `Send`, and pass frames through channels. + let (stop_tx, mut stop_rx) = tokio::sync::oneshot::channel::<()>(); let poll_task = tokio::task::spawn_blocking(move || { let rt = tokio::runtime::Builder::new_current_thread() .enable_all() @@ -670,9 +677,18 @@ impl BepPeer { tokio::select! { _ = notify.notified() => {} _ = interval.tick() => {} + // The peer was dropped: leave the loop so the runtime + // that spawned this blocking task can shut down. + _ = &mut stop_rx => break, } - while let Ok(frame) = host.device.rx.try_recv() { - host.device.pending.push_back(frame); + loop { + match host.device.rx.try_recv() { + Ok(frame) => host.device.pending.push_back(frame), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) => break, + // The reader pump is gone and no frame will ever + // arrive: the interface has nothing left to serve. + Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => return, + } } let elapsed = start.elapsed().as_millis() as i64; host.poll(Instant::from_millis(elapsed)); @@ -680,7 +696,11 @@ impl BepPeer { }); }); - Ok(Self { poll_task, pumps }) + Ok(Self { + poll_task, + stop: Some(stop_tx), + pumps, + }) } } From 901448d6ce17d9a9deaf6bd7976af631a5af6f04 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:56:05 +0000 Subject: [PATCH 4/7] feat(minvmd): keep the proxy peer silent, socketless, and pinned to the plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The design review's three conditions on the host-side stack peer: - Silence: the peer no longer announces itself at attach. `announce()` is gone, so the peer originates no unsolicited frame of any kind — no gratuitous ARP, no ARP probe, nothing on an idle lane — and answers only the box that addressed it. The module doc records why it cannot speak IPv6 either: the smoltcp feature set excludes `proto-ipv6`, so no neighbour solicitation, router solicitation, or MLD can leave it. `idle_peer_emits_no_frames` proves it against a stand-in switch socket. - No socket in this task: the interface polls an empty `SocketSet`; nothing is bound, and the module doc states T69 is the only task that may bind a socket on the peer, carrying the per-source caps and the gate rule. `peer_binds_no_socket` asserts the empty set. - One source for the run end: `lease_run_gate_run_and_rendered_pool_agree` pins the lease run, the gate's run, and the rendered static-lease pool against the switch crate's plan, with the reserved run above it in order — last PTask, proxy address, daemon address, host alias. --- crates/minvmd/src/net.rs | 106 ++++++++++++++++++++++++++++ crates/minvmd/src/net/bep_host.rs | 113 ++++++++++++++++++++++++------ 2 files changed, 198 insertions(+), 21 deletions(-) diff --git a/crates/minvmd/src/net.rs b/crates/minvmd/src/net.rs index 90d3c48db5..4096abf736 100644 --- a/crates/minvmd/src/net.rs +++ b/crates/minvmd/src/net.rs @@ -1080,6 +1080,112 @@ mod tests { assert!(yaml.contains("{}")); } + /// NET-132: one source for the run's end. The last PTask address and the + /// proxy's own address both come from the `switch` crate's plan, and every + /// consumer here reads them there — the lease run the registry hands box + /// addresses out of, the gate's answer to which sources the plan could + /// ever have handed out, and the static-lease pool the rendered switch + /// config serves. The three are pinned against each other here, so moving + /// the run's end or the proxy address fails this test instead of leaving a + /// consumer behind. `minimald`'s mirror of the same run — its + /// self-allocation reserve, the run's lower half — is pinned by its own + /// test against the same `switch` crate, and the switch crate's own test + /// pins the proxy address out of gvproxy's NAT and virtual-IP tables. + #[test] + fn lease_run_gate_run_and_rendered_pool_agree() { + let subnet = SwitchSubnet::default(); + let first = Ipv4Addr::from(subnet.first_ptask()); + let last = Ipv4Addr::from(subnet.last_ptask()); + + // The reserved run above the lease pool, in order: the proxy address + // the host-side peer owns (NET-132), the daemon's own tap, and the + // host alias gvproxy NATs — all above the run's end, all from the one + // plan. + assert!( + u32::from(last) < u32::from(subnet.box_egress_proxy_address()), + "the proxy address sits above the lease pool" + ); + assert!(u32::from(subnet.box_egress_proxy_address()) < u32::from(subnet.daemon_ip())); + assert!(u32::from(subnet.daemon_ip()) < u32::from(subnet.host_alias())); + + // The gate's run: the one set of sources the plan could ever hand a + // box — both ends of the lease run inside it, every reserved address + // and everything off the subnet outside it. + let registry = crate::box_registry::BoxRegistry::new(subnet); + let table = registry.table(); + assert!( + table.is_allocatable(first.octets()), + "the run's first address is a source the gate's run admits" + ); + assert!( + table.is_allocatable(last.octets()), + "the run's last address is a source the gate's run admits" + ); + for outside in [ + subnet.network(), + subnet.gateway(), + subnet.box_egress_proxy_address(), + subnet.daemon_ip(), + subnet.host_alias(), + subnet.broadcast(), + Ipv4Addr::new(203, 0, 113, 7), + ] { + assert!( + !table.is_allocatable(outside.octets()), + "the gate's run never admits the reserved {outside}" + ); + } + + // The lease book's range: the registry hands a registered box an + // address from the same run — inside its ends, never a reserved one. + for name in ["web", "db"] { + let row = registry + .register_client_box(crate::box_registry::ClientBoxSpec { + name: name.to_string(), + ingress_ports: Vec::new(), + egress: None, + }) + .expect("the default plan has hand-out addresses"); + let handed = row.switch_addr(); + assert!( + first <= handed && handed <= last, + "the lease book hands {handed} from inside the plan's run" + ); + } + + // The rendered pool: the config's static-lease table carries exactly + // the addresses it is handed — the run's own ends here — on the same + // subnet, and none of the reserved run above it. The lease map is the + // tail of the rendered YAML (`dhcpStaticLeases:` is its last key), so + // the alias's own NAT and DNS lines do not read as leases. + let pool = [ + (first, MacAddr::for_switch_ip(first)), + (last, MacAddr::for_switch_ip(last)), + ]; + let yaml = render_gvproxy_config(subnet, &pool); + assert!(yaml.contains(&format!("subnet: \"{subnet}\""))); + let lease_map = yaml + .split("dhcpStaticLeases:\n") + .nth(1) + .expect("the rendered config carries a static-lease map"); + for (ip, mac) in pool { + assert!(lease_map.contains(&format!(" \"{ip}\": \"{mac}\""))); + } + for reserved in [subnet.daemon_ip(), subnet.host_alias()] { + assert!( + !lease_map.contains(&format!("\"{reserved}\"")), + "{reserved} is reserved, never a lease in the rendered pool" + ); + } + // The proxy address is nowhere in the config at all: not a lease, not + // a virtual IP, not translated — the peer owns it, and the switch + // crate's own test pins the same from the other side. + assert!( + !yaml.contains(&format!("\"{}\"", subnet.box_egress_proxy_address())), + "the proxy address is the peer's, never gvproxy's" + ); + } + #[test] fn write_config_creates_parent_and_file() { let dir = tempfile::TempDir::new().expect("tempdir"); diff --git a/crates/minvmd/src/net/bep_host.rs b/crates/minvmd/src/net/bep_host.rs index b21d47464f..9e414c3d13 100644 --- a/crates/minvmd/src/net/bep_host.rs +++ b/crates/minvmd/src/net/bep_host.rs @@ -20,6 +20,20 @@ //! [`SwitchSubnet`], its MAC derived the switch's way //! ([`MacAddr::for_switch_ip`]), stepped with [`BepHost::poll`]. //! +//! The peer speaks only when spoken to. It originates no unsolicited frame — +//! no gratuitous ARP, no ARP probe, nothing on an idle lane — and every frame +//! it does send is an answer to the box that addressed it: an ARP reply for +//! its address, a TCP reset for a segment no socket holds, an ICMP +//! port-unreachable for a UDP datagram at the address. The smoltcp feature set +//! this workspace builds with excludes `proto-ipv6`, so the interface has no +//! IPv6 neighbour discovery either: no neighbour solicitation, no router +//! solicitation, no multicast listener discovery can leave it. +//! +//! No socket is bound here. The interface polls an empty [`SocketSet`]: this +//! task terminates nothing above the network layer, and **T69 is the only task +//! that may bind a socket on the peer** — it carries the per-source caps and +//! the gate rule before it does. +//! //! [`BepPeer`] dials the switch `-listen` socket once, upgrades the connection //! with the HyperKit `/connect` request, and runs the stack in a dedicated //! local task that is woken by inbound frames, by the outbound pump, and by a @@ -191,8 +205,10 @@ impl smoltcp::phy::TxToken for BepTxToken { /// The host leg's stack: one smoltcp [`Interface`] over a channel-backed /// [`BepDevice`], holding the leg's address on the switch's plan. /// -/// Built by [`BepHost::new`], stepped by [`BepHost::poll`]. The sockets the -/// proxy terminates come later, when the proxy's own work adds them; the set +/// Built by [`BepHost::new`], stepped by [`BepHost::poll`]. The socket set is +/// empty and stays empty in this task: the peer terminates nothing above the +/// network layer, and T69 is the only task that may bind a socket on the peer, +/// carrying the per-source caps and the gate rule before it does. The set /// exists here so the interface has something to poll against from the first /// frame on. pub struct BepHost { @@ -255,22 +271,6 @@ impl BepHost { self.mac } - /// Send a gratuitous ARP announcement for the leg's address. - /// - /// The peer originates no frame unless addressed, but at attach it tells - /// the switch and any neighbors its MAC for the proxy address so inbound - /// frames can be addressed to it. - pub fn announce(&self) { - let repr = ArpRepr::EthernetIpv4 { - operation: ArpOperation::Request, - source_hardware_addr: self.mac, - source_protocol_addr: self.ip, - target_hardware_addr: EthernetAddress::BROADCAST, - target_protocol_addr: self.ip, - }; - self.send_arp(repr, EthernetAddress::BROADCAST); - } - /// Step the stack at `now`: process every buffered inbound frame, then /// emit whatever the interface has to send. Both drain to quiescence, so /// one call is one full turn of the stack. @@ -549,8 +549,10 @@ impl BepPeer { /// socket at `switch_sock`. /// /// The peer dials the switch, upgrades the connection with the HyperKit - /// `/connect` request, sends one gratuitous ARP, and then runs three - /// cooperating tasks: + /// `/connect` request, and then runs three cooperating tasks. It announces + /// nothing: the switch steers frames to the peer's derived MAC + /// ([`MacAddr::for_switch_ip`]) without a round trip, so the peer waits to + /// be asked — see the module docs for the frames it answers with. /// /// - a socket reader that reads length-framed Ethernet frames from the /// switch and feeds them into the stack's inbound channel; @@ -668,7 +670,6 @@ impl BepPeer { .expect("current-thread runtime for BepHost"); rt.block_on(async move { let mut host = BepHost::new(device, subnet, proxy_ip); - host.announce(); host.poll(Instant::from_millis(0)); let start = tokio::time::Instant::now(); @@ -1049,4 +1050,74 @@ mod tests { .expect("read after peer drop should not error"); assert_eq!(n, 0, "peer socket must close when the peer is dropped"); } + + /// NET-132: the peer is silent until addressed. A stand-in switch socket + /// accepts the upgrade and holds the lane open; the peer, attached and + /// polled across several poll periods, must write nothing at all — no + /// gratuitous ARP, no ARP probe, no keepalive. Every frame it sends is an + /// answer, and nothing has asked it anything. + #[tokio::test] + async fn idle_peer_emits_no_frames() { + let dir = tempfile::tempdir().expect("tempdir"); + let sock = dir.path().join("switch.sock"); + let listener = tokio::net::UnixListener::bind(&sock).expect("bind stand-in switch socket"); + + // Accept the upgrade, then hold the lane open and collect every byte + // the peer writes after the upgrade head. + let accept_fut = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("peer connected"); + let mut head = vec![0u8; super::CONNECT_REQUEST.len()]; + stream + .read_exact(&mut head) + .await + .expect("read upgrade head"); + assert_eq!(&head, super::CONNECT_REQUEST); + let mut written = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + match stream.read(&mut buf).await { + Ok(0) => break, + Ok(n) => written.extend_from_slice(&buf[..n]), + Err(_) => break, + } + } + written + }); + + let peer = BepPeer::spawn(&sock, SwitchSubnet::default()) + .await + .expect("peer spawns against a listening switch socket"); + + // Several poll periods must pass with nothing to answer: the poll loop + // wakes at least every POLL_DELAY, so this window covers every periodic + // wake the stack has, with slack for a loaded runner. + tokio::time::sleep(POLL_DELAY * 5 + Duration::from_millis(250)).await; + + // Dropping the peer closes the lane, which ends the collector above. + drop(peer); + let written = accept_fut.await.expect("acceptor task completed"); + assert!( + written.is_empty(), + "an idle peer emitted {} bytes without being addressed", + written.len() + ); + } + + /// NET-132: the peer terminates nothing above the network layer. The + /// interface polls an empty `SocketSet` — no TCP, UDP, or ICMP socket is + /// bound, so the reset and the port-unreachable the tests above pin are + /// the stack's own answers, not a listener's. T69 is the only task that + /// may bind a socket on the peer, and it carries the per-source caps and + /// the gate rule before it does. + #[test] + fn peer_binds_no_socket() { + let subnet = SwitchSubnet::default(); + let (device, _ends) = BepDevice::pair(); + let host = BepHost::new(device, subnet, subnet.box_egress_proxy_address()); + assert_eq!( + host.sockets.iter().count(), + 0, + "the peer binds no socket; T69 is the only task that may" + ); + } } From 331342f98ecdddc63c2f202eb2364c9ee82170d0 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:25:03 +0000 Subject: [PATCH 5/7] feat(minvmd): rate-limit the proxy peer's answer lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The task's observability contract asks for one rate-limited debug line per reset and per port-unreachable, not one per event: a box hammering the proxy address with SYNs or datagrams would otherwise fill the log while being answered normally. The answer itself is never held back — only the line saying so is — and the events an interval swallows are counted and named by the next line that fires. `answer_log_rate_limits_a_burst` pins the limiter's behaviour. --- crates/minvmd/src/net/bep_host.rs | 119 +++++++++++++++++++++++++----- 1 file changed, 102 insertions(+), 17 deletions(-) diff --git a/crates/minvmd/src/net/bep_host.rs b/crates/minvmd/src/net/bep_host.rs index 9e414c3d13..9457f452b0 100644 --- a/crates/minvmd/src/net/bep_host.rs +++ b/crates/minvmd/src/net/bep_host.rs @@ -39,6 +39,7 @@ //! local task that is woken by inbound frames, by the outbound pump, and by a //! periodic `poll_delay`. +use std::cell::Cell; use std::collections::VecDeque; use std::fmt; use std::io; @@ -70,6 +71,39 @@ const CONNECT_REQUEST: &[u8] = b"POST /connect HTTP/1.0\r\nHost: localhost\r\n\r /// TCP timers still advance. const POLL_DELAY: Duration = Duration::from_millis(100); +/// Minimum interval between the debug lines the peer logs for one kind of +/// answer (a reset, a port-unreachable), so a box hammering the proxy address +/// cannot fill the log. The answers the interval holds back are counted and +/// named by the next line that does fire. +const ANSWER_LOG_INTERVAL_MS: i64 = 1_000; + +/// The rate limit on one kind of answer's debug line, stepped by the poll's +/// own clock (the peer has no wall clock to read and needs none): one line per +/// [`ANSWER_LOG_INTERVAL_MS`], the rest counted into the next line that fires. +#[derive(Debug, Default)] +struct AnswerLog { + last: Cell>, + suppressed: Cell, +} + +impl AnswerLog { + /// Whether an answer at `now` may log, and how many were held back since + /// the last line that fired. + fn gate(&self, now: i64) -> Option { + if self + .last + .get() + .is_none_or(|last| now >= last + ANSWER_LOG_INTERVAL_MS) + { + self.last.set(Some(now)); + Some(self.suppressed.replace(0)) + } else { + self.suppressed.set(self.suppressed.get().saturating_add(1)); + None + } + } +} + /// A channel-backed smoltcp [`Device`] over the switch's Ethernet lane. /// /// One unbounded channel in (frames arriving from the lane), one out (frames @@ -217,6 +251,12 @@ pub struct BepHost { sockets: SocketSet<'static>, ip: Ipv4Address, mac: EthernetAddress, + /// The rate limits on the two kinds of answer the peer logs: a reset for + /// a TCP segment no socket holds, an ICMP port-unreachable for a UDP + /// datagram. The frames themselves are never rate-limited — every one is + /// answered — only the debug line saying so is. + reset_log: AnswerLog, + unreachable_log: AnswerLog, } impl fmt::Debug for BepHost { @@ -256,6 +296,8 @@ impl BepHost { sockets: SocketSet::new(vec![]), ip, mac, + reset_log: AnswerLog::default(), + unreachable_log: AnswerLog::default(), } } @@ -276,12 +318,12 @@ impl BepHost { /// one call is one full turn of the stack. pub fn poll(&mut self, now: Instant) { while let Ok(frame) = self.device.rx.try_recv() { - self.handle_frame(&frame); + self.handle_frame(&frame, now); } self.iface.poll(now, &mut self.device, &mut self.sockets); } - fn handle_frame(&mut self, frame: &[u8]) { + fn handle_frame(&mut self, frame: &[u8], now: Instant) { let eth = match EthernetFrame::new_checked(frame) { Ok(eth) => eth, Err(_) => return, @@ -310,8 +352,12 @@ impl BepHost { } let src_mac = eth.src_addr(); match repr.next_header { - IpProtocol::Tcp => self.handle_tcp(src_mac, repr, ip.payload()), - IpProtocol::Udp => self.handle_udp(src_mac, repr, ip.payload()), + IpProtocol::Tcp => { + self.handle_tcp(src_mac, repr, ip.payload(), now); + } + IpProtocol::Udp => { + self.handle_udp(src_mac, repr, ip.payload(), now); + } _ => {} } } @@ -350,7 +396,13 @@ impl BepHost { self.send_arp(reply, source_hardware_addr); } - fn handle_tcp(&self, src_mac: EthernetAddress, ip_repr: Ipv4Repr, tcp_payload: &[u8]) { + fn handle_tcp( + &self, + src_mac: EthernetAddress, + ip_repr: Ipv4Repr, + tcp_payload: &[u8], + now: Instant, + ) { let tcp = match TcpPacket::new_checked(tcp_payload) { Ok(tcp) => tcp, Err(_) => return, @@ -396,14 +448,24 @@ impl BepHost { ); }, ); - tracing::debug!( - src = %Ipv4Addr::from(u32::from_be_bytes(ip_repr.src_addr.octets())), - dst_port = tcp.dst_port(), - "sent TCP reset for unlistened proxy port" - ); + // Rate-limited: the reset itself is never held back, only this line is. + if let Some(suppressed) = self.reset_log.gate(now.total_millis()) { + tracing::debug!( + src = %Ipv4Addr::from(u32::from_be_bytes(ip_repr.src_addr.octets())), + dst_port = tcp.dst_port(), + suppressed, + "sent TCP reset for unlistened proxy port" + ); + } } - fn handle_udp(&self, src_mac: EthernetAddress, ip_repr: Ipv4Repr, udp_payload: &[u8]) { + fn handle_udp( + &self, + src_mac: EthernetAddress, + ip_repr: Ipv4Repr, + udp_payload: &[u8], + now: Instant, + ) { let udp = match UdpPacket::new_checked(udp_payload) { Ok(udp) => udp, Err(_) => return, @@ -462,12 +524,17 @@ impl BepHost { icmp_buf[2..4].copy_from_slice(&checksum.to_be_bytes()); }, ); - tracing::debug!( - src = %Ipv4Addr::from(u32::from_be_bytes(original_src.octets())), - src_port, - dst_port, - "sent ICMP port-unreachable for UDP to proxy address" - ); + // Rate-limited: the port-unreachable itself is never held back, only + // this line is. + if let Some(suppressed) = self.unreachable_log.gate(now.total_millis()) { + tracing::debug!( + src = %Ipv4Addr::from(u32::from_be_bytes(original_src.octets())), + src_port, + dst_port, + suppressed, + "sent ICMP port-unreachable for UDP to proxy address" + ); + } } fn send_arp(&self, repr: ArpRepr, dst_mac: EthernetAddress) { @@ -1120,4 +1187,22 @@ mod tests { "the peer binds no socket; T69 is the only task that may" ); } + + /// The answer the peer logs is rate-limited, so a box hammering the proxy + /// address cannot fill the log: the first line fires at once, a burst is + /// held back, and the next line that fires names how many were held. + #[test] + fn answer_log_rate_limits_a_burst() { + let log = AnswerLog::default(); + assert_eq!(log.gate(0), Some(0), "the first answer logs at once"); + for now in 1..1_000 { + assert_eq!(log.gate(now), None, "the burst is held back"); + } + assert_eq!( + log.gate(1_000), + Some(999), + "the interval's end fires and names the answers it held back" + ); + assert_eq!(log.gate(1_001), None, "a fresh burst is held back again"); + } } From 84b2987a07ebb61383e7306b5f3d8726fca13fa6 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:33:58 +0000 Subject: [PATCH 6/7] test(e2e): drop the unused probe capture in the proxy-MAC case The BEP MAC case assigned the probe's stdout to a variable it never read; the verdict comes from the refusal the probe's stderr names, so the capture was dead weight shellcheck flagged. Behaviour is unchanged. --- scripts/session-e2e.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index b13642c929..49f17f732e 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -4792,10 +4792,11 @@ proof_switch_steers_proxy_mac_frames_to_the_host_stack() { fi # RST proves the peer received the frame and answered; a non-RST fast refusal - # would mean the switch dropped or mis-routed it. - rst="$(mnl session exec "$bep_sid" \ + # would mean the switch dropped or mis-routed it. The probe's body is not + # read: the verdict is the refusal its stderr names. + mnl session exec "$bep_sid" \ "python3 -c \"import socket; s=socket.socket(); s.settimeout(5); s.connect(('$proxy_ip', 443)); s.close()\"" \ - 2>"$WORK/bep-mac-probe.err" || true)" + 2>"$WORK/bep-mac-probe.err" || true # The python connect will raise ConnectionRefusedError when the RST arrives, # which on the exec path is a non-zero exit but no stdout body. if grep -q "ConnectionRefusedError\|Connection refused" "$WORK/bep-mac-probe.err" || \ From 878cac1260d67165a78a0836808fe09dbe7391e4 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:55:11 +0000 Subject: [PATCH 7/7] test(minimald): pin the tiny subnet's PTask run below the proxy address The proxy address the switch layer reserved at broadcast - 3 pushed last_ptask down to broadcast - 4, so a /29's PTask run is .2-.3, not .2-.4: allocator_exhausts_a_tiny_subnet still unwrapped a hand of .4 as if the run reached it. Pin the run the plan now draws and let the same hand refuse the proxy address as HandedAddressOutsidePlan, which is what the failed unwrap was hiding. Exhaustion on each side, no wrap and no reuse, are unchanged; the split-rule comment now counts the run's two addresses. --- crates/minimald/src/net/mod.rs | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/crates/minimald/src/net/mod.rs b/crates/minimald/src/net/mod.rs index 1cae50a5e1..2bb19c7bf6 100644 --- a/crates/minimald/src/net/mod.rs +++ b/crates/minimald/src/net/mod.rs @@ -894,19 +894,29 @@ mod tests { #[test] fn allocator_exhausts_a_tiny_subnet() { - // /29 => 8 addresses; network, gateway, daemon, host-alias, broadcast - // reserved, leaving exactly three allocatable hosts (.2 through .4). - // The run splits at its midpoint, so this daemon's reserve is .2 - // alone and the host's hand-out run is .3 and .4 — exhaustion is - // explicit on each side, with no wrap and no reuse. + // /29 => 8 addresses; network, gateway, proxy, daemon, host-alias, + // broadcast reserved, leaving exactly two allocatable hosts (.2 and + // .3). The run splits at its midpoint, so this daemon's reserve is .2 + // alone and the host's hand-out run is .3 alone — exhaustion is + // explicit on each side, with no wrap and no reuse. The proxy's + // address (.4, `broadcast - 3`) is infrastructure above the run, so + // it is refused as a handed address, exactly as one past the run + // would be. let subnet = SwitchSubnet::new(Ipv4Addr::new(10, 0, 0, 0), 29).unwrap(); + assert_eq!( + subnet.box_egress_proxy_address(), + Ipv4Addr::new(10, 0, 0, 4) + ); let mut a = IpAllocator::new(subnet); assert_eq!(a.allocate().unwrap().ip, Ipv4Addr::new(10, 0, 0, 2)); assert!(matches!(a.allocate(), Err(NetError::SubnetExhausted(_)))); // The reserve bounds only the daemon's draws: the run's remainder is // still there for the host's hand-outs, none of them ever drawn. a.hand(Ipv4Addr::new(10, 0, 0, 3)).unwrap(); - a.hand(Ipv4Addr::new(10, 0, 0, 4)).unwrap(); + assert!(matches!( + a.hand(Ipv4Addr::new(10, 0, 0, 4)), + Err(NetError::HandedAddressOutsidePlan { .. }) + )); assert!(matches!( a.hand(Ipv4Addr::new(10, 0, 0, 3)), Err(NetError::HandedAddressCollision { .. }) @@ -940,8 +950,8 @@ mod tests { "both sub-runs are inside the plan's PTask run, and neither is empty" ); // A tiny subnet splits in half, so both sides keep at least one - // address: a /29's three PTask addresses give the reserve .2 and the - // hand-out run .3 and .4. + // address: a /29's two PTask addresses give the reserve .2 and the + // hand-out run .3 alone (the proxy at .4 sits above the run). let tiny = SwitchSubnet::new(Ipv4Addr::new(10, 0, 0, 0), 29).unwrap(); assert_eq!( self_allocation_run(tiny),