diff --git a/crates/minimald/src/net/mod.rs b/crates/minimald/src/net/mod.rs index 49e0b0231e..2bb19c7bf6 100644 --- a/crates/minimald/src/net/mod.rs +++ b/crates/minimald/src/net/mod.rs @@ -894,19 +894,29 @@ mod tests { #[test] fn allocator_exhausts_a_tiny_subnet() { - // /29 => 8 addresses; network, gateway, daemon, host-alias, broadcast - // reserved, leaving exactly three allocatable hosts (.2 through .4). - // The run splits at its midpoint, so this daemon's reserve is .2 - // alone and the host's hand-out run is .3 and .4 — exhaustion is - // explicit on each side, with no wrap and no reuse. + // /29 => 8 addresses; network, gateway, proxy, daemon, host-alias, + // broadcast reserved, leaving exactly two allocatable hosts (.2 and + // .3). The run splits at its midpoint, so this daemon's reserve is .2 + // alone and the host's hand-out run is .3 alone — exhaustion is + // explicit on each side, with no wrap and no reuse. The proxy's + // address (.4, `broadcast - 3`) is infrastructure above the run, so + // it is refused as a handed address, exactly as one past the run + // would be. let subnet = SwitchSubnet::new(Ipv4Addr::new(10, 0, 0, 0), 29).unwrap(); + assert_eq!( + subnet.box_egress_proxy_address(), + Ipv4Addr::new(10, 0, 0, 4) + ); let mut a = IpAllocator::new(subnet); assert_eq!(a.allocate().unwrap().ip, Ipv4Addr::new(10, 0, 0, 2)); assert!(matches!(a.allocate(), Err(NetError::SubnetExhausted(_)))); // The reserve bounds only the daemon's draws: the run's remainder is // still there for the host's hand-outs, none of them ever drawn. a.hand(Ipv4Addr::new(10, 0, 0, 3)).unwrap(); - a.hand(Ipv4Addr::new(10, 0, 0, 4)).unwrap(); + assert!(matches!( + a.hand(Ipv4Addr::new(10, 0, 0, 4)), + Err(NetError::HandedAddressOutsidePlan { .. }) + )); assert!(matches!( a.hand(Ipv4Addr::new(10, 0, 0, 3)), Err(NetError::HandedAddressCollision { .. }) @@ -940,8 +950,8 @@ mod tests { "both sub-runs are inside the plan's PTask run, and neither is empty" ); // A tiny subnet splits in half, so both sides keep at least one - // address: a /29's three PTask addresses give the reserve .2 and the - // hand-out run .3 and .4. + // address: a /29's two PTask addresses give the reserve .2 and the + // hand-out run .3 alone (the proxy at .4 sits above the run). let tiny = SwitchSubnet::new(Ipv4Addr::new(10, 0, 0, 0), 29).unwrap(); assert_eq!( self_allocation_run(tiny), @@ -1104,6 +1114,9 @@ mod tests { assert!(cfg.contains("subnet: \"100.64.0.0/16\"")); assert!(cfg.contains("gatewayIP: \"100.64.0.1\"")); assert!(cfg.contains(&format!("\"{}\": \"{}\"", lease.ip, lease.mac))); + // NET-132: the box egress proxy address is infrastructure and must not be + // handled by gvproxy's NAT or virtual-IP machinery. + assert!(!cfg.contains("100.64.255.252")); // Host alias is NAT'd to loopback and never allocated. assert!(cfg.contains("\"100.64.255.254\": \"127.0.0.1\"")); // NET-003: the static zone entry the bundle's switch-configuration copy diff --git a/crates/minvmd/src/cmd/run.rs b/crates/minvmd/src/cmd/run.rs index acc9a3ce6a..dd982f0e25 100644 --- a/crates/minvmd/src/cmd/run.rs +++ b/crates/minvmd/src/cmd/run.rs @@ -507,7 +507,13 @@ fn run_foreground() -> Result<()> { &boxes, ) { Ok(gvproxy) => { - tracing::info!(pid = gvproxy.pid(), "host gvproxy switch up"); + let subnet = boxes.subnet(); + tracing::info!( + pid = gvproxy.pid(), + proxy_ip = %subnet.box_egress_proxy_address(), + proxy_mac = %subnet.bep_mac(), + "host gvproxy switch up; box egress proxy peer started", + ); Some(gvproxy) } // An own-IP VM cannot work without the switch: fail loudly. A diff --git a/crates/minvmd/src/net.rs b/crates/minvmd/src/net.rs index 5e1fcaf8aa..4096abf736 100644 --- a/crates/minvmd/src/net.rs +++ b/crates/minvmd/src/net.rs @@ -88,7 +88,7 @@ pub use shuttle::{VSOCK_GVPROXY_SHUTTLE_PORT, resolve_gate_sock, resolve_switch_ mod baseline; pub use baseline::{NodeBaselinePhase, NodePlaneBaseline}; pub(crate) mod bep_host; -pub use bep_host::{BepDevice, BepDeviceEnds, BepHost}; +pub use bep_host::{BepDevice, BepDeviceEnds, BepHost, BepPeer}; /// Default time to wait for gvproxy to exit on SIGTERM before escalating to /// SIGKILL. @@ -718,6 +718,7 @@ impl HostGvproxy { // its rows were compiled against — so a lease the gate checks a // frame's source by is one the switch actually routes. .with_subnet(registry.subnet()); + let subnet = registry.subnet(); let table = registry.table(); // The node-plane baseline set: the helper's built-in enumeration of // the categories the in-VM daemon's own traffic may reach (NET-130), @@ -726,7 +727,7 @@ impl HostGvproxy { // by it, beside the boxes' rows; announced — the shipped posture — // the allow-all interim node row still does, and the gate's // start-up line names the posture deciding. - let baseline = NodePlaneBaseline::built_in(registry.subnet()); + let baseline = NodePlaneBaseline::built_in(subnet); let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>(); let (ready_tx, ready_rx) = std::sync::mpsc::channel::>(); @@ -789,6 +790,26 @@ impl HostGvproxy { drop(ready_tx.send(Err(e))); return; } + // Start the Box Egress Proxy host stack peer after the switch + // socket is known to accept connections: the peer owns the + // proxy's infrastructure address on the subnet, speaks the + // same length-framed Ethernet the switch socket carries, and + // stops when this runtime drops at switch teardown. + let _bep_peer = match BepPeer::spawn(&sock, subnet).await { + Ok(peer) => peer, + Err(e) => { + tracing::error!(error = %e, "failed to start box egress proxy peer"); + switch.stop().await; + drop(ready_tx.send(Err(e))); + return; + } + }; + tracing::info!( + proxy_ip = %subnet.box_egress_proxy_address(), + proxy_mac = %subnet.bep_mac(), + "box egress proxy peer started", + ); + if ready_tx.send(Ok(pid)).is_err() { // Caller went away before learning the PID; tear down. switch.stop().await; @@ -799,6 +820,8 @@ impl HostGvproxy { tokio::select! { _ = stop_rx => { switch.stop().await; + drop(_bep_peer); + drop(_gate); } status = exit.recv() => { tracing::error!( @@ -806,7 +829,9 @@ impl HostGvproxy { code = status.and_then(|s| s.code()), "host gvproxy switch exited unexpectedly", ); - // gvproxy is already gone; drop the handle (no signal). + // gvproxy is already gone; drop the handles (no signal). + drop(_bep_peer); + drop(_gate); drop(switch); } } @@ -1055,6 +1080,112 @@ mod tests { assert!(yaml.contains("{}")); } + /// NET-132: one source for the run's end. The last PTask address and the + /// proxy's own address both come from the `switch` crate's plan, and every + /// consumer here reads them there — the lease run the registry hands box + /// addresses out of, the gate's answer to which sources the plan could + /// ever have handed out, and the static-lease pool the rendered switch + /// config serves. The three are pinned against each other here, so moving + /// the run's end or the proxy address fails this test instead of leaving a + /// consumer behind. `minimald`'s mirror of the same run — its + /// self-allocation reserve, the run's lower half — is pinned by its own + /// test against the same `switch` crate, and the switch crate's own test + /// pins the proxy address out of gvproxy's NAT and virtual-IP tables. + #[test] + fn lease_run_gate_run_and_rendered_pool_agree() { + let subnet = SwitchSubnet::default(); + let first = Ipv4Addr::from(subnet.first_ptask()); + let last = Ipv4Addr::from(subnet.last_ptask()); + + // The reserved run above the lease pool, in order: the proxy address + // the host-side peer owns (NET-132), the daemon's own tap, and the + // host alias gvproxy NATs — all above the run's end, all from the one + // plan. + assert!( + u32::from(last) < u32::from(subnet.box_egress_proxy_address()), + "the proxy address sits above the lease pool" + ); + assert!(u32::from(subnet.box_egress_proxy_address()) < u32::from(subnet.daemon_ip())); + assert!(u32::from(subnet.daemon_ip()) < u32::from(subnet.host_alias())); + + // The gate's run: the one set of sources the plan could ever hand a + // box — both ends of the lease run inside it, every reserved address + // and everything off the subnet outside it. + let registry = crate::box_registry::BoxRegistry::new(subnet); + let table = registry.table(); + assert!( + table.is_allocatable(first.octets()), + "the run's first address is a source the gate's run admits" + ); + assert!( + table.is_allocatable(last.octets()), + "the run's last address is a source the gate's run admits" + ); + for outside in [ + subnet.network(), + subnet.gateway(), + subnet.box_egress_proxy_address(), + subnet.daemon_ip(), + subnet.host_alias(), + subnet.broadcast(), + Ipv4Addr::new(203, 0, 113, 7), + ] { + assert!( + !table.is_allocatable(outside.octets()), + "the gate's run never admits the reserved {outside}" + ); + } + + // The lease book's range: the registry hands a registered box an + // address from the same run — inside its ends, never a reserved one. + for name in ["web", "db"] { + let row = registry + .register_client_box(crate::box_registry::ClientBoxSpec { + name: name.to_string(), + ingress_ports: Vec::new(), + egress: None, + }) + .expect("the default plan has hand-out addresses"); + let handed = row.switch_addr(); + assert!( + first <= handed && handed <= last, + "the lease book hands {handed} from inside the plan's run" + ); + } + + // The rendered pool: the config's static-lease table carries exactly + // the addresses it is handed — the run's own ends here — on the same + // subnet, and none of the reserved run above it. The lease map is the + // tail of the rendered YAML (`dhcpStaticLeases:` is its last key), so + // the alias's own NAT and DNS lines do not read as leases. + let pool = [ + (first, MacAddr::for_switch_ip(first)), + (last, MacAddr::for_switch_ip(last)), + ]; + let yaml = render_gvproxy_config(subnet, &pool); + assert!(yaml.contains(&format!("subnet: \"{subnet}\""))); + let lease_map = yaml + .split("dhcpStaticLeases:\n") + .nth(1) + .expect("the rendered config carries a static-lease map"); + for (ip, mac) in pool { + assert!(lease_map.contains(&format!(" \"{ip}\": \"{mac}\""))); + } + for reserved in [subnet.daemon_ip(), subnet.host_alias()] { + assert!( + !lease_map.contains(&format!("\"{reserved}\"")), + "{reserved} is reserved, never a lease in the rendered pool" + ); + } + // The proxy address is nowhere in the config at all: not a lease, not + // a virtual IP, not translated — the peer owns it, and the switch + // crate's own test pins the same from the other side. + assert!( + !yaml.contains(&format!("\"{}\"", subnet.box_egress_proxy_address())), + "the proxy address is the peer's, never gvproxy's" + ); + } + #[test] fn write_config_creates_parent_and_file() { let dir = tempfile::TempDir::new().expect("tempdir"); diff --git a/crates/minvmd/src/net/bep_host.rs b/crates/minvmd/src/net/bep_host.rs index 4c2cd2f19e..9457f452b0 100644 --- a/crates/minvmd/src/net/bep_host.rs +++ b/crates/minvmd/src/net/bep_host.rs @@ -8,8 +8,8 @@ //! see [`egress_gate`]'s relay). The host kernel knows nothing of the switch //! subnet, so the leg terminates those frames itself: one smoltcp //! [`Interface`] over a channel-backed [`Device`], answering ARP for the leg's -//! address (the proof the first test pins) and carrying the proxy's sockets -//! once the proxy's own work lands. +//! address, resetting TCP to unlistened proxy ports, and sending ICMP +//! port-unreachable for UDP at the address. //! //! [`BepDevice`] is that [`Device`]: a pair of unbounded tokio channels, one //! raw Ethernet frame per message on each. [`BepDevice::pair`] hands the @@ -19,16 +19,90 @@ //! [`BepHost`] builds the interface over it: the leg's address on a //! [`SwitchSubnet`], its MAC derived the switch's way //! ([`MacAddr::for_switch_ip`]), stepped with [`BepHost::poll`]. +//! +//! The peer speaks only when spoken to. It originates no unsolicited frame — +//! no gratuitous ARP, no ARP probe, nothing on an idle lane — and every frame +//! it does send is an answer to the box that addressed it: an ARP reply for +//! its address, a TCP reset for a segment no socket holds, an ICMP +//! port-unreachable for a UDP datagram at the address. The smoltcp feature set +//! this workspace builds with excludes `proto-ipv6`, so the interface has no +//! IPv6 neighbour discovery either: no neighbour solicitation, no router +//! solicitation, no multicast listener discovery can leave it. +//! +//! No socket is bound here. The interface polls an empty [`SocketSet`]: this +//! task terminates nothing above the network layer, and **T69 is the only task +//! that may bind a socket on the peer** — it carries the per-source caps and +//! the gate rule before it does. +//! +//! [`BepPeer`] dials the switch `-listen` socket once, upgrades the connection +//! with the HyperKit `/connect` request, and runs the stack in a dedicated +//! local task that is woken by inbound frames, by the outbound pump, and by a +//! periodic `poll_delay`. +use std::cell::Cell; +use std::collections::VecDeque; use std::fmt; +use std::io; use std::net::Ipv4Addr; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; use smoltcp::iface::{Config as InterfaceConfig, Interface, SocketSet}; -use smoltcp::phy::{Device, DeviceCapabilities, Medium, RxToken, TxToken}; +use smoltcp::phy::{Checksum, ChecksumCapabilities, Device, DeviceCapabilities, Medium}; use smoltcp::time::Instant; -use smoltcp::wire::{EthernetAddress, HardwareAddress, IpAddress, IpCidr, Ipv4Address}; +use smoltcp::wire::{ + ArpOperation, ArpPacket, ArpRepr, EthernetAddress, EthernetFrame, EthernetProtocol, + HardwareAddress, IPV4_HEADER_LEN, IpAddress, IpCidr, IpProtocol, Ipv4Address, Ipv4Packet, + Ipv4Repr, TcpControl, TcpPacket, TcpRepr, TcpSeqNumber, UdpPacket, +}; use switch::{DEFAULT_MTU, MacAddr, SwitchSubnet}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::UnixStream; +use tokio::sync::Notify; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; +use tokio::task::JoinSet; + +/// The HTTP upgrade request that turns a gvproxy control socket connection +/// into a raw Ethernet frame stream (the same head the guest shuttle uses). +const CONNECT_REQUEST: &[u8] = b"POST /connect HTTP/1.0\r\nHost: localhost\r\n\r\n"; + +/// Maximum interval between stack polls even when no traffic arrives, so smoltcp +/// TCP timers still advance. +const POLL_DELAY: Duration = Duration::from_millis(100); + +/// Minimum interval between the debug lines the peer logs for one kind of +/// answer (a reset, a port-unreachable), so a box hammering the proxy address +/// cannot fill the log. The answers the interval holds back are counted and +/// named by the next line that does fire. +const ANSWER_LOG_INTERVAL_MS: i64 = 1_000; + +/// The rate limit on one kind of answer's debug line, stepped by the poll's +/// own clock (the peer has no wall clock to read and needs none): one line per +/// [`ANSWER_LOG_INTERVAL_MS`], the rest counted into the next line that fires. +#[derive(Debug, Default)] +struct AnswerLog { + last: Cell>, + suppressed: Cell, +} + +impl AnswerLog { + /// Whether an answer at `now` may log, and how many were held back since + /// the last line that fired. + fn gate(&self, now: i64) -> Option { + if self + .last + .get() + .is_none_or(|last| now >= last + ANSWER_LOG_INTERVAL_MS) + { + self.last.set(Some(now)); + Some(self.suppressed.replace(0)) + } else { + self.suppressed.set(self.suppressed.get().saturating_add(1)); + None + } + } +} /// A channel-backed smoltcp [`Device`] over the switch's Ethernet lane. /// @@ -40,6 +114,8 @@ use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; pub struct BepDevice { rx: UnboundedReceiver>, tx: UnboundedSender>, + /// Frames the host loop has admitted and wants the interface to process. + pending: VecDeque>, } /// The peer ends of a [`BepDevice`]'s channels: what feeds the stack and what @@ -66,6 +142,7 @@ impl BepDevice { Self { rx: inbound_rx, tx: outbound_tx, + pending: VecDeque::new(), }, BepDeviceEnds { inbound: inbound_tx, @@ -73,6 +150,13 @@ impl BepDevice { }, ) } + + fn send_out(&self, frame: Vec) { + // A failed send means the leg's peer end is gone — the lane is closed + // and the frame has nowhere to go. Drop it rather than fail the poll; + // the lane's teardown is the leg's wiring's to log. + let _ = self.tx.send(frame); + } } impl Device for BepDevice { @@ -86,8 +170,7 @@ impl Device for BepDevice { Self: 'a; fn receive(&mut self, _timestamp: Instant) -> Option<(Self::RxToken<'_>, Self::TxToken<'_>)> { - // No buffered frame: nothing for the interface to process this poll. - let frame = self.rx.try_recv().ok()?; + let frame = self.pending.pop_front()?; Some(( BepRxToken { frame }, BepTxToken { @@ -111,6 +194,12 @@ impl Device for BepDevice { let mut caps = DeviceCapabilities::default(); caps.medium = Medium::Ethernet; caps.max_transmission_unit = usize::from(DEFAULT_MTU); + let mut checksum_caps = ChecksumCapabilities::default(); + checksum_caps.ipv4 = Checksum::Both; + checksum_caps.tcp = Checksum::Both; + checksum_caps.udp = Checksum::Both; + checksum_caps.icmpv4 = Checksum::Both; + caps.checksum = checksum_caps; caps } } @@ -120,7 +209,7 @@ pub struct BepRxToken { frame: Vec, } -impl RxToken for BepRxToken { +impl smoltcp::phy::RxToken for BepRxToken { fn consume(self, f: F) -> R where F: FnOnce(&[u8]) -> R, @@ -135,17 +224,14 @@ pub struct BepTxToken { tx: UnboundedSender>, } -impl TxToken for BepTxToken { +impl smoltcp::phy::TxToken for BepTxToken { fn consume(self, len: usize, f: F) -> R where F: FnOnce(&mut [u8]) -> R, { let mut frame = vec![0u8; len]; let result = f(&mut frame); - // A failed send means the leg's peer end is gone — the lane is closed - // and the frame has nowhere to go. Drop it rather than fail the poll; - // the lane's teardown is the leg's wiring's to log. - drop(self.tx.send(frame)); + let _ = self.tx.send(frame); result } } @@ -153,8 +239,10 @@ impl TxToken for BepTxToken { /// The host leg's stack: one smoltcp [`Interface`] over a channel-backed /// [`BepDevice`], holding the leg's address on the switch's plan. /// -/// Built by [`BepHost::new`], stepped by [`BepHost::poll`]. The sockets the -/// proxy terminates come later, when the proxy's own work adds them; the set +/// Built by [`BepHost::new`], stepped by [`BepHost::poll`]. The socket set is +/// empty and stays empty in this task: the peer terminates nothing above the +/// network layer, and T69 is the only task that may bind a socket on the peer, +/// carrying the per-source caps and the gate rule before it does. The set /// exists here so the interface has something to poll against from the first /// frame on. pub struct BepHost { @@ -163,6 +251,12 @@ pub struct BepHost { sockets: SocketSet<'static>, ip: Ipv4Address, mac: EthernetAddress, + /// The rate limits on the two kinds of answer the peer logs: a reset for + /// a TCP segment no socket holds, an ICMP port-unreachable for a UDP + /// datagram. The frames themselves are never rate-limited — every one is + /// answered — only the debug line saying so is. + reset_log: AnswerLog, + unreachable_log: AnswerLog, } impl fmt::Debug for BepHost { @@ -194,12 +288,16 @@ impl BepHost { .push(IpCidr::new(IpAddress::Ipv4(ip), subnet.prefix())) .expect("the leg holds one address and the interface's address capacity is four"); }); + // any_ip stays off: the interface owns this address and nothing else. + debug_assert!(!iface.any_ip()); Self { device, iface, sockets: SocketSet::new(vec![]), ip, mac, + reset_log: AnswerLog::default(), + unreachable_log: AnswerLog::default(), } } @@ -219,14 +317,467 @@ impl BepHost { /// emit whatever the interface has to send. Both drain to quiescence, so /// one call is one full turn of the stack. pub fn poll(&mut self, now: Instant) { + while let Ok(frame) = self.device.rx.try_recv() { + self.handle_frame(&frame, now); + } self.iface.poll(now, &mut self.device, &mut self.sockets); } + + fn handle_frame(&mut self, frame: &[u8], now: Instant) { + let eth = match EthernetFrame::new_checked(frame) { + Ok(eth) => eth, + Err(_) => return, + }; + let dst = eth.dst_addr(); + if dst != self.mac && dst != EthernetAddress::BROADCAST { + return; + } + + match eth.ethertype() { + EthernetProtocol::Arp => { + if let Ok(arp) = ArpPacket::new_checked(eth.payload()) + && let Ok(repr) = ArpRepr::parse(&arp) + { + self.handle_arp(repr); + } + } + EthernetProtocol::Ipv4 => { + if let Ok(ip) = Ipv4Packet::new_checked(eth.payload()) + && let Ok(repr) = Ipv4Repr::parse(&ip, &ChecksumCapabilities::ignored()) + { + if repr.dst_addr != self.ip { + // any_ip is off: traffic not for the proxy address + // is dropped without a reply. + return; + } + let src_mac = eth.src_addr(); + match repr.next_header { + IpProtocol::Tcp => { + self.handle_tcp(src_mac, repr, ip.payload(), now); + } + IpProtocol::Udp => { + self.handle_udp(src_mac, repr, ip.payload(), now); + } + _ => {} + } + } + } + _ => {} + } + } + + fn handle_arp(&self, repr: ArpRepr) { + let (operation, target_protocol_addr, source_hardware_addr, source_protocol_addr) = + match repr { + ArpRepr::EthernetIpv4 { + operation, + target_protocol_addr, + source_hardware_addr, + source_protocol_addr, + .. + } => ( + operation, + target_protocol_addr, + source_hardware_addr, + source_protocol_addr, + ), + _ => return, + }; + if operation != ArpOperation::Request || target_protocol_addr != self.ip { + return; + } + let reply = ArpRepr::EthernetIpv4 { + operation: ArpOperation::Reply, + source_hardware_addr: self.mac, + source_protocol_addr: self.ip, + target_hardware_addr: source_hardware_addr, + target_protocol_addr: source_protocol_addr, + }; + self.send_arp(reply, source_hardware_addr); + } + + fn handle_tcp( + &self, + src_mac: EthernetAddress, + ip_repr: Ipv4Repr, + tcp_payload: &[u8], + now: Instant, + ) { + let tcp = match TcpPacket::new_checked(tcp_payload) { + Ok(tcp) => tcp, + Err(_) => return, + }; + let seq = tcp.seq_number(); + let ack = tcp.ack_number(); + let (rst_seq, rst_ack, rst_ack_set) = if tcp.ack() { + (ack, TcpSeqNumber(0), false) + } else { + let len = tcp.payload().len() + if tcp.syn() || tcp.fin() { 1 } else { 0 }; + ( + TcpSeqNumber(0), + TcpSeqNumber(seq.0.wrapping_add(len as i32)), + true, + ) + }; + let rst = TcpRepr { + src_port: tcp.dst_port(), + dst_port: tcp.src_port(), + control: TcpControl::Rst, + seq_number: rst_seq, + ack_number: if rst_ack_set { Some(rst_ack) } else { None }, + window_len: 0, + window_scale: None, + max_seg_size: None, + sack_permitted: false, + sack_ranges: [None; 3], + timestamp: None, + payload: &[], + }; + self.send_ip_packet( + src_mac, + ip_repr.src_addr, + IpProtocol::Tcp, + rst.header_len(), + |tcp_buf| { + let mut tcp_packet = TcpPacket::new_unchecked(tcp_buf); + rst.emit( + &mut tcp_packet, + &IpAddress::Ipv4(self.ip), + &IpAddress::Ipv4(ip_repr.src_addr), + &self.device.capabilities().checksum, + ); + }, + ); + // Rate-limited: the reset itself is never held back, only this line is. + if let Some(suppressed) = self.reset_log.gate(now.total_millis()) { + tracing::debug!( + src = %Ipv4Addr::from(u32::from_be_bytes(ip_repr.src_addr.octets())), + dst_port = tcp.dst_port(), + suppressed, + "sent TCP reset for unlistened proxy port" + ); + } + } + + fn handle_udp( + &self, + src_mac: EthernetAddress, + ip_repr: Ipv4Repr, + udp_payload: &[u8], + now: Instant, + ) { + let udp = match UdpPacket::new_checked(udp_payload) { + Ok(udp) => udp, + Err(_) => return, + }; + let src_port = udp.src_port(); + let dst_port = udp.dst_port(); + let original_ip_header_len = 20_usize; + let original_total_len = (original_ip_header_len + udp.len() as usize) as u16; + let original_src = ip_repr.src_addr; + let original_dst = self.ip; + + // ICMP port-unreachable payload: original IPv4 header + first 8 bytes + // of the original datagram (the UDP header). + let returned_len = original_ip_header_len + 8; + self.send_ip_packet( + src_mac, + original_src, + IpProtocol::Icmp, + 8 + returned_len, + |icmp_buf| { + // Type 3, code 3, checksum, unused = 4 zero bytes. + icmp_buf[0] = 3; // Destination Unreachable + icmp_buf[1] = 3; // Port Unreachable + icmp_buf[2] = 0; + icmp_buf[3] = 0; + icmp_buf[4..8].fill(0); + + // Copy original IP header. + let mut orig_header = [0u8; 20]; + { + let mut orig_ip = Ipv4Packet::new_unchecked(&mut orig_header[..]); + orig_ip.set_version(4); + orig_ip.set_header_len(20); + orig_ip.set_dscp(0); + orig_ip.set_ecn(0); + orig_ip.set_total_len(original_total_len); + orig_ip.set_ident(0); + orig_ip.clear_flags(); + orig_ip.set_more_frags(false); + orig_ip.set_dont_frag(true); + orig_ip.set_frag_offset(0); + orig_ip.set_hop_limit(64); + orig_ip.set_next_header(IpProtocol::Udp); + orig_ip.set_src_addr(original_src); + orig_ip.set_dst_addr(original_dst); + orig_ip.fill_checksum(); + } + icmp_buf[8..28].copy_from_slice(&orig_header); + // Copy first 8 bytes of original UDP datagram (the UDP header). + let udp_header_len = core::cmp::min(8, udp_payload.len()); + icmp_buf[28..28 + udp_header_len].copy_from_slice(&udp_payload[..udp_header_len]); + + // Compute ICMP checksum over the ICMP message. + let checksum = smoltcp::wire::checksum::data(icmp_buf); + let checksum = !checksum; + icmp_buf[2..4].copy_from_slice(&checksum.to_be_bytes()); + }, + ); + // Rate-limited: the port-unreachable itself is never held back, only + // this line is. + if let Some(suppressed) = self.unreachable_log.gate(now.total_millis()) { + tracing::debug!( + src = %Ipv4Addr::from(u32::from_be_bytes(original_src.octets())), + src_port, + dst_port, + suppressed, + "sent ICMP port-unreachable for UDP to proxy address" + ); + } + } + + fn send_arp(&self, repr: ArpRepr, dst_mac: EthernetAddress) { + let len = EthernetFrame::<&[u8]>::buffer_len(repr.buffer_len()); + let mut buf = vec![0u8; len]; + let mut frame = EthernetFrame::new_unchecked(&mut buf); + frame.set_dst_addr(dst_mac); + frame.set_src_addr(self.mac); + frame.set_ethertype(EthernetProtocol::Arp); + let mut packet = ArpPacket::new_unchecked(frame.payload_mut()); + repr.emit(&mut packet); + self.device.send_out(buf); + } + + fn send_ip_packet( + &self, + dst_mac: EthernetAddress, + dst_ip: Ipv4Address, + protocol: IpProtocol, + payload_len: usize, + mut build_payload: F, + ) where + F: FnMut(&mut [u8]), + { + // Largest buffer we'll need for the payloads this peer emits. + let mut buf = vec![0u8; 14 + IPV4_HEADER_LEN + payload_len]; + let mut frame = EthernetFrame::new_unchecked(&mut buf); + frame.set_dst_addr(dst_mac); + frame.set_src_addr(self.mac); + frame.set_ethertype(EthernetProtocol::Ipv4); + + let ip_repr = Ipv4Repr { + src_addr: self.ip, + dst_addr: dst_ip, + next_header: protocol, + payload_len, + hop_limit: 64, + }; + { + let mut ip_packet = Ipv4Packet::new_unchecked(frame.payload_mut()); + ip_repr.emit(&mut ip_packet, &self.device.capabilities().checksum); + } + build_payload(&mut frame.payload_mut()[IPV4_HEADER_LEN..]); + self.device.send_out(buf); + } +} + +/// A running Box Egress Proxy host peer. +/// +/// It owns the upstream switch socket connection (the `POST /connect` +/// upgrade), the channel-backed [`BepDevice`] that talks to the smoltcp +/// [`BepHost`], the two flow pumps that move framed Ethernet between the socket +/// and the device channels, and the dedicated local task that polls the stack. +/// Dropping the handle aborts those tasks and closes the socket. +#[derive(Debug)] +#[must_use = "dropping BepPeer stops the host-side proxy stack"] +pub struct BepPeer { + /// The stack poll task. It runs on a blocking thread, which `abort` cannot + /// stop, so the loop watches `stop` and returns when the sender drops. + poll_task: tokio::task::JoinHandle<()>, + /// Dropped by `Drop`: the poll loop exits when this closes. Without it the + /// runtime that spawned the peer waits forever for the blocking task at + /// shutdown, which is how the switch runtime hung at stop. + stop: Option>, + /// Pump tasks moving frames between socket and device channels. + pumps: JoinSet<()>, +} + +impl Drop for BepPeer { + fn drop(&mut self) { + self.pumps.abort_all(); + drop(self.stop.take()); + self.poll_task.abort(); + } +} + +impl BepPeer { + /// Start a peer for `subnet.box_egress_proxy_address()` on the switch + /// socket at `switch_sock`. + /// + /// The peer dials the switch, upgrades the connection with the HyperKit + /// `/connect` request, and then runs three cooperating tasks. It announces + /// nothing: the switch steers frames to the peer's derived MAC + /// ([`MacAddr::for_switch_ip`]) without a round trip, so the peer waits to + /// be asked — see the module docs for the frames it answers with. + /// + /// - a socket reader that reads length-framed Ethernet frames from the + /// switch and feeds them into the stack's inbound channel; + /// - a socket writer that reads from the stack's outbound channel and + /// writes length-framed Ethernet frames onto the switch; + /// - a dedicated local task that owns the smoltcp [`BepHost`] and polls it + /// whenever inbound frames arrive, whenever the outbound pump has + /// drained, and at least every [`POLL_DELAY`]. + /// + /// Must be called inside a tokio runtime. + /// + /// # Errors + /// + /// Returns an I/O error if the switch socket cannot be connected to. + pub async fn spawn(switch_sock: &Path, subnet: SwitchSubnet) -> io::Result { + let proxy_ip = subnet.box_egress_proxy_address(); + let proxy_mac = MacAddr::for_switch_ip(proxy_ip); + + let mut stream = UnixStream::connect(switch_sock).await?; + stream.write_all(CONNECT_REQUEST).await?; + stream.flush().await?; + + tracing::info!( + switch_socket = %switch_sock.display(), + proxy_ip = %proxy_ip, + proxy_mac = %proxy_mac, + "box egress proxy peer attached" + ); + + // The socket is split so the reader and writer can run independently; + // the stack poll task cannot own the `UnixStream` because the smoltcp + // `Interface` is not `Send`. + let (mut read_half, write_half) = stream.into_split(); + + // Channel-backed device: raw Ethernet frames in both directions. The + // reader/reader tasks are `Send`; the `BepHost` built around the device + // lives in the local poll task. + let (device, ends) = BepDevice::pair(); + let inbound_tx = ends.inbound; + let mut outbound_rx = ends.outbound; + + // Shared wake source: the reader calls `notify_one` after every frame + // it enqueues, and the writer calls `notify_one` whenever it drains a + // frame so the stack can make progress while the outbound channel has + // capacity. Because the channels are unbounded, the writer waking the + // poll task is the back-pressure substitute the spec asks for. + let notify = Arc::new(Notify::new()); + + let mut pumps = JoinSet::new(); + + // Pump 1: switch socket -> stack. Reads a 2-byte little-endian length, + // then that many frame bytes, and feeds the device. + let reader_notify = Arc::clone(¬ify); + pumps.spawn(async move { + let mut len_buf = [0u8; 2]; + loop { + match read_half.read(&mut len_buf).await { + Ok(0) => return, + Ok(2) => {} + Ok(_) => { + // A short length read means the socket is closing. + return; + } + Err(_) => return, + } + let len = u16::from_le_bytes(len_buf) as usize; + if len == 0 || len > usize::from(DEFAULT_MTU) + 14 + 4 { + // Malformed length; drop the connection. + return; + } + let mut frame = vec![0u8; len]; + if read_half.read_exact(&mut frame).await.is_err() { + return; + } + if inbound_tx.send(frame).is_err() { + // The poll task is gone. + return; + } + reader_notify.notify_one(); + } + }); + + // Pump 2: stack -> switch socket. Reads outbound frames, prefixes them + // with a 2-byte little-endian length, and writes them. + let writer_notify = Arc::clone(¬ify); + pumps.spawn(async move { + let mut stream = write_half; + while let Some(frame) = outbound_rx.recv().await { + let len = frame.len() as u16; + if len == 0 { + continue; + } + let mut buf = Vec::with_capacity(2 + frame.len()); + buf.extend_from_slice(&len.to_le_bytes()); + buf.extend_from_slice(&frame); + if stream.write_all(&buf).await.is_err() { + return; + } + if stream.flush().await.is_err() { + return; + } + writer_notify.notify_one(); + } + }); + + // Poll task: the only task that owns the smoltcp `Interface`. It runs + // on the current-thread runtime so `BepHost` (which is not `Send`) can + // be held across `.await` points. We block_in_place in an async task + // that is itself `Send`, and pass frames through channels. + let (stop_tx, mut stop_rx) = tokio::sync::oneshot::channel::<()>(); + let poll_task = tokio::task::spawn_blocking(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime for BepHost"); + rt.block_on(async move { + let mut host = BepHost::new(device, subnet, proxy_ip); + host.poll(Instant::from_millis(0)); + + let start = tokio::time::Instant::now(); + let mut interval = tokio::time::interval(POLL_DELAY); + loop { + tokio::select! { + _ = notify.notified() => {} + _ = interval.tick() => {} + // The peer was dropped: leave the loop so the runtime + // that spawned this blocking task can shut down. + _ = &mut stop_rx => break, + } + loop { + match host.device.rx.try_recv() { + Ok(frame) => host.device.pending.push_back(frame), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) => break, + // The reader pump is gone and no frame will ever + // arrive: the interface has nothing left to serve. + Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => return, + } + } + let elapsed = start.elapsed().as_millis() as i64; + host.poll(Instant::from_millis(elapsed)); + } + }); + }); + + Ok(Self { + poll_task, + stop: Some(stop_tx), + pumps, + }) + } } #[cfg(test)] mod tests { use super::*; - use smoltcp::wire::{ArpOperation, ArpPacket, ArpRepr, EthernetFrame, EthernetProtocol}; + use smoltcp::wire::{ + ArpOperation, ArpPacket, EthernetFrame, EthernetProtocol, Icmpv4Packet, UdpRepr, + }; /// One ARP request frame, as a box on the plan would send it: broadcast, /// from the asker's switch-derived MAC, asking who has `target_ip`. @@ -252,10 +803,135 @@ mod tests { buf } - /// The skeleton's first proof: build an interface over an in-process - /// device and read its ARP reply. A box on the switch's plan asks who has - /// the leg's address; the stack answers for the address it was given, - /// from the switch-derived MAC, pointed back at the asker. + /// Build a minimal IPv4 frame carrying the given ethertype payload. + fn ip_frame( + src_mac: EthernetAddress, + dst_mac: EthernetAddress, + src_ip: Ipv4Address, + dst_ip: Ipv4Address, + protocol: IpProtocol, + payload: &[u8], + ) -> Vec { + let ip_repr = Ipv4Repr { + src_addr: src_ip, + dst_addr: dst_ip, + next_header: protocol, + payload_len: payload.len(), + hop_limit: 64, + }; + let mut buf = + vec![0u8; EthernetFrame::<&[u8]>::buffer_len(ip_repr.buffer_len() + payload.len())]; + let mut frame = EthernetFrame::new_unchecked(&mut buf); + frame.set_dst_addr(dst_mac); + frame.set_src_addr(src_mac); + frame.set_ethertype(EthernetProtocol::Ipv4); + let mut ip_packet = Ipv4Packet::new_unchecked(frame.payload_mut()); + ip_repr.emit(&mut ip_packet, &ChecksumCapabilities::ignored()); + // Fill IP header checksum explicitly since ignored caps skipped it. + ip_packet.fill_checksum(); + ip_packet.payload_mut().copy_from_slice(payload); + buf + } + + /// Build a TCP SYN from `src:src_port` to `dst:dst_port`. + fn tcp_syn(src_ip: Ipv4Addr, dst_ip: Ipv4Addr, src_port: u16, dst_port: u16) -> Vec { + let src_mac = EthernetAddress(MacAddr::for_switch_ip(src_ip).0); + let dst_mac = EthernetAddress(MacAddr::for_switch_ip(dst_ip).0); + let repr = TcpRepr { + src_port, + dst_port, + control: TcpControl::Syn, + seq_number: TcpSeqNumber(1_000_000), + ack_number: None, + window_len: 1024, + window_scale: None, + max_seg_size: None, + sack_permitted: false, + sack_ranges: [None; 3], + timestamp: None, + payload: &[], + }; + let mut tcp_buf = vec![0u8; repr.buffer_len()]; + let mut tcp = TcpPacket::new_unchecked(&mut tcp_buf); + repr.emit( + &mut tcp, + &IpAddress::Ipv4(src_ip), + &IpAddress::Ipv4(dst_ip), + &ChecksumCapabilities::ignored(), + ); + ip_frame(src_mac, dst_mac, src_ip, dst_ip, IpProtocol::Tcp, &tcp_buf) + } + + /// Build a UDP datagram from `src:src_port` to `dst:dst_port`. + fn udp_datagram( + src_ip: Ipv4Addr, + dst_ip: Ipv4Addr, + src_port: u16, + dst_port: u16, + payload: &[u8], + ) -> Vec { + let src_mac = EthernetAddress(MacAddr::for_switch_ip(src_ip).0); + let dst_mac = EthernetAddress(MacAddr::for_switch_ip(dst_ip).0); + let repr = UdpRepr { src_port, dst_port }; + let mut udp_buf = vec![0u8; 8 + payload.len()]; + let mut udp = UdpPacket::new_unchecked(&mut udp_buf); + repr.emit( + &mut udp, + &IpAddress::Ipv4(src_ip), + &IpAddress::Ipv4(dst_ip), + payload.len(), + |buf| buf.copy_from_slice(payload), + &ChecksumCapabilities::ignored(), + ); + ip_frame(src_mac, dst_mac, src_ip, dst_ip, IpProtocol::Udp, &udp_buf) + } + + fn expect_one_arp_reply( + ends: &mut BepDeviceEnds, + host_mac: EthernetAddress, + host_ip: Ipv4Addr, + peer_mac: EthernetAddress, + peer_ip: Ipv4Addr, + ) { + let reply = ends.outbound.try_recv().expect("expected an ARP reply"); + let frame = EthernetFrame::new_checked(&reply).expect("reply is ethernet"); + assert_eq!(frame.dst_addr(), peer_mac); + assert_eq!(frame.src_addr(), host_mac); + assert_eq!(frame.ethertype(), EthernetProtocol::Arp); + let arp = ArpPacket::new_checked(frame.payload()).unwrap(); + assert_eq!( + ArpRepr::parse(&arp).unwrap(), + ArpRepr::EthernetIpv4 { + operation: ArpOperation::Reply, + source_hardware_addr: host_mac, + source_protocol_addr: host_ip, + target_hardware_addr: peer_mac, + target_protocol_addr: peer_ip, + } + ); + } + + #[test] + fn proxy_address_answers_arp_from_the_host_stack() { + let subnet = SwitchSubnet::default(); + let ip = subnet.box_egress_proxy_address(); + let (device, mut ends) = BepDevice::pair(); + let mut host = BepHost::new(device, subnet, ip); + + let peer_ip = Ipv4Addr::from(subnet.first_ptask()); + let peer_mac = EthernetAddress(MacAddr::for_switch_ip(peer_ip).0); + ends.inbound + .send(arp_request(peer_mac, peer_ip, ip)) + .expect("send the ARP request into the device"); + host.poll(Instant::from_millis(0)); + + expect_one_arp_reply(&mut ends, host.mac(), ip, peer_mac, peer_ip); + assert!(matches!( + ends.outbound.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + )); + } + #[test] fn host_stack_answers_arp_for_its_address() { let subnet = SwitchSubnet::default(); @@ -270,26 +946,263 @@ mod tests { .expect("send the ARP request into the device"); host.poll(Instant::from_millis(0)); - let reply = ends.outbound.try_recv().expect("the stack answered"); - let frame = EthernetFrame::new_checked(&reply).expect("the reply is an Ethernet frame"); - assert_eq!(frame.ethertype(), EthernetProtocol::Arp); + expect_one_arp_reply(&mut ends, host.mac(), ip, peer_mac, peer_ip); + } + + #[test] + fn tcp_to_unlistened_proxy_port_is_reset() { + let subnet = SwitchSubnet::default(); + let proxy_ip = subnet.box_egress_proxy_address(); + let (device, mut ends) = BepDevice::pair(); + let mut host = BepHost::new(device, subnet, proxy_ip); + + let peer_ip = Ipv4Addr::from(subnet.first_ptask()); + ends.inbound + .send(tcp_syn(peer_ip, proxy_ip, 1234, 443)) + .expect("send TCP SYN"); + host.poll(Instant::from_millis(0)); + + let reply = ends.outbound.try_recv().expect("expected a TCP reset"); + let frame = EthernetFrame::new_checked(&reply).unwrap(); assert_eq!(frame.src_addr(), host.mac()); - assert_eq!(frame.dst_addr(), peer_mac); - let arp = ArpPacket::new_checked(frame.payload()).expect("the reply carries ARP"); + let ip = Ipv4Packet::new_checked(frame.payload()).unwrap(); + assert_eq!(ip.src_addr(), proxy_ip); + assert_eq!(ip.dst_addr(), peer_ip); + let tcp = TcpPacket::new_checked(ip.payload()).unwrap(); + assert!(tcp.rst()); + assert_eq!(tcp.dst_port(), 1234); + assert_eq!(tcp.src_port(), 443); + } + + #[test] + fn udp_to_proxy_address_gets_port_unreachable() { + let subnet = SwitchSubnet::default(); + let proxy_ip = subnet.box_egress_proxy_address(); + let (device, mut ends) = BepDevice::pair(); + let mut host = BepHost::new(device, subnet, proxy_ip); + + let peer_ip = Ipv4Addr::from(subnet.first_ptask()); + ends.inbound + .send(udp_datagram(peer_ip, proxy_ip, 1234, 53, b"query")) + .expect("send UDP datagram"); + host.poll(Instant::from_millis(0)); + + let reply = ends.outbound.try_recv().expect("expected an ICMP reply"); + let frame = EthernetFrame::new_checked(&reply).unwrap(); + assert_eq!(frame.src_addr(), host.mac()); + let ip = Ipv4Packet::new_checked(frame.payload()).unwrap(); + assert_eq!(ip.src_addr(), proxy_ip); + assert_eq!(ip.dst_addr(), peer_ip); + let icmp = Icmpv4Packet::new_checked(ip.payload()).unwrap(); assert_eq!( - ArpRepr::parse(&arp).expect("the reply parses as ARP"), - ArpRepr::EthernetIpv4 { - operation: ArpOperation::Reply, - source_hardware_addr: host.mac(), - source_protocol_addr: ip, - target_hardware_addr: peer_mac, - target_protocol_addr: peer_ip, + icmp.msg_type(), + smoltcp::wire::Icmpv4Message::DstUnreachable + ); + assert_eq!(icmp.msg_code(), 3); // port unreachable + assert!(icmp.verify_checksum()); + // The returned data must carry the original IP header + UDP header. + let returned = icmp.data(); + assert!(returned.len() >= 28); + let expected_dst = u32::from(proxy_ip).to_be_bytes(); + assert_eq!(&returned[16..20], &expected_dst); + let expected_src_port = 1234u16.to_be_bytes(); + let expected_dst_port = 53u16.to_be_bytes(); + assert_eq!(&returned[20..22], &expected_src_port); + assert_eq!(&returned[22..24], &expected_dst_port); + } + + #[test] + fn stack_peer_originates_frames_only_to_the_box_that_addressed_it() { + let subnet = SwitchSubnet::default(); + let proxy_ip = subnet.box_egress_proxy_address(); + let (device, mut ends) = BepDevice::pair(); + let mut host = BepHost::new(device, subnet, proxy_ip); + + let peer_a_ip = Ipv4Addr::from(subnet.first_ptask()); + let peer_a_mac = EthernetAddress(MacAddr::for_switch_ip(peer_a_ip).0); + let peer_b_ip = Ipv4Addr::from(subnet.first_ptask() + 1); + let peer_b_mac = EthernetAddress(MacAddr::for_switch_ip(peer_b_ip).0); + + // A asks for proxy ARP. + ends.inbound + .send(arp_request(peer_a_mac, peer_a_ip, proxy_ip)) + .unwrap(); + host.poll(Instant::from_millis(0)); + let reply = ends.outbound.try_recv().unwrap(); + let frame = EthernetFrame::new_checked(&reply).unwrap(); + assert_eq!(frame.dst_addr(), peer_a_mac); + assert!(frame.ethertype() == EthernetProtocol::Arp); + + // B asks for proxy ARP. + ends.inbound + .send(arp_request(peer_b_mac, peer_b_ip, proxy_ip)) + .unwrap(); + host.poll(Instant::from_millis(0)); + let reply = ends.outbound.try_recv().unwrap(); + let frame = EthernetFrame::new_checked(&reply).unwrap(); + assert_eq!(frame.dst_addr(), peer_b_mac); + assert!(frame.ethertype() == EthernetProtocol::Arp); + + // A sends a TCP SYN; the reply goes back to A, not B. + ends.inbound + .send(tcp_syn(peer_a_ip, proxy_ip, 1000, 80)) + .unwrap(); + host.poll(Instant::from_millis(0)); + let reply = ends.outbound.try_recv().unwrap(); + let frame = EthernetFrame::new_checked(&reply).unwrap(); + assert_eq!(frame.dst_addr(), peer_a_mac); + assert!(frame.ethertype() == EthernetProtocol::Ipv4); + + // A UDP from B to the proxy address goes back to B. + ends.inbound + .send(udp_datagram(peer_b_ip, proxy_ip, 1000, 53, b"x")) + .unwrap(); + host.poll(Instant::from_millis(0)); + let reply = ends.outbound.try_recv().unwrap(); + let frame = EthernetFrame::new_checked(&reply).unwrap(); + assert_eq!(frame.dst_addr(), peer_b_mac); + assert!( + matches!( + ends.outbound.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + ), + "no extra broadcast frames should be emitted" + ); + } + + /// NET-132: the host-side stack peer starts when the switch socket is ready + /// and its drop aborts the socket connection and the poll task. We stand in + /// for gvproxy with a bound listener that accepts the connect upgrade; the + /// peer dials it, sends the upgrade head, and then runs until dropped. + #[tokio::test] + async fn stack_peer_stops_with_the_switch() { + let dir = tempfile::tempdir().expect("tempdir"); + let sock = dir.path().join("switch.sock"); + let listener = tokio::net::UnixListener::bind(&sock).expect("bind stand-in switch socket"); + + // Accept the peer's connection on a separate task so `spawn` completes. + let accept_fut = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("peer connected"); + // The peer must send the HyperKit upgrade head verbatim. + let mut head = vec![0u8; super::CONNECT_REQUEST.len()]; + stream + .read_exact(&mut head) + .await + .expect("read upgrade head"); + assert_eq!(&head, super::CONNECT_REQUEST); + stream + }); + + let subnet = SwitchSubnet::default(); + let peer = BepPeer::spawn(&sock, subnet) + .await + .expect("peer spawns against a listening switch socket"); + + // Wait for the acceptor to confirm the upgrade head was received; once it + // has, the connection is live from the peer's side. Dropping the peer + // must then close that connection, which the acceptor observes as EOF. + let mut accepted = accept_fut.await.expect("acceptor task completed"); + + drop(peer); + + // Give the peer tasks a moment to be aborted and the socket closed. + tokio::time::sleep(Duration::from_millis(100)).await; + + // A read on the still-open accepted socket must now return EOF because + // the peer's side has gone away. + let mut buf = [0u8; 1]; + let n = accepted + .read(&mut buf) + .await + .expect("read after peer drop should not error"); + assert_eq!(n, 0, "peer socket must close when the peer is dropped"); + } + + /// NET-132: the peer is silent until addressed. A stand-in switch socket + /// accepts the upgrade and holds the lane open; the peer, attached and + /// polled across several poll periods, must write nothing at all — no + /// gratuitous ARP, no ARP probe, no keepalive. Every frame it sends is an + /// answer, and nothing has asked it anything. + #[tokio::test] + async fn idle_peer_emits_no_frames() { + let dir = tempfile::tempdir().expect("tempdir"); + let sock = dir.path().join("switch.sock"); + let listener = tokio::net::UnixListener::bind(&sock).expect("bind stand-in switch socket"); + + // Accept the upgrade, then hold the lane open and collect every byte + // the peer writes after the upgrade head. + let accept_fut = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("peer connected"); + let mut head = vec![0u8; super::CONNECT_REQUEST.len()]; + stream + .read_exact(&mut head) + .await + .expect("read upgrade head"); + assert_eq!(&head, super::CONNECT_REQUEST); + let mut written = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + match stream.read(&mut buf).await { + Ok(0) => break, + Ok(n) => written.extend_from_slice(&buf[..n]), + Err(_) => break, + } } + written + }); + + let peer = BepPeer::spawn(&sock, SwitchSubnet::default()) + .await + .expect("peer spawns against a listening switch socket"); + + // Several poll periods must pass with nothing to answer: the poll loop + // wakes at least every POLL_DELAY, so this window covers every periodic + // wake the stack has, with slack for a loaded runner. + tokio::time::sleep(POLL_DELAY * 5 + Duration::from_millis(250)).await; + + // Dropping the peer closes the lane, which ends the collector above. + drop(peer); + let written = accept_fut.await.expect("acceptor task completed"); + assert!( + written.is_empty(), + "an idle peer emitted {} bytes without being addressed", + written.len() ); - // One request, one reply: the stack emitted nothing else. - assert!(matches!( - ends.outbound.try_recv(), - Err(tokio::sync::mpsc::error::TryRecvError::Empty) - )); + } + + /// NET-132: the peer terminates nothing above the network layer. The + /// interface polls an empty `SocketSet` — no TCP, UDP, or ICMP socket is + /// bound, so the reset and the port-unreachable the tests above pin are + /// the stack's own answers, not a listener's. T69 is the only task that + /// may bind a socket on the peer, and it carries the per-source caps and + /// the gate rule before it does. + #[test] + fn peer_binds_no_socket() { + let subnet = SwitchSubnet::default(); + let (device, _ends) = BepDevice::pair(); + let host = BepHost::new(device, subnet, subnet.box_egress_proxy_address()); + assert_eq!( + host.sockets.iter().count(), + 0, + "the peer binds no socket; T69 is the only task that may" + ); + } + + /// The answer the peer logs is rate-limited, so a box hammering the proxy + /// address cannot fill the log: the first line fires at once, a burst is + /// held back, and the next line that fires names how many were held. + #[test] + fn answer_log_rate_limits_a_burst() { + let log = AnswerLog::default(); + assert_eq!(log.gate(0), Some(0), "the first answer logs at once"); + for now in 1..1_000 { + assert_eq!(log.gate(now), None, "the burst is held back"); + } + assert_eq!( + log.gate(1_000), + Some(999), + "the interval's end fires and names the answers it held back" + ); + assert_eq!(log.gate(1_001), None, "a fresh burst is held back again"); } } diff --git a/crates/switch/src/lib.rs b/crates/switch/src/lib.rs index 19d004a5dd..e2eaafffa9 100644 --- a/crates/switch/src/lib.rs +++ b/crates/switch/src/lib.rs @@ -19,6 +19,11 @@ pub const DEFAULT_MTU: u16 = 1500; /// Stable, locally-administered MAC for the switch gateway. pub const GATEWAY_MAC: MacAddr = MacAddr([0x5a, 0x94, 0xef, 0xe4, 0x0c, 0xdd]); +/// Stable, locally-administered MAC for the Box Egress Proxy on the default +/// [`DEFAULT_SUBNET`]. It matches [`MacAddr::for_switch_ip`] applied to +/// [`SwitchSubnet::box_egress_proxy_address`] on that subnet. +pub const BEP_MAC: MacAddr = MacAddr([0x52, 0x54, 0x00, 0x40, 0xff, 0xfc]); + /// AF_VSOCK CID of the host as seen from inside a libkrun guest. Well-known: /// `VMADDR_CID_HOST == 2`. The per-PTask shuttle dials this CID to reach the /// host gvproxy switch (DM1/3/4). @@ -197,8 +202,8 @@ impl SwitchSubnet { /// # Errors /// /// Returns [`InvalidPrefix`] for a prefix outside `8..=29`. A prefix narrower - /// than /29 has no room for the reserved network/gateway/host-alias/broadcast - /// addresses plus a PTask address. A prefix wider than /8 lets the high octet + /// than /29 has no room for the reserved network/gateway/proxy/daemon/ + /// host-alias/broadcast addresses plus a PTask address. A prefix wider than /8 lets the high octet /// vary, which [`MacAddr::for_switch_ip`] does not fold into the derived MAC, /// so two addresses differing only in that octet would collide. pub fn new(base: Ipv4Addr, prefix: u8) -> Result { @@ -275,6 +280,22 @@ impl SwitchSubnet { Ipv4Addr::from(u32::from(self.broadcast()) - 1) } + /// The Box Egress Proxy address (`broadcast - 3`): a second infrastructure + /// address above the PTask lease pool that the host-side stack peer owns + /// directly. Reserved, never handed to a PTask, and kept out of gvproxy's + /// NAT/virtual-IP tables so the peer can answer ARP and reset stray TCP + /// packets itself. + #[must_use] + pub fn box_egress_proxy_address(self) -> Ipv4Addr { + Ipv4Addr::from(u32::from(self.broadcast()) - 3) + } + + /// The locally-administered MAC for [`Self::box_egress_proxy_address`]. + #[must_use] + pub fn bep_mac(self) -> MacAddr { + MacAddr::for_switch_ip(self.box_egress_proxy_address()) + } + /// The daemon address (`broadcast - 2`): the guest root netns' primary tap, /// which gives `minimald` itself egress through the host gvproxy (so it can /// fetch upstream packages). Reserved from the top so the PTask range still @@ -291,12 +312,12 @@ impl SwitchSubnet { u32::from(self.network()) + 2 } - /// The last address that may be allocated to a PTask (`broadcast - 3`), - /// leaving the daemon address at `broadcast - 2` and the host alias at - /// `broadcast - 1` reserved. + /// The last address that may be allocated to a PTask (`broadcast - 4`), + /// leaving the proxy at `broadcast - 3`, the daemon at `broadcast - 2` + /// and the host alias at `broadcast - 1` reserved. #[must_use] pub fn last_ptask(self) -> u32 { - u32::from(self.broadcast()) - 3 + u32::from(self.broadcast()) - 4 } } @@ -582,8 +603,28 @@ mod tests { let net = SwitchSubnet::default(); assert_eq!(net.to_string(), "100.64.0.0/16"); assert_eq!(net.gateway(), Ipv4Addr::new(100, 64, 0, 1)); - assert_eq!(net.host_alias(), Ipv4Addr::new(100, 64, 255, 254)); + assert_eq!( + net.box_egress_proxy_address(), + Ipv4Addr::new(100, 64, 255, 252) + ); + assert_eq!(net.bep_mac(), BEP_MAC); + assert_eq!(BEP_MAC.to_string(), "52:54:00:40:ff:fc"); assert_eq!(net.daemon_ip(), Ipv4Addr::new(100, 64, 255, 253)); + assert_eq!(net.host_alias(), Ipv4Addr::new(100, 64, 255, 254)); + } + + #[test] + fn proxy_address_is_neither_virtual_ip_nor_translated() { + let yaml = render_gvproxy_config(SwitchSubnet::default(), &[]); + let net = SwitchSubnet::default(); + // The proxy address must not appear in gvproxy's NAT map or in its + // gatewayVirtualIPs list — the host-side stack peer owns it directly. + assert!(!yaml.contains(&format!("\"{}\"", net.box_egress_proxy_address()))); + assert!(!yaml.contains("100.64.255.252")); + // The daemon address is also infrastructure and should not be listed. + assert!(!yaml.contains("100.64.255.253")); + // The host alias, by contrast, is both NAT'd and a virtual IP. + assert!(yaml.contains("100.64.255.254")); } #[test] diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index 77b2127906..49f17f732e 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -4707,6 +4707,185 @@ while True: # needs no listener, and says what it did not run. # CI's native lane has neither conflict and runs every assertion. # +# --------------------------------------------------------------------------- +# NET-132: the host-side stack peer owns the proxy's infrastructure address. +# These two cases are gated like the own-IP proof: +# * a switch must exist (`MINVMD_GVPROXY_BIN` on the lane), and +# * inside the VM the peer's MAC and ARP behaviour are observable only where +# /dev/net/tun and userns let an own-address box attach. +# Where the prereqs are absent the case prints what it cannot assert and +# returns 0, so a lane that cannot run them reports honestly rather than green. +# +# What is proved: +# * switch_steers_proxy_mac_frames_to_the_host_stack: a frame addressed to the +# proxy's MAC (52:54:00:40:ff:fc) leaves the guest, reaches the host-side +# peer, and is answered (or reset/ICMP'd) back to the originating box, never +# to any other box. +# * switch_answers_no_arp_for_the_proxy_address: the gvproxy switch itself does +# not answer ARP for 100.64.255.252; the host stack peer does. +proof_switch_steers_proxy_mac_frames_to_the_host_stack() { + echo "::group::switch steers proxy-MAC frames to the host stack peer (NET-132)" + if [ -z "${MINVMD_GVPROXY_BIN:-}" ]; then + echo "switch_steers_proxy_mac_frames_to_the_host_stack SKIPPED (no MINVMD_GVPROXY_BIN: this target has no switch)" + echo "::endgroup::" + return 0 + fi + if [ ! -c /dev/net/tun ] || [ ! -r /dev/net/tun ]; then + echo "switch_steers_proxy_mac_frames_to_the_host_stack SKIPPED (/dev/net/tun not available; own-address boxes cannot attach on this host)" + echo "::endgroup::" + return 0 + fi + if [ ! -e /proc/sys/kernel/unprivileged_userns_clone ] && [ "$(id -u)" -ne 0 ]; then + echo "switch_steers_proxy_mac_frames_to_the_host_stack SKIPPED (unprivileged user namespaces unavailable and not root; own-address boxes cannot attach)" + echo "::endgroup::" + return 0 + fi + + # When we can run, the test is identical in shape to the own-IP proof: boot an + # own-address box and, from inside it, send an Ethernet frame addressed to the + # proxy MAC. Because the box has no raw socket privileges (NET-083), we instead + # route to the proxy's address: the in-box stack emits the frame with the right + # destination MAC and source MAC derived from the box's lease. A TCP SYN to the + # proxy address on an unlistened port proves the frame reached the peer (it + # returns a TCP RST) and that the answer comes back to this box only (no other + # guest sees it). The RST is what the BepHost unit test already pins; here we + # prove the end-to-end path through the switch handles the proxy MAC correctly. + local bep_sid bep_ip proxy_ip proxy_mac + proxy_ip="100.64.255.252" + proxy_mac="52:54:00:40:ff:fc" + + BEP_SEED_DIR="$(hook_mktemp /tmp/mnlbep.XXXXXX)" + hook_seed_preamble > "$BEP_SEED_DIR/minimal.toml" + mkdir "$BEP_SEED_DIR/.git" + + bep_sid="$(cd "$BEP_SEED_DIR" && mnl session activate . --no-prompt \ + --name e2e-bep-mac --network own_ip 2>"$WORK/bep-mac.err")" || { + echo "::error::'min session activate --network own_ip' failed for BEP MAC test" + cat "$WORK/bep-mac.err" 2>/dev/null || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + } + bep_sid="$(printf '%s\n' "$bep_sid" | tail -n1 | tr -d '\r')" + + if ! mnl session exec "$bep_sid" sh -c 'cat /proc/net/dev' >"$WORK/bep-dev.out" 2>"$WORK/bep-dev.err"; then + echo "::error::could not read /proc/net/dev from the own-IP box" + cat "$WORK/bep-dev.err" 2>/dev/null || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + fi + if [ "$(grep -c ':' "$WORK/bep-dev.out")" -lt 2 ]; then + echo "::error::own-IP box has no tap interface; the switch did not attach" + cat "$WORK/bep-dev.out" + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + fi + + bep_ip="$(mnl session exec "$bep_sid" sh -c 'ip -4 -o addr show | awk \!/127\.0\.0\.1/{print $4}' 2>/dev/null | head -n1 | cut -d/ -f1)" + if [ -z "$bep_ip" ]; then + echo "::error::could not determine the own-IP box's switch address" + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + fi + + # RST proves the peer received the frame and answered; a non-RST fast refusal + # would mean the switch dropped or mis-routed it. The probe's body is not + # read: the verdict is the refusal its stderr names. + mnl session exec "$bep_sid" \ + "python3 -c \"import socket; s=socket.socket(); s.settimeout(5); s.connect(('$proxy_ip', 443)); s.close()\"" \ + 2>"$WORK/bep-mac-probe.err" || true + # The python connect will raise ConnectionRefusedError when the RST arrives, + # which on the exec path is a non-zero exit but no stdout body. + if grep -q "ConnectionRefusedError\|Connection refused" "$WORK/bep-mac-probe.err" || \ + grep -q "Connection refused" "$WORK/bep-mac-probe.err"; then + echo "BEP MAC test OK: TCP SYN to $proxy_ip reached the host stack peer and returned RST" + else + echo "::error::TCP SYN to $proxy_ip did not produce a RST; proxy-MAC frame may not have reached the peer" + cat "$WORK/bep-mac-probe.err" 2>/dev/null || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + fi + + mnl session destroy --force "$bep_sid" >/dev/null 2>&1 || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + echo "switch steers proxy-MAC frames to the host stack peer OK" + echo "::endgroup::" +} + +proof_switch_answers_no_arp_for_the_proxy_address() { + echo "::group::switch answers no ARP for the proxy address (NET-132)" + if [ -z "${MINVMD_GVPROXY_BIN:-}" ]; then + echo "switch_answers_no_arp_for_the_proxy_address SKIPPED (no MINVMD_GVPROXY_BIN: this target has no switch)" + echo "::endgroup::" + return 0 + fi + if [ ! -c /dev/net/tun ] || [ ! -r /dev/net/tun ]; then + echo "switch_answers_no_arp_for_the_proxy_address SKIPPED (/dev/net/tun not available; own-address boxes cannot attach on this host)" + echo "::endgroup::" + return 0 + fi + if [ ! -e /proc/sys/kernel/unprivileged_userns_clone ] && [ "$(id -u)" -ne 0 ]; then + echo "switch_answers_no_arp_for_the_proxy_address SKIPPED (unprivileged user namespaces unavailable and not root; own-address boxes cannot attach)" + echo "::endgroup::" + return 0 + fi + + # The BepHost unit test pins this for the peer's stack; the e2e case checks + # that gvproxy itself does not answer. We boot an own-address box and arping + # the proxy address: the reply's MAC must be the peer's, not the gateway's. + local bep_sid proxy_ip proxy_mac + proxy_ip="100.64.255.252" + proxy_mac="52:54:00:40:ff:fc" + + BEP_SEED_DIR="$(hook_mktemp /tmp/mnlbep.XXXXXX)" + hook_seed_preamble > "$BEP_SEED_DIR/minimal.toml" + mkdir "$BEP_SEED_DIR/.git" + + bep_sid="$(cd "$BEP_SEED_DIR" && mnl session activate . --no-prompt \ + --name e2e-bep-arp --network own_ip 2>"$WORK/bep-arp.err")" || { + echo "::error::'min session activate --network own_ip' failed for BEP ARP test" + cat "$WORK/bep-arp.err" 2>/dev/null || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + } + bep_sid="$(printf '%s\n' "$bep_sid" | tail -n1 | tr -d '\r')" + + if ! mnl session exec "$bep_sid" sh -c 'test -x /usr/sbin/arping || test -x /sbin/arping' >/dev/null 2>&1; then + echo "::error::own-IP box has no arping utility; cannot verify ARP reply source" + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + fi + + # arping -c 1 -I ; success means a reply arrived. We then + # inspect the neighbor table to confirm the resolved MAC is the peer's MAC. + mnl session exec "$bep_sid" "sh -c 'iface=\$(ip -o link show | awk -F: \'!/lo/{print \$2}\' | head -n1 | tr -d \" \"); arping -c 1 -I \"\$iface\" $proxy_ip >/dev/null 2>&1; ip neigh show $proxy_ip'" \ + >"$WORK/bep-arp.out" 2>"$WORK/bep-arp-run.err" || true + + if grep -qF "$proxy_mac" "$WORK/bep-arp.out"; then + echo "BEP ARP test OK: proxy address $proxy_ip resolves to peer MAC $proxy_mac, not the gateway" + else + echo "::error::proxy address $proxy_ip did not resolve to the peer MAC $proxy_mac; the switch may be answering ARP itself" + echo "--- arp/neighbor output ---"; cat "$WORK/bep-arp.out" 2>/dev/null || true + echo "--- stderr ---"; cat "$WORK/bep-arp-run.err" 2>/dev/null || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + fail + fi + + mnl session destroy --force "$bep_sid" >/dev/null 2>&1 || true + rm -rf "$BEP_SEED_DIR" + BEP_SEED_DIR="" + echo "switch answers no ARP for the proxy address OK" + echo "::endgroup::" +} + # Ordered LAST in the whole-lane run on purpose: it restarts the daemon (see # the RUST_LOG note inside) and nothing after it depends on the one before. proof_min_internal_names_through_proxy() { @@ -6816,6 +6995,8 @@ case "${1:-}" in proof_min_internal_names_through_proxy proof_proxy_refuses_like_direct proof_retired_surfaces_gone + proof_switch_steers_proxy_mac_frames_to_the_host_stack + proof_switch_answers_no_arp_for_the_proxy_address ;; lifecycle | session_exec | session_outbound_request | own_ip | own_ip_egress_declared_and_enforced | task_run | hooks \ | skip_scaffold | sandbox | restart | fresh_install_own_ip_ingress_publishes_loopback \ @@ -6823,7 +7004,11 @@ case "${1:-}" in | hostnames_recover_and_two_daemons_route \ | min_internal_names_through_proxy | proxy_refuses_like_direct | retired_surfaces_gone \ | fresh_linux_kvm_activate_local_minvmd | fresh_arm64_kvm_activate_local_minvmd \ - | linux_stock_install_runs_vm_boxes) + | linux_stock_install_runs_vm_boxes \ + | switch_steers_proxy_mac_frames_to_the_host_stack | switch_answers_no_arp_for_the_proxy_address) + "proof_$1" + ;; + switch_steers_proxy_mac_frames_to_the_host_stack | switch_answers_no_arp_for_the_proxy_address) "proof_$1" ;; *) @@ -6836,6 +7021,7 @@ case "${1:-}" in echo " linux_stock_install_runs_vm_boxes" echo " hostnames_recover_and_two_daemons_route" echo " min_internal_names_through_proxy proxy_refuses_like_direct retired_surfaces_gone" + echo " switch_steers_proxy_mac_frames_to_the_host_stack switch_answers_no_arp_for_the_proxy_address" exit 2 ;; esac