From 71097dccba43c9ef6da906ddc0a57c964c2b7f57 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 08:12:22 +0000 Subject: [PATCH 1/3] feat(minimald): retry the hostname proxy startup with backoff A host-side proxy whose listen address is held at boot no longer stays down until a daemon restart. start_host_proxies now drives each proxy's bind - and, in a microVM, its host-loopback publish - on a detached task that retries with a doubling, capped backoff until it succeeds (NET-021), so a freed port is picked up in seconds without touching the daemon. bind_listener returns a BindFailure carrying the reason (address and OS error) and the one remedy that clears it, authored once in proxy.rs; the startup retry logs one warning per failed attempt with that report and the next retry delay, records it on the proxy_unavailable note the ListSessions RPC serves, and on recovery logs one info line and clears the note (NET-022). The mTLS proxy rides the same loop. The publish failure report gains its remedy (check the host gvproxy), and the publish attempt cap now bounds a single attempt rather than the whole startup, since the retry supplies the next attempt itself. --- crates/minimald/src/net/proxy.rs | 133 ++++---- crates/minimald/src/server.rs | 560 +++++++++++++++++++++++++------ 2 files changed, 518 insertions(+), 175 deletions(-) diff --git a/crates/minimald/src/net/proxy.rs b/crates/minimald/src/net/proxy.rs index 68783a3f90..9be4d51d4f 100644 --- a/crates/minimald/src/net/proxy.rs +++ b/crates/minimald/src/net/proxy.rs @@ -141,11 +141,38 @@ fn split_authority(authority: &str) -> (&str, Option) { (host, port) } -/// Binds the egress-proxy listener at `addr`, returning it on success. On a bind -/// failure it emits the `component = "dns-proxy"` reachability warning and -/// returns `None`: the proxy is unavailable and PTask hostnames will not route -/// until the address is free. This is the daemon-startup reachability check that -/// supersedes the former systemd-resolved probe (R3.4). +/// Why a host-side proxy listener could not bind its address, carrying the +/// reason and the one remedy that clears it. +/// +/// The pair is authored once here so every surface reads the same text: the +/// startup retry's warning, the daemon's `proxy_unavailable` note the +/// `ListSessions` RPC serves, and the warning `min ls` and +/// `min session activate` print (NET-020). +#[derive(Debug, Clone)] +pub struct BindFailure { + /// What failed, named for a human: the address and the OS error. + pub reason: String, + /// The remedy: free the listen address so the daemon's retry can bind it. + pub remedy: String, +} + +impl BindFailure { + /// The reason and remedy as the one report the daemon's unavailable note + /// and the CLI warning carry. + #[must_use] + pub fn reported(&self) -> String { + format!("{}. Remedy: {}", self.reason, self.remedy) + } +} + +/// Binds the egress-proxy listener at `addr`, returning it on success. On a +/// bind failure it returns a [`BindFailure`] carrying the reason (the address +/// and the OS error) and the remedy that clears it, and logs nothing: the +/// caller owns the failure's log line, because only it knows the retry +/// schedule that line reports — `server::start_host_proxies` retries with +/// backoff until the bind succeeds (NET-021), so one failed bind is one +/// warning, not a silent fallback. This is the daemon-startup reachability +/// check that supersedes the former systemd-resolved probe (R3.4). /// /// The returned listener is the caller's to either serve (via [`serve`]) or /// drop. The success event reports the address as `reachable` rather than @@ -154,7 +181,12 @@ fn split_authority(authority: &str) -> (&str, Option) { /// serve it (see `server::start_host_proxies`); this said otherwise, and reading /// it as a bind-and-drop probe is what made gominimal/inbox#560 look like a /// false alarm on macOS. -pub async fn bind_listener(addr: SocketAddr) -> Option { +/// +/// # Errors +/// +/// Returns a [`BindFailure`] when the address cannot be bound; the OS error is +/// carried inside the failure's reason. +pub async fn bind_listener(addr: SocketAddr) -> Result { match TcpListener::bind(addr).await { Ok(listener) => { tracing::info!( @@ -163,19 +195,15 @@ pub async fn bind_listener(addr: SocketAddr) -> Option { status = "reachable", "host-side egress proxy listen address is bindable" ); - Some(listener) - } - Err(error) => { - tracing::warn!( - component = "dns-proxy", - %addr, - status = "unavailable", - error = %error, - remedy = "free the listen address; PTask *.min.internal hostnames will not route until the egress proxy can bind", - "host-side egress proxy could not bind its listener" - ); - None + Ok(listener) } + Err(error) => Err(BindFailure { + reason: format!("the daemon could not bind {addr}: {error}"), + remedy: format!( + "free the listen address; `lsof -nP -iTCP:{} -sTCP:LISTEN` names the holder", + addr.port() + ), + }), } } @@ -595,35 +623,6 @@ mod tests { use std::sync::{Mutex, RwLock}; use sessions::SessionId; - use tracing_subscriber::fmt::MakeWriter; - - /// A `MakeWriter` accumulating everything written into a shared buffer, so a - /// test can assert on the structured fields a `tracing` event emitted. - #[derive(Clone, Default)] - struct CaptureWriter(Arc>>); - - impl CaptureWriter { - fn contents(&self) -> String { - String::from_utf8(self.0.lock().unwrap().clone()).unwrap() - } - } - - impl io::Write for CaptureWriter { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.0.lock().unwrap().extend_from_slice(buf); - Ok(buf.len()) - } - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } - - impl<'a> MakeWriter<'a> for CaptureWriter { - type Writer = CaptureWriter; - fn make_writer(&'a self) -> Self::Writer { - self.clone() - } - } /// Spawns a one-shot loopback backend that answers every connection with a /// fixed `200 OK` and closes, returning the port it listens on. @@ -723,32 +722,36 @@ mod tests { } /// Proof artifact 3 (R3.4 supersession): when the listen address cannot be - /// bound, the reachability check emits the `component = "dns-proxy"` - /// `status = "unavailable"` warning and yields no listener. + /// bound, the reachability check yields no listener and a failure carrying + /// the reason (address and OS error) and the remedy that clears it — the + /// text the startup retry warns with, the daemon records on its + /// unavailable note, and `min ls` / `min session activate` print. The + /// failure is logged by the caller (the startup retry in + /// `server::start_host_proxies`), not here, so nothing is asserted about + /// `tracing` output. #[tokio::test] - async fn bind_failure_warns_dns_proxy_unavailable() { + async fn bind_failure_carries_reason_and_remedy() { // Hold the address so the reachability bind fails deterministically. let held = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); let addr = held.local_addr().unwrap(); - let buf = CaptureWriter::default(); - let subscriber = tracing_subscriber::fmt() - .with_writer(buf.clone()) - .with_ansi(false) - .finish(); - let guard = tracing::subscriber::set_default(subscriber); - let listener = bind_listener(addr).await; - drop(guard); - - assert!(listener.is_none(), "a bind to a held address must fail"); - let logged = buf.contents(); + let failure = bind_listener(addr) + .await + .expect_err("a bind to a held address must fail"); + assert!( + failure.reason.contains("could not bind") && failure.reason.contains(&addr.to_string()), + "the reason must name the address, got: {}", + failure.reason + ); assert!( - logged.contains(r#"component="dns-proxy""#), - "expected the dns-proxy component field, got: {logged}" + failure.remedy.contains("free") && failure.remedy.contains(&addr.port().to_string()), + "the remedy must point at freeing the held port, got: {}", + failure.remedy ); assert!( - logged.contains(r#"status="unavailable""#), - "expected the unavailable status field, got: {logged}" + failure.reported().contains("Remedy"), + "the report must carry both parts, got: {}", + failure.reported() ); } diff --git a/crates/minimald/src/server.rs b/crates/minimald/src/server.rs index d57725a8e5..4543c7410c 100644 --- a/crates/minimald/src/server.rs +++ b/crates/minimald/src/server.rs @@ -6,6 +6,14 @@ use std::path::PathBuf; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::net::{UnixListener, UnixStream}; +// The host-side proxies and their startup retry live behind the Linux gate +// with the `net` module they route against. +#[cfg(target_os = "linux")] +use std::net::SocketAddr; +#[cfg(target_os = "linux")] +use std::time::Duration; +#[cfg(target_os = "linux")] +use tokio::net::TcpListener; use tokio::sync::Mutex; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; @@ -327,12 +335,27 @@ impl ServerStateHandle { self.0.lock().await.proxy_unavailable.clone() } + /// Clears the hostname-routing unavailability note: the proxy's startup + /// retry bound and published it, so `ListSessions` stops reporting the + /// reason and `min ls` stops printing the warning — without a daemon + /// restart (NET-022). + pub(crate) async fn clear_proxy_unavailable(&self) { + self.0.lock().await.proxy_unavailable = None; + } + /// Records why the mTLS reverse proxy is not serving. #[cfg_attr(not(feature = "networking-proxy"), expect(dead_code))] pub(crate) async fn set_mtls_unavailable(&self, reason: String) { self.0.lock().await.mtls_unavailable = Some(reason); } + /// Clears the mTLS unavailability note: the proxy's startup retry bound + /// and published it. + #[cfg_attr(not(feature = "networking-proxy"), expect(dead_code))] + pub(crate) async fn clear_mtls_unavailable(&self) { + self.0.lock().await.mtls_unavailable = None; + } + /// Why the mTLS reverse proxy is not serving, or `None` if it is. pub(crate) async fn mtls_unavailable(&self) -> Option { self.0.lock().await.mtls_unavailable.clone() @@ -739,15 +762,19 @@ async fn reap_unfinalized_sessions(state: &ServerStateHandle, ids: Vec<::session /// PTask hostname registry. In a microVM they bind the daemon's switch IP /// ([`DEFAULT_SUBNET`](crate::net::DEFAULT_SUBNET)`.daemon_ip()`) so the host /// gvproxy forward can reach them; on native Linux (DM2) they bind host loopback -/// directly. A bind failure warns and is skipped — the daemon keeps serving. The -/// serve loops run on detached tasks; this returns once the listeners are bound -/// and (DM1) exposed. +/// directly. +/// +/// Each proxy's startup — the bind, and in a microVM the host-loopback +/// publish — runs on a detached task that retries with backoff until it +/// succeeds ([`drive_proxy_until_serving`], NET-021) and then clears the +/// unavailable note `min ls` warns from (NET-022). Nothing here is awaited: +/// a proxy whose port some other process holds must not hold the SSH accept +/// loop hostage — the daemon starts serving regardless, reports the reason on +/// its state, and the proxy comes up on its own once the address frees. #[cfg(target_os = "linux")] async fn start_host_proxies(state: &ServerStateHandle, in_microvm: bool) { - use crate::net::proxy::{self, Router}; - use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + use std::net::{IpAddr, Ipv4Addr}; - let registry = state.sessions_manager().await.hostnames(); // DM1 (in-VM): bind 0.0.0.0 so the listener comes up regardless of whether // eth0 has finished coming up, then publish the port on the host loopback via // the gvproxy forwarder. DM2: bind host loopback directly, no host-expose. @@ -761,41 +788,12 @@ async fn start_host_proxies(state: &ServerStateHandle, in_microvm: bool) { // `.local.min.internal` not routing, so both are recorded on the // state where `ListSessions` can reach them — a daemon that keeps serving // without its proxy looks identical to a healthy one otherwise. - let egress_addr = SocketAddr::new(bind_base, proxy::EGRESS_PROXY_PORT); - let bound = proxy::bind_listener(egress_addr) - .await - .map(|listener| { - let router = Router::new(registry.clone()); - tokio::spawn(async move { - if let Err(error) = proxy::serve(listener, router).await { - tracing::error!(%error, "egress proxy accept loop exited"); - } - }) - }) - .is_some(); - - if !bound { - // DM2: something else on the host holds the port. - state - .set_proxy_unavailable(format!( - "the daemon could not bind {egress_addr}; another process is \ - holding it. Check with: lsof -nP -iTCP:{} -sTCP:LISTEN", - proxy::EGRESS_PROXY_PORT - )) - .await; - } else if in_microvm { - // DM1: the guest bind cannot collide with a host process, so the - // failure moves to the publish instead. Only publish a port whose - // listener actually bound. - if let Some(reason) = expose_proxy_on_host( - crate::net::DEFAULT_SUBNET.daemon_ip(), - proxy::EGRESS_PROXY_PORT, - ) - .await - { - state.set_proxy_unavailable(reason).await; - } - } + let egress_addr = SocketAddr::new(bind_base, crate::net::proxy::EGRESS_PROXY_PORT); + tokio::spawn(drive_proxy_until_serving( + state.clone(), + HostProxyStartup::Egress { addr: egress_addr }, + RetryBackoff::production(), + )); // B8 mTLS reverse proxy (:7655), under the networking-proxy feature. #[cfg(feature = "networking-proxy")] @@ -804,41 +802,20 @@ async fn start_host_proxies(state: &ServerStateHandle, in_microvm: bool) { // were silent: the TLS config failing to build, the bind failing, and // the publish failing. The daemon carries on in every case, so only a // reported reason distinguishes "no mTLS proxy configured" from "the - // mTLS proxy is broken". - let https_addr = SocketAddr::new(bind_base, proxy::HTTPS_PROXY_PORT); + // mTLS proxy is broken". A TLS config that cannot build is not + // retryable — the CA material does not change under the retry — so it + // is reported once, here, and only the bind and publish retry. + let https_addr = SocketAddr::new(bind_base, crate::net::proxy::HTTPS_PROXY_PORT); match state.cert_authority().await.build_server_config() { Ok(tls_config) => { - let bound = proxy::bind_listener(https_addr) - .await - .map(|listener| { - let router = Router::new(registry.clone()); - tokio::spawn(async move { - if let Err(error) = - proxy::serve_https(listener, router, tls_config).await - { - tracing::error!(%error, "mTLS proxy accept loop exited"); - } - }) - }) - .is_some(); - - if !bound { - state - .set_mtls_unavailable(format!( - "the daemon could not bind {https_addr}; another process is \ - holding it. Check with: lsof -nP -iTCP:{} -sTCP:LISTEN", - proxy::HTTPS_PROXY_PORT - )) - .await; - } else if in_microvm - && let Some(reason) = expose_proxy_on_host( - crate::net::DEFAULT_SUBNET.daemon_ip(), - proxy::HTTPS_PROXY_PORT, - ) - .await - { - state.set_mtls_unavailable(reason).await; - } + tokio::spawn(drive_proxy_until_serving( + state.clone(), + HostProxyStartup::Mtls { + addr: https_addr, + tls_config, + }, + RetryBackoff::production(), + )); } Err(error) => { tracing::warn!(%error, "could not build TLS config for the mTLS reverse proxy"); @@ -853,28 +830,267 @@ async fn start_host_proxies(state: &ServerStateHandle, in_microvm: bool) { } } -/// Upper bound on the best-effort host-loopback publish in +/// The retry schedule a host-side proxy's startup uses while its bind (or, in +/// a microVM, its host-loopback publish) keeps failing: each retry waits twice +/// as long as the one before, from `initial` up to `max`. Tests pass a +/// compressed schedule so a retry loop's worth of failures costs milliseconds +/// instead of seconds. +#[cfg(target_os = "linux")] +#[derive(Debug, Clone, Copy)] +pub struct RetryBackoff { + initial: Duration, + max: Duration, +} + +#[cfg(target_os = "linux")] +impl RetryBackoff { + /// Builds a schedule that doubles from `initial` and never waits longer + /// than `max`. + #[must_use] + pub const fn new(initial: Duration, max: Duration) -> Self { + Self { initial, max } + } + + /// The daemon's schedule: the first retry 1 s after the failure, doubling + /// to a 30 s ceiling. Fast enough that a port freed moments after boot is + /// serving again in seconds; slow enough that a port held for good costs + /// one warning per 30 s, not a spin. + #[must_use] + pub const fn production() -> Self { + Self::new(Duration::from_secs(1), Duration::from_secs(30)) + } + + /// The wait before retry `attempt` (0-based: the wait after the first + /// failure), doubling from `initial` and capped at `max`. + fn delay(&self, attempt: u32) -> Duration { + let factor = 1u32.checked_shl(attempt.min(30)).unwrap_or(u32::MAX); + self.initial + .checked_mul(factor) + .unwrap_or(self.max) + .min(self.max) + } +} + +/// Which host-side proxy a startup retry drives. The two differ in what serves +/// a bound listener and which state note a failure lands on; the retry loop +/// itself is shared. +#[cfg(target_os = "linux")] +#[derive(Debug)] +enum HostProxyStartup { + /// The B5 egress/DNS proxy: plain HTTP routing through the shared router. + Egress { addr: SocketAddr }, + /// The B8 mTLS reverse proxy: the same routing behind TLS termination. + #[cfg(feature = "networking-proxy")] + Mtls { + addr: SocketAddr, + tls_config: Arc, + }, +} + +#[cfg(target_os = "linux")] +impl HostProxyStartup { + /// The address the listener binds. + fn addr(&self) -> SocketAddr { + match self { + Self::Egress { addr } => *addr, + #[cfg(feature = "networking-proxy")] + Self::Mtls { addr, .. } => *addr, + } + } + + /// The `component` log field this proxy's startup events carry, matching + /// what its serve loop logs. + fn component(&self) -> &'static str { + match self { + Self::Egress { .. } => "dns-proxy", + #[cfg(feature = "networking-proxy")] + Self::Mtls { .. } => "https-proxy", + } + } + + /// Spawns the serve loop for a bound listener. Runs for the daemon's + /// lifetime; the startup retry never rebinds a bound-and-served listener. + async fn spawn_serve(&self, state: &ServerStateHandle, listener: TcpListener) { + use crate::net::proxy::{Router, serve}; + #[cfg(feature = "networking-proxy")] + use crate::net::proxy::serve_https; + + let router = Router::new(state.sessions_manager().await.hostnames()); + match self { + Self::Egress { .. } => { + tokio::spawn(async move { + if let Err(error) = serve(listener, router).await { + tracing::error!(%error, "egress proxy accept loop exited"); + } + }); + } + #[cfg(feature = "networking-proxy")] + Self::Mtls { tls_config, .. } => { + let tls_config = Arc::clone(tls_config); + tokio::spawn(async move { + if let Err(error) = serve_https(listener, router, tls_config).await { + tracing::error!(%error, "mTLS proxy accept loop exited"); + } + }); + } + } + } + + /// Records a failure's reason-and-remedy report on the state note the + /// `ListSessions` RPC serves (NET-020). + async fn record_unavailable(&self, state: &ServerStateHandle, report: String) { + match self { + Self::Egress { .. } => state.set_proxy_unavailable(report).await, + #[cfg(feature = "networking-proxy")] + Self::Mtls { .. } => state.set_mtls_unavailable(report).await, + } + } + + /// Clears the failure note: bound and published, the proxy is serving. + async fn clear_unavailable(&self, state: &ServerStateHandle) { + match self { + Self::Egress { .. } => state.clear_proxy_unavailable().await, + #[cfg(feature = "networking-proxy")] + Self::Mtls { .. } => state.clear_mtls_unavailable().await, + } + } +} + +/// Drives the hostname-routing proxy (the B5 egress proxy — the listener +/// `*.min.internal` hostnames route through) to serving: binds `addr`, +/// retrying bind — and, in a microVM, the host-loopback publish — with +/// `retry`'s backoff until both succeed, then keeps serving and clears the +/// daemon's `proxy_unavailable` note so `min ls` stops warning (NET-021, +/// NET-022). +/// +/// Detached for the daemon's lifetime by `start_host_proxies`; also spawned +/// directly by tests, which hold the address and watch the retry recover. +#[cfg(any(test, feature = "test-support"))] +#[cfg(target_os = "linux")] +pub async fn retry_hostname_proxy_until_serving( + state: ServerStateHandle, + addr: SocketAddr, + retry: RetryBackoff, +) { + drive_proxy_until_serving(state, HostProxyStartup::Egress { addr }, retry).await; +} + +/// Drives one host-side proxy to serving, retrying with backoff (NET-021). +/// +/// Two gates stand between daemon start and a serving proxy, in order: the +/// listener must bind, and — in a microVM (DM1), where the listener binds +/// inside the guest — the port must then be published on the host loopback via +/// the gvproxy forwarder. Each failed attempt logs one warning carrying the +/// failure's reason and remedy and the next retry delay, and records it on the +/// state note the `ListSessions` RPC serves, so `min ls` and +/// `min session activate` print it (NET-020). Once both gates pass the note is +/// cleared and — when any attempt failed — one info line marks the recovery +/// (NET-022): the warning disappears from `min ls` without a daemon restart. +/// +/// The serve loop starts as soon as the listener binds and stays up while the +/// publish retries; the bind gate never runs again once it has passed, so a +/// bound-and-served listener is never dropped and rebound. +#[cfg(target_os = "linux")] +async fn drive_proxy_until_serving( + state: ServerStateHandle, + proxy: HostProxyStartup, + retry: RetryBackoff, +) { + let addr = proxy.addr(); + let component = proxy.component(); + // DM1 only: the bind happens in-guest, so the host loopback is reachable + // only through the gvproxy forwarder's publish. DM2 binds host loopback + // directly and has no second gate. + let publish_port = state.in_microvm().await.then_some(addr.port()); + + let mut bound = false; + let mut attempt: u32 = 0; + let mut failed_before = false; + loop { + if !bound { + match crate::net::proxy::bind_listener(addr).await { + Ok(listener) => { + proxy.spawn_serve(&state, listener).await; + bound = true; + if publish_port.is_none() { + break; + } + } + Err(failure) => { + let report = failure.reported(); + let next_retry = retry.delay(attempt); + tracing::warn!( + component, + %addr, + status = "unavailable", + reason = %report, + next_retry = ?next_retry, + "host-side proxy could not bind its listener; retrying with backoff" + ); + proxy.record_unavailable(&state, report).await; + failed_before = true; + attempt += 1; + tokio::time::sleep(next_retry).await; + continue; + } + } + } + // Bound and serving. Only the host-loopback publish can still be + // pending: a bind success with no publish gate broke out above. + let Some(port) = publish_port else { + break; + }; + match expose_proxy_on_host(crate::net::DEFAULT_SUBNET.daemon_ip(), port).await { + None => break, + Some(report) => { + let next_retry = retry.delay(attempt); + tracing::warn!( + component, + %port, + status = "unavailable", + %report, + next_retry = ?next_retry, + "host-side proxy could not publish on the host loopback; retrying with backoff" + ); + proxy.record_unavailable(&state, report).await; + failed_before = true; + attempt += 1; + tokio::time::sleep(next_retry).await; + } + } + } + + if failed_before { + tracing::info!( + component, + %addr, + status = "recovered", + "host-side proxy is serving after retrying" + ); + } + proxy.clear_unavailable(&state).await; +} + +/// Upper bound on one host-loopback publish attempt in /// [`expose_proxy_on_host`]. Deliberately far below `post_json`'s gvproxy -/// control timeout: the publish is awaited on [`Server::run`]'s boot path -/// *before* the SSH accept loop starts serving. When the forwarder control -/// request does not complete promptly — it times out at the full 5 s even with a -/// host gvproxy present, since the forwarder control path is not reachable over -/// the shuttle in every deployment — the accept loop must not be held that long: -/// the cold `minimal ls` connect-retry deadline expires first and the first list -/// fails (`ssh connect: Disconnected`). A reachable forwarder answers in well -/// under this bound. +/// control timeout: the startup retry's failed attempts must not each cost the +/// full 5 s before the next try — when the forwarder control path is not +/// reachable over the shuttle in every deployment, a retry every 5 s would +/// spend nearly all its time waiting instead of checking. A reachable +/// forwarder answers in well under this bound. #[cfg(target_os = "linux")] const HOST_EXPOSE_PUBLISH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1); /// Publishes a guest-side proxy bound on `daemon_ip:port` onto the macOS host's /// loopback (`127.0.0.1:port`) via the host gvproxy forwarder, reached over the /// vsock shuttle (DM1). Best-effort in that it never fails the daemon, since -/// the host gvproxy may be absent. Capped at [`HOST_EXPOSE_PUBLISH_TIMEOUT`] so -/// it never stalls [`Server::run`]'s SSH accept loop. +/// the host gvproxy may be absent; capped at [`HOST_EXPOSE_PUBLISH_TIMEOUT`] +/// per attempt so a stalled forwarder cannot stretch the retry cadence. /// -/// Returns `Some(reason)` when the publish did not happen, so the caller can -/// tell a client rather than leaving the loss in the daemon log — on DM1 this -/// is the path a host process holding the port actually breaks. +/// Returns `Some(report)` when the publish did not happen — the reason and the +/// remedy as one text, which the startup retry logs with its next retry and +/// records on the state note for `min ls` / `min session activate` to print — +/// and `None` once published. #[cfg(target_os = "linux")] async fn expose_proxy_on_host(daemon_ip: std::net::Ipv4Addr, port: u16) -> Option { use crate::net::policy::{ControlChannel, ExposeRequest, post_json}; @@ -895,28 +1111,16 @@ async fn expose_proxy_on_host(daemon_ip: std::net::Ipv4Addr, port: u16) -> Optio .await { Ok(Ok(_)) => None, - Ok(Err(error)) => { - tracing::warn!( - %port, - %error, - "could not publish host-side proxy on the host loopback via gvproxy forwarder" - ); - Some(format!( - "the daemon could not publish port {port} on the host loopback \ - via the gvproxy forwarder: {error}" - )) - } - Err(_) => { - tracing::warn!( - %port, - timeout = ?HOST_EXPOSE_PUBLISH_TIMEOUT, - "host-side proxy publish did not complete in time; continuing (best-effort)" - ); - Some(format!( - "publishing port {port} on the host loopback did not complete within \ - {HOST_EXPOSE_PUBLISH_TIMEOUT:?}" - )) - } + Ok(Err(error)) => Some(format!( + "the daemon could not publish port {port} on the host loopback via \ + the gvproxy forwarder: {error}. Remedy: check that the host \ + gvproxy (minvmd) is running and reachable over the shuttle" + )), + Err(_) => Some(format!( + "publishing port {port} on the host loopback did not complete within \ + {HOST_EXPOSE_PUBLISH_TIMEOUT:?}. Remedy: check that the host \ + gvproxy (minvmd) is running and reachable over the shuttle" + )), } } @@ -1098,4 +1302,140 @@ mod tests { .await; let _ = tokio::time::timeout(Duration::from_secs(5), run).await; } + + /// A `MakeWriter` accumulating everything written into a shared buffer, so + /// a test can assert on the structured fields a `tracing` event emitted. + /// Local twin of the helper the `net::proxy` tests used before the bind + /// failure's log line moved to the startup retry that owns the schedule. + #[cfg(target_os = "linux")] + #[derive(Clone, Default)] + struct CaptureWriter(Arc>>); + + #[cfg(target_os = "linux")] + impl CaptureWriter { + fn contents(&self) -> String { + String::from_utf8(self.0.lock().unwrap().clone()).unwrap() + } + } + + #[cfg(target_os = "linux")] + impl std::io::Write for CaptureWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + #[cfg(target_os = "linux")] + impl tracing_subscriber::fmt::MakeWriter<'_> for CaptureWriter { + type Writer = CaptureWriter; + fn make_writer(&self) -> Self::Writer { + self.clone() + } + } + + /// The `next_retry=` values the startup retry's failure warnings + /// carried, in the order they were logged. + #[cfg(target_os = "linux")] + fn logged_next_retries(log: &str) -> Vec<&str> { + log.match_indices("next_retry=") + .map(|(start, _)| { + let rest = &log[start + "next_retry=".len()..]; + rest.split([' ', '\n', ',']).next().unwrap_or_default() + }) + .collect() + } + + /// The hostname proxy's startup retry keeps trying a held listen address: + /// each failed bind warns once with the reason, the remedy, and the next + /// (doubling, capped) retry delay and records the reason-and-remedy report + /// on the state note (NET-021); once the address frees, the next attempt + /// binds, the note clears, and the recovery is logged (NET-022) — no + /// daemon restart anywhere. + #[cfg(target_os = "linux")] + #[tokio::test] + async fn listener_retries_with_backoff() { + // Hold an address so the startup retry's binds fail deterministically. + let held = tokio::net::TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let addr = held.local_addr().unwrap(); + + let buf = CaptureWriter::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(buf.clone()) + .with_ansi(false) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + + let dir = TempDir::new().unwrap(); + let state = ServerStateHandle::new(test_config(&dir), None).await.unwrap(); + // A compressed schedule: first retry 5 ms after the failure, doubling + // to a 20 ms cap, so a loop's worth of failures costs milliseconds. + let retrier = tokio::spawn(drive_proxy_until_serving( + state.clone(), + HostProxyStartup::Egress { addr }, + RetryBackoff::new(Duration::from_millis(5), Duration::from_millis(20)), + )); + + // Three failures are enough to see the doubling and the cap. + let mut saw_three_warnings = false; + for _ in 0..200 { + if buf.contents().matches("could not bind its listener").count() >= 3 { + saw_three_warnings = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!( + saw_three_warnings, + "expected three failure warnings while the address is held, got: {}", + buf.contents() + ); + let note = state + .proxy_unavailable() + .await + .expect("a held address must be recorded as the unavailable note"); + assert!( + note.contains("could not bind") && note.contains("Remedy"), + "the note must carry the reason and the remedy, got: {note}" + ); + + // The retry delays double from the schedule's initial wait and cap at + // its max: 5 ms, 10 ms, then 20 ms for as long as the address stays + // held. + let logged = buf.contents(); + let delays = logged_next_retries(&logged); + assert_eq!( + &delays[..3.min(delays.len())], + &["5ms", "10ms", "20ms"][..3.min(delays.len())], + "retry delays must grow with backoff and cap, got: {delays:?}" + ); + assert!( + delays.iter().skip(3).all(|d| *d == "20ms"), + "retries past the cap must wait the cap, got: {delays:?}" + ); + + // Freeing the address lets the next attempt bind: the retry task + // resolves, the note clears, and the recovery is on the log. + drop(held); + tokio::time::timeout(Duration::from_secs(5), retrier) + .await + .expect("the retry must finish once the address frees") + .expect("the retry task must not panic"); + assert!( + state.proxy_unavailable().await.is_none(), + "recovery must clear the unavailable note without a restart" + ); + let logged = buf.contents(); + assert!( + logged.contains(r#"status="recovered""#), + "the recovery must be logged, got: {logged}" + ); + assert!( + logged.contains(r#"status="reachable""#), + "the successful bind must be logged as reachable, got: {logged}" + ); + } } From 6c19dde9964fd75f250470be82c55e227e31e348 Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 08:15:24 +0000 Subject: [PATCH 2/3] feat(minimal): the coder's uncommitted work as it stood when its 3000s ran out the coder's uncommitted work as it stood when its 3000s ran out The coder was stopped at the end of its time with these changes not committed. The runtime committed its working tree as it stood so the work is kept: this commit is a snapshot, not a finished step, and may not build or pass its tests. --- crates/minimal/src/cmd/list.rs | 26 ++++- crates/minimal/tests/cli.rs | 171 +++++++++++++++++++++++++++++++++ crates/minimald/src/server.rs | 17 +++- 3 files changed, 207 insertions(+), 7 deletions(-) diff --git a/crates/minimal/src/cmd/list.rs b/crates/minimal/src/cmd/list.rs index 50d7ce9dbe..de7b77a135 100644 --- a/crates/minimal/src/cmd/list.rs +++ b/crates/minimal/src/cmd/list.rs @@ -310,15 +310,35 @@ pub async fn cmd_ls(global: &GlobalArgs, args: LsArgs) -> Result<(), anyhow::Err Ok(()) } -/// Tells the user that `.local.min.internal` will not resolve, and why. +/// The warning `min ls` and `min session activate` print when the daemon +/// reports hostname routing down: the daemon's reason and remedy for the +/// failed bind or publish, plus what recovery looks like from here — the +/// daemon retries with backoff and clears the warning on its own once the +/// listener recovers (NET-020, NET-022). Pure, so tests can assert the wording +/// without capturing stderr. +#[must_use] +pub fn hostname_routing_warning(reason: &str) -> String { + format!( + "warning: session hostnames will not route: {reason}. The daemon retries \ + with backoff and clears this warning on its own once the listener \ + recovers; run `min ls` again to check." + ) +} + +/// Tells the user that `.local.min.internal` will not resolve, and why, +/// and what clears it. /// /// The daemon keeps serving without its host-side proxy, so nothing else the /// user sees is different: sessions activate, exec works, the list prints. The /// only other trace is a `warn!` in the daemon log, which is not where someone -/// watching curl fail is looking (gominimal/inbox#560). +/// watching curl fail is looking (gominimal/inbox#560). The daemon's report +/// carries the reason and the remedy for the cause (a held port, a failed +/// publish); the recovery is the daemon's job now — it retries with backoff +/// and the warning goes away by itself, so the remedy says to check again +/// rather than to restart anything. pub(crate) fn warn_if_hostname_routing_down(reason: Option<&str>) { if let Some(reason) = reason { - eprintln!("warning: session hostnames will not route: {reason}"); + eprintln!("{}", hostname_routing_warning(reason)); } } diff --git a/crates/minimal/tests/cli.rs b/crates/minimal/tests/cli.rs index 39f2125d98..0759218ab7 100644 --- a/crates/minimal/tests/cli.rs +++ b/crates/minimal/tests/cli.rs @@ -781,6 +781,177 @@ async fn session_policy_succeeds() { .unwrap(); } +// --- hostname routing warning (NET-020/NET-021/NET-022) --- +// +// The startup retry lives in `minimald::server` behind the Linux gate with the +// `net` module it drives, so this section is Linux-only too: it holds a port, +// runs the real retry loop against it, and reads what `min ls` and +// `min session activate` print while the proxy is down and after it recovers. + +/// Runs the compiled `min` with the harness daemon's `--minimal-dir`, an empty +/// `--config-dir` (the developer's own loadouts and policy stay out of the +/// run), and `--no-input`, plus `extra` as the command, and returns stderr. +#[cfg(target_os = "linux")] +async fn run_min_stderr(args: &GlobalArgs, extra: &[&str]) -> String { + let minimal_dir = args + .minimal_dir + .as_ref() + .expect("setup points at a tempdir"); + let config_dir = tempfile::TempDir::new().unwrap(); + let out = tokio::process::Command::new(env!("CARGO_BIN_EXE_min")) + .args(["--minimal-dir".as_ref(), minimal_dir.as_os_str()]) + .args(["--config-dir".as_ref(), config_dir.path().as_os_str()]) + .arg("--no-input") + .args(extra) + .output() + .await + .expect("the min binary should be invocable"); + String::from_utf8_lossy(&out.stderr).into_owned() +} + +/// Polls `ListSessions` until the daemon reports hostname routing down — the +/// startup retry's first failed bind has landed on its note — and returns the +/// reason. +#[cfg(target_os = "linux")] +async fn wait_for_routing_failure(args: &GlobalArgs) -> String { + use minimald_rpc::ListSessions; + + let mut client = connect_daemon(args).await.unwrap(); + for _ in 0..200 { + let resp = client.oneshot_rpc::(()).await.unwrap(); + if let Some(reason) = resp.hostname_routing_unavailable { + return reason; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + panic!("the daemon never reported hostname routing down"); +} + +/// Holds a fresh port and starts the daemon's hostname-proxy startup retry +/// against it — the real loop `start_host_proxies` spawns, with a compressed +/// backoff — so the note `min ls` warns from is set exactly the way an +/// occupied proxy port sets it, without holding the production `:7654`. +/// Dropping the held listener frees the port; the returned task resolves once +/// the proxy is serving again and the note is cleared. +#[cfg(target_os = "linux")] +async fn hold_proxy_port_and_start_retry( + state: &minimald::server::ServerStateHandle, +) -> (tokio::net::TcpListener, tokio::task::JoinHandle<()>) { + use minimald::server::{RetryBackoff, retry_hostname_proxy_until_serving}; + + let held = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = held.local_addr().unwrap(); + let retry = tokio::spawn(retry_hostname_proxy_until_serving( + state.clone(), + addr, + RetryBackoff::new( + std::time::Duration::from_millis(5), + std::time::Duration::from_millis(40), + ), + )); + (held, retry) +} + +/// NET-020: with the proxy port occupied, both `min ls` and `min session +/// activate` print the failure's reason and the remedy for it, and say the +/// daemon recovers on its own. Driven through the compiled binary so the +/// assertion is on what the user actually sees. +#[cfg(target_os = "linux")] +#[tokio::test] +async fn listener_failure_reported_with_remedy() { + let (daemon, args) = setup().await; + let (held, retry) = hold_proxy_port_and_start_retry(&daemon.server.state).await; + let reason = wait_for_routing_failure(&args).await; + + let ls_stderr = run_min_stderr(&args, &["ls"]).await; + assert!( + ls_stderr.contains("warning: session hostnames will not route"), + "ls must warn about hostname routing, got: {ls_stderr}" + ); + assert!( + ls_stderr.contains(&reason), + "ls must print the daemon's reason, got: {ls_stderr}" + ); + assert!( + ls_stderr.contains("Remedy"), + "ls must print the remedy, got: {ls_stderr}" + ); + assert!( + ls_stderr.contains("retries"), + "ls must say the daemon recovers on its own, got: {ls_stderr}" + ); + + // `min session activate` prints the same report on its path. + let project = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(project.path().join(".git")).unwrap(); + std::fs::write( + project.path().join("minimal.toml"), + "# test minimal.toml\n[upstream]\nrepo = \"https://github.com/gominimal/pkgs\"\nbranch = \"main\"\n\n[stack]\nuse = \"shell\"\n", + ) + .unwrap(); + let activate_stderr = run_min_stderr( + &args, + &[ + "session", + "activate", + project.path().to_str().unwrap(), + "--name", + "remedy-report", + "--sync", + "tarball", + "--no-prompt", + ], + ) + .await; + assert!( + activate_stderr.contains("warning: session hostnames will not route"), + "activate must warn about hostname routing, got: {activate_stderr}" + ); + assert!( + activate_stderr.contains(&reason), + "activate must print the daemon's reason, got: {activate_stderr}" + ); + assert!( + activate_stderr.contains("Remedy"), + "activate must print the remedy, got: {activate_stderr}" + ); + + drop(held); + let _ = tokio::time::timeout(std::time::Duration::from_secs(5), retry).await; +} + +/// NET-022: once the port frees and the listener recovers, `min ls` stops +/// carrying the warning — against the same daemon process, with no restart. +#[cfg(target_os = "linux")] +#[tokio::test] +async fn ls_warning_clears_on_recovery() { + let (daemon, args) = setup().await; + let (held, retry) = hold_proxy_port_and_start_retry(&daemon.server.state).await; + let _reason = wait_for_routing_failure(&args).await; + + let before = run_min_stderr(&args, &["ls"]).await; + assert!( + before.contains("warning: session hostnames will not route"), + "ls must warn while the port is held, got: {before}" + ); + + // Free the port: the startup retry binds (a native daemon has no + // host-loopback publish gate) and clears the note. + drop(held); + tokio::time::timeout(std::time::Duration::from_secs(5), retry) + .await + .expect("the startup retry must finish once the port frees") + .expect("the startup retry must not panic"); + + let after = run_min_stderr(&args, &["ls"]).await; + assert!( + !after.contains("warning: session hostnames will not route"), + "the warning must clear without a daemon restart, got: {after}" + ); +} + // --- helpers --- /// Creates a session whose workspace mfile declares a `[session.vars]` diff --git a/crates/minimald/src/server.rs b/crates/minimald/src/server.rs index 4543c7410c..9b85bee993 100644 --- a/crates/minimald/src/server.rs +++ b/crates/minimald/src/server.rs @@ -911,9 +911,9 @@ impl HostProxyStartup { /// Spawns the serve loop for a bound listener. Runs for the daemon's /// lifetime; the startup retry never rebinds a bound-and-served listener. async fn spawn_serve(&self, state: &ServerStateHandle, listener: TcpListener) { - use crate::net::proxy::{Router, serve}; #[cfg(feature = "networking-proxy")] use crate::net::proxy::serve_https; + use crate::net::proxy::{Router, serve}; let router = Router::new(state.sessions_manager().await.hostnames()); match self { @@ -1359,7 +1359,9 @@ mod tests { #[tokio::test] async fn listener_retries_with_backoff() { // Hold an address so the startup retry's binds fail deterministically. - let held = tokio::net::TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let held = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); let addr = held.local_addr().unwrap(); let buf = CaptureWriter::default(); @@ -1370,7 +1372,9 @@ mod tests { let _guard = tracing::subscriber::set_default(subscriber); let dir = TempDir::new().unwrap(); - let state = ServerStateHandle::new(test_config(&dir), None).await.unwrap(); + let state = ServerStateHandle::new(test_config(&dir), None) + .await + .unwrap(); // A compressed schedule: first retry 5 ms after the failure, doubling // to a 20 ms cap, so a loop's worth of failures costs milliseconds. let retrier = tokio::spawn(drive_proxy_until_serving( @@ -1382,7 +1386,12 @@ mod tests { // Three failures are enough to see the doubling and the cap. let mut saw_three_warnings = false; for _ in 0..200 { - if buf.contents().matches("could not bind its listener").count() >= 3 { + if buf + .contents() + .matches("could not bind its listener") + .count() + >= 3 + { saw_three_warnings = true; break; } From 6001231a84bd67c436ef2ae6a1f441a4d641745c Mon Sep 17 00:00:00 2001 From: "minimal-foundry[bot]" <327719912+minimal-foundry[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 00:43:16 +0000 Subject: [PATCH 3/3] fix(minimal): fit the hostname-routing recovery hint to its command The hostname-routing-down warning is shared by `min ls` and `min session activate`, but its recovery sentence hardcoded "run `min ls` again to check". A user mid-activate was told to switch to another command to watch the recovery, which reads as a non-sequitur. `hostname_routing_warning` now takes the command the warning rides on: each path names itself, and the sentence says to re-run the command the user is already in. Review finding on #1681 (low, api-design). --- crates/minimal/src/cmd/list.rs | 21 +++++++++++++-------- crates/minimal/src/cmd/session.rs | 5 ++++- crates/minimal/tests/cli.rs | 8 ++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/crates/minimal/src/cmd/list.rs b/crates/minimal/src/cmd/list.rs index de7b77a135..4c2d537822 100644 --- a/crates/minimal/src/cmd/list.rs +++ b/crates/minimal/src/cmd/list.rs @@ -304,7 +304,7 @@ pub async fn cmd_ls(global: &GlobalArgs, args: LsArgs) -> Result<(), anyhow::Err // fault this severe — `--raw` most of all, since a script parsing bare ids // is exactly what will go on using hostnames that no longer resolve — and // stdout stays clean for the parser either way. - warn_if_hostname_routing_down(resp.hostname_routing_unavailable.as_deref()); + warn_if_hostname_routing_down(resp.hostname_routing_unavailable.as_deref(), "min ls"); warn_if_mtls_proxy_down(resp.mtls_proxy_unavailable.as_deref()); format_ls(&mut std::io::stdout(), &args, &resp)?; Ok(()) @@ -314,14 +314,17 @@ pub async fn cmd_ls(global: &GlobalArgs, args: LsArgs) -> Result<(), anyhow::Err /// reports hostname routing down: the daemon's reason and remedy for the /// failed bind or publish, plus what recovery looks like from here — the /// daemon retries with backoff and clears the warning on its own once the -/// listener recovers (NET-020, NET-022). Pure, so tests can assert the wording -/// without capturing stderr. +/// listener recovers (NET-020, NET-022). `command` names the command the +/// warning rides on: re-running it shows the warning while the listener is +/// still down and nothing once it has recovered, so the recovery sentence +/// tells the user to re-run the command they are already in, not some other +/// one. Pure, so tests can assert the wording without capturing stderr. #[must_use] -pub fn hostname_routing_warning(reason: &str) -> String { +pub fn hostname_routing_warning(reason: &str, command: &str) -> String { format!( "warning: session hostnames will not route: {reason}. The daemon retries \ with backoff and clears this warning on its own once the listener \ - recovers; run `min ls` again to check." + recovers; run `{command}` again to check." ) } @@ -335,10 +338,12 @@ pub fn hostname_routing_warning(reason: &str) -> String { /// carries the reason and the remedy for the cause (a held port, a failed /// publish); the recovery is the daemon's job now — it retries with backoff /// and the warning goes away by itself, so the remedy says to check again -/// rather than to restart anything. -pub(crate) fn warn_if_hostname_routing_down(reason: Option<&str>) { +/// rather than to restart anything. `command` names the command this runs +/// from, so the recovery sentence fits it — `min ls` on the list, `min +/// session activate` at activation. +pub(crate) fn warn_if_hostname_routing_down(reason: Option<&str>, command: &str) { if let Some(reason) = reason { - eprintln!("{}", hostname_routing_warning(reason)); + eprintln!("{}", hostname_routing_warning(reason, command)); } } diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index 0488bb237c..b9f29d2c28 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -258,7 +258,10 @@ pub(crate) async fn activate_session( // loadout, and the finalize that #1251 died at, and with the session left // unfinalized for the daemon to reap when this connection drops. ensure_version_reported(created.daemon_version.as_deref())?; - warn_if_hostname_routing_down(created.hostname_routing_unavailable.as_deref()); + warn_if_hostname_routing_down( + created.hostname_routing_unavailable.as_deref(), + "min session activate", + ); warn_if_mtls_proxy_down(created.mtls_proxy_unavailable.as_deref()); let id = created.id; diff --git a/crates/minimal/tests/cli.rs b/crates/minimal/tests/cli.rs index 0759218ab7..45ca89224c 100644 --- a/crates/minimal/tests/cli.rs +++ b/crates/minimal/tests/cli.rs @@ -882,6 +882,10 @@ async fn listener_failure_reported_with_remedy() { ls_stderr.contains("retries"), "ls must say the daemon recovers on its own, got: {ls_stderr}" ); + assert!( + ls_stderr.contains("run `min ls` again to check"), + "ls must name the command its warning rides on, got: {ls_stderr}" + ); // `min session activate` prints the same report on its path. let project = tempfile::TempDir::new().unwrap(); @@ -917,6 +921,10 @@ async fn listener_failure_reported_with_remedy() { activate_stderr.contains("Remedy"), "activate must print the remedy, got: {activate_stderr}" ); + assert!( + activate_stderr.contains("run `min session activate` again to check"), + "activate must name the command its warning rides on, got: {activate_stderr}" + ); drop(held); let _ = tokio::time::timeout(std::time::Duration::from_secs(5), retry).await;