diff --git a/crates/minimal/src/cmd/list.rs b/crates/minimal/src/cmd/list.rs index 50d7ce9dbe..4c2d537822 100644 --- a/crates/minimal/src/cmd/list.rs +++ b/crates/minimal/src/cmd/list.rs @@ -304,21 +304,46 @@ pub async fn cmd_ls(global: &GlobalArgs, args: LsArgs) -> Result<(), anyhow::Err // fault this severe — `--raw` most of all, since a script parsing bare ids // is exactly what will go on using hostnames that no longer resolve — and // stdout stays clean for the parser either way. - warn_if_hostname_routing_down(resp.hostname_routing_unavailable.as_deref()); + warn_if_hostname_routing_down(resp.hostname_routing_unavailable.as_deref(), "min ls"); warn_if_mtls_proxy_down(resp.mtls_proxy_unavailable.as_deref()); format_ls(&mut std::io::stdout(), &args, &resp)?; Ok(()) } -/// Tells the user that `.local.min.internal` will not resolve, and why. +/// The warning `min ls` and `min session activate` print when the daemon +/// reports hostname routing down: the daemon's reason and remedy for the +/// failed bind or publish, plus what recovery looks like from here — the +/// daemon retries with backoff and clears the warning on its own once the +/// listener recovers (NET-020, NET-022). `command` names the command the +/// warning rides on: re-running it shows the warning while the listener is +/// still down and nothing once it has recovered, so the recovery sentence +/// tells the user to re-run the command they are already in, not some other +/// one. Pure, so tests can assert the wording without capturing stderr. +#[must_use] +pub fn hostname_routing_warning(reason: &str, command: &str) -> String { + format!( + "warning: session hostnames will not route: {reason}. The daemon retries \ + with backoff and clears this warning on its own once the listener \ + recovers; run `{command}` again to check." + ) +} + +/// Tells the user that `.local.min.internal` will not resolve, and why, +/// and what clears it. /// /// The daemon keeps serving without its host-side proxy, so nothing else the /// user sees is different: sessions activate, exec works, the list prints. The /// only other trace is a `warn!` in the daemon log, which is not where someone -/// watching curl fail is looking (gominimal/inbox#560). -pub(crate) fn warn_if_hostname_routing_down(reason: Option<&str>) { +/// watching curl fail is looking (gominimal/inbox#560). The daemon's report +/// carries the reason and the remedy for the cause (a held port, a failed +/// publish); the recovery is the daemon's job now — it retries with backoff +/// and the warning goes away by itself, so the remedy says to check again +/// rather than to restart anything. `command` names the command this runs +/// from, so the recovery sentence fits it — `min ls` on the list, `min +/// session activate` at activation. +pub(crate) fn warn_if_hostname_routing_down(reason: Option<&str>, command: &str) { if let Some(reason) = reason { - eprintln!("warning: session hostnames will not route: {reason}"); + eprintln!("{}", hostname_routing_warning(reason, command)); } } diff --git a/crates/minimal/src/cmd/session.rs b/crates/minimal/src/cmd/session.rs index 86b413ce40..30019339d5 100644 --- a/crates/minimal/src/cmd/session.rs +++ b/crates/minimal/src/cmd/session.rs @@ -258,7 +258,10 @@ pub(crate) async fn activate_session( // loadout, and the finalize that #1251 died at, and with the session left // unfinalized for the daemon to reap when this connection drops. ensure_version_reported(created.daemon_version.as_deref())?; - warn_if_hostname_routing_down(created.hostname_routing_unavailable.as_deref()); + warn_if_hostname_routing_down( + created.hostname_routing_unavailable.as_deref(), + "min session activate", + ); warn_if_mtls_proxy_down(created.mtls_proxy_unavailable.as_deref()); let id = created.id; diff --git a/crates/minimal/tests/cli.rs b/crates/minimal/tests/cli.rs index 39f2125d98..45ca89224c 100644 --- a/crates/minimal/tests/cli.rs +++ b/crates/minimal/tests/cli.rs @@ -781,6 +781,185 @@ async fn session_policy_succeeds() { .unwrap(); } +// --- hostname routing warning (NET-020/NET-021/NET-022) --- +// +// The startup retry lives in `minimald::server` behind the Linux gate with the +// `net` module it drives, so this section is Linux-only too: it holds a port, +// runs the real retry loop against it, and reads what `min ls` and +// `min session activate` print while the proxy is down and after it recovers. + +/// Runs the compiled `min` with the harness daemon's `--minimal-dir`, an empty +/// `--config-dir` (the developer's own loadouts and policy stay out of the +/// run), and `--no-input`, plus `extra` as the command, and returns stderr. +#[cfg(target_os = "linux")] +async fn run_min_stderr(args: &GlobalArgs, extra: &[&str]) -> String { + let minimal_dir = args + .minimal_dir + .as_ref() + .expect("setup points at a tempdir"); + let config_dir = tempfile::TempDir::new().unwrap(); + let out = tokio::process::Command::new(env!("CARGO_BIN_EXE_min")) + .args(["--minimal-dir".as_ref(), minimal_dir.as_os_str()]) + .args(["--config-dir".as_ref(), config_dir.path().as_os_str()]) + .arg("--no-input") + .args(extra) + .output() + .await + .expect("the min binary should be invocable"); + String::from_utf8_lossy(&out.stderr).into_owned() +} + +/// Polls `ListSessions` until the daemon reports hostname routing down — the +/// startup retry's first failed bind has landed on its note — and returns the +/// reason. +#[cfg(target_os = "linux")] +async fn wait_for_routing_failure(args: &GlobalArgs) -> String { + use minimald_rpc::ListSessions; + + let mut client = connect_daemon(args).await.unwrap(); + for _ in 0..200 { + let resp = client.oneshot_rpc::(()).await.unwrap(); + if let Some(reason) = resp.hostname_routing_unavailable { + return reason; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + panic!("the daemon never reported hostname routing down"); +} + +/// Holds a fresh port and starts the daemon's hostname-proxy startup retry +/// against it — the real loop `start_host_proxies` spawns, with a compressed +/// backoff — so the note `min ls` warns from is set exactly the way an +/// occupied proxy port sets it, without holding the production `:7654`. +/// Dropping the held listener frees the port; the returned task resolves once +/// the proxy is serving again and the note is cleared. +#[cfg(target_os = "linux")] +async fn hold_proxy_port_and_start_retry( + state: &minimald::server::ServerStateHandle, +) -> (tokio::net::TcpListener, tokio::task::JoinHandle<()>) { + use minimald::server::{RetryBackoff, retry_hostname_proxy_until_serving}; + + let held = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = held.local_addr().unwrap(); + let retry = tokio::spawn(retry_hostname_proxy_until_serving( + state.clone(), + addr, + RetryBackoff::new( + std::time::Duration::from_millis(5), + std::time::Duration::from_millis(40), + ), + )); + (held, retry) +} + +/// NET-020: with the proxy port occupied, both `min ls` and `min session +/// activate` print the failure's reason and the remedy for it, and say the +/// daemon recovers on its own. Driven through the compiled binary so the +/// assertion is on what the user actually sees. +#[cfg(target_os = "linux")] +#[tokio::test] +async fn listener_failure_reported_with_remedy() { + let (daemon, args) = setup().await; + let (held, retry) = hold_proxy_port_and_start_retry(&daemon.server.state).await; + let reason = wait_for_routing_failure(&args).await; + + let ls_stderr = run_min_stderr(&args, &["ls"]).await; + assert!( + ls_stderr.contains("warning: session hostnames will not route"), + "ls must warn about hostname routing, got: {ls_stderr}" + ); + assert!( + ls_stderr.contains(&reason), + "ls must print the daemon's reason, got: {ls_stderr}" + ); + assert!( + ls_stderr.contains("Remedy"), + "ls must print the remedy, got: {ls_stderr}" + ); + assert!( + ls_stderr.contains("retries"), + "ls must say the daemon recovers on its own, got: {ls_stderr}" + ); + assert!( + ls_stderr.contains("run `min ls` again to check"), + "ls must name the command its warning rides on, got: {ls_stderr}" + ); + + // `min session activate` prints the same report on its path. + let project = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(project.path().join(".git")).unwrap(); + std::fs::write( + project.path().join("minimal.toml"), + "# test minimal.toml\n[upstream]\nrepo = \"https://github.com/gominimal/pkgs\"\nbranch = \"main\"\n\n[stack]\nuse = \"shell\"\n", + ) + .unwrap(); + let activate_stderr = run_min_stderr( + &args, + &[ + "session", + "activate", + project.path().to_str().unwrap(), + "--name", + "remedy-report", + "--sync", + "tarball", + "--no-prompt", + ], + ) + .await; + assert!( + activate_stderr.contains("warning: session hostnames will not route"), + "activate must warn about hostname routing, got: {activate_stderr}" + ); + assert!( + activate_stderr.contains(&reason), + "activate must print the daemon's reason, got: {activate_stderr}" + ); + assert!( + activate_stderr.contains("Remedy"), + "activate must print the remedy, got: {activate_stderr}" + ); + assert!( + activate_stderr.contains("run `min session activate` again to check"), + "activate must name the command its warning rides on, got: {activate_stderr}" + ); + + drop(held); + let _ = tokio::time::timeout(std::time::Duration::from_secs(5), retry).await; +} + +/// NET-022: once the port frees and the listener recovers, `min ls` stops +/// carrying the warning — against the same daemon process, with no restart. +#[cfg(target_os = "linux")] +#[tokio::test] +async fn ls_warning_clears_on_recovery() { + let (daemon, args) = setup().await; + let (held, retry) = hold_proxy_port_and_start_retry(&daemon.server.state).await; + let _reason = wait_for_routing_failure(&args).await; + + let before = run_min_stderr(&args, &["ls"]).await; + assert!( + before.contains("warning: session hostnames will not route"), + "ls must warn while the port is held, got: {before}" + ); + + // Free the port: the startup retry binds (a native daemon has no + // host-loopback publish gate) and clears the note. + drop(held); + tokio::time::timeout(std::time::Duration::from_secs(5), retry) + .await + .expect("the startup retry must finish once the port frees") + .expect("the startup retry must not panic"); + + let after = run_min_stderr(&args, &["ls"]).await; + assert!( + !after.contains("warning: session hostnames will not route"), + "the warning must clear without a daemon restart, got: {after}" + ); +} + // --- helpers --- /// Creates a session whose workspace mfile declares a `[session.vars]` diff --git a/crates/minimald/src/net/proxy.rs b/crates/minimald/src/net/proxy.rs index f8861c2f91..5d2e7ebcb3 100644 --- a/crates/minimald/src/net/proxy.rs +++ b/crates/minimald/src/net/proxy.rs @@ -158,11 +158,38 @@ fn split_authority(authority: &str) -> (&str, Option) { (host, port) } -/// Binds the egress-proxy listener at `addr`, returning it on success. On a bind -/// failure it emits the `component = "dns-proxy"` reachability warning and -/// returns `None`: the proxy is unavailable and PTask hostnames will not route -/// until the address is free. This is the daemon-startup reachability check that -/// supersedes the former systemd-resolved probe (R3.4). +/// Why a host-side proxy listener could not bind its address, carrying the +/// reason and the one remedy that clears it. +/// +/// The pair is authored once here so every surface reads the same text: the +/// startup retry's warning, the daemon's `proxy_unavailable` note the +/// `ListSessions` RPC serves, and the warning `min ls` and +/// `min session activate` print (NET-020). +#[derive(Debug, Clone)] +pub struct BindFailure { + /// What failed, named for a human: the address and the OS error. + pub reason: String, + /// The remedy: free the listen address so the daemon's retry can bind it. + pub remedy: String, +} + +impl BindFailure { + /// The reason and remedy as the one report the daemon's unavailable note + /// and the CLI warning carry. + #[must_use] + pub fn reported(&self) -> String { + format!("{}. Remedy: {}", self.reason, self.remedy) + } +} + +/// Binds the egress-proxy listener at `addr`, returning it on success. On a +/// bind failure it returns a [`BindFailure`] carrying the reason (the address +/// and the OS error) and the remedy that clears it, and logs nothing: the +/// caller owns the failure's log line, because only it knows the retry +/// schedule that line reports — `server::start_host_proxies` retries with +/// backoff until the bind succeeds (NET-021), so one failed bind is one +/// warning, not a silent fallback. This is the daemon-startup reachability +/// check that supersedes the former systemd-resolved probe (R3.4). /// /// The returned listener is the caller's to either serve (via [`serve`]) or /// drop. The success event reports the address as `reachable` rather than @@ -171,7 +198,12 @@ fn split_authority(authority: &str) -> (&str, Option) { /// serve it (see `server::start_host_proxies`); this said otherwise, and reading /// it as a bind-and-drop probe is what made gominimal/inbox#560 look like a /// false alarm on macOS. -pub async fn bind_listener(addr: SocketAddr) -> Option { +/// +/// # Errors +/// +/// Returns a [`BindFailure`] when the address cannot be bound; the OS error is +/// carried inside the failure's reason. +pub async fn bind_listener(addr: SocketAddr) -> Result { match TcpListener::bind(addr).await { Ok(listener) => { tracing::info!( @@ -180,19 +212,15 @@ pub async fn bind_listener(addr: SocketAddr) -> Option { status = "reachable", "host-side egress proxy listen address is bindable" ); - Some(listener) - } - Err(error) => { - tracing::warn!( - component = "dns-proxy", - %addr, - status = "unavailable", - error = %error, - remedy = "free the listen address; PTask *.min.internal hostnames will not route until the egress proxy can bind", - "host-side egress proxy could not bind its listener" - ); - None + Ok(listener) } + Err(error) => Err(BindFailure { + reason: format!("the daemon could not bind {addr}: {error}"), + remedy: format!( + "free the listen address; `lsof -nP -iTCP:{} -sTCP:LISTEN` names the holder", + addr.port() + ), + }), } } @@ -1058,15 +1086,16 @@ mod tests { let listener = bind_listener(addr).await; drop(guard); - assert!(listener.is_none(), "a bind to a held address must fail"); - let logged = buf.contents(); + let failure = listener.expect_err("a bind to a held address must fail"); assert!( - logged.contains(r#"component="dns-proxy""#), - "expected the dns-proxy component field, got: {logged}" + failure.reported().contains("could not bind"), + "the failure must name the bind failure, got: {}", + failure.reported() ); + let logged = buf.contents(); assert!( - logged.contains(r#"status="unavailable""#), - "expected the unavailable status field, got: {logged}" + logged.is_empty(), + "bind_listener must not log on failure; the caller owns the log line, got: {logged}" ); } diff --git a/crates/minimald/src/server.rs b/crates/minimald/src/server.rs index adb84816b8..12bf4f5689 100644 --- a/crates/minimald/src/server.rs +++ b/crates/minimald/src/server.rs @@ -6,6 +6,14 @@ use std::path::PathBuf; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::net::{UnixListener, UnixStream}; +// The host-side proxies and their startup retry live behind the Linux gate +// with the `net` module they route against. +#[cfg(target_os = "linux")] +use std::net::SocketAddr; +#[cfg(target_os = "linux")] +use std::time::Duration; +#[cfg(target_os = "linux")] +use tokio::net::TcpListener; use tokio::sync::Mutex; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; @@ -327,12 +335,27 @@ impl ServerStateHandle { self.0.lock().await.proxy_unavailable.clone() } + /// Clears the hostname-routing unavailability note: the proxy's startup + /// retry bound and published it, so `ListSessions` stops reporting the + /// reason and `min ls` stops printing the warning — without a daemon + /// restart (NET-022). + pub(crate) async fn clear_proxy_unavailable(&self) { + self.0.lock().await.proxy_unavailable = None; + } + /// Records why the mTLS reverse proxy is not serving. #[cfg_attr(not(feature = "networking-proxy"), expect(dead_code))] pub(crate) async fn set_mtls_unavailable(&self, reason: String) { self.0.lock().await.mtls_unavailable = Some(reason); } + /// Clears the mTLS unavailability note: the proxy's startup retry bound + /// and published it. + #[cfg_attr(not(feature = "networking-proxy"), expect(dead_code))] + pub(crate) async fn clear_mtls_unavailable(&self) { + self.0.lock().await.mtls_unavailable = None; + } + /// Why the mTLS reverse proxy is not serving, or `None` if it is. pub(crate) async fn mtls_unavailable(&self) -> Option { self.0.lock().await.mtls_unavailable.clone() @@ -739,15 +762,19 @@ async fn reap_unfinalized_sessions(state: &ServerStateHandle, ids: Vec<::session /// PTask hostname registry. In a microVM they bind the daemon's switch IP /// ([`DEFAULT_SUBNET`](crate::net::DEFAULT_SUBNET)`.daemon_ip()`) so the host /// gvproxy forward can reach them; on native Linux (DM2) they bind host loopback -/// directly. A bind failure warns and is skipped — the daemon keeps serving. The -/// serve loops run on detached tasks; this returns once the listeners are bound -/// and (DM1) exposed. +/// directly. +/// +/// Each proxy's startup — the bind, and in a microVM the host-loopback +/// publish — runs on a detached task that retries with backoff until it +/// succeeds ([`drive_proxy_until_serving`], NET-021) and then clears the +/// unavailable note `min ls` warns from (NET-022). Nothing here is awaited: +/// a proxy whose port some other process holds must not hold the SSH accept +/// loop hostage — the daemon starts serving regardless, reports the reason on +/// its state, and the proxy comes up on its own once the address frees. #[cfg(target_os = "linux")] async fn start_host_proxies(state: &ServerStateHandle, in_microvm: bool) { - use crate::net::proxy::{self, Router}; - use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + use std::net::{IpAddr, Ipv4Addr}; - let registry = state.sessions_manager().await.hostnames(); // DM1 (in-VM): bind 0.0.0.0 so the listener comes up regardless of whether // eth0 has finished coming up, then publish the port on the host loopback via // the gvproxy forwarder. DM2: bind host loopback directly, no host-expose. @@ -761,41 +788,12 @@ async fn start_host_proxies(state: &ServerStateHandle, in_microvm: bool) { // `.min.internal` not routing, so both are recorded on the // state where `ListSessions` can reach them — a daemon that keeps serving // without its proxy looks identical to a healthy one otherwise. - let egress_addr = SocketAddr::new(bind_base, proxy::EGRESS_PROXY_PORT); - let bound = proxy::bind_listener(egress_addr) - .await - .map(|listener| { - let router = Router::new(registry.clone()); - tokio::spawn(async move { - if let Err(error) = proxy::serve(listener, router).await { - tracing::error!(%error, "egress proxy accept loop exited"); - } - }) - }) - .is_some(); - - if !bound { - // DM2: something else on the host holds the port. - state - .set_proxy_unavailable(format!( - "the daemon could not bind {egress_addr}; another process is \ - holding it. Check with: lsof -nP -iTCP:{} -sTCP:LISTEN", - proxy::EGRESS_PROXY_PORT - )) - .await; - } else if in_microvm { - // DM1: the guest bind cannot collide with a host process, so the - // failure moves to the publish instead. Only publish a port whose - // listener actually bound. - if let Some(reason) = expose_proxy_on_host( - crate::net::DEFAULT_SUBNET.daemon_ip(), - proxy::EGRESS_PROXY_PORT, - ) - .await - { - state.set_proxy_unavailable(reason).await; - } - } + let egress_addr = SocketAddr::new(bind_base, crate::net::proxy::EGRESS_PROXY_PORT); + tokio::spawn(drive_proxy_until_serving( + state.clone(), + HostProxyStartup::Egress { addr: egress_addr }, + RetryBackoff::production(), + )); // B8 mTLS reverse proxy (:7655), under the networking-proxy feature. #[cfg(feature = "networking-proxy")] @@ -804,41 +802,20 @@ async fn start_host_proxies(state: &ServerStateHandle, in_microvm: bool) { // were silent: the TLS config failing to build, the bind failing, and // the publish failing. The daemon carries on in every case, so only a // reported reason distinguishes "no mTLS proxy configured" from "the - // mTLS proxy is broken". - let https_addr = SocketAddr::new(bind_base, proxy::HTTPS_PROXY_PORT); + // mTLS proxy is broken". A TLS config that cannot build is not + // retryable — the CA material does not change under the retry — so it + // is reported once, here, and only the bind and publish retry. + let https_addr = SocketAddr::new(bind_base, crate::net::proxy::HTTPS_PROXY_PORT); match state.cert_authority().await.build_server_config() { Ok(tls_config) => { - let bound = proxy::bind_listener(https_addr) - .await - .map(|listener| { - let router = Router::new(registry.clone()); - tokio::spawn(async move { - if let Err(error) = - proxy::serve_https(listener, router, tls_config).await - { - tracing::error!(%error, "mTLS proxy accept loop exited"); - } - }) - }) - .is_some(); - - if !bound { - state - .set_mtls_unavailable(format!( - "the daemon could not bind {https_addr}; another process is \ - holding it. Check with: lsof -nP -iTCP:{} -sTCP:LISTEN", - proxy::HTTPS_PROXY_PORT - )) - .await; - } else if in_microvm - && let Some(reason) = expose_proxy_on_host( - crate::net::DEFAULT_SUBNET.daemon_ip(), - proxy::HTTPS_PROXY_PORT, - ) - .await - { - state.set_mtls_unavailable(reason).await; - } + tokio::spawn(drive_proxy_until_serving( + state.clone(), + HostProxyStartup::Mtls { + addr: https_addr, + tls_config, + }, + RetryBackoff::production(), + )); } Err(error) => { tracing::warn!(%error, "could not build TLS config for the mTLS reverse proxy"); @@ -853,28 +830,267 @@ async fn start_host_proxies(state: &ServerStateHandle, in_microvm: bool) { } } -/// Upper bound on the best-effort host-loopback publish in +/// The retry schedule a host-side proxy's startup uses while its bind (or, in +/// a microVM, its host-loopback publish) keeps failing: each retry waits twice +/// as long as the one before, from `initial` up to `max`. Tests pass a +/// compressed schedule so a retry loop's worth of failures costs milliseconds +/// instead of seconds. +#[cfg(target_os = "linux")] +#[derive(Debug, Clone, Copy)] +pub struct RetryBackoff { + initial: Duration, + max: Duration, +} + +#[cfg(target_os = "linux")] +impl RetryBackoff { + /// Builds a schedule that doubles from `initial` and never waits longer + /// than `max`. + #[must_use] + pub const fn new(initial: Duration, max: Duration) -> Self { + Self { initial, max } + } + + /// The daemon's schedule: the first retry 1 s after the failure, doubling + /// to a 30 s ceiling. Fast enough that a port freed moments after boot is + /// serving again in seconds; slow enough that a port held for good costs + /// one warning per 30 s, not a spin. + #[must_use] + pub const fn production() -> Self { + Self::new(Duration::from_secs(1), Duration::from_secs(30)) + } + + /// The wait before retry `attempt` (0-based: the wait after the first + /// failure), doubling from `initial` and capped at `max`. + fn delay(&self, attempt: u32) -> Duration { + let factor = 1u32.checked_shl(attempt.min(30)).unwrap_or(u32::MAX); + self.initial + .checked_mul(factor) + .unwrap_or(self.max) + .min(self.max) + } +} + +/// Which host-side proxy a startup retry drives. The two differ in what serves +/// a bound listener and which state note a failure lands on; the retry loop +/// itself is shared. +#[cfg(target_os = "linux")] +#[derive(Debug)] +enum HostProxyStartup { + /// The B5 egress/DNS proxy: plain HTTP routing through the shared router. + Egress { addr: SocketAddr }, + /// The B8 mTLS reverse proxy: the same routing behind TLS termination. + #[cfg(feature = "networking-proxy")] + Mtls { + addr: SocketAddr, + tls_config: Arc, + }, +} + +#[cfg(target_os = "linux")] +impl HostProxyStartup { + /// The address the listener binds. + fn addr(&self) -> SocketAddr { + match self { + Self::Egress { addr } => *addr, + #[cfg(feature = "networking-proxy")] + Self::Mtls { addr, .. } => *addr, + } + } + + /// The `component` log field this proxy's startup events carry, matching + /// what its serve loop logs. + fn component(&self) -> &'static str { + match self { + Self::Egress { .. } => "dns-proxy", + #[cfg(feature = "networking-proxy")] + Self::Mtls { .. } => "https-proxy", + } + } + + /// Spawns the serve loop for a bound listener. Runs for the daemon's + /// lifetime; the startup retry never rebinds a bound-and-served listener. + async fn spawn_serve(&self, state: &ServerStateHandle, listener: TcpListener) { + #[cfg(feature = "networking-proxy")] + use crate::net::proxy::serve_https; + use crate::net::proxy::{Router, serve}; + + let router = Router::new(state.sessions_manager().await.hostnames()); + match self { + Self::Egress { .. } => { + tokio::spawn(async move { + if let Err(error) = serve(listener, router).await { + tracing::error!(%error, "egress proxy accept loop exited"); + } + }); + } + #[cfg(feature = "networking-proxy")] + Self::Mtls { tls_config, .. } => { + let tls_config = Arc::clone(tls_config); + tokio::spawn(async move { + if let Err(error) = serve_https(listener, router, tls_config).await { + tracing::error!(%error, "mTLS proxy accept loop exited"); + } + }); + } + } + } + + /// Records a failure's reason-and-remedy report on the state note the + /// `ListSessions` RPC serves (NET-020). + async fn record_unavailable(&self, state: &ServerStateHandle, report: String) { + match self { + Self::Egress { .. } => state.set_proxy_unavailable(report).await, + #[cfg(feature = "networking-proxy")] + Self::Mtls { .. } => state.set_mtls_unavailable(report).await, + } + } + + /// Clears the failure note: bound and published, the proxy is serving. + async fn clear_unavailable(&self, state: &ServerStateHandle) { + match self { + Self::Egress { .. } => state.clear_proxy_unavailable().await, + #[cfg(feature = "networking-proxy")] + Self::Mtls { .. } => state.clear_mtls_unavailable().await, + } + } +} + +/// Drives the hostname-routing proxy (the B5 egress proxy — the listener +/// `*.min.internal` hostnames route through) to serving: binds `addr`, +/// retrying bind — and, in a microVM, the host-loopback publish — with +/// `retry`'s backoff until both succeed, then keeps serving and clears the +/// daemon's `proxy_unavailable` note so `min ls` stops warning (NET-021, +/// NET-022). +/// +/// Detached for the daemon's lifetime by `start_host_proxies`; also spawned +/// directly by tests, which hold the address and watch the retry recover. +#[cfg(any(test, feature = "test-support"))] +#[cfg(target_os = "linux")] +pub async fn retry_hostname_proxy_until_serving( + state: ServerStateHandle, + addr: SocketAddr, + retry: RetryBackoff, +) { + drive_proxy_until_serving(state, HostProxyStartup::Egress { addr }, retry).await; +} + +/// Drives one host-side proxy to serving, retrying with backoff (NET-021). +/// +/// Two gates stand between daemon start and a serving proxy, in order: the +/// listener must bind, and — in a microVM (DM1), where the listener binds +/// inside the guest — the port must then be published on the host loopback via +/// the gvproxy forwarder. Each failed attempt logs one warning carrying the +/// failure's reason and remedy and the next retry delay, and records it on the +/// state note the `ListSessions` RPC serves, so `min ls` and +/// `min session activate` print it (NET-020). Once both gates pass the note is +/// cleared and — when any attempt failed — one info line marks the recovery +/// (NET-022): the warning disappears from `min ls` without a daemon restart. +/// +/// The serve loop starts as soon as the listener binds and stays up while the +/// publish retries; the bind gate never runs again once it has passed, so a +/// bound-and-served listener is never dropped and rebound. +#[cfg(target_os = "linux")] +async fn drive_proxy_until_serving( + state: ServerStateHandle, + proxy: HostProxyStartup, + retry: RetryBackoff, +) { + let addr = proxy.addr(); + let component = proxy.component(); + // DM1 only: the bind happens in-guest, so the host loopback is reachable + // only through the gvproxy forwarder's publish. DM2 binds host loopback + // directly and has no second gate. + let publish_port = state.in_microvm().await.then_some(addr.port()); + + let mut bound = false; + let mut attempt: u32 = 0; + let mut failed_before = false; + loop { + if !bound { + match crate::net::proxy::bind_listener(addr).await { + Ok(listener) => { + proxy.spawn_serve(&state, listener).await; + bound = true; + if publish_port.is_none() { + break; + } + } + Err(failure) => { + let report = failure.reported(); + let next_retry = retry.delay(attempt); + tracing::warn!( + component, + %addr, + status = "unavailable", + reason = %report, + next_retry = ?next_retry, + "host-side proxy could not bind its listener; retrying with backoff" + ); + proxy.record_unavailable(&state, report).await; + failed_before = true; + attempt += 1; + tokio::time::sleep(next_retry).await; + continue; + } + } + } + // Bound and serving. Only the host-loopback publish can still be + // pending: a bind success with no publish gate broke out above. + let Some(port) = publish_port else { + break; + }; + match expose_proxy_on_host(crate::net::DEFAULT_SUBNET.daemon_ip(), port).await { + None => break, + Some(report) => { + let next_retry = retry.delay(attempt); + tracing::warn!( + component, + %port, + status = "unavailable", + %report, + next_retry = ?next_retry, + "host-side proxy could not publish on the host loopback; retrying with backoff" + ); + proxy.record_unavailable(&state, report).await; + failed_before = true; + attempt += 1; + tokio::time::sleep(next_retry).await; + } + } + } + + if failed_before { + tracing::info!( + component, + %addr, + status = "recovered", + "host-side proxy is serving after retrying" + ); + } + proxy.clear_unavailable(&state).await; +} + +/// Upper bound on one host-loopback publish attempt in /// [`expose_proxy_on_host`]. Deliberately far below `post_json`'s gvproxy -/// control timeout: the publish is awaited on [`Server::run`]'s boot path -/// *before* the SSH accept loop starts serving. When the forwarder control -/// request does not complete promptly — it times out at the full 5 s even with a -/// host gvproxy present, since the forwarder control path is not reachable over -/// the shuttle in every deployment — the accept loop must not be held that long: -/// the cold `minimal ls` connect-retry deadline expires first and the first list -/// fails (`ssh connect: Disconnected`). A reachable forwarder answers in well -/// under this bound. +/// control timeout: the startup retry's failed attempts must not each cost the +/// full 5 s before the next try — when the forwarder control path is not +/// reachable over the shuttle in every deployment, a retry every 5 s would +/// spend nearly all its time waiting instead of checking. A reachable +/// forwarder answers in well under this bound. #[cfg(target_os = "linux")] const HOST_EXPOSE_PUBLISH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1); /// Publishes a guest-side proxy bound on `daemon_ip:port` onto the macOS host's /// loopback (`127.0.0.1:port`) via the host gvproxy forwarder, reached over the /// vsock shuttle (DM1). Best-effort in that it never fails the daemon, since -/// the host gvproxy may be absent. Capped at [`HOST_EXPOSE_PUBLISH_TIMEOUT`] so -/// it never stalls [`Server::run`]'s SSH accept loop. +/// the host gvproxy may be absent; capped at [`HOST_EXPOSE_PUBLISH_TIMEOUT`] +/// per attempt so a stalled forwarder cannot stretch the retry cadence. /// -/// Returns `Some(reason)` when the publish did not happen, so the caller can -/// tell a client rather than leaving the loss in the daemon log — on DM1 this -/// is the path a host process holding the port actually breaks. +/// Returns `Some(report)` when the publish did not happen — the reason and the +/// remedy as one text, which the startup retry logs with its next retry and +/// records on the state note for `min ls` / `min session activate` to print — +/// and `None` once published. #[cfg(target_os = "linux")] async fn expose_proxy_on_host(daemon_ip: std::net::Ipv4Addr, port: u16) -> Option { use crate::net::policy::{ControlChannel, ExposeRequest, post_json}; @@ -895,28 +1111,16 @@ async fn expose_proxy_on_host(daemon_ip: std::net::Ipv4Addr, port: u16) -> Optio .await { Ok(Ok(_)) => None, - Ok(Err(error)) => { - tracing::warn!( - %port, - %error, - "could not publish host-side proxy on the host loopback via gvproxy forwarder" - ); - Some(format!( - "the daemon could not publish port {port} on the host loopback \ - via the gvproxy forwarder: {error}" - )) - } - Err(_) => { - tracing::warn!( - %port, - timeout = ?HOST_EXPOSE_PUBLISH_TIMEOUT, - "host-side proxy publish did not complete in time; continuing (best-effort)" - ); - Some(format!( - "publishing port {port} on the host loopback did not complete within \ - {HOST_EXPOSE_PUBLISH_TIMEOUT:?}" - )) - } + Ok(Err(error)) => Some(format!( + "the daemon could not publish port {port} on the host loopback via \ + the gvproxy forwarder: {error}. Remedy: check that the host \ + gvproxy (minvmd) is running and reachable over the shuttle" + )), + Err(_) => Some(format!( + "publishing port {port} on the host loopback did not complete within \ + {HOST_EXPOSE_PUBLISH_TIMEOUT:?}. Remedy: check that the host \ + gvproxy (minvmd) is running and reachable over the shuttle" + )), } } @@ -1098,4 +1302,149 @@ mod tests { .await; let _ = tokio::time::timeout(Duration::from_secs(5), run).await; } + + /// A `MakeWriter` accumulating everything written into a shared buffer, so + /// a test can assert on the structured fields a `tracing` event emitted. + /// Local twin of the helper the `net::proxy` tests used before the bind + /// failure's log line moved to the startup retry that owns the schedule. + #[cfg(target_os = "linux")] + #[derive(Clone, Default)] + struct CaptureWriter(Arc>>); + + #[cfg(target_os = "linux")] + impl CaptureWriter { + fn contents(&self) -> String { + String::from_utf8(self.0.lock().unwrap().clone()).unwrap() + } + } + + #[cfg(target_os = "linux")] + impl std::io::Write for CaptureWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + #[cfg(target_os = "linux")] + impl tracing_subscriber::fmt::MakeWriter<'_> for CaptureWriter { + type Writer = CaptureWriter; + fn make_writer(&self) -> Self::Writer { + self.clone() + } + } + + /// The `next_retry=` values the startup retry's failure warnings + /// carried, in the order they were logged. + #[cfg(target_os = "linux")] + fn logged_next_retries(log: &str) -> Vec<&str> { + log.match_indices("next_retry=") + .map(|(start, _)| { + let rest = &log[start + "next_retry=".len()..]; + rest.split([' ', '\n', ',']).next().unwrap_or_default() + }) + .collect() + } + + /// The hostname proxy's startup retry keeps trying a held listen address: + /// each failed bind warns once with the reason, the remedy, and the next + /// (doubling, capped) retry delay and records the reason-and-remedy report + /// on the state note (NET-021); once the address frees, the next attempt + /// binds, the note clears, and the recovery is logged (NET-022) — no + /// daemon restart anywhere. + #[cfg(target_os = "linux")] + #[tokio::test] + async fn listener_retries_with_backoff() { + // Hold an address so the startup retry's binds fail deterministically. + let held = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = held.local_addr().unwrap(); + + let buf = CaptureWriter::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(buf.clone()) + .with_ansi(false) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + + let dir = TempDir::new().unwrap(); + let state = ServerStateHandle::new(test_config(&dir), None) + .await + .unwrap(); + // A compressed schedule: first retry 5 ms after the failure, doubling + // to a 20 ms cap, so a loop's worth of failures costs milliseconds. + let retrier = tokio::spawn(drive_proxy_until_serving( + state.clone(), + HostProxyStartup::Egress { addr }, + RetryBackoff::new(Duration::from_millis(5), Duration::from_millis(20)), + )); + + // Three failures are enough to see the doubling and the cap. + let mut saw_three_warnings = false; + for _ in 0..200 { + if buf + .contents() + .matches("could not bind its listener") + .count() + >= 3 + { + saw_three_warnings = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!( + saw_three_warnings, + "expected three failure warnings while the address is held, got: {}", + buf.contents() + ); + let note = state + .proxy_unavailable() + .await + .expect("a held address must be recorded as the unavailable note"); + assert!( + note.contains("could not bind") && note.contains("Remedy"), + "the note must carry the reason and the remedy, got: {note}" + ); + + // The retry delays double from the schedule's initial wait and cap at + // its max: 5 ms, 10 ms, then 20 ms for as long as the address stays + // held. + let logged = buf.contents(); + let delays = logged_next_retries(&logged); + assert_eq!( + &delays[..3.min(delays.len())], + &["5ms", "10ms", "20ms"][..3.min(delays.len())], + "retry delays must grow with backoff and cap, got: {delays:?}" + ); + assert!( + delays.iter().skip(3).all(|d| *d == "20ms"), + "retries past the cap must wait the cap, got: {delays:?}" + ); + + // Freeing the address lets the next attempt bind: the retry task + // resolves, the note clears, and the recovery is on the log. + drop(held); + tokio::time::timeout(Duration::from_secs(5), retrier) + .await + .expect("the retry must finish once the address frees") + .expect("the retry task must not panic"); + assert!( + state.proxy_unavailable().await.is_none(), + "recovery must clear the unavailable note without a restart" + ); + let logged = buf.contents(); + assert!( + logged.contains(r#"status="recovered""#), + "the recovery must be logged, got: {logged}" + ); + assert!( + logged.contains(r#"status="reachable""#), + "the successful bind must be logged as reachable, got: {logged}" + ); + } }