From 5e083ab408cc436883c060a3ccb29cbfe362c47a Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:16:57 +0000 Subject: [PATCH 1/6] fix(minimald): bound host mailbox sends against a wedged host HostHandle::attach and HostHandle::kill posted to the session host mailbox with an unbounded send. Once the host loop stops draining the mailbox (capacity HOST_MAILBOX_CAPACITY), it fills within a few polls and the next send parks forever: min session attach hangs with no timeout, and min stop stalls daemon shutdown because stop_running awaits that kill and then the wedged loop's task. Bound both sends with the HOST_PROBE_TIMEOUT already applied to the host probes. A missed attach deadline flows into the existing host-error branch that re-mints the host, and stop_running aborts the host task when the kill cannot land so shutdown proceeds. Co-Authored-By: Claude Opus 4.8 --- crates/minimald/src/session.rs | 14 +++++-- crates/minimald/src/session_host.rs | 59 ++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index d3a7d5ebb0..7fbc0340eb 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -1629,9 +1629,15 @@ impl Session { if let Some((host, task)) = host { // Signal the process to die, then await the runtime loop so the // sandbox files backing its rootfs are released before the caller - // removes the session's directory tree. - let _ = host.kill(for_shutdown).await; - let _ = task.await; + // removes the session's directory tree. The kill is bounded: a + // wedged loop never accepts it, and awaiting such a loop would + // park daemon shutdown behind it forever, so a kill that does + // not land aborts the loop rather than waiting on it. + if host.kill(for_shutdown).await.is_ok() { + let _ = task.await; + } else { + task.abort(); + } } } } @@ -1873,7 +1879,7 @@ impl Session { { Ok(()) => Ok(()), Err((channel, sz)) => { - // session host is dead + // The host is gone, or wedged past the attach deadline. self.mint_session_host( session_hnd, conn_username, diff --git a/crates/minimald/src/session_host.rs b/crates/minimald/src/session_host.rs index df51bbb717..dedf34f136 100644 --- a/crates/minimald/src/session_host.rs +++ b/crates/minimald/src/session_host.rs @@ -18,7 +18,7 @@ use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; use std::time::SystemTime; use tokio::io::AsyncWriteExt; use tokio::io::unix::AsyncFd; -use tokio::sync::mpsc::error::SendError; +use tokio::sync::mpsc::error::{SendError, SendTimeoutError}; use tokio::sync::{mpsc, oneshot}; use tokio::task::JoinHandle; use tracing::Instrument as _; @@ -1350,9 +1350,20 @@ impl HostHandle { } pub async fn kill(&self, for_shutdown: bool) -> Result<(), ()> { - match self.sender.send(Message::Kill(for_shutdown)).await { + match self + .sender + .send_timeout( + Message::Kill(for_shutdown), + crate::session::HOST_PROBE_TIMEOUT, + ) + .await + { Ok(()) => Ok(()), - Err(_e) => Err(()), // closed + // A closed channel, or a wedged loop that never drains its mailbox + // so the send cannot be queued before the deadline: either way the + // kill did not land, and the caller must not block awaiting a loop + // that will never observe it. + Err(_e) => Err(()), } } /// Binds `c` to this host, carrying the attaching terminal's facts. @@ -1373,11 +1384,18 @@ impl HostHandle { ) -> Result<(), (Channel, WinSize)> { match self .sender - .send(Message::Attach(c, sz, connection, keys)) + .send_timeout( + Message::Attach(c, sz, connection, keys), + crate::session::HOST_PROBE_TIMEOUT, + ) .await { Ok(()) => Ok(()), - Err(SendError(Message::Attach(c, sz, _, _))) => Err((c, sz)), + // A wedged loop that never drains its mailbox (the send times out) + // and a loop that has gone (the channel is closed) both hand the + // channel back; the caller re-mints a host from it either way. + Err(SendTimeoutError::Timeout(Message::Attach(c, sz, _, _))) + | Err(SendTimeoutError::Closed(Message::Attach(c, sz, _, _))) => Err((c, sz)), Err(e) => unreachable!("{:?}", e), } } @@ -4531,6 +4549,37 @@ mod tests { ); } + /// A kill aimed at a host whose loop has stopped draining its mailbox + /// gives up on its own deadline instead of parking forever. Left + /// unbounded, this send blocked once the mailbox filled — the wedged-host + /// hang behind a stuck `min stop` and, via the same send, `min session + /// attach`. + #[tokio::test(start_paused = true)] + async fn kill_to_a_wedged_host_gives_up_instead_of_parking() { + // Holding the mailbox is what makes the host wedged: it accepts + // messages up to its capacity and never drains one. + let (host, _mailbox) = HostHandle::wedged(); + + // Queue past the mailbox so the next send has to block trying to + // enqueue at all — the case an unbounded send never returned from. + for _ in 0..HOST_MAILBOX_CAPACITY { + host.kill(false) + .await + .expect("queuing into an open mailbox should succeed"); + } + + // Far longer than the send's own deadline, so under the paused clock + // the send's timeout is the one that fires; reaching this outer bound + // would mean the send had no deadline at all. + let outcome = tokio::time::timeout(Duration::from_secs(600), host.kill(false)) + .await + .expect("a bounded kill must return on its own deadline, not park forever"); + assert!( + outcome.is_err(), + "a kill that cannot be queued before the deadline must report failure", + ); + } + /// A [`sandbox2::NetGuard`] that records whether its teardown ran, so a test /// can assert the session's network is released exactly when the shell /// process ends — and left up while it is merely detached. From 348f2f359c88cf82dfa6c0f220a35bc969e3cb33 Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:26:41 +0000 Subject: [PATCH 2/6] chore(worker-iterate): address CodeRabbit findings on PR #1369 --- crates/minimald/src/session.rs | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index 7fbc0340eb..588ba38e1d 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -1626,16 +1626,33 @@ impl Session { for s in sops.drain(..) { s.shutdown().await; } - if let Some((host, task)) = host { + if let Some((host, mut task)) = host { // Signal the process to die, then await the runtime loop so the // sandbox files backing its rootfs are released before the caller - // removes the session's directory tree. The kill is bounded: a - // wedged loop never accepts it, and awaiting such a loop would - // park daemon shutdown behind it forever, so a kill that does - // not land aborts the loop rather than waiting on it. - if host.kill(for_shutdown).await.is_ok() { - let _ = task.await; - } else { + // removes the session's directory tree. Both the kill and the + // wait for the loop to act on it are bounded: `send_timeout` + // only bounds the wait for mailbox *capacity*, so a loop parked + // mid-`step()` (mailbox nearly empty) queues the kill yet never + // processes it, and awaiting that loop unbounded would park + // daemon shutdown behind it forever. So a kill that cannot be + // queued, or a loop that does not finish within + // `HOST_PROBE_TIMEOUT` of accepting it, aborts the loop instead + // of waiting on it. + // + // Aborting drops the loop at its await point, skipping the + // awaited `NetGuard` teardown in `Host::mainloop`; the wedged + // host's sandbox process and network are orphaned rather than + // reclaimed here. That leak is the acknowledged follow-up in + // this PR's description ("Directly reclaiming a wedged host's + // sandbox process ... is left as follow-up"): cancellation-safe + // ownership of the `NetGuard` is out of scope for bounding the + // shutdown wait, which is what actually unblocks `min stop`. + let killed = host.kill(for_shutdown).await.is_ok(); + if !killed + || tokio::time::timeout(HOST_PROBE_TIMEOUT, &mut task) + .await + .is_err() + { task.abort(); } } From 5b9bec7688a293ea88371215f2aa110cf9483c60 Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:24:59 +0000 Subject: [PATCH 3/6] chore(worker-iterate): address CodeRabbit findings on PR #1369 --- crates/minimald/src/session.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index 588ba38e1d..9617d4ca30 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -1639,14 +1639,11 @@ impl Session { // `HOST_PROBE_TIMEOUT` of accepting it, aborts the loop instead // of waiting on it. // - // Aborting drops the loop at its await point, skipping the - // awaited `NetGuard` teardown in `Host::mainloop`; the wedged - // host's sandbox process and network are orphaned rather than - // reclaimed here. That leak is the acknowledged follow-up in - // this PR's description ("Directly reclaiming a wedged host's - // sandbox process ... is left as follow-up"): cancellation-safe - // ownership of the `NetGuard` is out of scope for bounding the - // shutdown wait, which is what actually unblocks `min stop`. + // Aborting drops the loop at its await point, so the awaited + // `NetGuard` teardown in `Host::mainloop` is skipped and the + // wedged host's sandbox process and network are orphaned + // rather than reclaimed here; reclamation is a tracked + // follow-up. let killed = host.kill(for_shutdown).await.is_ok(); if !killed || tokio::time::timeout(HOST_PROBE_TIMEOUT, &mut task) From 47fab2f1225f51eb5911c895aa2656073ae0a254 Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:20:20 +0000 Subject: [PATCH 4/6] chore(worker-iterate): address CodeRabbit findings on PR #1369 Bound the attach-time host replacement wait and abort the replaced host task in mint_session_host, mirroring stop_running, so a wedged-but-alive host cannot park attach or leak its loop task/sandbox/NetGuard. --- crates/minimald/src/session.rs | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index 9617d4ca30..3ba7bbd53d 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -1860,9 +1860,23 @@ impl Session { tracing::info!( "replacing the hook-launched session shell with one minted for the attaching terminal" ); - let (handle, join) = slot.take().expect("matched on Some"); - let _ = handle.kill(false).await; - let _ = join.await; + let (handle, mut join) = slot.take().expect("matched on Some"); + // Bound the wait for the hook host to wind down, mirroring + // `stop_running`: `kill` can time out against a wedged loop, and an + // unbounded `join.await` would then park the attach forever — the + // very "attach hangs with no timeout" symptom this change removes. + // A kill that cannot land, or a loop that does not finish within + // `HOST_PROBE_TIMEOUT` of accepting it, aborts the loop instead of + // waiting on it (leaving the same acknowledged process-reclaim gap + // as `stop_running`). + let killed = handle.kill(false).await.is_ok(); + if !killed + || tokio::time::timeout(HOST_PROBE_TIMEOUT, &mut join) + .await + .is_err() + { + join.abort(); + } } let host = match &mut self.inner { @@ -2158,7 +2172,16 @@ impl Session { let SessionInner::Active { host, .. } = &mut self.inner else { unreachable!("mint_session_host is only reachable from the Active state"); }; - *host = Some(launched); + // A wedged-but-alive host can reach here via attach's re-mint branch + // (`session_host.rs` `SendTimeoutError::Timeout`): its loop task is + // still running. Dropping the replaced `(HostHandle, JoinHandle)` + // would detach that task, leaking the old sandbox process, pty master, + // and `NetGuard` for the daemon's lifetime — one leak per attach to the + // same wedged host. Abort the task being replaced, mirroring + // `stop_running` (a no-op when the old loop had already exited). + if let Some((_, task)) = host.replace(launched) { + task.abort(); + } // Minted by an attach: its environment describes the terminal that is // here, so nothing may replace it out from under that client. self.host_origin = HostOrigin::Interactive; From 257879c3ad75702beab62fa55c608e5c902e489c Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 21:34:12 +0000 Subject: [PATCH 5/6] chore(worker-iterate): address CodeRabbit findings on PR #1369 --- crates/minimald/src/session.rs | 121 ++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 41 deletions(-) diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index 3ba7bbd53d..9f5aa553b9 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -1611,6 +1611,35 @@ impl Session { Ok(rx) } + /// Kills `host` and stops its runtime loop `task`, bounded so a wedged + /// host cannot park the caller. Both the kill and the wait for the loop to + /// act on it are bounded: `HostHandle::kill`'s `send_timeout` only bounds + /// the wait for mailbox *capacity*, so a loop parked mid-`step()` (mailbox + /// nearly empty) queues the kill yet never processes it, and awaiting that + /// loop unbounded would park the caller behind it forever. So a kill that + /// cannot be queued, or a loop that does not finish within + /// `HOST_PROBE_TIMEOUT` of accepting it, aborts the loop instead of + /// waiting on it. + /// + /// Aborting drops the loop at its await point, so the awaited `NetGuard` + /// teardown in `Host::mainloop` is skipped and the wedged host's sandbox + /// process and network are orphaned rather than reclaimed here; + /// reclamation is a tracked follow-up. + async fn kill_and_stop_loop( + host: &session_host::HostHandle, + task: &mut JoinHandle>, + for_shutdown: bool, + ) { + let killed = host.kill(for_shutdown).await.is_ok(); + if !killed + || tokio::time::timeout(HOST_PROBE_TIMEOUT, &mut *task) + .await + .is_err() + { + task.abort(); + } + } + /// Tears down any runtime objects, such as the host or side ops. Shutdown /// of these objects is complete once awaited. /// @@ -1627,31 +1656,12 @@ impl Session { s.shutdown().await; } if let Some((host, mut task)) = host { - // Signal the process to die, then await the runtime loop so the - // sandbox files backing its rootfs are released before the caller - // removes the session's directory tree. Both the kill and the - // wait for the loop to act on it are bounded: `send_timeout` - // only bounds the wait for mailbox *capacity*, so a loop parked - // mid-`step()` (mailbox nearly empty) queues the kill yet never - // processes it, and awaiting that loop unbounded would park - // daemon shutdown behind it forever. So a kill that cannot be - // queued, or a loop that does not finish within - // `HOST_PROBE_TIMEOUT` of accepting it, aborts the loop instead - // of waiting on it. - // - // Aborting drops the loop at its await point, so the awaited - // `NetGuard` teardown in `Host::mainloop` is skipped and the - // wedged host's sandbox process and network are orphaned - // rather than reclaimed here; reclamation is a tracked - // follow-up. - let killed = host.kill(for_shutdown).await.is_ok(); - if !killed - || tokio::time::timeout(HOST_PROBE_TIMEOUT, &mut task) - .await - .is_err() - { - task.abort(); - } + // Signal the process to die, then stop its runtime loop so the + // sandbox files backing its rootfs are released before the + // caller removes the session's directory tree. The wait is + // bounded and aborts a wedged loop rather than parking shutdown + // behind it — see [`Session::kill_and_stop_loop`]. + Self::kill_and_stop_loop(&host, &mut task, for_shutdown).await; } } } @@ -1861,22 +1871,11 @@ impl Session { "replacing the hook-launched session shell with one minted for the attaching terminal" ); let (handle, mut join) = slot.take().expect("matched on Some"); - // Bound the wait for the hook host to wind down, mirroring - // `stop_running`: `kill` can time out against a wedged loop, and an - // unbounded `join.await` would then park the attach forever — the - // very "attach hangs with no timeout" symptom this change removes. - // A kill that cannot land, or a loop that does not finish within - // `HOST_PROBE_TIMEOUT` of accepting it, aborts the loop instead of - // waiting on it (leaving the same acknowledged process-reclaim gap - // as `stop_running`). - let killed = handle.kill(false).await.is_ok(); - if !killed - || tokio::time::timeout(HOST_PROBE_TIMEOUT, &mut join) - .await - .is_err() - { - join.abort(); - } + // Bound the wait for the hook host to wind down, aborting a wedged + // loop rather than parking the attach behind it — the very "attach + // hangs with no timeout" symptom this change removes. Same bounded + // kill-and-stop as shutdown; see [`Session::kill_and_stop_loop`]. + Self::kill_and_stop_loop(&handle, &mut join, false).await; } let host = match &mut self.inner { @@ -3056,6 +3055,46 @@ mod tests { ); } + /// The bounded stop path gives up on a host whose loop accepted the kill + /// but never winds down, aborting the loop task instead of awaiting it + /// forever. + /// + /// This is the shape `HostHandle::kill`'s `send_timeout` alone cannot + /// cover: the mailbox has room, so the kill enqueues (`kill` returns + /// `Ok`) and the `!killed` short-circuit does not fire, yet the runtime + /// loop is parked mid-`step()` and never processes the queued kill. The + /// *join* bound — not the kill's own deadline — is what has to return the + /// caller and abort the loop. `kill_to_a_wedged_host_gives_up_instead_of_parking` + /// covers the saturated-mailbox sibling, where the kill itself times out. + #[tokio::test(start_paused = true)] + async fn stopping_a_wedged_host_aborts_its_loop_instead_of_parking() { + // Mailbox held but not saturated: the kill enqueues within its + // deadline, so the join bound is the branch under test. + let (host, _mailbox) = HostHandle::wedged(); + + // A loop that accepted the kill but never resolves — models the + // mainloop parked mid-`step()`. + let mut task = tokio::spawn(std::future::pending::>()); + + // Far past HOST_PROBE_TIMEOUT: under the paused clock the join bound is + // the deadline that returns this call. Reaching GIVE_UP would mean the + // wait was unbounded. + tokio::time::timeout( + GIVE_UP, + super::Session::kill_and_stop_loop(&host, &mut task, true), + ) + .await + .expect("the bounded stop path must return, not park on a wedged loop"); + + // The loop task is torn down, not left running behind a detached + // handle: awaiting it yields a cancelled join. + let outcome = task.await; + assert!( + outcome.is_err_and(|e| e.is_cancelled()), + "a loop that never wound down within the deadline must be aborted", + ); + } + /// The `AcceptEnv` allowlist keeps locale + timezone vars and drops /// everything else — critically the control-plane vars, which must never /// reach the shell environment. From 6cc7493a74e73debe6b63c77f2b98b493fcaaa61 Mon Sep 17 00:00:00 2001 From: "gominimal-aw-bot[bot]" <281738952+gominimal-aw-bot[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 23:06:58 +0000 Subject: [PATCH 6/6] chore(worker-iterate): address CodeRabbit findings on PR #1369 --- crates/minimald/src/session.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/minimald/src/session.rs b/crates/minimald/src/session.rs index 9f5aa553b9..3c880593ed 100644 --- a/crates/minimald/src/session.rs +++ b/crates/minimald/src/session.rs @@ -1872,8 +1872,7 @@ impl Session { ); let (handle, mut join) = slot.take().expect("matched on Some"); // Bound the wait for the hook host to wind down, aborting a wedged - // loop rather than parking the attach behind it — the very "attach - // hangs with no timeout" symptom this change removes. Same bounded + // loop rather than parking the attach behind it. Same bounded // kill-and-stop as shutdown; see [`Session::kill_and_stop_loop`]. Self::kill_and_stop_loop(&handle, &mut join, false).await; }