diff --git a/Cargo.lock b/Cargo.lock index feb3574e5c..0a652e6bd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4155,6 +4155,7 @@ dependencies = [ "tracing-subscriber", "uuid", "version", + "vsock", "vt100", "walkdir", "zstd", @@ -4212,6 +4213,7 @@ dependencies = [ "russh", "russh-sftp", "serde", + "serde_json", "serde_json_lenient", "serial_test", "sessions", diff --git a/Cargo.toml b/Cargo.toml index 8ffe6c2bac..5b1e48bbc5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -228,6 +228,9 @@ tempfile = "3.27" tokio = { version = "1.53", features = ["full"] } tokio-stream = "0.1" tokio-vsock = "0.7" +# The blocking AF_VSOCK stream the guest daemon's telemetry sender writes +# the host over (`minimald::telemetry_forward`), tokio-vsock's own base. +vsock = "0.5" tokio-util = "0.7" toml = "1.1" toml_edit = "0.25" diff --git a/crates/minimal/src/diag/collect.rs b/crates/minimal/src/diag/collect.rs index d5f8ffd3fc..76fe243f10 100644 --- a/crates/minimal/src/diag/collect.rs +++ b/crates/minimal/src/diag/collect.rs @@ -441,7 +441,23 @@ pub async fn spool( ) -> Result<(), anyhow::Error> { let dir = spool_dir(paths, std::env::var_os("MINIMAL_OTEL_SPOOL_DIR")); let headers = diagnostics::redact::exporter_header_values(|n| std::env::var(n).ok()); - spool_in(w, &dir, tail_bytes, &headers).await + spool_in(w, &dir, tail_bytes, &headers).await?; + // The VMs' guest spools, `/guest//`, each bounded on its own + // (`mlog::otel::spool_guest_line`): collected beside the host's. `guest` + // itself is taken only as a real directory, the no-follow discipline + // `diagnostics::spool::collect` applies to each directory it is handed. + let guest = dir.join("guest"); + let guest_is_dir = tokio::fs::symlink_metadata(&guest) + .await + .is_ok_and(|m| m.is_dir()); + if guest_is_dir && let Ok(mut vms) = tokio::fs::read_dir(&guest).await { + while let Ok(Some(vm)) = vms.next_entry().await { + if vm.file_type().await.is_ok_and(|t| t.is_dir()) { + spool_in(w, &vm.path(), tail_bytes, &headers).await?; + } + } + } + Ok(()) } /// The spool directory `min bug` reads: `override_dir`, the value of diff --git a/crates/minimald/Cargo.toml b/crates/minimald/Cargo.toml index 6eb0248938..bc317c4fed 100644 --- a/crates/minimald/Cargo.toml +++ b/crates/minimald/Cargo.toml @@ -119,6 +119,7 @@ tracing-opentelemetry.workspace = true # The guest (initramfs pid-1) boot path is Linux-only. [target.'cfg(target_os = "linux")'.dependencies] tokio-vsock.workspace = true +vsock.workspace = true [lints] workspace = true diff --git a/crates/minimald/src/lib.rs b/crates/minimald/src/lib.rs index 019ae9a30d..5492160f29 100644 --- a/crates/minimald/src/lib.rs +++ b/crates/minimald/src/lib.rs @@ -28,6 +28,8 @@ pub mod session_sop; mod sessions; mod sftp; mod store; +#[cfg(target_os = "linux")] +pub mod telemetry_forward; #[cfg(any(test, feature = "test-support"))] pub mod test_harness; pub mod traced; diff --git a/crates/minimald/src/main.rs b/crates/minimald/src/main.rs index 7d7d425048..58b6d1cb87 100644 --- a/crates/minimald/src/main.rs +++ b/crates/minimald/src/main.rs @@ -424,6 +424,16 @@ fn main() -> Result<(), MainError> { // `GetRecord`). Bounded, so a stuck task cannot hold the exit. runtime.shutdown_timeout(std::time::Duration::from_secs(2)); mlog::otel::shutdown(std::time::Duration::from_secs(5)); + // The guest's forwarded records (TEL-034): give the sender a moment to + // write the spans that just ended to the host before the VM goes down. + #[cfg(target_os = "linux")] + if is_minimal_microvm() + && !minimald::telemetry_forward::drain(std::time::Duration::from_millis(500)) + { + tracing::warn!( + "telemetry forward: records still queued at shutdown; the host spool keeps what arrived" + ); + } // Only now, with the flush done (or given up on), take the egress down. egress.tear_down(); @@ -705,6 +715,13 @@ async fn async_main(egress: &mut GuestEgress) -> Result<(), MainError> { LogMode::Console }; let logger = DaemonLogger::install(log_mode)?; + // The guest's telemetry sender (TEL-034): started once telemetry is + // initialised (in the logger), a no-op unless the boot line named a + // forward destination. + #[cfg(target_os = "linux")] + if is_minimal_microvm() { + minimald::telemetry_forward::start(); + } let listen_args = cli.listen_args().unwrap(); diff --git a/crates/minimald/src/telemetry_forward.rs b/crates/minimald/src/telemetry_forward.rs new file mode 100644 index 0000000000..708ded397d --- /dev/null +++ b/crates/minimald/src/telemetry_forward.rs @@ -0,0 +1,351 @@ +//! The guest daemon's vsock sender for its telemetry (spec 25 TEL-034, +//! `docs/specs/25-spec-telemetry/guest-vsock.md`): inside a microVM the +//! daemon has no network for telemetry, so the records `mlog` queues for +//! forwarding (each one the spool's OTLP-JSON line) go to the VM host over +//! AF_VSOCK, to the port `MINIMAL_OTEL_FORWARD=vsock:` names on the +//! host (CID 2), where `minvmd` receives them and exports them under the +//! host's switches. +//! +//! One dedicated thread owns the connection. It connects when the first +//! record arrives, writes each burst of queued records as length-delimited +//! frames (a 4-byte big-endian length, then the line), and reconnects after +//! [`RETRY_EVERY`] when a write fails or the host is not there. Nothing on +//! the daemon's request paths waits on it: the queue between `mlog` and +//! this thread is bounded, and while the host is away the records are +//! dropped and counted, not held. The spool is written before any of this +//! and is never affected (TEL-014, TEL-015). + +use std::io::{self, Write}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::Receiver; +use std::time::{Duration, Instant}; + +/// How long the sender waits after a failed connect or write before it +/// tries the host again (TEL-046). +pub const RETRY_EVERY: Duration = Duration::from_secs(5); + +/// A write that blocks this long is a host that stopped reading: the +/// connection is dropped and made again, rather than the thread held. +const WRITE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Records written at most per frame burst: one `write_all` each. +const BURST_LINES: usize = 64; + +/// Lines taken off the queue so far (sent or dropped here). +static TAKEN: AtomicU64 = AtomicU64::new(0); +/// Lines written to the host so far. +static SENT: AtomicU64 = AtomicU64::new(0); +/// Lines dropped here, with no connection to write them on, or too long +/// (or empty) for the host to take as a frame. +static DROPPED: AtomicU64 = AtomicU64::new(0); +/// Of [`DROPPED`], the lines no frame could carry: longer than +/// `mlog::otel::FORWARD_MAX_FRAME_BYTES`, or empty. +static UNFRAMEABLE: AtomicU64 = AtomicU64::new(0); +/// Connections made so far (one, plus one per reconnect). +static CONNECTS: AtomicU64 = AtomicU64::new(0); + +/// Start the sender, once, after `mlog::otel::init`: a no-op when that +/// init forwards nothing (no `MINIMAL_OTEL_FORWARD`, telemetry off, or a +/// destination it refused), or when the sender already runs. +pub fn start() { + let Some(rx) = mlog::otel::take_forward_receiver() else { + return; + }; + let Some(mlog::otel::ForwardDestination::Vsock { port }) = + std::env::var(mlog::otel::FORWARD_ENV) + .ok() + .and_then(|v| mlog::otel::ForwardDestination::parse(&v)) + else { + return; + }; + let spawned = std::thread::Builder::new() + .name("otel-forward".into()) + .spawn(move || run(rx, move || connect_vsock(port), RETRY_EVERY)); + match spawned { + Ok(_) => tracing::info!(port, "telemetry forward: sending records to the VM host"), + Err(error) => tracing::warn!( + %error, + "telemetry forward: cannot start the sender thread; records are not forwarded" + ), + } +} + +/// Wait up to `timeout` for the sender to settle every record queued so +/// far (written to the host, or dropped and counted): the daemon's last +/// spans (its shutdown) are written to the host before the VM goes down. +/// Settled, not taken: the sender counts a burst taken before its write, +/// so a drain on the taken count returns while the last burst is still +/// being written, and the `reboot(2)` after it cuts that write. +/// Returns whether the queue drained. +pub fn drain(timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + loop { + let queued = mlog::otel::forward_queued(); + if settled() >= queued { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(10)); + } +} + +/// Lines the sender is done with: written to the host, or dropped. +fn settled() -> u64 { + SENT.load(Ordering::Relaxed) + DROPPED.load(Ordering::Relaxed) +} + +/// Lines written to the host so far. +pub fn sent() -> u64 { + SENT.load(Ordering::Relaxed) +} + +/// Lines this sender dropped: for want of a connection, or because no +/// frame could carry them. The queue's own drops (a full queue) are +/// `mlog::otel::forward_dropped`. +pub fn dropped() -> u64 { + DROPPED.load(Ordering::Relaxed) +} + +/// A connection to the host's receiver at `port`, with a write timeout. +fn connect_vsock(port: u32) -> io::Result> { + let stream = vsock::VsockStream::connect_with_cid_port(libc::VMADDR_CID_HOST, port)?; + stream.set_write_timeout(Some(WRITE_TIMEOUT))?; + Ok(Box::new(stream)) +} + +/// The sender's loop: for every burst of queued lines, connect when there +/// is no connection and the last attempt is `retry` ago, write the burst +/// as frames, and on a failed write drop the connection. Returns when the +/// queue's writers are gone. `connect` is the one I/O choice, so the loop +/// is tested over a socket pair. +pub(crate) fn run( + rx: Receiver, + mut connect: impl FnMut() -> io::Result>, + retry: Duration, +) { + let mut conn: Option> = None; + let mut next_try = Instant::now(); + let mut warned = false; + while let Ok(first) = rx.recv() { + let mut burst = vec![first]; + while burst.len() < BURST_LINES + && let Ok(more) = rx.try_recv() + { + burst.push(more); + } + let n = burst.len() as u64; + TAKEN.fetch_add(n, Ordering::Relaxed); + if conn.is_none() && Instant::now() >= next_try { + match connect() { + Ok(c) => { + conn = Some(c); + warned = false; + let connects = CONNECTS.fetch_add(1, Ordering::Relaxed) + 1; + tracing::info!( + connects, + dropped = DROPPED.load(Ordering::Relaxed), + sent = SENT.load(Ordering::Relaxed), + "telemetry forward: connected to the VM host" + ); + } + Err(error) => { + next_try = Instant::now() + retry; + if !warned { + warned = true; + tracing::warn!( + %error, + "telemetry forward: cannot reach the VM host; records are dropped \ + until it answers (retried every {} s)", + retry.as_secs() + ); + } + } + } + } + match conn.as_mut() { + Some(w) => match write_frames(w, &burst) { + Ok(skipped) => { + let skipped = skipped as u64; + SENT.fetch_add(n - skipped, Ordering::Relaxed); + if skipped > 0 { + DROPPED.fetch_add(skipped, Ordering::Relaxed); + let before = UNFRAMEABLE.fetch_add(skipped, Ordering::Relaxed); + if before == 0 { + tracing::warn!( + max_bytes = mlog::otel::FORWARD_MAX_FRAME_BYTES, + "telemetry forward: a record too long for one frame was dropped \ + (it stays in the guest's spool; later ones are counted only)" + ); + } + } + } + Err(error) => { + conn = None; + next_try = Instant::now() + retry; + DROPPED.fetch_add(n, Ordering::Relaxed); + tracing::warn!( + %error, + connects = CONNECTS.load(Ordering::Relaxed), + dropped = DROPPED.load(Ordering::Relaxed), + "telemetry forward: the VM host connection failed; reconnecting in {} s", + retry.as_secs() + ); + } + }, + None => { + DROPPED.fetch_add(n, Ordering::Relaxed); + } + } + } +} + +/// Write `lines` as frames: each a 4-byte big-endian length, then the line +/// without a trailing newline. One write for the burst. A line the host +/// would refuse as a frame (longer than `mlog::otel::FORWARD_MAX_FRAME_BYTES`, +/// or empty) is left out rather than sent: the host ends the connection on +/// such a frame, which would cost the burst and a [`RETRY_EVERY`] of +/// records. Returns how many lines were left out. +pub(crate) fn write_frames(w: &mut impl Write, lines: &[String]) -> io::Result { + let mut buf = Vec::with_capacity(lines.iter().map(|l| l.len() + 4).sum()); + let mut skipped = 0; + for line in lines { + let line = line.trim_end_matches(['\n', '\r']); + let len = match u32::try_from(line.len()) { + Ok(len) if !line.is_empty() && line.len() <= mlog::otel::FORWARD_MAX_FRAME_BYTES => len, + _ => { + skipped += 1; + continue; + } + }; + buf.extend_from_slice(&len.to_be_bytes()); + buf.extend_from_slice(line.as_bytes()); + } + if !buf.is_empty() { + w.write_all(&buf)?; + w.flush()?; + } + Ok(skipped) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read as _; + use std::os::unix::net::UnixStream; + use std::sync::mpsc::sync_channel; + + /// The frames a reader parses out of `bytes`. + fn frames(mut bytes: &[u8]) -> Vec { + let mut out = Vec::new(); + while !bytes.is_empty() { + let (len, rest) = bytes.split_at(4); + let n = u32::from_be_bytes(len.try_into().unwrap()) as usize; + let (line, rest) = rest.split_at(n); + out.push(String::from_utf8(line.to_vec()).unwrap()); + bytes = rest; + } + out + } + + #[test] + fn frames_are_length_prefixed_lines_without_the_newline() { + let mut buf = Vec::new(); + let skipped = write_frames( + &mut buf, + &["{\"a\":1}\n".to_string(), "{\"b\":2}".to_string()], + ) + .unwrap(); + assert_eq!(skipped, 0); + assert_eq!(buf[..4], 7u32.to_be_bytes()); + assert_eq!(frames(&buf), vec!["{\"a\":1}", "{\"b\":2}"]); + } + + /// A record no frame can carry (over the host's frame bound, or empty) + /// is left out and counted, and the records around it still go: the + /// host would end the connection on that frame. + #[test] + fn a_record_over_the_frame_bound_is_dropped_not_sent() { + let max = mlog::otel::FORWARD_MAX_FRAME_BYTES; + let at_bound = "x".repeat(max); + let mut buf = Vec::new(); + let skipped = write_frames( + &mut buf, + &[ + "{\"a\":1}".to_string(), + "y".repeat(max + 1), + String::new(), + "\n".to_string(), + at_bound.clone(), + "{\"b\":2}".to_string(), + ], + ) + .unwrap(); + assert_eq!(skipped, 3); + assert_eq!( + frames(&buf), + vec!["{\"a\":1}".to_string(), at_bound, "{\"b\":2}".to_string()] + ); + + // A burst of nothing but such records writes nothing at all. + let mut buf = Vec::new(); + assert_eq!(write_frames(&mut buf, &["z".repeat(max + 1)]).unwrap(), 1); + assert!(buf.is_empty()); + } + + /// With the host away the sender drops and counts; once a connect + /// succeeds the later records arrive as frames; a failed write drops the + /// connection and the burst, and the next record reconnects. + #[test] + fn the_sender_drops_while_the_host_is_away_and_reconnects() { + let (tx, rx) = sync_channel::(16); + let (mut host, guest) = UnixStream::pair().unwrap(); + let (mut host2, guest2) = UnixStream::pair().unwrap(); + let ends = std::sync::Mutex::new(vec![Some(guest2), Some(guest)]); + let attempts = std::sync::atomic::AtomicUsize::new(0); + let (sent_before, dropped_before) = (sent(), dropped()); + std::thread::scope(|s| { + s.spawn(|| { + run( + rx, + || { + let n = attempts.fetch_add(1, Ordering::Relaxed); + // The first attempt fails; the next ones hand out the + // pairs' guest ends in order. + if n == 0 { + return Err(io::Error::other("no host")); + } + let end = ends.lock().unwrap().pop().flatten(); + end.map(|e| Box::new(e) as Box) + .ok_or_else(|| io::Error::other("no more ends")) + }, + Duration::ZERO, + ); + }); + tx.send("lost".into()).unwrap(); + // Let the sender take the record with no host. + while TAKEN.load(Ordering::Relaxed) == 0 { + std::thread::sleep(Duration::from_millis(5)); + } + tx.send("one".into()).unwrap(); + let mut buf = [0u8; 7]; + host.read_exact(&mut buf).unwrap(); + assert_eq!(frames(&buf), vec!["one"]); + // The host closes: the next write fails, that burst is dropped, + // and the one after it goes out on the fresh connection. + drop(host); + tx.send("two".into()).unwrap(); + while dropped() - dropped_before < 2 { + std::thread::sleep(Duration::from_millis(5)); + } + tx.send("three".into()).unwrap(); + let mut buf = [0u8; 9]; + host2.read_exact(&mut buf).unwrap(); + assert_eq!(frames(&buf), vec!["three"]); + drop(tx); + }); + assert_eq!(sent() - sent_before, 2); + assert_eq!(dropped() - dropped_before, 2); + } +} diff --git a/crates/minvmd/Cargo.toml b/crates/minvmd/Cargo.toml index 42218c1263..0503e5e5f3 100644 --- a/crates/minvmd/Cargo.toml +++ b/crates/minvmd/Cargo.toml @@ -29,6 +29,10 @@ libc.workspace = true serde = { workspace = true } thiserror.workspace = true serde_json_lenient.workspace = true +# The strict parser the guest telemetry door reads the guest's lines with +# (`guest_telemetry::accept_line`): no comments, no trailing commas, so the +# host forwards only what a strict collector reads the same way. +serde_json.workspace = true toml.workspace = true # Cross-platform per-process resource sampling (CPU%, RSS, disk I/O) of the VMM # child PID, and host capacity (cores/memory) for `config set` validation (#747). diff --git a/crates/minvmd/src/cmd/run.rs b/crates/minvmd/src/cmd/run.rs index 490bafb559..9beedd4285 100644 --- a/crates/minvmd/src/cmd/run.rs +++ b/crates/minvmd/src/cmd/run.rs @@ -655,6 +655,27 @@ fn run_foreground(root: tracing::span::EnteredSpan) -> Result<()> { &proxy_publish, )?; + // The guest telemetry door (TEL-034): bound only with telemetry on, + // and handed to the VMM child the same way, so the guest daemon's + // records reach this process's exporter and spool over the vsock + // bridge and the guest needs no endpoint. Best-effort like the doors + // above: a bind failure is warned and the VM boots without it, and the + // guest then spools alone (TEL-046). + let guest_telemetry_door = if mlog::otel::telemetry_enabled() { + crate::control::resolve_control_sock() + .and_then(|sock_path| crate::guest_telemetry::spawn_guest_telemetry_door(&sock_path)) + .inspect_err(|error| { + tracing::warn!( + %error, + "failed to bind the guest telemetry door; the guest daemon's telemetry \ + stays in the guest's own spool" + ); + }) + .ok() + } else { + None + }; + // The host answerer (NET-138): the box zone's answerer on the host // loopback, answering from this host-authored table — the same // semantics the native daemon's answerer gives, over the same shared @@ -953,6 +974,9 @@ fn run_foreground(root: tracing::span::EnteredSpan) -> Result<()> { // names. A redraw hands the same door to the fresh boot. let (name, path) = &guest_report_door; cmd.env(name, path); + if let Some(path) = guest_telemetry_door.as_ref() { + cmd.env(crate::guest_telemetry::GUEST_TELEMETRY_SOCK_ENV, path); + } cmd.env(MARKER_SOCK_ENV, &marker_sock_path) // The node's proxy port travels to the guest through the VMM child's // env: the child is a separate process (like the marker socket path), @@ -1268,6 +1292,13 @@ fn run_foreground(root: tracing::span::EnteredSpan) -> Result<()> { // ── Phase 3: Supervise until VMM child exits ───────────────────────────── let status = child.wait().context("waiting for VMM child")?; + // The VM is gone, so the guest telemetry door's connection ends: let + // the door hand the guest's last records (its shutdown) to the host's + // endpoint before this process exits, bounded (TEL-034). The spool + // already holds them either way. + if guest_telemetry_door.is_some() { + crate::guest_telemetry::drain_at_exit(crate::guest_telemetry::DRAIN_AT_EXIT); + } // The VM is gone, so no answer to a pending ask can still matter: each // is cancelled and audited before the supervisor exits (NET-045). crate::control::stop_pending_asks(&boxes); diff --git a/crates/minvmd/src/cmd/stop.rs b/crates/minvmd/src/cmd/stop.rs index 7d0c8f4e8d..c58d804444 100644 --- a/crates/minvmd/src/cmd/stop.rs +++ b/crates/minvmd/src/cmd/stop.rs @@ -1,9 +1,11 @@ //! `minvmd stop` subcommand (R4.4, R2.3). //! //! Reads `vmm_pid` from `minvmd.toml`, asks the in-VM minimald to shut down -//! (drain sessions + quiesce the data volume) over the bridge UDS, then sends -//! `SIGTERM` to the VMM child, waits up to 5 s, escalates to `SIGKILL` on -//! timeout, then resets `minvmd.toml` to `Stopped`. +//! (drain sessions + quiesce the data volume) over the bridge UDS, lets an +//! acknowledging guest power the VM off by itself for up to +//! `GUEST_POWEROFF_GRACE` (its exit path carries its last telemetry), then +//! sends `SIGTERM` to the VMM child if it is still there, waits up to 5 s, +//! escalates to `SIGKILL` on timeout, then resets `minvmd.toml` to `Stopped`. //! //! The command is idempotent: if the daemon is already stopped (or has never //! been provisioned), it returns successfully with no action. Stale active @@ -213,10 +215,76 @@ fn quiesce_then_signal(quiesce: Option, pid: u32, grace: Duration) } } }); - signal_and_wait(pid, grace)?; + settle_vmm(pid, acknowledged, GUEST_POWEROFF_GRACE, grace)?; Ok(acknowledged) } +/// How long a stop lets an acknowledged guest power the VM off by itself +/// before it SIGTERMs the VMM (TEL-034, guest-vsock.md). The ack is not the +/// guest's last word: its `rpc` span for the Shutdown (the guest's one child +/// of `guest.shutdown`) ends only once the ack is written, and the guest's +/// exit path (`minimald` `main`: drain, telemetry shutdown, the vsock +/// sender's drain, then `reboot(2)`) is what carries that span and the +/// handler's last records to the host. A SIGTERM at the ack races that path +/// and, on a loaded host, wins (lab 715k on debian-12). The guest powering +/// off is what makes the VMM exit, as on a `min stop`; a guest that does not +/// within this bound gets the SIGTERM as before. Typical exit paths take a +/// few hundred milliseconds, so a healthy stop pays that, not the bound. +const GUEST_POWEROFF_GRACE: Duration = Duration::from_secs(3); + +/// Take the VMM child `pid` down after the quiesce. When the guest +/// acknowledged the Shutdown RPC (`acked`), first wait up to `poweroff_grace` +/// (never past `grace`, a stop's whole budget for this child) for the VM to +/// exit by itself, so the guest's exit-time telemetry flush completes before +/// the VMM goes; then (or at once, without an ack) [`signal_and_wait`] with +/// what is left of `grace`. +fn settle_vmm(pid: u32, acked: bool, poweroff_grace: Duration, grace: Duration) -> Result<()> { + if !acked { + return signal_and_wait(pid, grace); + } + let started = Instant::now(); + let pid_t = libc::pid_t::try_from(pid) + .ok() + .filter(|p| *p > 0) + .ok_or_else(|| anyhow::anyhow!("invalid vmm_pid {pid} in state"))?; + if wait_for_exit(pid_t, poweroff_grace.min(grace)) { + tracing::debug!(pid, "the guest powered the VM off after its ack"); + return Ok(()); + } + tracing::info!( + pid, + grace_ms = u64::try_from(poweroff_grace.as_millis()).unwrap_or(u64::MAX), + "the guest acknowledged Shutdown but did not power off within the grace; \ + signalling the VMM (its last telemetry may not reach the host)" + ); + signal_and_wait(pid, grace.saturating_sub(started.elapsed())) +} + +/// Whether `pid` exists. Errors other than ESRCH count as alive (best effort, +/// as the SIGTERM wait has always treated them). +fn pid_alive(pid: libc::pid_t) -> bool { + // SAFETY: kill(pid, 0) checks for process existence without delivering a + // signal; `pid` is positive (checked by the caller). + if unsafe { libc::kill(pid, 0) } == 0 { + return true; + } + std::io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH) +} + +/// Poll `pid` every 50 ms for up to `timeout`; whether it is gone. +fn wait_for_exit(pid: libc::pid_t, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + loop { + if !pid_alive(pid) { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(50)); + } +} + /// Send `SIGTERM` to `pid`; wait up to `grace`; escalate to `SIGKILL` on /// timeout. fn signal_and_wait(pid: u32, grace: Duration) -> Result<()> { @@ -651,6 +719,62 @@ mod tests { child.wait().unwrap(); } + /// A stand-in VMM: `sh -c