diff --git a/Cargo.lock b/Cargo.lock
index feb3574e5c..0a652e6bd5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4155,6 +4155,7 @@ dependencies = [
"tracing-subscriber",
"uuid",
"version",
+ "vsock",
"vt100",
"walkdir",
"zstd",
@@ -4212,6 +4213,7 @@ dependencies = [
"russh",
"russh-sftp",
"serde",
+ "serde_json",
"serde_json_lenient",
"serial_test",
"sessions",
diff --git a/Cargo.toml b/Cargo.toml
index 8ffe6c2bac..5b1e48bbc5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -228,6 +228,9 @@ tempfile = "3.27"
tokio = { version = "1.53", features = ["full"] }
tokio-stream = "0.1"
tokio-vsock = "0.7"
+# The blocking AF_VSOCK stream the guest daemon's telemetry sender writes
+# the host over (`minimald::telemetry_forward`), tokio-vsock's own base.
+vsock = "0.5"
tokio-util = "0.7"
toml = "1.1"
toml_edit = "0.25"
diff --git a/crates/minimal/src/diag/collect.rs b/crates/minimal/src/diag/collect.rs
index d5f8ffd3fc..76fe243f10 100644
--- a/crates/minimal/src/diag/collect.rs
+++ b/crates/minimal/src/diag/collect.rs
@@ -441,7 +441,23 @@ pub async fn spool(
) -> Result<(), anyhow::Error> {
let dir = spool_dir(paths, std::env::var_os("MINIMAL_OTEL_SPOOL_DIR"));
let headers = diagnostics::redact::exporter_header_values(|n| std::env::var(n).ok());
- spool_in(w, &dir, tail_bytes, &headers).await
+ spool_in(w, &dir, tail_bytes, &headers).await?;
+ // The VMs' guest spools, `
/guest//`, each bounded on its own
+ // (`mlog::otel::spool_guest_line`): collected beside the host's. `guest`
+ // itself is taken only as a real directory, the no-follow discipline
+ // `diagnostics::spool::collect` applies to each directory it is handed.
+ let guest = dir.join("guest");
+ let guest_is_dir = tokio::fs::symlink_metadata(&guest)
+ .await
+ .is_ok_and(|m| m.is_dir());
+ if guest_is_dir && let Ok(mut vms) = tokio::fs::read_dir(&guest).await {
+ while let Ok(Some(vm)) = vms.next_entry().await {
+ if vm.file_type().await.is_ok_and(|t| t.is_dir()) {
+ spool_in(w, &vm.path(), tail_bytes, &headers).await?;
+ }
+ }
+ }
+ Ok(())
}
/// The spool directory `min bug` reads: `override_dir`, the value of
diff --git a/crates/minimald/Cargo.toml b/crates/minimald/Cargo.toml
index 6eb0248938..bc317c4fed 100644
--- a/crates/minimald/Cargo.toml
+++ b/crates/minimald/Cargo.toml
@@ -119,6 +119,7 @@ tracing-opentelemetry.workspace = true
# The guest (initramfs pid-1) boot path is Linux-only.
[target.'cfg(target_os = "linux")'.dependencies]
tokio-vsock.workspace = true
+vsock.workspace = true
[lints]
workspace = true
diff --git a/crates/minimald/src/lib.rs b/crates/minimald/src/lib.rs
index 019ae9a30d..5492160f29 100644
--- a/crates/minimald/src/lib.rs
+++ b/crates/minimald/src/lib.rs
@@ -28,6 +28,8 @@ pub mod session_sop;
mod sessions;
mod sftp;
mod store;
+#[cfg(target_os = "linux")]
+pub mod telemetry_forward;
#[cfg(any(test, feature = "test-support"))]
pub mod test_harness;
pub mod traced;
diff --git a/crates/minimald/src/main.rs b/crates/minimald/src/main.rs
index 7d7d425048..58b6d1cb87 100644
--- a/crates/minimald/src/main.rs
+++ b/crates/minimald/src/main.rs
@@ -424,6 +424,16 @@ fn main() -> Result<(), MainError> {
// `GetRecord`). Bounded, so a stuck task cannot hold the exit.
runtime.shutdown_timeout(std::time::Duration::from_secs(2));
mlog::otel::shutdown(std::time::Duration::from_secs(5));
+ // The guest's forwarded records (TEL-034): give the sender a moment to
+ // write the spans that just ended to the host before the VM goes down.
+ #[cfg(target_os = "linux")]
+ if is_minimal_microvm()
+ && !minimald::telemetry_forward::drain(std::time::Duration::from_millis(500))
+ {
+ tracing::warn!(
+ "telemetry forward: records still queued at shutdown; the host spool keeps what arrived"
+ );
+ }
// Only now, with the flush done (or given up on), take the egress down.
egress.tear_down();
@@ -705,6 +715,13 @@ async fn async_main(egress: &mut GuestEgress) -> Result<(), MainError> {
LogMode::Console
};
let logger = DaemonLogger::install(log_mode)?;
+ // The guest's telemetry sender (TEL-034): started once telemetry is
+ // initialised (in the logger), a no-op unless the boot line named a
+ // forward destination.
+ #[cfg(target_os = "linux")]
+ if is_minimal_microvm() {
+ minimald::telemetry_forward::start();
+ }
let listen_args = cli.listen_args().unwrap();
diff --git a/crates/minimald/src/telemetry_forward.rs b/crates/minimald/src/telemetry_forward.rs
new file mode 100644
index 0000000000..708ded397d
--- /dev/null
+++ b/crates/minimald/src/telemetry_forward.rs
@@ -0,0 +1,351 @@
+//! The guest daemon's vsock sender for its telemetry (spec 25 TEL-034,
+//! `docs/specs/25-spec-telemetry/guest-vsock.md`): inside a microVM the
+//! daemon has no network for telemetry, so the records `mlog` queues for
+//! forwarding (each one the spool's OTLP-JSON line) go to the VM host over
+//! AF_VSOCK, to the port `MINIMAL_OTEL_FORWARD=vsock:` names on the
+//! host (CID 2), where `minvmd` receives them and exports them under the
+//! host's switches.
+//!
+//! One dedicated thread owns the connection. It connects when the first
+//! record arrives, writes each burst of queued records as length-delimited
+//! frames (a 4-byte big-endian length, then the line), and reconnects after
+//! [`RETRY_EVERY`] when a write fails or the host is not there. Nothing on
+//! the daemon's request paths waits on it: the queue between `mlog` and
+//! this thread is bounded, and while the host is away the records are
+//! dropped and counted, not held. The spool is written before any of this
+//! and is never affected (TEL-014, TEL-015).
+
+use std::io::{self, Write};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::mpsc::Receiver;
+use std::time::{Duration, Instant};
+
+/// How long the sender waits after a failed connect or write before it
+/// tries the host again (TEL-046).
+pub const RETRY_EVERY: Duration = Duration::from_secs(5);
+
+/// A write that blocks this long is a host that stopped reading: the
+/// connection is dropped and made again, rather than the thread held.
+const WRITE_TIMEOUT: Duration = Duration::from_secs(5);
+
+/// Records written at most per frame burst: one `write_all` each.
+const BURST_LINES: usize = 64;
+
+/// Lines taken off the queue so far (sent or dropped here).
+static TAKEN: AtomicU64 = AtomicU64::new(0);
+/// Lines written to the host so far.
+static SENT: AtomicU64 = AtomicU64::new(0);
+/// Lines dropped here, with no connection to write them on, or too long
+/// (or empty) for the host to take as a frame.
+static DROPPED: AtomicU64 = AtomicU64::new(0);
+/// Of [`DROPPED`], the lines no frame could carry: longer than
+/// `mlog::otel::FORWARD_MAX_FRAME_BYTES`, or empty.
+static UNFRAMEABLE: AtomicU64 = AtomicU64::new(0);
+/// Connections made so far (one, plus one per reconnect).
+static CONNECTS: AtomicU64 = AtomicU64::new(0);
+
+/// Start the sender, once, after `mlog::otel::init`: a no-op when that
+/// init forwards nothing (no `MINIMAL_OTEL_FORWARD`, telemetry off, or a
+/// destination it refused), or when the sender already runs.
+pub fn start() {
+ let Some(rx) = mlog::otel::take_forward_receiver() else {
+ return;
+ };
+ let Some(mlog::otel::ForwardDestination::Vsock { port }) =
+ std::env::var(mlog::otel::FORWARD_ENV)
+ .ok()
+ .and_then(|v| mlog::otel::ForwardDestination::parse(&v))
+ else {
+ return;
+ };
+ let spawned = std::thread::Builder::new()
+ .name("otel-forward".into())
+ .spawn(move || run(rx, move || connect_vsock(port), RETRY_EVERY));
+ match spawned {
+ Ok(_) => tracing::info!(port, "telemetry forward: sending records to the VM host"),
+ Err(error) => tracing::warn!(
+ %error,
+ "telemetry forward: cannot start the sender thread; records are not forwarded"
+ ),
+ }
+}
+
+/// Wait up to `timeout` for the sender to settle every record queued so
+/// far (written to the host, or dropped and counted): the daemon's last
+/// spans (its shutdown) are written to the host before the VM goes down.
+/// Settled, not taken: the sender counts a burst taken before its write,
+/// so a drain on the taken count returns while the last burst is still
+/// being written, and the `reboot(2)` after it cuts that write.
+/// Returns whether the queue drained.
+pub fn drain(timeout: Duration) -> bool {
+ let deadline = Instant::now() + timeout;
+ loop {
+ let queued = mlog::otel::forward_queued();
+ if settled() >= queued {
+ return true;
+ }
+ if Instant::now() >= deadline {
+ return false;
+ }
+ std::thread::sleep(Duration::from_millis(10));
+ }
+}
+
+/// Lines the sender is done with: written to the host, or dropped.
+fn settled() -> u64 {
+ SENT.load(Ordering::Relaxed) + DROPPED.load(Ordering::Relaxed)
+}
+
+/// Lines written to the host so far.
+pub fn sent() -> u64 {
+ SENT.load(Ordering::Relaxed)
+}
+
+/// Lines this sender dropped: for want of a connection, or because no
+/// frame could carry them. The queue's own drops (a full queue) are
+/// `mlog::otel::forward_dropped`.
+pub fn dropped() -> u64 {
+ DROPPED.load(Ordering::Relaxed)
+}
+
+/// A connection to the host's receiver at `port`, with a write timeout.
+fn connect_vsock(port: u32) -> io::Result> {
+ let stream = vsock::VsockStream::connect_with_cid_port(libc::VMADDR_CID_HOST, port)?;
+ stream.set_write_timeout(Some(WRITE_TIMEOUT))?;
+ Ok(Box::new(stream))
+}
+
+/// The sender's loop: for every burst of queued lines, connect when there
+/// is no connection and the last attempt is `retry` ago, write the burst
+/// as frames, and on a failed write drop the connection. Returns when the
+/// queue's writers are gone. `connect` is the one I/O choice, so the loop
+/// is tested over a socket pair.
+pub(crate) fn run(
+ rx: Receiver,
+ mut connect: impl FnMut() -> io::Result>,
+ retry: Duration,
+) {
+ let mut conn: Option> = None;
+ let mut next_try = Instant::now();
+ let mut warned = false;
+ while let Ok(first) = rx.recv() {
+ let mut burst = vec![first];
+ while burst.len() < BURST_LINES
+ && let Ok(more) = rx.try_recv()
+ {
+ burst.push(more);
+ }
+ let n = burst.len() as u64;
+ TAKEN.fetch_add(n, Ordering::Relaxed);
+ if conn.is_none() && Instant::now() >= next_try {
+ match connect() {
+ Ok(c) => {
+ conn = Some(c);
+ warned = false;
+ let connects = CONNECTS.fetch_add(1, Ordering::Relaxed) + 1;
+ tracing::info!(
+ connects,
+ dropped = DROPPED.load(Ordering::Relaxed),
+ sent = SENT.load(Ordering::Relaxed),
+ "telemetry forward: connected to the VM host"
+ );
+ }
+ Err(error) => {
+ next_try = Instant::now() + retry;
+ if !warned {
+ warned = true;
+ tracing::warn!(
+ %error,
+ "telemetry forward: cannot reach the VM host; records are dropped \
+ until it answers (retried every {} s)",
+ retry.as_secs()
+ );
+ }
+ }
+ }
+ }
+ match conn.as_mut() {
+ Some(w) => match write_frames(w, &burst) {
+ Ok(skipped) => {
+ let skipped = skipped as u64;
+ SENT.fetch_add(n - skipped, Ordering::Relaxed);
+ if skipped > 0 {
+ DROPPED.fetch_add(skipped, Ordering::Relaxed);
+ let before = UNFRAMEABLE.fetch_add(skipped, Ordering::Relaxed);
+ if before == 0 {
+ tracing::warn!(
+ max_bytes = mlog::otel::FORWARD_MAX_FRAME_BYTES,
+ "telemetry forward: a record too long for one frame was dropped \
+ (it stays in the guest's spool; later ones are counted only)"
+ );
+ }
+ }
+ }
+ Err(error) => {
+ conn = None;
+ next_try = Instant::now() + retry;
+ DROPPED.fetch_add(n, Ordering::Relaxed);
+ tracing::warn!(
+ %error,
+ connects = CONNECTS.load(Ordering::Relaxed),
+ dropped = DROPPED.load(Ordering::Relaxed),
+ "telemetry forward: the VM host connection failed; reconnecting in {} s",
+ retry.as_secs()
+ );
+ }
+ },
+ None => {
+ DROPPED.fetch_add(n, Ordering::Relaxed);
+ }
+ }
+ }
+}
+
+/// Write `lines` as frames: each a 4-byte big-endian length, then the line
+/// without a trailing newline. One write for the burst. A line the host
+/// would refuse as a frame (longer than `mlog::otel::FORWARD_MAX_FRAME_BYTES`,
+/// or empty) is left out rather than sent: the host ends the connection on
+/// such a frame, which would cost the burst and a [`RETRY_EVERY`] of
+/// records. Returns how many lines were left out.
+pub(crate) fn write_frames(w: &mut impl Write, lines: &[String]) -> io::Result {
+ let mut buf = Vec::with_capacity(lines.iter().map(|l| l.len() + 4).sum());
+ let mut skipped = 0;
+ for line in lines {
+ let line = line.trim_end_matches(['\n', '\r']);
+ let len = match u32::try_from(line.len()) {
+ Ok(len) if !line.is_empty() && line.len() <= mlog::otel::FORWARD_MAX_FRAME_BYTES => len,
+ _ => {
+ skipped += 1;
+ continue;
+ }
+ };
+ buf.extend_from_slice(&len.to_be_bytes());
+ buf.extend_from_slice(line.as_bytes());
+ }
+ if !buf.is_empty() {
+ w.write_all(&buf)?;
+ w.flush()?;
+ }
+ Ok(skipped)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::io::Read as _;
+ use std::os::unix::net::UnixStream;
+ use std::sync::mpsc::sync_channel;
+
+ /// The frames a reader parses out of `bytes`.
+ fn frames(mut bytes: &[u8]) -> Vec {
+ let mut out = Vec::new();
+ while !bytes.is_empty() {
+ let (len, rest) = bytes.split_at(4);
+ let n = u32::from_be_bytes(len.try_into().unwrap()) as usize;
+ let (line, rest) = rest.split_at(n);
+ out.push(String::from_utf8(line.to_vec()).unwrap());
+ bytes = rest;
+ }
+ out
+ }
+
+ #[test]
+ fn frames_are_length_prefixed_lines_without_the_newline() {
+ let mut buf = Vec::new();
+ let skipped = write_frames(
+ &mut buf,
+ &["{\"a\":1}\n".to_string(), "{\"b\":2}".to_string()],
+ )
+ .unwrap();
+ assert_eq!(skipped, 0);
+ assert_eq!(buf[..4], 7u32.to_be_bytes());
+ assert_eq!(frames(&buf), vec!["{\"a\":1}", "{\"b\":2}"]);
+ }
+
+ /// A record no frame can carry (over the host's frame bound, or empty)
+ /// is left out and counted, and the records around it still go: the
+ /// host would end the connection on that frame.
+ #[test]
+ fn a_record_over_the_frame_bound_is_dropped_not_sent() {
+ let max = mlog::otel::FORWARD_MAX_FRAME_BYTES;
+ let at_bound = "x".repeat(max);
+ let mut buf = Vec::new();
+ let skipped = write_frames(
+ &mut buf,
+ &[
+ "{\"a\":1}".to_string(),
+ "y".repeat(max + 1),
+ String::new(),
+ "\n".to_string(),
+ at_bound.clone(),
+ "{\"b\":2}".to_string(),
+ ],
+ )
+ .unwrap();
+ assert_eq!(skipped, 3);
+ assert_eq!(
+ frames(&buf),
+ vec!["{\"a\":1}".to_string(), at_bound, "{\"b\":2}".to_string()]
+ );
+
+ // A burst of nothing but such records writes nothing at all.
+ let mut buf = Vec::new();
+ assert_eq!(write_frames(&mut buf, &["z".repeat(max + 1)]).unwrap(), 1);
+ assert!(buf.is_empty());
+ }
+
+ /// With the host away the sender drops and counts; once a connect
+ /// succeeds the later records arrive as frames; a failed write drops the
+ /// connection and the burst, and the next record reconnects.
+ #[test]
+ fn the_sender_drops_while_the_host_is_away_and_reconnects() {
+ let (tx, rx) = sync_channel::(16);
+ let (mut host, guest) = UnixStream::pair().unwrap();
+ let (mut host2, guest2) = UnixStream::pair().unwrap();
+ let ends = std::sync::Mutex::new(vec![Some(guest2), Some(guest)]);
+ let attempts = std::sync::atomic::AtomicUsize::new(0);
+ let (sent_before, dropped_before) = (sent(), dropped());
+ std::thread::scope(|s| {
+ s.spawn(|| {
+ run(
+ rx,
+ || {
+ let n = attempts.fetch_add(1, Ordering::Relaxed);
+ // The first attempt fails; the next ones hand out the
+ // pairs' guest ends in order.
+ if n == 0 {
+ return Err(io::Error::other("no host"));
+ }
+ let end = ends.lock().unwrap().pop().flatten();
+ end.map(|e| Box::new(e) as Box)
+ .ok_or_else(|| io::Error::other("no more ends"))
+ },
+ Duration::ZERO,
+ );
+ });
+ tx.send("lost".into()).unwrap();
+ // Let the sender take the record with no host.
+ while TAKEN.load(Ordering::Relaxed) == 0 {
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ tx.send("one".into()).unwrap();
+ let mut buf = [0u8; 7];
+ host.read_exact(&mut buf).unwrap();
+ assert_eq!(frames(&buf), vec!["one"]);
+ // The host closes: the next write fails, that burst is dropped,
+ // and the one after it goes out on the fresh connection.
+ drop(host);
+ tx.send("two".into()).unwrap();
+ while dropped() - dropped_before < 2 {
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ tx.send("three".into()).unwrap();
+ let mut buf = [0u8; 9];
+ host2.read_exact(&mut buf).unwrap();
+ assert_eq!(frames(&buf), vec!["three"]);
+ drop(tx);
+ });
+ assert_eq!(sent() - sent_before, 2);
+ assert_eq!(dropped() - dropped_before, 2);
+ }
+}
diff --git a/crates/minvmd/Cargo.toml b/crates/minvmd/Cargo.toml
index 42218c1263..0503e5e5f3 100644
--- a/crates/minvmd/Cargo.toml
+++ b/crates/minvmd/Cargo.toml
@@ -29,6 +29,10 @@ libc.workspace = true
serde = { workspace = true }
thiserror.workspace = true
serde_json_lenient.workspace = true
+# The strict parser the guest telemetry door reads the guest's lines with
+# (`guest_telemetry::accept_line`): no comments, no trailing commas, so the
+# host forwards only what a strict collector reads the same way.
+serde_json.workspace = true
toml.workspace = true
# Cross-platform per-process resource sampling (CPU%, RSS, disk I/O) of the VMM
# child PID, and host capacity (cores/memory) for `config set` validation (#747).
diff --git a/crates/minvmd/src/cmd/run.rs b/crates/minvmd/src/cmd/run.rs
index 490bafb559..9beedd4285 100644
--- a/crates/minvmd/src/cmd/run.rs
+++ b/crates/minvmd/src/cmd/run.rs
@@ -655,6 +655,27 @@ fn run_foreground(root: tracing::span::EnteredSpan) -> Result<()> {
&proxy_publish,
)?;
+ // The guest telemetry door (TEL-034): bound only with telemetry on,
+ // and handed to the VMM child the same way, so the guest daemon's
+ // records reach this process's exporter and spool over the vsock
+ // bridge and the guest needs no endpoint. Best-effort like the doors
+ // above: a bind failure is warned and the VM boots without it, and the
+ // guest then spools alone (TEL-046).
+ let guest_telemetry_door = if mlog::otel::telemetry_enabled() {
+ crate::control::resolve_control_sock()
+ .and_then(|sock_path| crate::guest_telemetry::spawn_guest_telemetry_door(&sock_path))
+ .inspect_err(|error| {
+ tracing::warn!(
+ %error,
+ "failed to bind the guest telemetry door; the guest daemon's telemetry \
+ stays in the guest's own spool"
+ );
+ })
+ .ok()
+ } else {
+ None
+ };
+
// The host answerer (NET-138): the box zone's answerer on the host
// loopback, answering from this host-authored table — the same
// semantics the native daemon's answerer gives, over the same shared
@@ -953,6 +974,9 @@ fn run_foreground(root: tracing::span::EnteredSpan) -> Result<()> {
// names. A redraw hands the same door to the fresh boot.
let (name, path) = &guest_report_door;
cmd.env(name, path);
+ if let Some(path) = guest_telemetry_door.as_ref() {
+ cmd.env(crate::guest_telemetry::GUEST_TELEMETRY_SOCK_ENV, path);
+ }
cmd.env(MARKER_SOCK_ENV, &marker_sock_path)
// The node's proxy port travels to the guest through the VMM child's
// env: the child is a separate process (like the marker socket path),
@@ -1268,6 +1292,13 @@ fn run_foreground(root: tracing::span::EnteredSpan) -> Result<()> {
// ── Phase 3: Supervise until VMM child exits ─────────────────────────────
let status = child.wait().context("waiting for VMM child")?;
+ // The VM is gone, so the guest telemetry door's connection ends: let
+ // the door hand the guest's last records (its shutdown) to the host's
+ // endpoint before this process exits, bounded (TEL-034). The spool
+ // already holds them either way.
+ if guest_telemetry_door.is_some() {
+ crate::guest_telemetry::drain_at_exit(crate::guest_telemetry::DRAIN_AT_EXIT);
+ }
// The VM is gone, so no answer to a pending ask can still matter: each
// is cancelled and audited before the supervisor exits (NET-045).
crate::control::stop_pending_asks(&boxes);
diff --git a/crates/minvmd/src/cmd/stop.rs b/crates/minvmd/src/cmd/stop.rs
index 7d0c8f4e8d..c58d804444 100644
--- a/crates/minvmd/src/cmd/stop.rs
+++ b/crates/minvmd/src/cmd/stop.rs
@@ -1,9 +1,11 @@
//! `minvmd stop` subcommand (R4.4, R2.3).
//!
//! Reads `vmm_pid` from `minvmd.toml`, asks the in-VM minimald to shut down
-//! (drain sessions + quiesce the data volume) over the bridge UDS, then sends
-//! `SIGTERM` to the VMM child, waits up to 5 s, escalates to `SIGKILL` on
-//! timeout, then resets `minvmd.toml` to `Stopped`.
+//! (drain sessions + quiesce the data volume) over the bridge UDS, lets an
+//! acknowledging guest power the VM off by itself for up to
+//! `GUEST_POWEROFF_GRACE` (its exit path carries its last telemetry), then
+//! sends `SIGTERM` to the VMM child if it is still there, waits up to 5 s,
+//! escalates to `SIGKILL` on timeout, then resets `minvmd.toml` to `Stopped`.
//!
//! The command is idempotent: if the daemon is already stopped (or has never
//! been provisioned), it returns successfully with no action. Stale active
@@ -213,10 +215,76 @@ fn quiesce_then_signal(quiesce: Option, pid: u32, grace: Duration)
}
}
});
- signal_and_wait(pid, grace)?;
+ settle_vmm(pid, acknowledged, GUEST_POWEROFF_GRACE, grace)?;
Ok(acknowledged)
}
+/// How long a stop lets an acknowledged guest power the VM off by itself
+/// before it SIGTERMs the VMM (TEL-034, guest-vsock.md). The ack is not the
+/// guest's last word: its `rpc` span for the Shutdown (the guest's one child
+/// of `guest.shutdown`) ends only once the ack is written, and the guest's
+/// exit path (`minimald` `main`: drain, telemetry shutdown, the vsock
+/// sender's drain, then `reboot(2)`) is what carries that span and the
+/// handler's last records to the host. A SIGTERM at the ack races that path
+/// and, on a loaded host, wins (lab 715k on debian-12). The guest powering
+/// off is what makes the VMM exit, as on a `min stop`; a guest that does not
+/// within this bound gets the SIGTERM as before. Typical exit paths take a
+/// few hundred milliseconds, so a healthy stop pays that, not the bound.
+const GUEST_POWEROFF_GRACE: Duration = Duration::from_secs(3);
+
+/// Take the VMM child `pid` down after the quiesce. When the guest
+/// acknowledged the Shutdown RPC (`acked`), first wait up to `poweroff_grace`
+/// (never past `grace`, a stop's whole budget for this child) for the VM to
+/// exit by itself, so the guest's exit-time telemetry flush completes before
+/// the VMM goes; then (or at once, without an ack) [`signal_and_wait`] with
+/// what is left of `grace`.
+fn settle_vmm(pid: u32, acked: bool, poweroff_grace: Duration, grace: Duration) -> Result<()> {
+ if !acked {
+ return signal_and_wait(pid, grace);
+ }
+ let started = Instant::now();
+ let pid_t = libc::pid_t::try_from(pid)
+ .ok()
+ .filter(|p| *p > 0)
+ .ok_or_else(|| anyhow::anyhow!("invalid vmm_pid {pid} in state"))?;
+ if wait_for_exit(pid_t, poweroff_grace.min(grace)) {
+ tracing::debug!(pid, "the guest powered the VM off after its ack");
+ return Ok(());
+ }
+ tracing::info!(
+ pid,
+ grace_ms = u64::try_from(poweroff_grace.as_millis()).unwrap_or(u64::MAX),
+ "the guest acknowledged Shutdown but did not power off within the grace; \
+ signalling the VMM (its last telemetry may not reach the host)"
+ );
+ signal_and_wait(pid, grace.saturating_sub(started.elapsed()))
+}
+
+/// Whether `pid` exists. Errors other than ESRCH count as alive (best effort,
+/// as the SIGTERM wait has always treated them).
+fn pid_alive(pid: libc::pid_t) -> bool {
+ // SAFETY: kill(pid, 0) checks for process existence without delivering a
+ // signal; `pid` is positive (checked by the caller).
+ if unsafe { libc::kill(pid, 0) } == 0 {
+ return true;
+ }
+ std::io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH)
+}
+
+/// Poll `pid` every 50 ms for up to `timeout`; whether it is gone.
+fn wait_for_exit(pid: libc::pid_t, timeout: Duration) -> bool {
+ let deadline = Instant::now() + timeout;
+ loop {
+ if !pid_alive(pid) {
+ return true;
+ }
+ if Instant::now() >= deadline {
+ return false;
+ }
+ std::thread::sleep(Duration::from_millis(50));
+ }
+}
+
/// Send `SIGTERM` to `pid`; wait up to `grace`; escalate to `SIGKILL` on
/// timeout.
fn signal_and_wait(pid: u32, grace: Duration) -> Result<()> {
@@ -651,6 +719,62 @@ mod tests {
child.wait().unwrap();
}
+ /// A stand-in VMM: `sh -c