diff --git a/Cargo.lock b/Cargo.lock index 099066aaaf..b43d0fa173 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3785,6 +3785,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" name = "minimald" version = "0.5.0-rc1" dependencies = [ + "anyhow", "args", "async-compression", "async-dialog", @@ -3794,11 +3795,13 @@ dependencies = [ "bytes", "camino", "check", + "chrono", "clap", "clap_complete", "common", "constcat", "decode", + "diagnostics", "either", "fd-lock", "futures", diff --git a/crates/diagnostics/src/bundle.rs b/crates/diagnostics/src/bundle.rs index 464f9f937e..7938ad994a 100644 --- a/crates/diagnostics/src/bundle.rs +++ b/crates/diagnostics/src/bundle.rs @@ -97,6 +97,10 @@ pub struct BundleWriter { collected: Vec, skipped: Vec, errors: Vec, + /// Set for the duration of a tar write and cleared once it completes. + /// A writer observed with this still set was dropped mid-record — see + /// [`BundleWriter::append`]. + writing: bool, } impl BundleWriter { @@ -143,6 +147,7 @@ impl BundleWriter { collected: Vec::new(), skipped: Vec::new(), errors: Vec::new(), + writing: false, } } @@ -242,24 +247,58 @@ impl BundleWriter { skipped: std::mem::take(&mut self.skipped), errors: std::mem::take(&mut self.errors), }; - let json = serde_json::to_vec_pretty(&manifest).context("serializing manifest")?; - self.append("manifest.json", &json).await?; + // No `?` until the Builder has been consumed. `async_tar`'s Drop panics + // on a Builder that was never finalized, so an early return here — a + // full disk, a broken pipe, an abandoned stream — would abort the + // process instead of surfacing the error. Record the outcome, close the + // archive either way, and report afterwards. + let recorded = match serde_json::to_vec_pretty(&manifest) { + Ok(json) => self.append("manifest.json", &json).await, + Err(e) => Err(anyhow::Error::new(e).context("serializing manifest")), + }; - let mut encoder = self - .tar - .into_inner() - .await - .context("finalizing tar archive")?; - encoder.shutdown().await.context("flushing zstd encoder")?; - encoder - .into_inner() - .finalize() - .await - .context("finalizing bundle writer")?; - Ok(()) + let closed = async { + let mut encoder = self + .tar + .into_inner() + .await + .context("finalizing tar archive")?; + encoder.shutdown().await.context("flushing zstd encoder")?; + encoder + .into_inner() + .finalize() + .await + .context("finalizing bundle writer")?; + Ok::<(), anyhow::Error>(()) + } + .await; + + // The manifest failure is the more informative of the two. + recorded.and(closed) } + /// Appends one tar record. + /// + /// `append_data` writes the header and the body in a single `await`. When + /// the sink is a bounded pipe — the daemon streams into one — a client that + /// stops reading fills it and parks this write mid-record. If the caller is + /// racing a deadline (`collect_step!` is) the future is then dropped with a + /// header, and possibly part of a body, already on the wire; appending the + /// next entry after that splices a second record into the first one's + /// declared body and the archive silently stops being readable. + /// + /// A dropped future runs no code, so the guard is inverted: `writing` is + /// raised before the await and lowered after it, and a raised flag on entry + /// means the previous write never returned. That converts silent corruption + /// into a refusal — the bundle ends short and says why, instead of arriving + /// intact-looking and unparseable. async fn append(&mut self, path: &str, bytes: &[u8]) -> Result<(), anyhow::Error> { + anyhow::ensure!( + !self.writing, + "archive stream abandoned: a previous entry write was cancelled \ + part-way through its tar record, so nothing further can be \ + appended without corrupting it" + ); let mut header = Header::new_gnu(); header.set_size(bytes.len() as u64); header.set_mode(0o644); @@ -269,10 +308,24 @@ impl BundleWriter { Some(root) => format!("{root}/{path}"), None => path.to_string(), }; - self.tar - .append_data(&mut header, &full, bytes) - .await - .with_context(|| format!("adding {full}")) + self.writing = true; + let written = self.tar.append_data(&mut header, &full, bytes).await; + self.writing = false; + written.with_context(|| format!("adding {full}")) + } +} + +/// Joins a caller-supplied group with a collector's fixed relative path. +/// +/// Collectors are parameterized by a `dest` group so the same mechanic can +/// serve a CLI bundle (`"host"`) and a daemon one. An **empty** group means +/// the entry sits at the bundle root, which the daemon's rootless stream uses +/// — joining it blindly would emit an absolute `/net/routes.txt` tar path. +pub(crate) fn scoped(dest: &str, rel: &str) -> String { + if dest.is_empty() { + rel.to_string() + } else { + format!("{dest}/{rel}") } } @@ -481,4 +534,56 @@ pub(crate) mod tests { .await .unwrap(); } + + /// A write cancelled part-way through its tar record must not be followed + /// by another one: the second record would be spliced into the first's + /// declared body and the archive would decode as garbage rather than + /// failing. Reproduces the daemon's shape — a bounded pipe whose reader + /// has stopped — instead of asserting on the flag directly. + #[tokio::test] + async fn a_cancelled_write_refuses_further_entries() { + // Held, never read: writes fill the pipe and park rather than erroring. + let (tx, rx) = tokio::io::duplex(64); + let mut w = BundleWriter::stream(tx, "test"); + + // Incompressible, so zstd cannot swallow it into its own buffer and the + // write genuinely reaches the full pipe. + let payload: Vec = (0..2u32 << 20) + .map(|i| (i.wrapping_mul(2_654_435_761) >> 16) as u8) + .collect(); + + let parked = tokio::time::timeout( + std::time::Duration::from_millis(200), + w.add_bytes("first.txt", &payload, Redaction::None), + ) + .await; + assert!( + parked.is_err(), + "expected the write to park on the unread pipe" + ); + + let err = w + .add_bytes("second.txt", b"after", Redaction::None) + .await + .expect_err("appending after a cancelled record must fail"); + assert!( + format!("{err:#}").contains("abandoned"), + "error should name the abandoned stream: {err:#}" + ); + + // `finish` writes the manifest through the same path, so it refuses too + // rather than emitting a manifest that describes a corrupt archive. + // + // The reader is dropped first, exactly as the daemon's pump does when + // its deadline expires: `finish` must still consume the tar Builder on + // this error path, and consuming it writes the trailer. Against a live + // but unread pipe that write would park forever — dropping the reader + // turns it into `BrokenPipe` and lets the error surface. + drop(rx); + let err = w + .finish(chrono::Utc::now(), std::time::Duration::ZERO) + .await + .expect_err("finish must refuse a poisoned archive"); + assert!(format!("{err:#}").contains("abandoned"), "{err:#}"); + } } diff --git a/crates/diagnostics/src/kmsg.rs b/crates/diagnostics/src/kmsg.rs new file mode 100644 index 0000000000..bc1eef8a31 --- /dev/null +++ b/crates/diagnostics/src/kmsg.rs @@ -0,0 +1,247 @@ +//! The kernel ring buffer, read straight from `/dev/kmsg`. +//! +//! A guest bundle fetched on its own carries no kernel evidence otherwise. +//! Kernel messages reach the host only over the `console=hvc0` stream the +//! *host* collector captures into `boot.log`, and that file is recreated per +//! boot and tail-capped — so the banner naming the running kernel, written +//! first, is the first thing a long-lived VM drops. The microVM rootfs ships +//! no `dmesg` either, which is fine: `/dev/kmsg` needs no tool, just one +//! `read(2)` per record. +//! +//! Mechanics only, per this crate's split — where the capture lands and how +//! much of it is kept are the caller's policy. + +#[cfg(any(target_os = "linux", test))] +use std::collections::VecDeque; + +use crate::bundle::{BundleSink, BundleWriter, scoped}; +#[cfg(target_os = "linux")] +use crate::manifest::Redaction; + +/// One `read(2)` on `/dev/kmsg` returns exactly one record, and fails with +/// `EINVAL` rather than truncating when the buffer cannot hold it whole. The +/// kernel's own record limit is `PRINTK_MESSAGE_MAX` (2 KiB); 8 KiB leaves +/// room for the dictionary half a record may carry. +#[cfg(target_os = "linux")] +const RECORD_BUF: usize = 8 * 1024; + +/// Bound on records consumed in one pass. The read stops at `EAGAIN` — the end +/// of the buffer — but a kernel logging faster than this loop drains it would +/// never produce one, so the pass is bounded by count as well. +#[cfg(target_os = "linux")] +const RECORDS_MAX: usize = 64 * 1024; + +/// `/logs/kmsg.txt`: the tail of the kernel ring buffer, at most `cap` +/// bytes, in the kernel's own order and format. +/// +/// Raw records, not decoded — the `priority,sequence,timestamp,flags;message` +/// prefix is data (the sequence numbers show what the buffer dropped), and +/// fidelity beats prettiness here for the same reason it does in +/// [`crate::net::proc_net_tables`]. +/// +/// Absence, unreadability and capping are each recorded rather than silent: on +/// a restricted or stripped host "no kernel evidence" is itself the finding, +/// and a reader must be able to tell it from a collector that never ran. +pub async fn kmsg_tail( + w: &mut BundleWriter, + dest: &str, + cap: u64, +) -> Result<(), anyhow::Error> { + let path = scoped(dest, "logs/kmsg.txt"); + #[cfg(not(target_os = "linux"))] + { + let _ = cap; + w.skip(path, "the kernel ring buffer needs /dev/kmsg"); + Ok(()) + } + #[cfg(target_os = "linux")] + { + use anyhow::Context as _; + // Up to `RECORDS_MAX` synchronous reads — off the caller's runtime, the + // same discipline the `/proc` walks use. `O_NONBLOCK` means no single + // read waits on the kernel, but the pass is still a syscall loop. + let read = tokio::task::spawn_blocking(move || read_kmsg(cap)) + .await + .context("kmsg worker")?; + match read { + Ok((bytes, dropped)) => { + let redaction = if dropped { + Redaction::TailCapped + } else { + Redaction::None + }; + w.add_bytes(&path, &bytes, redaction).await + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + w.skip( + path, + "no /dev/kmsg — this kernel does not expose the ring buffer here", + ); + Ok(()) + } + // Most often `EACCES`/`EPERM`: not root, or `kernel.dmesg_restrict` + // is set. Either way the errno is the answer, so it travels as-is. + Err(e) => { + w.skip(path, format!("unreadable: {e}")); + Ok(()) + } + } + } +} + +/// Reads the ring buffer to its end, keeping the last `cap` bytes of it. +/// +/// `O_NONBLOCK` is what makes "read to the end" terminate at all: without it +/// the read that exhausts the buffer blocks until the kernel logs something +/// new, which on the quiet wedged guest this capture exists for may be never. +/// The end of the buffer is therefore `EAGAIN`, not EOF. +/// +/// Returns the bytes and whether anything older than them is missing. +#[cfg(target_os = "linux")] +fn read_kmsg(cap: u64) -> std::io::Result<(Vec, bool)> { + use std::io::Read as _; + use std::os::unix::fs::OpenOptionsExt as _; + + let mut file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NONBLOCK) + .open("/dev/kmsg")?; + + let mut buf = vec![0u8; RECORD_BUF]; + let mut tail = RecordTail::new(cap); + let mut overwritten = false; + let mut drained = false; + for _ in 0..RECORDS_MAX { + match file.read(&mut buf) { + Ok(0) => { + drained = true; + break; + } + Ok(n) => tail.push(buf[..n].to_vec()), + // `EAGAIN`: the buffer is drained, which is the only end there is. + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + drained = true; + break; + } + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + // `EPIPE`: records were overwritten between reads. The file + // position has already advanced to the oldest surviving record, so + // carry on — what was lost is exactly what a tail drops anyway. + Err(e) if e.raw_os_error() == Some(libc::EPIPE) => overwritten = true, + Err(e) => return Err(e), + } + } + // Hitting the count bound truncates the *newest* end, the opposite of what + // a tail promises, so it is stated in the capture rather than inferred. + if !drained { + tail.push( + format!("\n") + .into_bytes(), + ); + } + let (bytes, dropped) = tail.into_bytes(); + Ok((bytes, dropped || overwritten || !drained)) +} + +/// A bounded tail of whole records: pushing past `cap` drops the oldest. +/// +/// Whole records only. The cap is a size bound, not a line budget, and half a +/// kmsg record is a corrupt line rather than a shorter one — so the newest +/// record is kept even when it alone exceeds the cap. +#[cfg(any(target_os = "linux", test))] +struct RecordTail { + kept: VecDeque>, + held: u64, + cap: u64, + dropped: bool, +} + +#[cfg(any(target_os = "linux", test))] +impl RecordTail { + fn new(cap: u64) -> Self { + Self { + kept: VecDeque::new(), + held: 0, + cap, + dropped: false, + } + } + + fn push(&mut self, record: Vec) { + self.held += record.len() as u64; + self.kept.push_back(record); + while self.held > self.cap && self.kept.len() > 1 { + let oldest = self.kept.pop_front().expect("len > 1 was just checked"); + self.held -= oldest.len() as u64; + self.dropped = true; + } + } + + /// The kept records concatenated, and whether anything older was dropped. + fn into_bytes(self) -> (Vec, bool) { + (self.kept.into_iter().flatten().collect(), self.dropped) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn record_tail_drops_whole_records_from_the_oldest_end() { + let mut tail = RecordTail::new(10); + tail.push(b"aaaa".to_vec()); + tail.push(b"bbbb".to_vec()); + tail.push(b"cccc".to_vec()); + let (bytes, dropped) = tail.into_bytes(); + assert_eq!(bytes, b"bbbbcccc", "the oldest record goes, not half of it"); + assert!(dropped, "the loss is reported, not silent"); + } + + #[test] + fn record_tail_under_the_cap_drops_nothing() { + let mut tail = RecordTail::new(64); + tail.push(b"one\n".to_vec()); + tail.push(b"two\n".to_vec()); + assert_eq!(tail.into_bytes(), (b"one\ntwo\n".to_vec(), false)); + } + + /// A single record over the cap is kept whole: half a record is a corrupt + /// line, and dropping the newest message defeats the point of the capture. + #[test] + fn record_tail_keeps_a_record_larger_than_the_cap() { + let mut tail = RecordTail::new(4); + tail.push(b"older".to_vec()); + tail.push(b"a record longer than the cap".to_vec()); + let (bytes, dropped) = tail.into_bytes(); + assert_eq!(bytes, b"a record longer than the cap"); + assert!(dropped); + } + + /// Whatever the host permits, the bundle explains itself: the buffer is + /// either collected or the manifest says why it is not. + #[cfg(target_os = "linux")] + #[tokio::test] + async fn kmsg_is_either_collected_or_explained() { + let tmp = tempfile::TempDir::new().unwrap(); + let out = tmp.path().join("b.tar.zst"); + let mut w = BundleWriter::create(&out, "r", "v").await.unwrap(); + kmsg_tail(&mut w, "", 64 * 1024).await.unwrap(); + w.finish(chrono::Utc::now(), std::time::Duration::ZERO) + .await + .unwrap(); + + let files = crate::bundle::tests::unpack_bundle(&out, "r").await; + let manifest: serde_json::Value = serde_json::from_slice(&files["manifest.json"]).unwrap(); + let recorded = format!("{}{}", manifest["collected"], manifest["skipped"]); + assert!( + recorded.contains("logs/kmsg.txt"), + "present or explained, never silent: {recorded}" + ); + assert!( + manifest["errors"].as_array().unwrap().is_empty(), + "an absent or restricted ring buffer is a skip, not a collector failure: {}", + manifest["errors"] + ); + } +} diff --git a/crates/diagnostics/src/lib.rs b/crates/diagnostics/src/lib.rs index 43f0363807..a4888163e8 100644 --- a/crates/diagnostics/src/lib.rs +++ b/crates/diagnostics/src/lib.rs @@ -15,6 +15,7 @@ //! - [`capture`] — bounded subprocess output capture for command-shaped //! collectors //! - [`logs`] — newest-first selection of rotated log files +//! - [`kmsg`] — the kernel ring buffer, read without `dmesg` //! - [`disk`] — filesystem capacity probes //! - [`system`] — host identity/capability probe (OS, kernel, KVM, disks) //! @@ -26,6 +27,8 @@ pub mod bundle; pub mod capture; #[cfg(unix)] pub mod disk; +#[cfg(unix)] +pub mod kmsg; pub mod listing; pub mod logs; pub mod manifest; @@ -39,6 +42,43 @@ pub mod redact; #[cfg(unix)] pub mod system; +/// Per-collector deadline. Generous enough for a slow disk, small enough that +/// one wedged collector cannot stall a whole bundle — and a wedged host is +/// exactly when the bundle is wanted. +pub const COLLECTOR_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + +/// Runs one collector future under [`COLLECTOR_TIMEOUT`], recording a failure +/// or timeout in the manifest and letting the run continue. Failure isolation +/// is the contract every producer shares: no collector may abort the bundle. +/// +/// A macro rather than a function so the future's borrow of the writer ends +/// before the error-recording arms re-borrow it. +/// +/// The macro itself emits no log event — a bundle producer that wants one owns +/// that decision, because the same event is a background note in a daemon and +/// terminal output in a CLI. The four-argument form takes +/// `FnOnce(&str /* collector */, &str /* error */)`, called before the manifest +/// is written. +#[macro_export] +macro_rules! collect_step { + ($w:expr, $name:expr, $fut:expr) => { + $crate::collect_step!($w, $name, $fut, |_: &str, _: &str| {}) + }; + ($w:expr, $name:expr, $fut:expr, $on_error:expr) => {{ + let name: String = $name.into(); + let started = std::time::Instant::now(); + let failure = match tokio::time::timeout($crate::COLLECTOR_TIMEOUT, $fut).await { + Ok(Ok(())) => None, + Ok(Err(e)) => Some(format!("{e:#}")), + Err(_) => Some(format!("timed out after {:?}", $crate::COLLECTOR_TIMEOUT)), + }; + if let Some(error) = failure { + ($on_error)(name.as_str(), error.as_str()); + $w.error(name, error, started.elapsed()); + } + }}; +} + pub use bundle::{ BundleSink, BundleWriter, LOG_TAIL_CAP, MAX_LOG_TAIL_BYTES, open_regular_nofollow, }; diff --git a/crates/diagnostics/src/net.rs b/crates/diagnostics/src/net.rs index cb7a8f407c..9deb18748b 100644 --- a/crates/diagnostics/src/net.rs +++ b/crates/diagnostics/src/net.rs @@ -9,7 +9,7 @@ use std::time::Duration; -use crate::bundle::{BundleSink, BundleWriter}; +use crate::bundle::{BundleSink, BundleWriter, scoped}; use crate::capture::{CaptureError, command_stdout}; use crate::manifest::Redaction; @@ -46,7 +46,7 @@ pub async fn listening_sockets( #[cfg(target_os = "linux")] Err(cmd_err) => format!( "(commands unavailable: {cmd_err})\n{}", - proc_net_listeners().await + proc_net_text(PROC_NET_SOCKET_TABLES).await ), #[cfg(not(target_os = "linux"))] Err(cmd_err) => return Err(cmd_err.into()), @@ -59,6 +59,33 @@ pub async fn listening_sockets( .await } +/// The `/proc/net` socket tables, in the order a reader wants them. +pub const PROC_NET_SOCKET_TABLES: &[&str] = &["tcp", "tcp6", "udp", "udp6", "unix"]; + +/// `/net/.txt`: the named raw `/proc/net` tables, concatenated +/// verbatim behind a `=== /proc/net/ ===` banner each. +/// +/// The tool-based captures above are the host's picture; this is the picture a +/// process has when there are no tools at all — the microVM rootfs ships no +/// `ss`, `ip`, or `netstat`, so `/proc/net` *is* the network state there. Which +/// tables land where is the caller's policy (`("routes", ["route", +/// "fib_trie"])` is the binary-free routes-and-addresses capture); an +/// unreadable table is recorded inline as data rather than failing the run. +pub async fn proc_net_tables( + w: &mut BundleWriter, + dest: &str, + name: &str, + tables: &[&str], +) -> Result<(), anyhow::Error> { + let text = proc_net_text(tables).await; + w.add_bytes( + &scoped(dest, &format!("net/{name}.txt")), + text.as_bytes(), + Redaction::None, + ) + .await +} + /// `/net/interfaces.txt`: interface addresses, MACs masked to OUI. The /// IPs and subnets are the diagnostic payload (own-IP rides the 100.64/16 /// CGNAT block VPNs also use, so collisions must be visible); MACs carry no @@ -150,11 +177,10 @@ fn mask_macs(text: &str) -> String { } /// Raw `/proc/net` tables — not parsed; the dev team decodes hex -/// address:port pairs, and fidelity beats prettiness in a fallback. -#[cfg(target_os = "linux")] -async fn proc_net_listeners() -> String { +/// address:port pairs, and fidelity beats prettiness. +async fn proc_net_text(tables: &[&str]) -> String { let mut text = String::new(); - for table in ["tcp", "tcp6", "udp", "udp6", "unix"] { + for table in tables { let path = format!("/proc/net/{table}"); text.push_str(&format!("=== {path} ===\n")); match tokio::fs::read_to_string(&path).await { diff --git a/crates/diagnostics/src/procs.rs b/crates/diagnostics/src/procs.rs index 42b873ff03..cbbfaa03c3 100644 --- a/crates/diagnostics/src/procs.rs +++ b/crates/diagnostics/src/procs.rs @@ -15,7 +15,7 @@ use std::borrow::Cow; use std::time::Duration; -use crate::bundle::{BundleSink, BundleWriter}; +use crate::bundle::{BundleSink, BundleWriter, scoped}; use crate::capture::command_stdout; use crate::manifest::Redaction; use crate::redact::is_sensitive_key; @@ -75,7 +75,11 @@ fn scrub_arg(arg: &str) -> Cow<'_, str> { /// than a half-masked secret in a bundle that gets mailed out. Where the real /// boundaries survive — the `/proc` scrape, whose argv is NUL-separated — /// [`scrub_arg`] is used per element instead and nothing is over-redacted. -fn scrub_flattened(line: &str) -> String { +/// +/// Public because a process argv is not the only space-joined command line a +/// bundle carries: the kernel boot line (`/proc/cmdline`) is one too, with the +/// same ambiguity, and it must travel under the same rule. +pub fn scrub_flattened(line: &str) -> String { let mut out = String::with_capacity(line.len()); for (i, tok) in line.split(' ').enumerate() { if i > 0 { @@ -102,7 +106,7 @@ pub async fn process_tree( ) -> Result<(), anyhow::Error> { let (text, pids) = process_table(markers).await?; w.add_bytes( - &format!("{dest}/process-tree.txt"), + &scoped(dest, "process-tree.txt"), text.as_bytes(), Redaction::Keys, ) @@ -119,7 +123,7 @@ pub async fn process_tree( .flatten(); if let Some(status) = status { w.add_bytes( - &format!("{dest}/proc/{pid}.status"), + &scoped(dest, &format!("proc/{pid}.status")), status.as_bytes(), Redaction::None, ) @@ -131,6 +135,100 @@ pub async fn process_tree( Ok(()) } +/// `/proc.txt`: the whole process table — pid, ppid, RSS, command — with +/// full (scrubbed) argv for marker-matched processes and `comm` alone for +/// everyone else. +/// +/// [`process_tree`] answers "where is our family"; this answers "what else is +/// running", which is the question when the wedge is a task child or a +/// neighbour eating the machine. Unrelated processes contribute their name and +/// never their arguments — a support bundle is not a keylogger. Rows are +/// ordered by pid, so two bundles from one incident diff cleanly. +pub async fn all_processes( + w: &mut BundleWriter, + dest: &str, + markers: &[&str], +) -> Result<(), anyhow::Error> { + let path = scoped(dest, "proc.txt"); + #[cfg(not(target_os = "linux"))] + { + let _ = markers; + w.skip(path, "a full process table needs /proc"); + Ok(()) + } + #[cfg(target_os = "linux")] + { + use anyhow::Context as _; + // Hundreds of synchronous /proc reads on a busy host — off the caller's + // runtime, for the same reason as the scrape in `process_table`. + let owned: Vec = markers.iter().map(|m| (*m).to_string()).collect(); + let text = tokio::task::spawn_blocking(move || { + let markers: Vec<&str> = owned.iter().map(String::as_str).collect(); + proc_table_text(&markers) + }) + .await + .context("proc table worker")??; + w.add_bytes(&path, text.as_bytes(), Redaction::Keys).await + } +} + +/// The `/proc` walk behind [`all_processes`]. +#[cfg(target_os = "linux")] +fn proc_table_text(markers: &[&str]) -> Result { + use anyhow::Context as _; + use std::fmt::Write as _; + + let mut rows: Vec<(u32, String)> = std::fs::read_dir("/proc") + .context("reading /proc")? + .flatten() + .filter_map(|entry| { + let pid = entry.file_name().to_str()?.parse::().ok()?; + // NUL-separated, so the true argv boundaries are intact and each + // element can be scrubbed on its own. + let argv: Vec = std::fs::read(format!("/proc/{pid}/cmdline")) + .unwrap_or_default() + .split(|b| *b == 0) + .filter(|part| !part.is_empty()) + .map(|part| String::from_utf8_lossy(part).into_owned()) + .collect(); + let ours = argv + .first() + .is_some_and(|argv0| argv0_matches(argv0, markers)); + let cmd = if ours { + argv.iter() + .map(|arg| scrub_arg(arg)) + .collect::>() + .join(" ") + } else { + // `comm` is the process name only, never its arguments. + std::fs::read_to_string(format!("/proc/{pid}/comm")) + .map(|comm| comm.trim_end().to_string()) + .unwrap_or_default() + }; + let status = std::fs::read_to_string(format!("/proc/{pid}/status")).unwrap_or_default(); + let field = |key: &str| { + status + .lines() + .find(|line| line.starts_with(key)) + .and_then(|line| line.split_whitespace().nth(1)) + .unwrap_or("-") + .to_string() + }; + Some(( + pid, + format!("{pid}\t{}\t{}\t{cmd}", field("PPid:"), field("VmRSS:")), + )) + }) + .collect(); + rows.sort_unstable_by_key(|(pid, _)| *pid); + + let mut text = String::from("pid\tppid\tvmrss_kb\tcmd\n"); + for (_, row) in rows { + let _ = writeln!(text, "{row}"); + } + Ok(text) +} + /// Hang triage for the matched family (capped at [`HANG_TRIAGE_PIDS_MAX`]): /// thread samples on macOS, wait-channel + kernel stack + open fds on Linux, /// and one `lsof` over the set. This is the evidence "it's frozen" reports run @@ -141,11 +239,27 @@ pub async fn hang_triage( dest: &str, markers: &[&str], ) -> Result<(), anyhow::Error> { - let (_, pids) = process_table(markers).await?; - let pids: Vec = pids.into_iter().take(HANG_TRIAGE_PIDS_MAX).collect(); + hang_triage_including(w, dest, markers, &[]).await +} + +/// [`hang_triage`] over the marker-matched family *plus* `always` — pids the +/// caller knows first-hand are its own. +/// +/// A daemon triaging itself has no marker to match on: its argv0 is whatever +/// it was exec'd as (an installed `minimald`, a renamed binary, a test +/// harness), and unlike a scraped pid its identity is not in question — it is +/// the process running this code. So `always` pids are captured first, ahead +/// of the cap, and skip the re-pin check the table snapshot needs. +pub async fn hang_triage_including( + w: &mut BundleWriter, + dest: &str, + markers: &[&str], + always: &[u32], +) -> Result<(), anyhow::Error> { + let pids = triage_pids(markers, always).await?; if pids.is_empty() { w.skip( - format!("{dest}/proc/"), + scoped(dest, "proc/"), "no marker-matched processes to hang-triage", ); return Ok(()); @@ -159,7 +273,7 @@ pub async fn hang_triage( #[cfg(target_os = "macos")] { let mut set = tokio::task::JoinSet::new(); - for &pid in &pids { + for &(pid, _) in &pids { set.spawn(async move { let pid_s = pid.to_string(); let out = command_stdout("sample", &[&pid_s, "1", "10"], SAMPLE_TIMEOUT).await; @@ -172,13 +286,13 @@ pub async fn hang_triage( Ok(sample) => samples.push(sample), // A panicked sample task is a bug, not a diagnostic outcome — // record it rather than dropping the pid silently. - Err(e) => w.skip(format!("{dest}/proc/"), format!("sample task failed: {e}")), + Err(e) => w.skip(scoped(dest, "proc/"), format!("sample task failed: {e}")), } } // Completion order is nondeterministic; keep bundle order by pid. samples.sort_by_key(|(pid, _)| *pid); for (pid, result) in samples { - let path = format!("{dest}/proc/{pid}.sample.txt"); + let path = scoped(dest, &format!("proc/{pid}.sample.txt")); match result { Ok(text) => w.add_bytes(&path, text.as_bytes(), Redaction::None).await?, Err(e) => w.skip(path, format!("sample failed: {e}")), @@ -186,25 +300,26 @@ pub async fn hang_triage( } } - // The pid set actually captured. On Linux each pid is re-checked against - // its own cmdline immediately before its state is read: the table snapshot - // is stale the moment it is taken, and a pid recycled in between would put - // an unrelated process's kernel state and open files into the bundle — - // precisely the "someone else's activity" this collector filters out. + // The pid set actually captured. On Linux a pid taken from the table is + // re-checked against its own cmdline immediately before its state is read: + // the snapshot is stale the moment it is taken, and a pid recycled in + // between would put an unrelated process's kernel state and open files into + // the bundle — precisely the "someone else's activity" this collector + // filters out. #[cfg(target_os = "linux")] let mut captured: Vec = Vec::with_capacity(pids.len()); #[cfg(target_os = "linux")] - for &pid in &pids { - if !still_matches(pid, markers).await { + for &(pid, from_snapshot) in &pids { + if from_snapshot && !still_matches(pid, markers).await { w.skip( - format!("{dest}/proc/{pid}.stack.txt"), + scoped(dest, &format!("proc/{pid}.stack.txt")), "pid no longer matches a marker (exited or recycled since the snapshot)", ); continue; } let text = linux_park_state(pid).await; w.add_bytes( - &format!("{dest}/proc/{pid}.stack.txt"), + &scoped(dest, &format!("proc/{pid}.stack.txt")), text.as_bytes(), Redaction::None, ) @@ -212,11 +327,11 @@ pub async fn hang_triage( captured.push(pid); } #[cfg(not(target_os = "linux"))] - let captured = pids; + let captured: Vec = pids.iter().map(|&(pid, _)| pid).collect(); if captured.is_empty() { w.skip( - format!("{dest}/proc/lsof.txt"), + scoped(dest, "proc/lsof.txt"), "no marker-matched pids still live to inspect", ); return Ok(()); @@ -226,7 +341,7 @@ pub async fn hang_triage( .map(u32::to_string) .collect::>() .join(","); - let path = format!("{dest}/proc/lsof.txt"); + let path = scoped(dest, "proc/lsof.txt"); match command_stdout("lsof", &["-nP", "-p", &pid_list], LSOF_TIMEOUT).await { Ok(text) => w.add_bytes(&path, text.as_bytes(), Redaction::None).await?, Err(e) => w.skip(path, format!("lsof unavailable: {e}")), @@ -234,6 +349,136 @@ pub async fn hang_triage( Ok(()) } +/// The pid set a per-process capture covers: `always` first, then the +/// marker-matched table, capped at [`HANG_TRIAGE_PIDS_MAX`]. The flag on each +/// pid is "this identity came from a stale table snapshot", i.e. whether it +/// must be re-pinned before its kernel state is read. +async fn triage_pids(markers: &[&str], always: &[u32]) -> Result, anyhow::Error> { + let (_, matched) = process_table(markers).await?; + let mut pids: Vec<(u32, bool)> = always.iter().map(|&pid| (pid, false)).collect(); + pids.extend( + matched + .into_iter() + .filter(|pid| !always.contains(pid)) + .map(|pid| (pid, true)), + ); + pids.truncate(HANG_TRIAGE_PIDS_MAX); + Ok(pids) +} + +/// `/proc/sockets.txt`: which of our processes holds which socket, in +/// which state — every `socket:[inode]` fd of the family joined against the +/// `/proc/net` tables. +/// +/// This is the binary-free `lsof` equivalent, and on a microVM it is the only +/// one: the guest rootfs ships no `lsof`, yet "who still holds the transport +/// socket, and is anyone reading it" is the question a partial wedge turns on. +/// The joined row is the raw table line, not a re-serialization of it. +pub async fn open_sockets( + w: &mut BundleWriter, + dest: &str, + markers: &[&str], + always: &[u32], +) -> Result<(), anyhow::Error> { + socket_join(w, &scoped(dest, "proc/sockets.txt"), markers, always).await +} + +#[cfg(not(target_os = "linux"))] +async fn socket_join( + w: &mut BundleWriter, + path: &str, + _markers: &[&str], + _always: &[u32], +) -> Result<(), anyhow::Error> { + w.skip(path, "fd→socket-inode joins need /proc"); + Ok(()) +} + +#[cfg(target_os = "linux")] +async fn socket_join( + w: &mut BundleWriter, + path: &str, + markers: &[&str], + always: &[u32], +) -> Result<(), anyhow::Error> { + use std::fmt::Write as _; + + let pids = triage_pids(markers, always).await?; + if pids.is_empty() { + w.skip(path, "no marker-matched processes holding sockets"); + return Ok(()); + } + let mut tables = Vec::with_capacity(crate::net::PROC_NET_SOCKET_TABLES.len()); + for table in crate::net::PROC_NET_SOCKET_TABLES { + if let Ok(text) = tokio::fs::read_to_string(format!("/proc/net/{table}")).await { + tables.push((*table, text)); + } + } + let sockets = index_by_inode(&tables); + + let mut text = String::from("pid\tfd\tinode\ttable\tentry\n"); + for (pid, _) in pids { + let Ok(mut entries) = tokio::fs::read_dir(format!("/proc/{pid}/fd")).await else { + let _ = writeln!(text, "{pid}\t-\t-\t-\t"); + continue; + }; + while let Ok(Some(entry)) = entries.next_entry().await { + let Ok(target) = tokio::fs::read_link(entry.path()).await else { + continue; + }; + let Some(inode) = socket_inode(&target.to_string_lossy()) else { + continue; // not a socket fd — files and pipes are `stack.txt`'s job + }; + let fd = entry.file_name().to_string_lossy().into_owned(); + let (table, row) = sockets + .get(&inode) + .map(|(t, l)| (t.as_str(), l.as_str())) + // A socket with no table row is itself a finding: it belongs to + // a family /proc/net does not list (netlink) or to a network + // namespace this process cannot see. + .unwrap_or(("?", "")); + let _ = writeln!(text, "{pid}\t{fd}\t{inode}\t{table}\t{row}"); + } + } + w.add_bytes(path, text.as_bytes(), Redaction::None).await +} + +/// The socket inode named by an fd's readlink target, e.g. `socket:[12345]`. +#[cfg(any(target_os = "linux", test))] +fn socket_inode(link_target: &str) -> Option { + link_target + .strip_prefix("socket:[")? + .strip_suffix(']')? + .parse() + .ok() +} + +/// The field index holding the socket inode in a `/proc/net` table row. +/// `unix` has its own column layout; the IP tables share one. +#[cfg(any(target_os = "linux", test))] +fn inode_column(table: &str) -> usize { + if table == "unix" { 6 } else { 9 } +} + +/// Indexes `(table, contents)` pairs by socket inode, keeping each row +/// verbatim. Header rows fall out for free — their inode column is a label, +/// not a number. +#[cfg(any(target_os = "linux", test))] +fn index_by_inode(tables: &[(&str, String)]) -> std::collections::BTreeMap { + tables + .iter() + .flat_map(|(table, text)| { + text.lines().filter_map(move |line| { + let inode = line.split_whitespace().nth(inode_column(table))?; + Some(( + inode.parse().ok()?, + ((*table).to_string(), line.trim().to_string()), + )) + }) + }) + .collect() +} + /// Filtered `ps` output plus the matched pids; on Linux a `/proc` scrape stands /// in when `ps` is unavailable. The header and a total count are kept so /// "nothing matched" is distinguishable from "ps saw nothing". @@ -460,4 +705,47 @@ mod tests { let line = "minimald run --detach --instance-num 0"; assert_eq!(scrub_flattened(line), line); } + + #[test] + fn socket_inode_reads_only_socket_links() { + assert_eq!(socket_inode("socket:[12345]"), Some(12345)); + assert_eq!(socket_inode("/run/minimald/ssh.sock"), None); + assert_eq!(socket_inode("pipe:[999]"), None); + assert_eq!(socket_inode("anon_inode:[eventpoll]"), None); + } + + /// The join keys on the inode column, which differs between the IP tables + /// and `unix`; header rows must not become entries. + #[test] + fn index_by_inode_keys_both_table_layouts() { + let tcp = "\ + sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode + 0: 00000000:0016 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 41001 1 ffff 100 0 0 10 0 +" + .to_string(); + let unix = "\ +Num RefCount Protocol Flags Type St Inode Path +ffff8880: 00000002 00000000 00010000 0001 01 41002 /run/minimald/ssh.sock +" + .to_string(); + let index = index_by_inode(&[("tcp", tcp), ("unix", unix)]); + + assert_eq!(index.len(), 2, "headers are not entries: {index:?}"); + let (table, row) = &index[&41001]; + assert_eq!(table, "tcp"); + assert!(row.contains("00000000:0016"), "row kept verbatim: {row}"); + let (table, row) = &index[&41002]; + assert_eq!(table, "unix"); + assert!(row.ends_with("/run/minimald/ssh.sock"), "{row}"); + } + + #[tokio::test] + async fn triage_pids_puts_caller_pids_first_and_never_repins_them() { + // No process can match this marker, so the whole set comes from + // `always` — the daemon-triaging-itself case, where argv0 says nothing. + let pids = triage_pids(&["\u{1}no-such-binary"], &[4242]) + .await + .unwrap(); + assert_eq!(pids, vec![(4242, false)]); + } } diff --git a/crates/diagnostics/src/redact.rs b/crates/diagnostics/src/redact.rs index c3d41a6cb5..03f39ab422 100644 --- a/crates/diagnostics/src/redact.rs +++ b/crates/diagnostics/src/redact.rs @@ -44,6 +44,20 @@ pub fn is_sensitive_key(key: &str) -> bool { .any(|part| stripped.contains(part)) } +/// Resolves an env-var name against a caller-supplied allowlist of exact names +/// and name prefixes, fail-closed: a sensitive-shaped name always loses, even +/// when the allowlist admits it. `MINIMALD_TOKEN` matches a project prefix but +/// must never leave the machine. +/// +/// The *mechanic* is here so every bundle producer resolves the allowlist the +/// same way; the *data* — which names a given producer considers safe — stays +/// with the caller, per this crate's mechanics-vs-policy split. +#[must_use] +pub fn is_env_value_allowlisted(name: &str, exact: &[&str], prefixes: &[&str]) -> bool { + !is_sensitive_key(name) + && (exact.contains(&name) || prefixes.iter().any(|p| name.starts_with(p))) +} + /// Returns true when a table/object with this name holds env-var style values /// that must be masked wholesale. pub fn is_env_table_name(name: &str) -> bool { diff --git a/crates/minimal/src/diag/mod.rs b/crates/minimal/src/diag/mod.rs index ff7f8216da..d3a8bbcd6d 100644 --- a/crates/minimal/src/diag/mod.rs +++ b/crates/minimal/src/diag/mod.rs @@ -8,7 +8,6 @@ //! daemons — diagnosing a wedged system must not change it. use std::path::PathBuf; -use std::time::Duration; use anyhow::Context as _; use clap::Args; @@ -58,10 +57,6 @@ fn parse_log_tail_bytes(raw: &str) -> Result { } } -/// A collector timeout generous enough for slow disks, small enough that a -/// hung collector can't stall the whole run. -const COLLECTOR_TIMEOUT: Duration = Duration::from_secs(30); - /// Process names (argv0 basenames) that mark a process as one of ours. This is /// minimal's policy — the *data*; the matching mechanic lives in /// [`diagnostics::procs`], which takes this list as an argument so the same @@ -75,26 +70,12 @@ const PROCESS_MARKERS: &[&str] = &[ "gvproxy", ]; -/// Runs one collector future with [`COLLECTOR_TIMEOUT`]; failure or timeout -/// is recorded in the manifest and the run continues. A macro (not a -/// function) so the future's borrow of the writer ends before the -/// error-recording arms re-borrow it. -macro_rules! collect_step { - ($w:expr, $name:expr, $fut:expr) => {{ - let name: String = $name.into(); - let started = std::time::Instant::now(); - let result = tokio::time::timeout(COLLECTOR_TIMEOUT, $fut).await; - match result { - Ok(Ok(())) => {} - Ok(Err(e)) => $w.error(name, format!("{e:#}"), started.elapsed()), - Err(_) => $w.error( - name, - format!("timed out after {COLLECTOR_TIMEOUT:?}"), - started.elapsed(), - ), - } - }}; -} +// The collector-step contract — the deadline, the manifest recording, the +// failure isolation — is `diagnostics::collect_step!`, shared with the daemon +// bundle so the two cannot drift. The CLI takes the silent form: a collector +// failure is already in the manifest and in the summary line `cmd_bug` prints, +// and this binary's tracing goes to the user's terminal. +use diagnostics::collect_step; pub async fn cmd_bug(global: &GlobalArgs, args: BugArgs) -> Result<(), anyhow::Error> { let created_at = chrono::Utc::now(); diff --git a/crates/minimal/src/diag/redact.rs b/crates/minimal/src/diag/redact.rs index 837fada7ac..a48eb10c73 100644 --- a/crates/minimal/src/diag/redact.rs +++ b/crates/minimal/src/diag/redact.rs @@ -5,8 +5,6 @@ //! applies the same behavior; what belongs to the CLI is only this policy: //! the allowlist of env names whose values are safe and useful verbatim. -use diagnostics::redact::is_sensitive_key; - /// Env vars whose *values* are safe and useful to include verbatim. Everything /// else is reported by name only. const ENV_VALUE_ALLOWLIST_EXACT: &[&str] = &["RUST_LOG", "HOME", "SHELL", "TERM", "PATH"]; @@ -16,13 +14,11 @@ const ENV_VALUE_ALLOWLIST_PREFIXES: &[&str] = &["XDG_", "MINIMAL_", "MINVMD_", " /// A sensitive-shaped name always loses to the allowlist — `MINIMAL_AUTH_TOKEN` /// matches the project prefix but must never leave the machine. pub fn is_env_value_allowlisted(name: &str) -> bool { - if is_sensitive_key(name) { - return false; - } - ENV_VALUE_ALLOWLIST_EXACT.contains(&name) - || ENV_VALUE_ALLOWLIST_PREFIXES - .iter() - .any(|p| name.starts_with(p)) + diagnostics::redact::is_env_value_allowlisted( + name, + ENV_VALUE_ALLOWLIST_EXACT, + ENV_VALUE_ALLOWLIST_PREFIXES, + ) } #[cfg(test)] diff --git a/crates/minimald-rpc/src/lib.rs b/crates/minimald-rpc/src/lib.rs index 827cab8f34..b292f56a4b 100644 --- a/crates/minimald-rpc/src/lib.rs +++ b/crates/minimald-rpc/src/lib.rs @@ -635,11 +635,94 @@ impl OneshotSshRpc for GetMeshStatus { type Response = MeshStatus; } +// --------------------------------------------------------------------------- +// Diagnostic bundle (`min bug`). +// --------------------------------------------------------------------------- + +/// Streaming RPC subsystem: the daemon's contribution to a `min bug` +/// diagnostic bundle. +/// +/// Not an [`OneshotSshRpc`]: the client writes one JSON-encoded +/// [`DiagBundleRequest`] and half-closes, then the daemon streams back a +/// zstd-compressed tar archive of its diagnostic bundle and closes. Errors hit +/// before streaming starts are relayed over extended-data stream 1, so a client +/// that reads zero payload bytes should surface the extended data as the +/// failure reason. +/// +/// Served identically by the native Linux minimald and the in-VM minimald +/// behind the minvmd bridge — the archive's `meta.json` says which one +/// answered. +pub const DIAG_BUNDLE_SUBSYSTEM: &str = constcat::concat!(RPC_SUBSYSTEM_PREFIX, "DiagBundleTarZst"); + +/// Request body for [`DIAG_BUNDLE_SUBSYSTEM`]. +#[non_exhaustive] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct DiagBundleRequest { + /// Per-log-file tail cap in bytes; `0` means the daemon's default. The + /// daemon clamps this to its own ceiling — the value is caller-controlled. + #[serde(default)] + pub log_tail_bytes: u64, + /// Include the recursive state-dir listing (names/sizes only). + #[serde(default = "default_true")] + pub include_state_listing: bool, +} + +impl Default for DiagBundleRequest { + fn default() -> Self { + Self { + log_tail_bytes: 0, + include_state_listing: true, + } + } +} + +/// The type is `#[non_exhaustive]`, so other crates cannot write a struct +/// literal for it; these are how a client departs from the defaults. +impl DiagBundleRequest { + #[must_use] + pub fn with_log_tail_bytes(mut self, bytes: u64) -> Self { + self.log_tail_bytes = bytes; + self + } + + #[must_use] + pub fn with_state_listing(mut self, include: bool) -> Self { + self.include_state_listing = include; + self + } +} + +fn default_true() -> bool { + true +} + #[cfg(test)] mod tests { use super::*; use sessions::wire::request::{ContributionResponse, WireContribution}; + /// An empty request body must decode with the documented defaults so a + /// bare `{}` probe (or an older client) still gets a full bundle. + #[test] + fn diag_bundle_request_defaults_from_empty_object() { + let req: DiagBundleRequest = serde_json::from_str("{}").expect("deserialize"); + assert_eq!(req, DiagBundleRequest::default()); + assert_eq!(req.log_tail_bytes, 0); + assert!(req.include_state_listing); + } + + #[test] + fn diag_bundle_request_round_trips() { + let req = DiagBundleRequest::default() + .with_log_tail_bytes(1024) + .with_state_listing(false); + assert_eq!(round_trip(&req), req); + assert_eq!( + DIAG_BUNDLE_SUBSYSTEM, "minimald-v1-DiagBundleTarZst", + "subsystem name is wire contract; changing it breaks old clients" + ); + } + #[test] fn policy_types_are_present_and_serializable() { // PortMapping construction and round-trip diff --git a/crates/minimald/Cargo.toml b/crates/minimald/Cargo.toml index 400f844c4c..e93e6338cf 100644 --- a/crates/minimald/Cargo.toml +++ b/crates/minimald/Cargo.toml @@ -24,10 +24,12 @@ ssh-forward = [] test-support = [] [dependencies] +anyhow.workspace = true async-compression.workspace = true async-tar.workspace = true bytes.workspace = true camino.workspace = true +chrono.workspace = true clap.workspace = true clap_complete.workspace = true constcat.workspace = true @@ -62,6 +64,7 @@ args.workspace = true async-dialog.workspace = true check.workspace = true common.workspace = true +diagnostics.workspace = true graph.workspace = true mctx.workspace = true mfile.workspace = true diff --git a/crates/minimald/src/diag.rs b/crates/minimald/src/diag.rs new file mode 100644 index 0000000000..4e91f4bb6f --- /dev/null +++ b/crates/minimald/src/diag.rs @@ -0,0 +1,1271 @@ +//! The daemon's contribution to a `min bug` diagnostic bundle. +//! +//! Serves the `DiagBundleTarZst` streaming subsystem: reads one JSON +//! [`DiagBundleRequest`], then streams back a zstd-compressed tar of the +//! daemon's own view — logs, state listings, session records (redacted), +//! process/network/disk state. Served identically by the native Linux daemon +//! and the in-VM pid-1 instance; `meta.json` says which one answered. +//! +//! The archive is written through [`diagnostics::BundleWriter`] in stream +//! (rootless) mode, so the guest bundle carries the same `manifest.json` as the +//! host one: collector failures never abort the stream, they land in the +//! manifest's error list. A half-broken daemon still reports everything it can, +//! plus why the rest is missing. +//! +//! What lives here is policy — which paths, which markers, which env names, +//! where entries land in the archive. Every generic mechanic (the `/proc` +//! walks, the redaction, the tail caps, the socket join) comes from the +//! `diagnostics` crate, which the CLI-side collector shares. + +use std::path::Path; +use std::time::Duration; + +use anyhow::Context as _; +use minimald_rpc::DiagBundleRequest; +use russh::{Channel as RuChannel, server::Msg}; +use serde::Serialize; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + +use diagnostics::redact::{masked_process_env, redact_json}; +use diagnostics::{BundleSink, BundleWriter, LOG_TAIL_CAP, Redaction}; + +use crate::ChannelConfig; +use crate::server::ServerStateHandle; + +/// Longest request body accepted before the read is abandoned. The body is a +/// two-field JSON object; anything larger is a client bug or an attempt to +/// make the daemon buffer for free. +const MAX_REQUEST_BYTES: u64 = 8 * 1024; +/// Longest wait for that body to arrive and the channel to half-close. +/// +/// Bounding the *size* of the request is not the same as bounding the *wait* +/// for it: a client that writes half an object and then holds the channel open +/// sends nothing more and never signals EOF, so the read below would park +/// forever. Nothing else reclaims it — the daemon deliberately runs with +/// `inactivity_timeout: None` (a live attach must not be reaped), and the SSH +/// keepalives it uses instead only detect a peer that has *stopped answering*, +/// not one that answers and stays silent. The body is a two-field object +/// written immediately after the subsystem opens, so this is a generous +/// allowance for a slow link rather than a working deadline. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +/// Ceiling on the caller-controlled per-log tail. +const MAX_LOG_TAIL_BYTES: u64 = 64 * 1024 * 1024; +/// Ceiling on a single session file read into memory (`index.json`, a +/// `record.json`). Both are daemon-written JSON — a record is a few kilobytes +/// and the index scales with the session count — so this is orders of +/// magnitude of headroom, and its job is only to keep a file the daemon did +/// not write and cannot bound from becoming an allocation it cannot survive. +const MAX_SESSION_FILE_BYTES: u64 = 8 * 1024 * 1024; +/// Rotated log files bundled, newest first. +const LOG_FILES_MAX: usize = 5; +/// Entry cap for the recursive state-dir listing. +const LISTING_MAX_ENTRIES: usize = 100_000; +/// Ceiling on the whole client-facing transfer. Every collector is deadlined +/// individually, but those deadlines only bound work — a client that holds the +/// channel open and stops reading it parks the pump instead, and through the +/// bounded duplex that stalls collection and finalization behind it. This is +/// the bound on the peer rather than on ourselves. +const STREAM_TIMEOUT: Duration = Duration::from_secs(300); +/// Duplex buffer between the bundle writer and the channel pump. +const PIPE_BUF: usize = 256 * 1024; +/// The gvproxy management endpoint, reachable only from inside the switch +/// fabric (i.e. from the guest). +const GVPROXY_MGMT: (&str, u16) = ("192.168.127.254", 8080); + +/// argv0 basenames marking a process as part of *this daemon's* family. +/// +/// The daemon's policy, not the CLI's: inside the microVM there is no `minvmd`, +/// no `__krun-vmm` and no `gvproxy` to find — those run on the host and are the +/// host collector's business — while a task child running `min` in the guest is +/// this daemon's business. The matching mechanic (argv0 basename, never a +/// substring) lives in [`diagnostics::procs`], which takes this list as an +/// argument. +const PROC_MARKERS: &[&str] = &["minimald", "min", "minimal"]; + +/// Env vars whose *values* are safe and useful to include verbatim; everything +/// else is reported by name only. Answers "which mode booted, and why is +/// logging off". +/// +/// The daemon owns this list separately from the CLI's because the two are +/// independent policies, free to diverge as `PROC_MARKERS` already has — they +/// happen to hold the same names today. The resolving mechanic, including the +/// fail-closed sensitive-name override, is +/// [`diagnostics::redact::is_env_value_allowlisted`]. +const ENV_VALUE_ALLOWLIST_EXACT: &[&str] = &["RUST_LOG", "HOME", "PATH", "TERM", "SHELL"]; +const ENV_VALUE_ALLOWLIST_PREFIXES: &[&str] = &["MINIMAL_", "MINIMALD_", "MINVMD_", "XDG_"]; + +/// Returns true when the named env var's value may be captured verbatim. +/// A sensitive-shaped name always loses to the allowlist — `MINIMALD_TOKEN` +/// matches the project prefix but must never leave the machine. +fn is_env_value_allowlisted(name: &str) -> bool { + diagnostics::redact::is_env_value_allowlisted( + name, + ENV_VALUE_ALLOWLIST_EXACT, + ENV_VALUE_ALLOWLIST_PREFIXES, + ) +} + +/// Logs a failed collector. The daemon streams its bundle to a remote client, +/// so its own log is the only place an operator watching this host sees that a +/// collector fell over. +fn warn_collector(collector: &str, error: &str) { + tracing::warn!(%collector, %error, "diag collector failed"); +} + +/// The daemon's [`diagnostics::collect_step!`] — the shared deadline, manifest +/// recording, and failure isolation, with the daemon's logging attached. +macro_rules! collect_step { + ($w:expr, $name:expr, $fut:expr) => { + diagnostics::collect_step!($w, $name, $fut, warn_collector) + }; +} + +pub(crate) async fn serve_stream_diag_bundle( + s: ServerStateHandle, + _config: ChannelConfig, + mut c: RuChannel, +) { + if let Err(msg) = stream_diag_bundle(&s, &mut c).await { + let _ = c.extended_data_bytes(1, msg).await; + } + let _ = c.close().await; +} + +/// Reads the request, then streams the bundle. On failure returns the +/// human-readable message to relay over the channel's extended-data stream. +/// +/// Everything that can fail *before* the first payload byte — the request read +/// and its parse — happens up front, so a client that reads zero payload bytes +/// can treat the extended data as the whole story. +async fn stream_diag_bundle(s: &ServerStateHandle, c: &mut RuChannel) -> Result<(), String> { + let req = read_request(c.make_reader(), REQUEST_TIMEOUT).await?; + + // `async_tar::Builder` requires `W: Sync`; the channel writer is not. Same + // duplex-pipe pattern as the workspace-files upload, reversed direction. + let (tx, mut rx) = tokio::io::duplex(PIPE_BUF); + let build = { + let s = s.clone(); + tokio::task::spawn(async move { build_bundle(&s, &req, tx).await }) + }; + + let mut writer = c.make_writer(); + // Bounded here rather than around the build: cancelling the pump is safe, + // cancelling the builder is not (see `build_bundle`). On expiry the `drop` + // below turns the build task's next write into `BrokenPipe`, so it unwinds + // through its own error path instead of parking on the full duplex. + let copy_result = + match tokio::time::timeout(STREAM_TIMEOUT, tokio::io::copy(&mut rx, &mut writer)).await { + Ok(r) => r, + Err(_) => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + format!("client stopped reading the bundle for {STREAM_TIMEOUT:?}"), + )), + }; + let shutdown_result = writer.shutdown().await; + // The read half must be gone before waiting on the build task: if the copy + // failed mid-stream (client vanished) the task is still writing, and only a + // dropped `rx` turns its next write into `BrokenPipe` instead of an + // indefinite block on the full duplex buffer. + drop(rx); + copy_result.map_err(|e| format!("streaming diag bundle: {e}"))?; + + match build.await { + Ok(Ok(())) => {} + Ok(Err(e)) => return Err(format!("building diag bundle: {e}")), + Err(e) => return Err(format!("diag bundle task panicked: {e}")), + } + shutdown_result.map_err(|e| format!("closing diag stream: {e}"))?; + Ok(()) +} + +/// Reads the request body, bounded in both dimensions: one byte past the size +/// limit is enough to know the body is oversized, and [`REQUEST_TIMEOUT`] +/// bounds the wait, so neither an unbounded client write nor an unbounded +/// client silence can become an unbounded daemon commitment. +/// +/// Takes the reader rather than the channel so the deadline is exercisable +/// against an arbitrary `AsyncRead`; the caller supplies `deadline` for the +/// same reason. Cancelling this future is safe — unlike the archive writes it +/// precedes, it owns nothing that must be finalized. +async fn read_request( + reader: impl tokio::io::AsyncRead, + deadline: Duration, +) -> Result { + let mut buf = Vec::with_capacity(256); + // Both halves need their own binding, not a chained temporary: `pin!` + // anchors the reader (so callers owe no `Unpin`), and `read_to_end` borrows + // the capped reader, which must therefore outlive the future built from it. + let reader = std::pin::pin!(reader); + let mut capped = reader.take(MAX_REQUEST_BYTES + 1); + let read = capped.read_to_end(&mut buf); + tokio::time::timeout(deadline, read) + .await + .map_err(|_| format!("diag request not received within {deadline:?}"))? + .map_err(|e| format!("reading diag request: {e}"))?; + if buf.len() as u64 > MAX_REQUEST_BYTES { + return Err(format!( + "diag request too large: over {MAX_REQUEST_BYTES} bytes" + )); + } + // An empty body is a valid request for the documented defaults. + if buf.is_empty() { + return Ok(DiagBundleRequest::default()); + } + serde_json::from_slice(&buf).map_err(|e| format!("parsing diag request: {e}")) +} + +/// The per-log tail cap this run honors: the daemon default when the request +/// leaves it at 0, otherwise the caller's value clamped to the server ceiling. +/// +/// The clamp is not politeness. `BundleWriter::add_file_tail` seeks +/// `-(cap as i64)` from the end, so an unclamped `u64` is both an unbounded +/// read and — past `i64::MAX` — a nonsense seek. +fn log_tail_cap(req: &DiagBundleRequest) -> u64 { + match req.log_tail_bytes { + 0 => LOG_TAIL_CAP, + requested => requested.min(MAX_LOG_TAIL_BYTES), + } +} + +/// Runs every collector into a rootless streaming bundle, finishing with the +/// manifest. Individual collectors fail into the manifest; only a failure of +/// the archive itself aborts. +async fn build_bundle( + s: &ServerStateHandle, + req: &DiagBundleRequest, + tx: tokio::io::DuplexStream, +) -> Result<(), String> { + let created_at = chrono::Utc::now(); + let started = std::time::Instant::now(); + let state_dir = s + .minimal_state_dir() + .await + .as_utf8_path() + .as_std_path() + .to_path_buf(); + // The daemon is always in its own family whatever argv0 says, so it is + // named to the per-process captures explicitly rather than matched. + let self_pid = [std::process::id()]; + let mut w = BundleWriter::stream(tx, version::LONG_VERSION); + + collect_step!(w, "meta", meta(&mut w, s)); + collect_step!(w, "logs", logs(&mut w, &state_dir, log_tail_cap(req))); + // The kernel's own log, beside the daemon's and under the same caller cap. + // A guest bundle has no other kernel evidence: `console=hvc0` lands in the + // *host's* `boot.log`, which is recreated per boot and tail-capped, so on a + // long-lived VM the banner is gone from there before it is ever wanted. + collect_step!( + w, + "kmsg", + diagnostics::kmsg::kmsg_tail(&mut w, "", log_tail_cap(req)) + ); + collect_step!( + w, + "state-listing", + state_listing(&mut w, &state_dir, req.include_state_listing) + ); + collect_step!(w, "sessions", sessions(&mut w, &state_dir)); + collect_step!(w, "env", env(&mut w)); + // Incident captures: pure `/proc`, since the microVM rootfs has no `ps`, + // `lsof`, `ss` or `ip`. The mechanics are the ones the host bundle uses. + collect_step!( + w, + "proc", + diagnostics::procs::all_processes(&mut w, "", PROC_MARKERS) + ); + collect_step!( + w, + "proc.hang-triage", + diagnostics::procs::hang_triage_including(&mut w, "", PROC_MARKERS, &self_pid) + ); + collect_step!( + w, + "proc.sockets", + diagnostics::procs::open_sockets(&mut w, "", PROC_MARKERS, &self_pid) + ); + collect_step!( + w, + "net.listening", + diagnostics::net::proc_net_tables( + &mut w, + "", + "listening", + diagnostics::net::PROC_NET_SOCKET_TABLES + ) + ); + collect_step!( + w, + "net.interfaces", + diagnostics::net::proc_net_tables(&mut w, "", "interfaces", &["dev"]) + ); + // Routing *and* addresses: `fib_trie` carries the local address picture the + // host's `ip addr` gives on the other side of the switch. + collect_step!( + w, + "net.routes", + diagnostics::net::proc_net_tables(&mut w, "", "routes", &["route", "fib_trie"]) + ); + if s.in_microvm().await { + collect_step!(w, "net.gvproxy", gvproxy_probe(&mut w)); + } + collect_step!(w, "disk", disk(&mut w, &state_dir)); + + // Deliberately not wrapped in a timeout. `finish` owns the `async_tar` + // Builder, whose Drop panics when it has not been finalized, so cancelling + // this future would abort the task rather than end it. The bound that stops + // a non-reading client from parking this write forever is on the pump in + // `stream_diag_bundle`: when it expires the read half is dropped and these + // writes fail fast with `BrokenPipe` instead of blocking. + w.finish(created_at, started.elapsed()) + .await + .map_err(|e| format!("{e:#}")) +} + +// ── collectors ─────────────────────────────────────────────────────────────── + +#[derive(Serialize)] +struct Meta { + version: &'static str, + long_version: &'static str, + stdlib_version: &'static str, + pid: u32, + uptime_secs: Option, + in_microvm: bool, + state_volume_mounted: bool, + state_dir: String, + /// `/proc/version` verbatim — the guest kernel's identity, which nothing + /// else in any bundle carries. `diagnostics::system_info` has a `kernel` + /// field, but it is a *host* collector and shells out to `uname`, a binary + /// the microVM rootfs does not ship. + kernel: Option, + /// The release parsed out of [`Meta::kernel`], for diffing two bundles. + kernel_release: Option, + /// `/proc/cmdline`, key-scrubbed — see [`scrubbed_cmdline`]. + kernel_cmdline: Option, +} + +async fn meta( + w: &mut BundleWriter, + s: &ServerStateHandle, +) -> Result<(), anyhow::Error> { + let uptime = tokio::fs::read_to_string("/proc/uptime").await.ok(); + let kernel = proc_line("/proc/version").await; + let info = Meta { + version: version::VERSION, + long_version: version::LONG_VERSION, + stdlib_version: stdlib::VERSION, + pid: std::process::id(), + uptime_secs: uptime.and_then(|s| s.split_whitespace().next()?.parse().ok()), + in_microvm: s.in_microvm().await, + state_volume_mounted: s.state_volume_mounted().await, + state_dir: s.minimal_state_dir().await.to_string(), + kernel_release: kernel + .as_deref() + .and_then(kernel_release) + .map(str::to_owned), + kernel, + kernel_cmdline: proc_line("/proc/cmdline") + .await + .as_deref() + .map(scrubbed_cmdline), + }; + let json = serde_json::to_vec_pretty(&info).context("serializing meta")?; + // `Keys`, not `None`: `kernel_cmdline` goes through the same key-based + // scrub a process argv does, so this is no longer a verbatim capture. + w.add_bytes("meta.json", &json, Redaction::Keys).await +} + +/// A one-line `/proc` file, trimmed; `None` when it cannot be read (a non-Linux +/// host, a restricted `procfs`) — the same best-effort shape every other field +/// in [`Meta`] has. +async fn proc_line(path: &str) -> Option { + tokio::fs::read_to_string(path) + .await + .ok() + .map(|text| text.trim_end().to_string()) +} + +/// The release out of a `/proc/version` banner: `6.12.94` from +/// `Linux version 6.12.94 (u@h) (gcc ...) #1 SMP PREEMPT ...`. +/// +/// The kernel writes that banner from one fixed format string +/// (`linux_proc_banner`, `"%s version %s ..."` over `utsname()->sysname` and +/// `->release`), so the third token is the release whenever the second is +/// literally `version`; anything else is not a banner this understands and +/// yields `None`. The parse rides *beside* the verbatim banner rather than +/// replacing it — a wrong parse can then only cost a convenience, never +/// evidence — and it earns its place because "which release is this" is the +/// question a bundle-to-bundle diff asks, and eyeballing two banners full of +/// toolchain and build-time noise is how that question gets answered wrongly. +fn kernel_release(banner: &str) -> Option<&str> { + let mut tokens = banner.split_whitespace(); + match (tokens.next(), tokens.next(), tokens.next()) { + (Some(_sysname), Some("version"), release) => release, + _ => None, + } +} + +/// The kernel boot line as it may travel: key-scrubbed exactly as a process +/// argv is. +/// +/// The guest boot line today is `console=hvc0` plus an optional forwarded +/// `RUST_LOG=`, and nothing secret can reach it — minvmd rejects a +/// value with whitespace and documents the invariant, because `/proc/cmdline` +/// is world-readable inside the guest. But that is an invariant about *this +/// machine*, and the bundle leaves it: a boot parameter added later that +/// happens to be secret-shaped would otherwise ship verbatim to whoever the +/// archive is mailed to. So the same fail-closed `key=value` rule the +/// flattened-argv capture uses applies here — it costs nothing on a clean line +/// and withholds the tail of a dirty one. +fn scrubbed_cmdline(raw: &str) -> String { + diagnostics::procs::scrub_flattened(raw) +} + +async fn logs( + w: &mut BundleWriter, + state_dir: &Path, + cap: u64, +) -> Result<(), anyhow::Error> { + let log_dir = state_dir.join("logs"); + // Absence and inaccessibility are different facts: "no log directory" may + // only be claimed on a real NotFound, never on EACCES or an I/O error. + // + // `symlink_metadata`, not `metadata`: guest tasks can write this volume, so + // the directory itself is attacker-controlled. `add_file_tail`'s O_NOFOLLOW + // refuses a symlinked *log file*, but it cannot see that the whole `logs/` + // directory was swapped for a link elsewhere — every regular file named + // `minimald.log.*` under the target would then be a legitimate open, and + // someone else's files would ride out in the bundle. + match tokio::fs::symlink_metadata(&log_dir).await { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + w.skip( + "logs/", + "no log directory — this daemon has never run detached", + ); + return Ok(()); + } + Err(e) => { + w.skip("logs/", format!("unreadable: {e}")); + return Ok(()); + } + Ok(m) if !m.is_dir() => { + w.skip( + "logs/", + "not a directory — refusing to follow it out of the state dir", + ); + return Ok(()); + } + Ok(_) => {} + } + // Only this daemon's own rotated logs: the state dir can be shared with + // minvmd (native installs), whose files belong to the host-side collector, + // not the daemon's self-view. + let files = diagnostics::newest_rotated(&log_dir, "minimald.log", LOG_FILES_MAX).await; + if files.is_empty() { + w.skip("logs/minimald.log*", "no files with this prefix"); + return Ok(()); + } + for path in files { + let name = path.file_name().unwrap_or_default().to_string_lossy(); + let dest = format!("logs/{name}"); + // `add_file_tail` opens O_NOFOLLOW and fstats the descriptor: the in-VM + // daemon reads a volume guest tasks can write, so a symlink named + // `minimald.log.*` must not steer an unrelated target into the bundle. + if let Err(e) = w.add_file_tail(&dest, &path, cap).await { + w.skip(&dest, format!("unreadable: {e:#}")); + } + } + Ok(()) +} + +async fn state_listing( + w: &mut BundleWriter, + state_dir: &Path, + include: bool, +) -> Result<(), anyhow::Error> { + if !include { + w.skip("state-listing.txt", "not requested"); + return Ok(()); + } + // Up to 100k synchronous fs ops — keep them off the RPC-serving runtime, + // so a wedged filesystem strands a blocking thread and not the worker whose + // collect_step! timeout is the failsafe. + let dir = state_dir.to_path_buf(); + let listing = + tokio::task::spawn_blocking(move || diagnostics::listing(&dir, LISTING_MAX_ENTRIES)) + .await + .context("listing worker")? + .context("listing state dir")?; + if listing.truncated { + w.skip( + "state-listing.txt (tail)", + format!("listing cap of {LISTING_MAX_ENTRIES} entries reached"), + ); + } + w.add_bytes( + "state-listing.txt", + listing.text.as_bytes(), + Redaction::ListingOnly, + ) + .await +} + +async fn sessions( + w: &mut BundleWriter, + state_dir: &Path, +) -> Result<(), anyhow::Error> { + let sessions_dir = state_dir.join("sessions"); + // The index maps short directory names to session UUIDs — no user data, so + // it travels verbatim. + match read_session_file(&sessions_dir.join("index.json")).await { + Ok(Some(bytes)) => { + w.add_bytes("sessions/index.json", &bytes, Redaction::None) + .await? + } + // A missing index doesn't mean we can skip the walk: orphan session + // records may still exist (e.g., an interrupted write), so note the + // missing file and continue to the directory enumeration. + Ok(None) => w.skip("sessions/index.json", "not present"), + // An unreadable index is not a reason to withhold the records it + // indexes, so this notes the omission and carries on. + Err(e) => w.skip("sessions/index.json", format!("unreadable: {e:#}")), + } + // Enumeration failing is a fact about the bundle, not an empty directory: + // a `sessions/` that denies listing (execute without read) still answers + // an open of `index.json` by name, so silently returning here would ship + // the index with every record missing and nothing saying why. + let mut entries = match tokio::fs::read_dir(&sessions_dir).await { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => { + w.skip( + "sessions/*/record.json", + format!("session directory could not be listed: {e}"), + ); + return Ok(()); + } + }; + loop { + let entry = match entries.next_entry().await { + Ok(Some(entry)) => entry, + Ok(None) => break, + // Same contract mid-walk: the records already collected stand, and + // the ones that will now never be reached are accounted for. + Err(e) => { + w.skip( + "sessions/*/record.json (remainder)", + format!("listing stopped early: {e}"), + ); + break; + } + }; + let short = entry.file_name().to_string_lossy().into_owned(); + // `file_type` here is the dirent's own — a symlink reports as a + // symlink, never as the directory it points at. + match entry.file_type().await { + Ok(t) if t.is_dir() => {} + // `index.json` and anything else that is not a session directory. + Ok(_) => continue, + Err(e) => { + w.skip(format!("sessions/{short}/"), format!("unstattable: {e}")); + continue; + } + } + // An absent file is always explainable (the manifest contract): a + // genuinely-missing record is silent, but a read failure or an + // unparseable-and-withheld record leaves a note in its place rather + // than vanishing. + let raw = match read_session_file(&entry.path().join("record.json")).await { + Ok(Some(raw)) => raw, + Ok(None) => continue, + Err(e) => { + w.skip( + format!("sessions/{short}/record.json"), + format!("unreadable: {e:#}"), + ); + continue; + } + }; + // Records can hold user env-var values (loadout vars); mask + // secret-shaped content before it leaves the machine. + match serde_json::from_slice::(&raw) { + Ok(mut value) => { + redact_json(&mut value); + let json = serde_json::to_vec_pretty(&value).context("serializing record")?; + w.add_bytes( + &format!("sessions/{short}/record.json.redacted"), + &json, + Redaction::Keys, + ) + .await?; + } + // Withheld: it cannot be redacted safely, so it does not travel. + Err(e) => w.skip( + format!("sessions/{short}/record.json"), + format!("does not parse as JSON, cannot redact safely: {e}"), + ), + } + } + Ok(()) +} + +/// Reads one session JSON file whole, bounded by [`MAX_SESSION_FILE_BYTES`] +/// and without following a symlink. `Ok(None)` means genuinely absent; +/// everything else — over the cap, a symlink, a directory, an I/O error — is an +/// `Err` the caller turns into a manifest note. +/// +/// Session records are the one place this collector reads a whole file into +/// memory, and `add_file_tail`'s discipline does not apply: a record has to +/// arrive complete to be parsed and redacted, so it cannot be tail-capped. The +/// cap is therefore a refusal rather than a truncation — the daemon is +/// answering a remote request, and a file it did not write should cost it a +/// skipped entry, not the process. +async fn read_session_file(path: &Path) -> Result>, anyhow::Error> { + let (file, _meta) = match diagnostics::open_regular_nofollow(path).await { + Ok(open) => open, + Err(e) if io_kind(&e) == Some(std::io::ErrorKind::NotFound) => return Ok(None), + Err(e) => return Err(e), + }; + // One byte past the cap: enough to know it was exceeded, and the read + // never allocates more than that whatever the file claims to be. + let mut bytes = Vec::new(); + file.take(MAX_SESSION_FILE_BYTES + 1) + .read_to_end(&mut bytes) + .await + .with_context(|| format!("reading {}", path.display()))?; + anyhow::ensure!( + bytes.len() as u64 <= MAX_SESSION_FILE_BYTES, + "over the {MAX_SESSION_FILE_BYTES}-byte session-file cap" + ); + Ok(Some(bytes)) +} + +/// The `io::ErrorKind` behind an [`anyhow::Error`], if one is in its chain. +/// The `diagnostics` helpers attach context, so the kind is a source rather +/// than the error itself. +fn io_kind(e: &anyhow::Error) -> Option { + e.chain() + .find_map(|cause| cause.downcast_ref::()) + .map(std::io::Error::kind) +} + +/// The daemon's own environment, values masked by the allowlist policy above. +/// Answers "which mode booted, why is logging off" — `RUST_LOG`, the +/// `MINIMALD_*` detach markers, the resolved dirs. +async fn env(w: &mut BundleWriter) -> Result<(), anyhow::Error> { + let env = masked_process_env(is_env_value_allowlisted); + let json = serde_json::to_vec_pretty(&env).context("serializing env")?; + w.add_bytes("env.json", &json, Redaction::Keys).await +} + +#[derive(Serialize)] +struct GvproxyProbe { + endpoint: String, + connect: String, + http_head: Option, +} + +/// Reachability of the gvproxy management endpoint from inside the VM — the +/// single NAT hop every guest egress depends on. +async fn gvproxy_probe(w: &mut BundleWriter) -> Result<(), anyhow::Error> { + let (host, port) = GVPROXY_MGMT; + let endpoint = format!("{host}:{port}"); + let probe = match tokio::time::timeout( + Duration::from_secs(2), + tokio::net::TcpStream::connect((host, port)), + ) + .await + { + Ok(Ok(mut stream)) => { + let http_head = async { + stream.write_all(b"GET / HTTP/1.0\r\n\r\n").await.ok()?; + let mut head = vec![0u8; 1024]; + let n = stream.read(&mut head).await.ok()?; + Some(String::from_utf8_lossy(&head[..n]).into_owned()) + }; + let http_head = tokio::time::timeout(Duration::from_secs(2), http_head) + .await + .ok() + .flatten(); + GvproxyProbe { + endpoint, + connect: "ok".to_string(), + http_head, + } + } + Ok(Err(e)) => GvproxyProbe { + endpoint, + connect: e.to_string(), + http_head: None, + }, + Err(_) => GvproxyProbe { + endpoint, + connect: "timed out after 2s".to_string(), + http_head: None, + }, + }; + let json = serde_json::to_vec_pretty(&probe).context("serializing gvproxy probe")?; + w.add_bytes("net/gvproxy.json", &json, Redaction::None) + .await +} + +#[derive(Serialize)] +struct Filesystem { + path: String, + total_bytes: Option, + free_bytes: Option, +} + +#[derive(Serialize)] +struct Disk { + filesystems: Vec, + mounts: String, +} + +async fn disk( + w: &mut BundleWriter, + state_dir: &Path, +) -> Result<(), anyhow::Error> { + let info = Disk { + filesystems: [state_dir, Path::new("/")] + .into_iter() + .map(|path| { + let usage = diagnostics::disk_usage(path); + Filesystem { + path: path.display().to_string(), + total_bytes: usage.map(|u| u.total_bytes), + free_bytes: usage.map(|u| u.free_bytes), + } + }) + .collect(), + mounts: tokio::fs::read_to_string("/proc/mounts") + .await + .unwrap_or_else(|e| format!("")), + }; + let json = serde_json::to_vec_pretty(&info).context("serializing disk info")?; + w.add_bytes("disk.json", &json, Redaction::None).await +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use minimald_rpc::DIAG_BUNDLE_SUBSYSTEM; + use russh::ChannelMsg; + + use super::*; + use crate::test_harness::TestServer; + + /// Drains a diag-bundle channel, returning the payload and the extended + /// data (stream 1) separately — the two halves of the R6.4 contract. + async fn drain(mut channel: russh::Channel) -> (Vec, String) { + let mut payload = Vec::new(); + let mut stderr = Vec::new(); + while let Some(msg) = channel.wait().await { + match msg { + ChannelMsg::Data { data } => payload.extend_from_slice(&data), + ChannelMsg::ExtendedData { data, ext: 1 } => stderr.extend_from_slice(&data), + _ => {} + } + } + (payload, String::from_utf8_lossy(&stderr).into_owned()) + } + + /// Requests a bundle over the real subsystem with `body` as the raw + /// request, returning the streamed payload and any extended data. + async fn request_bundle(server: &TestServer, body: &[u8]) -> (Vec, String) { + let mut client = server.connect().await; + let channel = client.open_subsystem(DIAG_BUNDLE_SUBSYSTEM, &[]).await; + channel.data(body).await.unwrap(); + channel.eof().await.unwrap(); + drain(channel).await + } + + /// Fetches a default bundle and decodes it to `path -> contents`. + async fn fetch_bundle(server: &TestServer) -> BTreeMap> { + let body = serde_json::to_vec(&DiagBundleRequest::default()).unwrap(); + let (payload, stderr) = request_bundle(server, &body).await; + assert!(!payload.is_empty(), "no bundle streamed; stderr: {stderr}"); + + let decoder = async_compression::tokio::bufread::ZstdDecoder::new(&payload[..]); + let out = tempfile::TempDir::new().unwrap(); + async_tar::Archive::new(decoder) + .unpack(out.path().to_path_buf()) + .await + .unwrap(); + + let mut files = BTreeMap::new(); + collect_files(out.path(), out.path(), &mut files); + files + } + + fn collect_files(base: &Path, dir: &Path, files: &mut BTreeMap>) { + for entry in std::fs::read_dir(dir).unwrap().flatten() { + let path = entry.path(); + if path.is_dir() { + collect_files(base, &path, files); + } else { + let rel = path.strip_prefix(base).unwrap().to_string_lossy().into(); + files.insert(rel, std::fs::read(&path).unwrap()); + } + } + } + + fn manifest(files: &BTreeMap>) -> serde_json::Value { + serde_json::from_slice(&files["manifest.json"]).expect("manifest is valid JSON") + } + + #[tokio::test] + async fn diag_bundle_streams_meta_and_manifest() { + let server = TestServer::new().await; + let files = fetch_bundle(&server).await; + + let meta: serde_json::Value = serde_json::from_slice(&files["meta.json"]).unwrap(); + assert_eq!(meta["version"], version::VERSION); + assert_eq!(meta["in_microvm"], false); + assert_eq!(meta["pid"], std::process::id()); + + // The bundle describes itself: every entry above is manifested, and a + // healthy daemon collects without errors. + let manifest = manifest(&files); + assert_eq!(manifest["schema_version"], 1); + assert_eq!(manifest["version"], version::LONG_VERSION); + assert!( + manifest["errors"].as_array().unwrap().is_empty(), + "healthy test server should collect cleanly, got: {}", + manifest["errors"] + ); + let collected: Vec<&str> = manifest["collected"] + .as_array() + .unwrap() + .iter() + .map(|e| e["path"].as_str().unwrap()) + .collect(); + assert!(collected.contains(&"meta.json"), "{collected:?}"); + + for entry in [ + "state-listing.txt", + "proc.txt", + "net/listening.txt", + "disk.json", + "env.json", + ] { + assert!( + files.contains_key(entry), + "missing {entry}: {:?}", + files.keys() + ); + } + // Not in a microVM: no gvproxy probe entry. + assert!(!files.contains_key("net/gvproxy.json")); + } + + /// R6.6: the guest-side incident trio, all of it binary-free `/proc`. + #[tokio::test] + async fn diag_bundle_captures_the_incident_trio_and_env() { + let server = TestServer::new().await; + // Keep a live connection so the daemon's own socket fds are open while + // the bundle is collected. + let _client = server.connect().await; + let files = fetch_bundle(&server).await; + let pid = std::process::id(); + + let routes = String::from_utf8_lossy(&files["net/routes.txt"]).into_owned(); + assert!(routes.contains("/proc/net/route"), "{routes}"); + assert!( + routes.contains("/proc/net/fib_trie"), + "addresses ride along with routes: {routes}" + ); + + let stack = files + .get(&format!("proc/{pid}.stack.txt")) + .expect("the daemon hang-triages itself whatever its argv0 says"); + let stack = String::from_utf8_lossy(stack); + assert!(stack.contains("=== wchan ==="), "{stack}"); + assert!(stack.contains("=== fds ==="), "{stack}"); + + let sockets = String::from_utf8_lossy(&files["proc/sockets.txt"]).into_owned(); + let joined: Vec<&str> = sockets + .lines() + .skip(1) + .filter(|line| line.starts_with(&format!("{pid}\t"))) + .collect(); + assert!(!joined.is_empty(), "no fds for our own pid: {sockets}"); + assert!( + joined + .iter() + .any(|line| line.split('\t').nth(3) == Some("unix")), + "the in-process ssh transport is a unix socket and must resolve \ + against /proc/net/unix: {joined:?}" + ); + + // Every value the policy does not allow is masked, names always kept. + let env: BTreeMap = serde_json::from_slice(&files["env.json"]).unwrap(); + assert!(!env.is_empty()); + for (name, value) in &env { + assert!( + is_env_value_allowlisted(name) || value.starts_with(">() + ); + } + + /// A symlink planted in the log dir must not steer its target into the + /// bundle — and the manifest must say so rather than staying silent. + #[tokio::test] + async fn diag_bundle_refuses_symlinked_logs() { + let server = TestServer::new().await; + let state_dir = server.state.minimal_state_dir().await; + let log_dir = state_dir.as_utf8_path().as_std_path().join("logs"); + std::fs::create_dir_all(&log_dir).unwrap(); + let secret = log_dir.join("secret.key"); + std::fs::write(&secret, "private key material").unwrap(); + std::os::unix::fs::symlink(&secret, log_dir.join("minimald.log.evil")).unwrap(); + + let files = fetch_bundle(&server).await; + assert!( + !files.contains_key("logs/minimald.log.evil"), + "symlinked log must not travel: {:?}", + files.keys().collect::>() + ); + let skipped = manifest(&files)["skipped"].to_string(); + assert!( + skipped.contains("minimald.log.evil"), + "the refusal is recorded, not silent: {skipped}" + ); + } + + #[tokio::test] + async fn diag_bundle_redacts_session_records() { + let server = TestServer::new().await; + { + let mut client = server.connect().await; + crate::test_harness::create_configured_session(&mut client, "diag-test", "/tmp").await; + } + let files = fetch_bundle(&server).await; + + assert!(files.contains_key("sessions/index.json")); + let record = files + .iter() + .find(|(path, _)| path.ends_with("record.json.redacted")) + .map(|(_, contents)| contents) + .expect("a redacted session record should be bundled"); + let value: serde_json::Value = serde_json::from_slice(record).unwrap(); + assert_eq!(value["name"], "diag-test", "non-secret fields survive"); + } + + /// An unreadable `sessions/` is a fact about the bundle, not an empty + /// directory. Execute-without-read still resolves `index.json` by name, so + /// without this the bundle ships the index with every record it points at + /// missing and nothing saying why. + #[cfg(unix)] + #[tokio::test] + async fn an_unlistable_sessions_dir_is_recorded_not_silent() { + use std::os::unix::fs::PermissionsExt as _; + + let server = TestServer::new().await; + let state_dir = server.state.minimal_state_dir().await; + let sessions_dir = state_dir.as_utf8_path().as_std_path().join("sessions"); + std::fs::create_dir_all(&sessions_dir).unwrap(); + std::fs::write(sessions_dir.join("index.json"), b"{}").unwrap(); + std::fs::set_permissions(&sessions_dir, std::fs::Permissions::from_mode(0o111)).unwrap(); + // Root ignores the mode bits, so on a root test runner the fixture is + // not a fixture and there is nothing to assert. + let reproduced = std::fs::read_dir(&sessions_dir).is_err(); + + let files = fetch_bundle(&server).await; + // Restore before any assertion can leave the tempdir unremovable. + std::fs::set_permissions(&sessions_dir, std::fs::Permissions::from_mode(0o755)).unwrap(); + if !reproduced { + return; + } + + assert!( + files.contains_key("sessions/index.json"), + "the index opens by name even when the directory will not list" + ); + let skipped = manifest(&files)["skipped"].to_string(); + assert!( + skipped.contains("could not be listed"), + "the omission is explained, not silent: {skipped}" + ); + } + + /// A `record.json` the daemon did not write costs a manifest note, never an + /// allocation: it cannot be tail-capped (a record must arrive whole to be + /// redacted), so the bound is a refusal. + #[tokio::test] + async fn an_oversized_session_record_is_refused_not_buffered() { + let server = TestServer::new().await; + let state_dir = server.state.minimal_state_dir().await; + let sessions_dir = state_dir.as_utf8_path().as_std_path().join("sessions"); + let dir = sessions_dir.join("bigsess"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(sessions_dir.join("index.json"), b"{}").unwrap(); + std::fs::write( + dir.join("record.json"), + vec![b'x'; (MAX_SESSION_FILE_BYTES + 1) as usize], + ) + .unwrap(); + + let files = fetch_bundle(&server).await; + assert!( + !files.keys().any(|k| k.starts_with("sessions/bigsess")), + "the oversized record must not travel: {:?}", + files.keys().collect::>() + ); + let skipped = manifest(&files)["skipped"].to_string(); + assert!( + skipped.contains("session-file cap"), + "the refusal names its bound: {skipped}" + ); + } + + /// The same read is no-follow, so a `record.json` replaced by a symlink + /// cannot steer an unrelated daemon-readable file into the bundle. + #[cfg(unix)] + #[tokio::test] + async fn a_symlinked_session_record_is_refused() { + /// Distinctive enough that finding it anywhere in the archive is + /// proof the target was read, not a coincidence. + const MARKER: &str = "symlink-target-must-not-travel"; + + let server = TestServer::new().await; + let state_dir = server.state.minimal_state_dir().await; + let sessions_dir = state_dir.as_utf8_path().as_std_path().join("sessions"); + let dir = sessions_dir.join("linksess"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(sessions_dir.join("index.json"), b"{}").unwrap(); + let secret = state_dir.as_utf8_path().as_std_path().join("host.key"); + std::fs::write(&secret, MARKER).unwrap(); + std::os::unix::fs::symlink(&secret, dir.join("record.json")).unwrap(); + + let files = fetch_bundle(&server).await; + assert!( + !files + .values() + .any(|c| c.windows(MARKER.len()).any(|w| w == MARKER.as_bytes())), + "the symlink target must not travel" + ); + let skipped = manifest(&files)["skipped"].to_string(); + assert!( + skipped.contains("linksess/record.json"), + "the refusal is recorded: {skipped}" + ); + } + + /// R6.3, the other dimension: bounding the request's *size* does not bound + /// the *wait* for it. A client that writes half an object and then holds + /// the channel open never sends EOF, and nothing else reclaims the task — + /// the daemon runs `inactivity_timeout: None` by design, and its keepalives + /// only notice a peer that has stopped answering, not one that answers and + /// stays silent. + #[tokio::test] + async fn a_request_that_never_ends_is_abandoned_on_the_deadline() { + // Kept alive for the whole call: dropping it would deliver the EOF + // whose absence is the point. + let (mut client, server) = tokio::io::duplex(64); + client.write_all(b"{\"log_tail_bytes\":").await.unwrap(); + + let err = read_request(server, Duration::from_millis(200)) + .await + .expect_err("an unterminated body must not be waited on forever"); + assert!(err.contains("not received within"), "got: {err}"); + } + + /// R6.4: a request that fails before streaming starts yields extended data + /// and zero payload bytes, never a truncated archive. + #[tokio::test] + async fn malformed_request_reports_on_extended_data_with_no_payload() { + let server = TestServer::new().await; + let (payload, stderr) = request_bundle(&server, b"{not json").await; + assert!(payload.is_empty(), "no payload may precede a request error"); + assert!(stderr.contains("parsing diag request"), "got: {stderr}"); + } + + /// R6.3: an oversized body is refused rather than buffered. + #[tokio::test] + async fn oversized_request_body_is_rejected() { + let server = TestServer::new().await; + let body = vec![b' '; (MAX_REQUEST_BYTES + 4096) as usize]; + let (payload, stderr) = request_bundle(&server, &body).await; + assert!(payload.is_empty(), "no payload for a refused request"); + assert!(stderr.contains("diag request too large"), "got: {stderr}"); + } + + /// R6.3: the caller-controlled tail is clamped to the server's ceiling; 0 + /// means the daemon's default. + #[test] + fn log_tail_is_clamped_to_the_server_cap() { + let with = |bytes| log_tail_cap(&DiagBundleRequest::default().with_log_tail_bytes(bytes)); + assert_eq!(with(0), LOG_TAIL_CAP, "0 means the daemon default"); + assert_eq!(with(4096), 4096, "a modest request is honored"); + assert_eq!(with(u64::MAX), MAX_LOG_TAIL_BYTES, "clamped, never wrapped"); + assert!( + MAX_LOG_TAIL_BYTES < i64::MAX as u64, + "the cap must keep the tail seek representable" + ); + } + + /// `include_state_listing: false` must skip the listing and say so. + #[tokio::test] + async fn state_listing_is_optional() { + let server = TestServer::new().await; + let body = + serde_json::to_vec(&DiagBundleRequest::default().with_state_listing(false)).unwrap(); + let (payload, stderr) = request_bundle(&server, &body).await; + assert!(!payload.is_empty(), "stderr: {stderr}"); + + let decoder = async_compression::tokio::bufread::ZstdDecoder::new(&payload[..]); + let out = tempfile::TempDir::new().unwrap(); + async_tar::Archive::new(decoder) + .unpack(out.path().to_path_buf()) + .await + .unwrap(); + let mut files = BTreeMap::new(); + collect_files(out.path(), out.path(), &mut files); + + assert!(!files.contains_key("state-listing.txt")); + assert!( + manifest(&files)["skipped"] + .to_string() + .contains("state-listing.txt"), + "an omission is recorded" + ); + } +} diff --git a/crates/minimald/src/lib.rs b/crates/minimald/src/lib.rs index 7c7ebaf71c..0f8d714447 100644 --- a/crates/minimald/src/lib.rs +++ b/crates/minimald/src/lib.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; pub mod channel_progress; pub mod connection; +mod diag; pub mod env; mod exec; #[cfg(target_os = "linux")] diff --git a/crates/minimald/src/rpc.rs b/crates/minimald/src/rpc.rs index 69b2987a5b..661e2ec63d 100644 --- a/crates/minimald/src/rpc.rs +++ b/crates/minimald/src/rpc.rs @@ -606,6 +606,7 @@ pub async fn handle_ssh_rpc( | GetSessionPolicy::NAME | GetMeshStatus::NAME | STREAM_WORKSPACE_FILES + | minimald_rpc::DIAG_BUNDLE_SUBSYSTEM | IssueClientCert::NAME => { let mut conn_lock = c.lock().await; let c_hnd = match conn_lock.take(id) { @@ -679,6 +680,9 @@ pub async fn handle_ssh_rpc( GetSessionPolicy::NAME => serve!(serve_get_session_policy(s, channel)), GetMeshStatus::NAME => serve!(serve_get_mesh_status(s, channel)), STREAM_WORKSPACE_FILES => serve!(serve_stream_workspace_files(s, config, channel)), + minimald_rpc::DIAG_BUNDLE_SUBSYSTEM => { + serve!(crate::diag::serve_stream_diag_bundle(s, config, channel)) + } IssueClientCert::NAME => { #[cfg(feature = "networking-proxy")] serve!(serve_issue_client_cert(s, channel)); diff --git a/crates/minimald/src/server.rs b/crates/minimald/src/server.rs index deb40d4bb9..205d5b5975 100644 --- a/crates/minimald/src/server.rs +++ b/crates/minimald/src/server.rs @@ -296,12 +296,19 @@ impl ServerStateHandle { self.0.lock().await.config.state_volume_mounted } - /// The configured state dir (the quiesce target when in a microVM). - #[cfg(target_os = "linux")] + /// The configured state dir (the quiesce target when in a microVM, and the + /// root every diagnostic collector reads from). pub(crate) async fn minimal_state_dir(&self) -> DaemonAbsPath { self.0.lock().await.config.minimal_state_dir.clone() } + /// Whether this daemon is the in-VM instance rather than a native one. + /// Recorded in the diagnostic bundle's `meta.json` so a reader knows which + /// of the two answered, and gates the guest-only gvproxy probe. + pub(crate) async fn in_microvm(&self) -> bool { + self.0.lock().await.config.in_microvm + } + /// Returns the daemon's TLS certificate authority (only with /// `networking-proxy` feature). Used by the `IssueClientCert` RPC handler. #[cfg(feature = "networking-proxy")]