diff --git a/Cargo.lock b/Cargo.lock index f533f82e60..46fb98b6fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1504,6 +1504,24 @@ dependencies = [ "cipher 0.5.2", ] +[[package]] +name = "diagnostics" +version = "0.0.1" +dependencies = [ + "anyhow", + "async-compression", + "async-tar", + "chrono", + "libc", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "walkdir", +] + [[package]] name = "diff" version = "0.1.13" diff --git a/Cargo.toml b/Cargo.toml index ec0fbbf7d6..e17d4e185a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/checkouts", "crates/common", "crates/decode", + "crates/diagnostics", "crates/graph", "crates/mctx", "crates/mfile", @@ -157,6 +158,7 @@ check = { version = "0.0.1", path = "crates/check" } checkouts = { version = "0.0.1", path = "crates/checkouts" } common = { version = "0.0.1", path = "crates/common" } decode = { version = "0.0.1", path = "crates/decode" } +diagnostics = { version = "0.0.1", path = "crates/diagnostics" } mctx = { version = "0.0.1", path = "crates/mctx" } minimald-rpc = { version = "0.0.1", path = "crates/minimald-rpc" } mfile = { version = "0.0.1", path = "crates/mfile" } diff --git a/crates/diagnostics/Cargo.toml b/crates/diagnostics/Cargo.toml new file mode 100644 index 0000000000..abd171ba69 --- /dev/null +++ b/crates/diagnostics/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "diagnostics" +version = "0.0.1" +rust-version = "1.91" +license = "MIT OR Apache-2.0" +edition.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +async-compression.workspace = true +async-tar.workspace = true +chrono.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true +walkdir.workspace = true + +[target.'cfg(unix)'.dependencies] +libc.workspace = true + +[dev-dependencies] +tempfile.workspace = true +tokio-stream.workspace = true diff --git a/crates/diagnostics/src/bundle.rs b/crates/diagnostics/src/bundle.rs new file mode 100644 index 0000000000..77e496b2e3 --- /dev/null +++ b/crates/diagnostics/src/bundle.rs @@ -0,0 +1,445 @@ +//! Tar+zstd writer for diagnostic bundles. +//! +//! One writer, two modes. File-backed mode (`create`) is for a CLI writing a +//! local artifact: entries land under a single top-level directory (the +//! bundle name) so extraction is tidy, and the file is created owner-only. +//! Stream mode (`stream`) is for a daemon writing through a pipe: entries sit +//! at the archive top level (rootless — nested-bundle consumers do exact-key +//! lookups). Both modes append the manifest last from the accumulated record, +//! so even a run whose collectors all failed produces a valid archive with a +//! complete error report. + +use std::path::Path; + +use anyhow::Context as _; +use async_compression::tokio::write::ZstdEncoder; +use async_tar::{Builder, EntryType, Header}; +use tokio::io::{AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _}; + +use crate::manifest::{CollectedEntry, CollectorError, Manifest, Redaction, SkippedEntry}; + +/// Default per-file tail cap for bundled logs. +pub const LOG_TAIL_CAP: u64 = 5 * 1024 * 1024; + +mod sealed { + pub trait Sealed {} + impl Sealed for tokio::fs::File {} + impl Sealed for tokio::io::DuplexStream {} +} + +/// A writer a [`BundleWriter`] can finalize. +/// +/// Sealed: the finalize semantics are mode decisions owned by this crate. +/// `Sync` is load-bearing — `async_tar::Builder` requires it, which is why a +/// bundle can never wrap a non-`Sync` transport writer directly (daemons pump +/// through a [`tokio::io::DuplexStream`] instead). +pub trait BundleSink: AsyncWrite + Unpin + Send + Sync + sealed::Sealed { + #[doc(hidden)] + fn finalize(self) -> impl Future> + Send + where + Self: Sized; +} + +impl BundleSink for tokio::fs::File { + /// Durability matters for a file artifact: flush and fsync. + async fn finalize(mut self) -> std::io::Result<()> { + self.flush().await?; + self.sync_all().await + } +} + +impl BundleSink for tokio::io::DuplexStream { + /// A pipe has no durability to guarantee; flush and half-close so the + /// reading side sees EOF. + async fn finalize(mut self) -> std::io::Result<()> { + self.flush().await?; + self.shutdown().await + } +} + +pub struct BundleWriter { + tar: Builder>, + /// Top-level directory for every entry; `None` in stream mode (rootless). + root: Option, + /// Producer version recorded in the manifest — the crate is app-agnostic, + /// so the caller supplies it. + version: String, + collected: Vec, + skipped: Vec, + errors: Vec, +} + +impl BundleWriter { + /// Creates the output file at `out_path`; `root` becomes the single + /// top-level directory inside the archive, and `version` is recorded + /// in the manifest. + pub async fn create(out_path: &Path, root: &str, version: &str) -> Result { + // 0600: the bundle carries unredacted log tails, session records, and + // process state, and lands in the cwd by default — not world-readable. + let mut opts = tokio::fs::OpenOptions::new(); + opts.write(true).create(true).truncate(true); + #[cfg(unix)] + opts.mode(0o600); + let file = opts + .open(out_path) + .await + .with_context(|| format!("creating {}", out_path.display()))?; + // mode() only applies when create() makes the file; a pre-existing + // 0644 file keeps its bits, so pin them explicitly. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + file.set_permissions(std::fs::Permissions::from_mode(0o600)) + .await + .with_context(|| format!("securing {}", out_path.display()))?; + } + Ok(Self::new(file, Some(root.to_string()), version)) + } +} + +impl BundleWriter { + /// Streams the archive through `writer` with entries at the top level + /// (no root directory) — the daemon-side mode, where the nested bundle's + /// consumers do exact-key lookups like `meta.json`. + pub fn stream(writer: W, version: &str) -> Self { + Self::new(writer, None, version) + } + + fn new(writer: W, root: Option, version: &str) -> Self { + Self { + tar: Builder::new(ZstdEncoder::new(writer)), + root, + version: version.to_string(), + collected: Vec::new(), + skipped: Vec::new(), + errors: Vec::new(), + } + } + + /// Adds `bytes` at `path` (bundle-relative) and records it in the + /// manifest with the given redaction level. + pub async fn add_bytes( + &mut self, + path: &str, + bytes: &[u8], + redaction: Redaction, + ) -> Result<(), anyhow::Error> { + self.append(path, bytes).await?; + self.collected.push(CollectedEntry { + path: path.to_string(), + redaction, + bytes: bytes.len() as u64, + }); + Ok(()) + } + + /// Copies up to the last `cap` bytes of `src` to `path`. Files over the + /// cap are recorded as tail-capped; smaller ones as unredacted copies. + pub async fn add_file_tail( + &mut self, + path: &str, + src: &Path, + cap: u64, + ) -> Result<(), anyhow::Error> { + // Open without following symlinks: a symlink planted in a tool-owned + // dir must not steer an unrelated target (a host key, another user's + // log) into a bundle the user then shares. O_NOFOLLOW refuses the + // link at open time — no check-then-open window to race. (It guards + // the final component only; intermediate directories are the tool's + // own, per the threat model.) + let mut opts = tokio::fs::OpenOptions::new(); + opts.read(true); + #[cfg(unix)] + opts.custom_flags(libc::O_NOFOLLOW); + let mut file = opts + .open(src) + .await + .with_context(|| format!("opening {}", src.display()))?; + // fstat the descriptor we actually read from: rejects directories and + // devices, and doubles as the symlink guard where O_NOFOLLOW is + // unavailable. + let meta = file + .metadata() + .await + .with_context(|| format!("statting {}", src.display()))?; + if !meta.is_file() { + anyhow::bail!("{} is not a regular file", src.display()); + } + let len = meta.len(); + let capped = len > cap; + if capped { + use tokio::io::AsyncSeekExt as _; + file.seek(std::io::SeekFrom::End(-(cap as i64))) + .await + .with_context(|| format!("seeking {}", src.display()))?; + } + // Bound the read at the reader, not just the seek: the file may be a + // live log appended to between the stat and the read. + let mut contents = Vec::with_capacity(len.min(cap) as usize); + (&mut file) + .take(cap) + .read_to_end(&mut contents) + .await + .with_context(|| format!("reading {}", src.display()))?; + let redaction = if capped { + Redaction::TailCapped + } else { + Redaction::None + }; + self.add_bytes(path, &contents, redaction).await + } + + /// Records something deliberately withheld from the bundle. + pub fn skip(&mut self, what: impl Into, reason: impl Into) { + self.skipped.push(SkippedEntry { + what: what.into(), + reason: reason.into(), + }); + } + + /// Records a failed collector; the bundle carries on without it. + pub fn error( + &mut self, + collector: impl Into, + error: impl std::fmt::Display, + duration: std::time::Duration, + ) { + self.errors.push(CollectorError { + collector: collector.into(), + error: error.to_string(), + duration_ms: duration.as_millis() as u64, + }); + } + + pub fn error_count(&self) -> usize { + self.errors.len() + } + + pub fn entry_count(&self) -> usize { + self.collected.len() + } + + /// Writes `manifest.json` from the accumulated record and finalizes the + /// archive. Must be the last call. + pub async fn finish( + mut self, + created_at: chrono::DateTime, + duration: std::time::Duration, + ) -> Result<(), anyhow::Error> { + let manifest = Manifest { + schema_version: crate::manifest::SCHEMA_VERSION, + created_at, + version: std::mem::take(&mut self.version), + duration_ms: duration.as_millis() as u64, + collected: std::mem::take(&mut self.collected), + skipped: std::mem::take(&mut self.skipped), + errors: std::mem::take(&mut self.errors), + }; + let json = serde_json::to_vec_pretty(&manifest).context("serializing manifest")?; + self.append("manifest.json", &json).await?; + + let mut encoder = self + .tar + .into_inner() + .await + .context("finalizing tar archive")?; + encoder.shutdown().await.context("flushing zstd encoder")?; + encoder + .into_inner() + .finalize() + .await + .context("finalizing bundle writer")?; + Ok(()) + } + + async fn append(&mut self, path: &str, bytes: &[u8]) -> Result<(), anyhow::Error> { + let mut header = Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_entry_type(EntryType::Regular); + header.set_cksum(); + let full = match &self.root { + Some(root) => format!("{root}/{path}"), + None => path.to_string(), + }; + self.tar + .append_data(&mut header, &full, bytes) + .await + .with_context(|| format!("adding {full}")) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use std::collections::BTreeMap; + + /// Decodes a bundle and returns its entries in archive order as + /// `(archive path, contents)`. + pub(crate) async fn bundle_entries(bytes: &[u8]) -> Vec<(String, Vec)> { + use tokio_stream::StreamExt as _; + let decoder = async_compression::tokio::bufread::ZstdDecoder::new(bytes); + let archive = async_tar::Archive::new(decoder); + let mut out = Vec::new(); + let mut entries = archive.entries().unwrap(); + while let Some(entry) = entries.next().await { + let mut entry = entry.unwrap(); + let path = entry.path().unwrap().to_string_lossy().into_owned(); + let mut contents = Vec::new(); + entry.read_to_end(&mut contents).await.unwrap(); + out.push((path, contents)); + } + out + } + + /// Unpacks a bundle file and returns `bundle-relative path -> contents`, + /// stripping `root/` when given. + pub(crate) async fn unpack_bundle(path: &Path, root: &str) -> BTreeMap> { + let bytes = tokio::fs::read(path).await.unwrap(); + let prefix = if root.is_empty() { + String::new() + } else { + format!("{root}/") + }; + bundle_entries(&bytes) + .await + .into_iter() + .map(|(p, c)| (p.strip_prefix(&prefix).unwrap_or(&p).to_string(), c)) + .collect() + } + + #[tokio::test] + async fn bundle_round_trips_with_manifest_last() { + let tmp = tempfile::TempDir::new().unwrap(); + let out = tmp.path().join("b.tar.zst"); + + let mut w = BundleWriter::create(&out, "minimal-diag-test", "test-version") + .await + .unwrap(); + w.add_bytes("host/system.json", b"{}", Redaction::None) + .await + .unwrap(); + w.skip("config/client.key", "private key material"); + w.error( + "net.listening", + "ss not found", + std::time::Duration::from_millis(3), + ); + w.finish(chrono::Utc::now(), std::time::Duration::from_secs(1)) + .await + .unwrap(); + + let bytes = tokio::fs::read(&out).await.unwrap(); + let entries = bundle_entries(&bytes).await; + assert_eq!( + entries.last().unwrap().0, + "minimal-diag-test/manifest.json", + "file-backed manifest lands last, under the root" + ); + + let files = unpack_bundle(&out, "minimal-diag-test").await; + assert_eq!(files["host/system.json"], b"{}"); + + let manifest: serde_json::Value = serde_json::from_slice(&files["manifest.json"]).unwrap(); + assert_eq!(manifest["schema_version"], 1); + assert_eq!(manifest["version"], "test-version"); + assert_eq!(manifest["collected"][0]["path"], "host/system.json"); + assert_eq!(manifest["collected"][0]["redaction"], "none"); + assert_eq!(manifest["skipped"][0]["what"], "config/client.key"); + assert_eq!(manifest["errors"][0]["collector"], "net.listening"); + } + + #[tokio::test] + async fn stream_mode_is_rootless_and_matches_file_mode() { + // File mode. + let tmp = tempfile::TempDir::new().unwrap(); + let out = tmp.path().join("f.tar.zst"); + let mut w = BundleWriter::create(&out, "r", "v").await.unwrap(); + w.add_bytes("meta.json", b"{}", Redaction::None) + .await + .unwrap(); + let created = chrono::Utc::now(); + w.finish(created, std::time::Duration::ZERO).await.unwrap(); + let file_entries = bundle_entries(&tokio::fs::read(&out).await.unwrap()).await; + + // Stream mode through a duplex pipe (the daemon shape). + let (tx, mut rx) = tokio::io::duplex(64 * 1024); + let reader = tokio::spawn(async move { + let mut buf = Vec::new(); + rx.read_to_end(&mut buf).await.unwrap(); + buf + }); + let mut w = BundleWriter::stream(tx, "v"); + w.add_bytes("meta.json", b"{}", Redaction::None) + .await + .unwrap(); + w.finish(created, std::time::Duration::ZERO).await.unwrap(); + let stream_entries = bundle_entries(&reader.await.unwrap()).await; + + // Rootless: exact top-level keys, manifest last. + assert_eq!(stream_entries[0].0, "meta.json"); + assert_eq!(stream_entries.last().unwrap().0, "manifest.json"); + // Identical modulo the root prefix. + let stripped: Vec<_> = file_entries + .iter() + .map(|(p, c)| (p.strip_prefix("r/").unwrap().to_string(), c.clone())) + .collect(); + assert_eq!(stripped, stream_entries); + } + + #[tokio::test] + async fn add_file_tail_caps_large_files() { + let tmp = tempfile::TempDir::new().unwrap(); + let big = tmp.path().join("big.log"); + std::fs::write(&big, [b'a'; 100].as_slice()).unwrap(); + let out = tmp.path().join("b.tar.zst"); + + let mut w = BundleWriter::create(&out, "r", "test-version") + .await + .unwrap(); + w.add_file_tail("logs/big.log", &big, 10).await.unwrap(); + w.finish(chrono::Utc::now(), std::time::Duration::ZERO) + .await + .unwrap(); + + let files = unpack_bundle(&out, "r").await; + assert_eq!(files["logs/big.log"].len(), 10); + let manifest: serde_json::Value = serde_json::from_slice(&files["manifest.json"]).unwrap(); + assert_eq!(manifest["collected"][0]["redaction"], "tail-capped"); + } + + #[cfg(unix)] + #[tokio::test] + async fn add_file_tail_rejects_symlinks_at_open() { + let tmp = tempfile::TempDir::new().unwrap(); + let target = tmp.path().join("target.log"); + std::fs::write(&target, "sensitive").unwrap(); + let link = tmp.path().join("minimald.log.evil"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + let out = tmp.path().join("b.tar.zst"); + let mut w = BundleWriter::create(&out, "r", "v").await.unwrap(); + let err = w.add_file_tail("logs/evil", &link, 1024).await.unwrap_err(); + assert!( + err.to_string().contains("opening"), + "symlink must be refused at open time, got: {err:#}" + ); + w.finish(chrono::Utc::now(), std::time::Duration::ZERO) + .await + .unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn bundle_file_is_owner_only() { + use std::os::unix::fs::PermissionsExt as _; + let tmp = tempfile::TempDir::new().unwrap(); + let out = tmp.path().join("b.tar.zst"); + let w = BundleWriter::create(&out, "r", "v").await.unwrap(); + let mode = std::fs::metadata(&out).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + w.finish(chrono::Utc::now(), std::time::Duration::ZERO) + .await + .unwrap(); + } +} diff --git a/crates/diagnostics/src/capture.rs b/crates/diagnostics/src/capture.rs new file mode 100644 index 0000000000..0607f52a1f --- /dev/null +++ b/crates/diagnostics/src/capture.rs @@ -0,0 +1,157 @@ +//! Bounded subprocess output capture for command-shaped collectors. +//! +//! One helper, one contract: run a command, capture stdout/stderr/status, +//! never outlive the timeout, never leak a child past the collection run. +//! On timeout the child is killed and the output captured so far is returned +//! inside the error — partial evidence from a hung tool is still evidence. + +use std::process::{ExitStatus, Stdio}; +use std::time::Duration; + +use tokio::io::AsyncReadExt as _; +use tokio::process::Command; + +/// Everything a finished (or killed) command left behind. +#[derive(Debug, Default)] +#[non_exhaustive] +pub struct Capture { + pub stdout: Vec, + pub stderr: Vec, + /// `None` when the child was killed before exiting (timeout). + pub status: Option, +} + +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum CaptureError { + #[error("spawning `{cmd}`")] + Spawn { + cmd: String, + #[source] + source: std::io::Error, + }, + #[error("reading `{cmd}` output")] + Io { + cmd: String, + #[source] + source: std::io::Error, + }, + /// The command outlived its deadline and was killed; `partial` retains + /// whatever it wrote first. + #[error("`{cmd}` timed out after {timeout:?}")] + Timeout { + cmd: String, + timeout: Duration, + partial: Capture, + }, +} + +/// Runs `cmd args…` with stdin closed, capturing stdout and stderr until the +/// process exits or `timeout` elapses. A timed-out child is killed +/// (`kill_on_drop` backstops the explicit kill) and the bytes captured so far +/// ride back in [`CaptureError::Timeout`]. +pub async fn command_capture( + cmd: &str, + args: &[&str], + timeout: Duration, +) -> Result { + let spawn_err = |source| CaptureError::Spawn { + cmd: cmd.to_string(), + source, + }; + let io_err = |source| CaptureError::Io { + cmd: cmd.to_string(), + source, + }; + + let mut child = Command::new(cmd) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .map_err(spawn_err)?; + let mut stdout_pipe = child.stdout.take().expect("stdout was piped"); + let mut stderr_pipe = child.stderr.take().expect("stderr was piped"); + + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let mut out_buf = [0u8; 8192]; + let mut err_buf = [0u8; 8192]; + let (mut out_done, mut err_done) = (false, false); + let deadline = tokio::time::sleep(timeout); + tokio::pin!(deadline); + + let status = loop { + tokio::select! { + _ = &mut deadline => { + let _ = child.start_kill(); + return Err(CaptureError::Timeout { + cmd: cmd.to_string(), + timeout, + partial: Capture { stdout, stderr, status: None }, + }); + } + n = stdout_pipe.read(&mut out_buf), if !out_done => { + let n = n.map_err(io_err)?; + if n == 0 { out_done = true; } else { stdout.extend_from_slice(&out_buf[..n]); } + } + n = stderr_pipe.read(&mut err_buf), if !err_done => { + let n = n.map_err(io_err)?; + if n == 0 { err_done = true; } else { stderr.extend_from_slice(&err_buf[..n]); } + } + status = child.wait(), if out_done && err_done => { + break status.map_err(io_err)?; + } + } + }; + Ok(Capture { + stdout, + stderr, + status: Some(status), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn captures_stdout_stderr_and_status() { + let cap = command_capture( + "sh", + &["-c", "echo out; echo err >&2; exit 3"], + Duration::from_secs(10), + ) + .await + .unwrap(); + assert_eq!(cap.stdout, b"out\n"); + assert_eq!(cap.stderr, b"err\n"); + assert_eq!(cap.status.unwrap().code(), Some(3)); + } + + #[tokio::test] + async fn timeout_kills_the_child_and_retains_partial_output() { + let err = command_capture( + "sh", + &["-c", "echo partial; sleep 30"], + Duration::from_millis(300), + ) + .await + .unwrap_err(); + let CaptureError::Timeout { partial, .. } = err else { + panic!("expected timeout, got: {err}"); + }; + assert_eq!(partial.stdout, b"partial\n", "pre-timeout output retained"); + assert_eq!(partial.status, None, "killed child has no exit status"); + } + + #[tokio::test] + async fn missing_binary_is_a_spawn_error() { + let err = command_capture("definitely-not-a-real-binary", &[], Duration::from_secs(1)) + .await + .unwrap_err(); + assert!(matches!(err, CaptureError::Spawn { .. }), "got: {err}"); + } +} diff --git a/crates/diagnostics/src/lib.rs b/crates/diagnostics/src/lib.rs new file mode 100644 index 0000000000..d61e1e22db --- /dev/null +++ b/crates/diagnostics/src/lib.rs @@ -0,0 +1,27 @@ +//! App-agnostic machinery for diagnostic support bundles. +//! +//! The application-specific collectors (which need a live daemon client, +//! resolved paths, and the process/network state of the machine) live in +//! their own crates; what lives here is the reusable core those collectors +//! build on: +//! +//! - [`bundle`] — the tar+zstd [`BundleWriter`] every collector writes +//! through, file-backed (CLI) or streaming (daemon), which keeps the +//! manifest complete by construction +//! - [`manifest`] — the bundle's self-describing manifest types +//! - [`redact`] — key-based masking of secret-shaped values, applied +//! identically by every producer +//! - [`listing`] — bounded recursive directory listing (names/sizes/metadata) +//! - [`capture`] — bounded subprocess output capture for command-shaped +//! collectors + +pub mod bundle; +pub mod capture; +pub mod listing; +pub mod manifest; +pub mod redact; + +pub use bundle::{BundleSink, BundleWriter, LOG_TAIL_CAP}; +pub use capture::{Capture, CaptureError, command_capture}; +pub use listing::{Listing, listing}; +pub use manifest::{CollectedEntry, CollectorError, Manifest, Redaction, SkippedEntry}; diff --git a/crates/diagnostics/src/listing.rs b/crates/diagnostics/src/listing.rs new file mode 100644 index 0000000000..0c3a81fb93 --- /dev/null +++ b/crates/diagnostics/src/listing.rs @@ -0,0 +1,165 @@ +//! Metadata-only directory listings for diagnostic bundles. +//! +//! Shared by host-side collectors and the daemon diagnostic RPC: both need to +//! show *what exists* (names, sizes, types, modes) without ever copying file +//! contents. + +use std::path::Path; + +use anyhow::Context as _; + +/// A completed listing. Structured so a caller can tell failure from +/// emptiness and record truncation in its manifest. +#[derive(Debug)] +#[non_exhaustive] +pub struct Listing { + /// One `type\tsize\tmtime\tmode\tpath` line per entry, header first. + pub text: String, + /// True when the entry cap was hit; `text` ends with an explicit marker. + pub truncated: bool, +} + +/// Lists everything under `root`, depth-first with sorted siblings +/// (deterministic, diffable), capped at `max_entries`. +/// +/// An unreadable `root` is an error — the caller records it in the manifest +/// instead of shipping a listing indistinguishable from an empty directory. +/// Per-entry read failures are recorded inline; hitting the cap appends an +/// explicit trailing truncation marker. The walk is iterative (walkdir), so a +/// pathologically deep tree cannot blow the stack, and symlinks are never +/// followed. +pub fn listing(root: &Path, max_entries: usize) -> Result { + use std::fmt::Write as _; + use std::os::unix::fs::MetadataExt as _; + + std::fs::read_dir(root).with_context(|| format!("listing {}", root.display()))?; + let mut text = String::from("type\tsize\tmtime\tmode\tpath\n"); + let mut truncated = false; + let walk = walkdir::WalkDir::new(root).min_depth(1).sort_by_file_name(); + for (seen, entry) in walk.into_iter().enumerate() { + if seen >= max_entries { + truncated = true; + text.push_str("\n"); + break; + } + match entry { + Ok(entry) => { + let rel = entry.path().strip_prefix(root).unwrap_or(entry.path()); + // metadata() is lstat-shaped here (follow_links is off), so a + // symlink reports itself, never its target. + let (kind, size, mtime, mode) = match entry.metadata() { + Ok(m) => ( + file_kind(&m), + m.len().to_string(), + m.mtime().to_string(), + format!("{:o}", m.mode() & 0o7777), + ), + Err(_) => ('?', "-".into(), "-".into(), "-".into()), + }; + let _ = writeln!(text, "{kind}\t{size}\t{mtime}\t{mode}\t{}", rel.display()); + } + Err(err) => { + let rel = err + .path() + .map(|p| p.strip_prefix(root).unwrap_or(p).display().to_string()) + .unwrap_or_else(|| "?".into()); + let _ = writeln!(text, "?\t-\t-\t-\t{rel} "); + } + } + } + Ok(Listing { text, truncated }) +} + +fn file_kind(meta: &std::fs::Metadata) -> char { + let ft = meta.file_type(); + if ft.is_dir() { + 'd' + } else if ft.is_symlink() { + 'l' + } else if ft.is_file() { + 'f' + } else { + 's' // socket/fifo/device — "special" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn listing_is_metadata_only_and_sorted() { + let tmp = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(tmp.path().join("b-dir")).unwrap(); + std::fs::write(tmp.path().join("b-dir/inner.txt"), "contents!").unwrap(); + std::fs::write(tmp.path().join("a.txt"), "secret-data").unwrap(); + + let listing = listing(tmp.path(), 100_000).unwrap(); + assert!(!listing.truncated); + let lines: Vec<&str> = listing.text.lines().collect(); + assert!(lines[1].ends_with("\ta.txt"), "got: {}", lines[1]); + assert!(lines[1].starts_with("f\t11\t"), "got: {}", lines[1]); + assert!(lines[2].ends_with("\tb-dir"), "got: {}", lines[2]); + assert!(lines[3].ends_with("b-dir/inner.txt"), "got: {}", lines[3]); + assert!( + !listing.text.contains("secret-data") && !listing.text.contains("contents!"), + "listing must never include file contents" + ); + } + + #[test] + fn listing_truncates_at_the_cap() { + let tmp = tempfile::TempDir::new().unwrap(); + for i in 0..10 { + std::fs::write(tmp.path().join(format!("f{i}")), "").unwrap(); + } + let listing = listing(tmp.path(), 3).unwrap(); + assert!(listing.truncated); + assert_eq!( + listing.text.lines().count(), + 1 + 3 + 1, + "header + 3 + marker" + ); + assert!(listing.text.ends_with("\n")); + } + + #[test] + fn exactly_at_the_cap_is_not_truncated() { + let tmp = tempfile::TempDir::new().unwrap(); + for i in 0..3 { + std::fs::write(tmp.path().join(format!("f{i}")), "").unwrap(); + } + let listing = listing(tmp.path(), 3).unwrap(); + assert!(!listing.truncated, "a complete listing is not truncated"); + assert_eq!(listing.text.lines().count(), 1 + 3, "header + 3, no marker"); + } + + #[test] + fn symlinks_report_themselves_not_their_targets() { + let tmp = tempfile::TempDir::new().unwrap(); + std::fs::write(tmp.path().join("target"), "x").unwrap(); + std::os::unix::fs::symlink(tmp.path().join("target"), tmp.path().join("z-link")).unwrap(); + + let listing = listing(tmp.path(), 10).unwrap(); + let link_line = listing + .text + .lines() + .find(|l| l.ends_with("z-link")) + .unwrap(); + assert!(link_line.starts_with('l'), "got: {link_line}"); + } + + #[test] + fn unreadable_root_is_an_error_not_an_empty_listing() { + let err = listing(Path::new("/nonexistent/diag-listing-root"), 10).unwrap_err(); + assert!(err.to_string().contains("listing"), "got: {err:#}"); + } + + #[test] + fn empty_root_is_a_valid_empty_listing() { + let tmp = tempfile::TempDir::new().unwrap(); + let listing = listing(tmp.path(), 10).unwrap(); + assert!(!listing.truncated); + assert_eq!(listing.text.lines().count(), 1, "header only"); + } +} diff --git a/crates/diagnostics/src/manifest.rs b/crates/diagnostics/src/manifest.rs new file mode 100644 index 0000000000..cac73ac7a0 --- /dev/null +++ b/crates/diagnostics/src/manifest.rs @@ -0,0 +1,71 @@ +//! The diagnostic bundle's self-description. +//! +//! `manifest.json` is the first thing a bundle reader opens: it records what +//! was collected (and how it was transformed), what was deliberately +//! withheld, and which collectors failed — so an absent file is always +//! explainable. + +use serde::Serialize; + +/// How a bundled file's content relates to the original. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case")] +#[non_exhaustive] +pub enum Redaction { + /// Copied verbatim. + None, + /// Secret-shaped values masked by key; structure preserved. + Keys, + /// Only names/sizes/metadata captured, never contents. + ListingOnly, + /// Only the trailing portion of a large file was captured. + TailCapped, + /// A streamed source exceeded its cap; only the leading portion was + /// captured and the rest was discarded mid-stream. + Truncated, +} + +/// Top-level manifest, serialized as the bundle's `manifest.json`. +#[derive(Debug, Serialize)] +#[non_exhaustive] +pub struct Manifest { + /// Bump when the bundle layout changes incompatibly. + pub schema_version: u32, + pub created_at: chrono::DateTime, + /// Version of the producing binary (CLI or daemon). + pub version: String, + /// Wall-clock duration of the whole collection run. + pub duration_ms: u64, + pub collected: Vec, + pub skipped: Vec, + pub errors: Vec, +} + +pub const SCHEMA_VERSION: u32 = 1; + +/// One file that made it into the bundle. +#[derive(Debug, Serialize)] +#[non_exhaustive] +pub struct CollectedEntry { + /// Path inside the bundle (relative to the bundle root). + pub path: String, + pub redaction: Redaction, + pub bytes: u64, +} + +/// Something deliberately not collected. +#[derive(Debug, Serialize)] +#[non_exhaustive] +pub struct SkippedEntry { + pub what: String, + pub reason: String, +} + +/// A collector that failed; the bundle carries on without its output. +#[derive(Debug, Serialize)] +#[non_exhaustive] +pub struct CollectorError { + pub collector: String, + pub error: String, + pub duration_ms: u64, +} diff --git a/crates/diagnostics/src/redact.rs b/crates/diagnostics/src/redact.rs new file mode 100644 index 0000000000..4fc0ea2bdf --- /dev/null +++ b/crates/diagnostics/src/redact.rs @@ -0,0 +1,197 @@ +//! Key-based redaction of secret-shaped values in structured data. +//! +//! Shared by the CLI-side diagnostic bundler and the daemon-side +//! diagnostic RPC so both apply identical rules. Redaction is +//! deliberately conservative: false positives (masking a harmless value) are +//! acceptable, false negatives (leaking a secret) are not. + +use serde_json::Value; + +/// Key substrings that mark a value as sensitive, matched case-insensitively. +const SENSITIVE_KEY_PARTS: &[&str] = &[ + "token", + "secret", + "password", + "passwd", + "credential", + "auth", + "bearer", + "private", + "api_key", + "apikey", + "key", +]; + +/// Table/object names whose *entire* contents are environment-variable style +/// user data and must be masked regardless of key names. +const ENV_TABLE_NAMES: &[&str] = &["vars", "vars_lenient", "env", "environment"]; + +/// Returns true when a key looks like it names secret material. +/// +/// `public_key` is exempt (`public_key`, `wg_public_key` must survive +/// redaction — mesh diagnostics depend on them), but only the `public_key` +/// token itself: a key that pairs it with a sensitive marker +/// (`public_key_token`, `private_public_key`) stays sensitive. Removing the +/// exempt token before the marker scan keeps that fail-closed. +pub fn is_sensitive_key(key: &str) -> bool { + let stripped = key.to_ascii_lowercase().replace("public_key", ""); + SENSITIVE_KEY_PARTS + .iter() + .any(|part| stripped.contains(part)) +} + +/// Returns true when a table/object with this name holds env-var style values +/// that must be masked wholesale. +pub fn is_env_table_name(name: &str) -> bool { + ENV_TABLE_NAMES.iter().any(|t| name.eq_ignore_ascii_case(t)) +} + +/// The placeholder a redacted value is replaced with. Records the original +/// (serialized) length so size-related issues stay diagnosable. +pub fn redaction_placeholder(original: &Value) -> Value { + let len = match original { + Value::String(s) => s.len(), + other => other.to_string().len(), + }; + Value::String(format!("")) +} + +/// Recursively masks sensitive values in `value` in place. +/// +/// A leaf is masked when its own key is sensitive per [`is_sensitive_key`], +/// or when any ancestor was named like an env table ([`is_env_table_name`]) +/// or like secret material — inside those, every leaf is masked, so a +/// `tokens` object can't smuggle its members out under harmless inner keys. +pub fn redact_json(value: &mut Value) { + redact_json_inner(value, false); +} + +fn redact_json_inner(value: &mut Value, mask_all: bool) { + match value { + Value::Object(map) => { + for (key, child) in map.iter_mut() { + let child_mask = mask_all || is_env_table_name(key) || is_sensitive_key(key); + if child.is_object() || child.is_array() { + redact_json_inner(child, child_mask); + } else if child_mask || is_sensitive_key(key) { + *child = redaction_placeholder(child); + } + } + } + Value::Array(items) => items + .iter_mut() + .for_each(|item| redact_json_inner(item, mask_all)), + leaf => { + if mask_all { + *leaf = redaction_placeholder(leaf); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn sensitive_keys_match_case_insensitively() { + for key in [ + "token", + "GITHUB_TOKEN", + "api_key", + "ApiKey", + "ssh_private_key", + "PASSWORD", + "authorization", + "my-secret-thing", + ] { + assert!(is_sensitive_key(key), "{key} should be sensitive"); + } + } + + #[test] + fn public_keys_are_exempt() { + assert!(!is_sensitive_key("public_key")); + assert!(!is_sensitive_key("WG_PUBLIC_KEY")); + assert!(!is_sensitive_key("name")); + assert!(!is_sensitive_key("packages")); + } + + #[test] + fn public_exemption_is_anchored_to_public_key() { + assert!(is_sensitive_key("publication_secret")); + assert!(is_sensitive_key("public_password")); + assert!(is_sensitive_key("republic_token")); + // The exempt token must not launder a sensitive marker sitting + // beside it. + assert!(is_sensitive_key("public_key_token")); + assert!(is_sensitive_key("public_key_password")); + assert!(is_sensitive_key("private_public_key")); + } + + #[test] + fn sensitive_keys_mask_container_values_wholesale() { + let mut v = json!({ + "tokens": { "github": "ghp_xxx", "gitlab": "glpat_yyy" }, + "api_token": ["primary", "secondary"], + "credentials": { "aws": { "access_key_id": "AKIA" } }, + }); + redact_json(&mut v); + assert_eq!(v["tokens"]["github"], ""); + assert_eq!(v["tokens"]["gitlab"], ""); + assert_eq!(v["api_token"][0], ""); + assert_eq!(v["api_token"][1], ""); + assert_eq!(v["credentials"]["aws"]["access_key_id"], ""); + } + + #[test] + fn redacts_sensitive_values_and_env_tables() { + let mut v = json!({ + "name": "dev", + "api_token": "abc123", + "vars": { "EDITOR": "vim", "HOME": "/home/u" }, + "nested": { "list": [ { "password": "hunter2", "port": 80 } ] }, + }); + redact_json(&mut v); + assert_eq!(v["name"], "dev"); + assert_eq!(v["api_token"], ""); + assert_eq!(v["vars"]["EDITOR"], ""); + assert_eq!(v["vars"]["HOME"], ""); + assert_eq!(v["nested"]["list"][0]["password"], ""); + assert_eq!(v["nested"]["list"][0]["port"], 80); + } + + #[test] + fn env_table_masks_nested_structures_wholesale() { + let mut v = json!({ + "vars": { "TERM": { "inherit": true, "default": "xterm" } }, + }); + redact_json(&mut v); + assert_eq!(v["vars"]["TERM"]["inherit"], ""); + assert_eq!(v["vars"]["TERM"]["default"], ""); + } + + #[test] + fn redaction_is_idempotent() { + let mut v = json!({ "secret": "s3cr3t", "vars": { "A": "b" } }); + redact_json(&mut v); + let once = v.clone(); + // Placeholders re-redact to placeholders of the placeholder's length; + // assert full equality instead by redacting a fresh copy. + let mut again = once.clone(); + redact_json(&mut again); + assert_eq!( + again["vars"]["A"], "", + "re-redaction only rewrites placeholders, never restores data" + ); + assert_eq!(once["secret"], ""); + } + + #[test] + fn non_string_sensitive_values_are_masked() { + let mut v = json!({ "auth_retries": 3 }); + redact_json(&mut v); + assert_eq!(v["auth_retries"], ""); + } +}