diff --git a/crates/minimal/src/diag/collect.rs b/crates/minimal/src/diag/collect.rs index a0903d48ed..d5f8ffd3fc 100644 --- a/crates/minimal/src/diag/collect.rs +++ b/crates/minimal/src/diag/collect.rs @@ -1785,6 +1785,202 @@ mod tests { assert_eq!(refused.traces.endpoint, None); } + /// The bundle path the child of + /// [`a_bundle_collected_with_telemetry_on_carries_no_secret`] writes; + /// set only in that child. + const T18_OUT: &str = "DIAG_T18_BUNDLE_OUT"; + /// The state and config directories that child collects from. + const T18_STATE: &str = "DIAG_T18_STATE_DIR"; + const T18_CONFIG: &str = "DIAG_T18_CONFIG_DIR"; + + /// Not a test on its own: the process + /// [`a_bundle_collected_with_telemetry_on_carries_no_secret`] spawns (this + /// test binary, with telemetry settings in its real environment) to run + /// `min bug`'s collection, [`crate::diag::cmd_bug`], host side only. Run + /// on its own it returns at once. + #[tokio::test] + #[ignore = "the child process of a_bundle_collected_with_telemetry_on_carries_no_secret"] + async fn telemetry_on_bundle_child() { + let Some(out) = std::env::var_os(T18_OUT) else { + return; + }; + let dir = |name| PathBuf::from(std::env::var_os(name).unwrap()); + let global = crate::GlobalArgs { + repo_dir: None, + minimal_dir: Some(dir(T18_STATE)), + config_dir: Some(dir(T18_CONFIG)), + provider: None, + no_input: true, + vm: None, + }; + let args = crate::diag::BugArgs { + output: Some(PathBuf::from(out)), + no_guest: true, + guest_timeout_secs: 5, + log_tail_bytes: diagnostics::LOG_TAIL_CAP, + upload: false, + portal: crate::diag::PortalArgs { + context: None, + endpoint: crate::diag::upload::DEFAULT_ENDPOINT.to_owned(), + }, + }; + // Boxed: `cmd_bug` is a large future (clippy::large_futures). + Box::pin(crate::diag::cmd_bug(&global, args)).await.unwrap(); + } + + /// Plan T18 (spec 25 TEL-044, TEL-051): `min bug` run with telemetry on + /// carries the spool, names the telemetry settings without their values, + /// and lets no secret through. The collection runs in a child process + /// whose real environment holds the settings (an endpoint with + /// credentials, a tenant path and a query; exporter headers, one + /// percent-encoded), because `min bug` reads its own environment; the + /// spool holds a header value in free text, a secret-shaped attribute, a + /// header-like attribute and a remote URL with a token in it. Every file + /// in the bundle is then opened and searched for each secret. + #[tokio::test] + async fn a_bundle_collected_with_telemetry_on_carries_no_secret() { + const SECRETS: &[&str] = &[ + "ingestPASSWORD9", + "TENANTsecret77", + "QUERYsecret55", + "hcaik_HEADERsecret01", + "BEARERsecret02", + "opensesame-api-k3y", + "AUTHZsecret03", + "ghp_REMOTEtoken1234567890abcdefABCDEF12", + ]; + let tmp = tempfile::TempDir::new().unwrap(); + let state = tmp.path().join("state"); + let config = tmp.path().join("config"); + let spool_dir = tmp.path().join("spool"); + for d in [&state, &config, &spool_dir] { + std::fs::create_dir_all(d).unwrap(); + } + std::fs::write( + spool_dir.join("minimal-cli-1-1700000000000.jsonl"), + concat!( + r#"{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"minimal-cli"}}]},"#, + r#""scopeSpans":[{"spans":[{"traceId":"0af7651916cd43dd8448eb211c80319c","spanId":"b7ad6b7169203331","#, + r#""name":"cmd","endTimeUnixNano":"200","attributes":["#, + r#"{"key":"cmd.name","value":{"stringValue":"ls"}},"#, + r#"{"key":"note","value":{"stringValue":"sent hcaik_HEADERsecret01 to the collector"}},"#, + r#"{"key":"vendor.api_key","value":{"stringValue":"opensesame-api-k3y"}},"#, + r#"{"key":"http.request.header.authorization","value":{"stringValue":"Bearer AUTHZsecret03"}},"#, + r#"{"key":"vcs.remote","value":{"stringValue":"https://x-access-token:ghp_REMOTEtoken1234567890abcdefABCDEF12@github.com/o/r.git"}}"#, + r#"]}]}]}]}"#, + "\n", + r#"{"resourceLogs":[{"scopeLogs":[{"logRecords":[{"body":{"stringValue":"export failed: BEARERsecret02 refused"}}]}]}]}"#, + "\n" + ), + ) + .unwrap(); + let out = tmp.path().join("bundle.tar.zst"); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()); + child.args([ + "--exact", + "diag::collect::tests::telemetry_on_bundle_child", + "--ignored", + ]); + for (k, _) in std::env::vars_os() { + let name = k.to_string_lossy(); + if name.starts_with("OTEL_") + || name.starts_with("MINIMAL_") + || name == "DO_NOT_TRACK" + || name == "TRACEPARENT" + { + child.env_remove(&k); + } + } + child + .env(T18_OUT, &out) + .env(T18_STATE, &state) + .env(T18_CONFIG, &config) + .env("MINIMAL_TELEMETRY", "1") + .env("MINIMAL_OTEL_SPOOL_DIR", &spool_dir) + .env( + "MINIMAL_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "https://ingest:ingestPASSWORD9@collector.example:4318/TENANTsecret77/v1/traces", + ) + .env( + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + "https://collector.example:4318/v1/logs?key=QUERYsecret55", + ) + .env( + "MINIMAL_OTEL_EXPORTER_OTLP_HEADERS", + "x-honeycomb-team=hcaik_HEADERsecret01,authorization=Bearer%20BEARERsecret02", + ); + let status = child.status().unwrap(); + assert!(status.success(), "the collecting child failed: {status}"); + + let files = { + let bytes = std::fs::read(&out).unwrap(); + let decoder = async_compression::tokio::bufread::ZstdDecoder::new(&bytes[..]); + let mut entries = async_tar::Archive::new(decoder).entries().unwrap(); + let mut files = BTreeMap::new(); + while let Some(entry) = entries.next().await { + let mut entry = entry.unwrap(); + let path = entry.path().unwrap().to_string_lossy().into_owned(); + let mut contents = Vec::new(); + entry.read_to_end(&mut contents).await.unwrap(); + // Drop the bundle's root directory. + let rel = path + .split_once('/') + .map_or(path.clone(), |(_, r)| r.to_owned()); + files.insert(rel, contents); + } + files + }; + + // The spool travels, its harmless attribute intact. + let spool = files + .get("telemetry/spool/minimal-cli-1-1700000000000.jsonl") + .unwrap_or_else(|| panic!("no spool file in {:?}", files.keys())); + let spool = String::from_utf8_lossy(spool); + assert!( + spool.contains("\"cmd.name\"") && spool.contains("\"ls\""), + "{spool}" + ); + // Every secret-bearing value is still there, redacted in place: + // scrubbed, not dropped with its record. + for kept in [ + "\"vendor.api_key\"", + "\"vcs.remote\"", + "@github.com/o/r.git", + "export failed:", + ] { + assert!(spool.contains(kept), "{kept} missing: {spool}"); + } + assert!(spool.matches("= 5, "{spool}"); + + // The settings are named, their values are not. + let telemetry: serde_json_lenient::Value = + serde_json_lenient::from_slice(&files["host/telemetry.json"]).unwrap(); + assert_eq!(telemetry["enabled"], true, "{telemetry}"); + assert_eq!( + telemetry["traces"]["endpoint"], + "https://collector.example:4318" + ); + let env = String::from_utf8_lossy(&files["host/env.json"]); + for name in [ + "MINIMAL_TELEMETRY", + "MINIMAL_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "MINIMAL_OTEL_EXPORTER_OTLP_HEADERS", + ] { + assert!( + env.contains(name), + "{name} is not named in host/env.json: {env}" + ); + } + + // No secret anywhere in the bundle. + for (path, bytes) in &files { + let text = String::from_utf8_lossy(bytes); + for secret in SECRETS { + assert!(!text.contains(secret), "{secret} in {path}:\n{text}"); + } + } + } + /// TEL-051: the bundle reads one spool directory, the one `min bug`'s /// own environment names, and says so. A spool a producer pinned through /// its own `MINIMAL_OTEL_SPOOL_DIR` stays out, and `host/telemetry.json` diff --git a/crates/minimal/tests/kani_workflow.rs b/crates/minimal/tests/kani_workflow.rs index eb6f891ec0..1e917e56db 100644 --- a/crates/minimal/tests/kani_workflow.rs +++ b/crates/minimal/tests/kani_workflow.rs @@ -16,19 +16,111 @@ fn repo_root() -> PathBuf { .to_path_buf() } -#[test] -fn kani_workflow_filter_names_bep_crate() { +/// The workflow's path and its `- "crates/..."` filter lines, trimmed. +fn kani_filter_lines() -> (PathBuf, Vec) { let path = repo_root().join(".github/workflows/ci-kani.yml"); let workflow = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())); - let filter_lines: Vec<&str> = workflow + let lines = workflow .lines() .map(str::trim) .filter(|l| l.starts_with("- \"crates/")) + .map(str::to_owned) .collect(); + (path, lines) +} + +#[test] +fn kani_workflow_filter_names_bep_crate() { + let (path, filter_lines) = kani_filter_lines(); assert!( - filter_lines.contains(&"- \"crates/bep/**\""), + filter_lines.iter().any(|l| l == "- \"crates/bep/**\""), "the kani path filter in {} does not name crates/bep/**; found: {filter_lines:?}", path.display() ); } + +/// The telemetry crate and the VM host daemon carry Kani harnesses of their +/// own (the switches' opt-in and endpoint rules and the spool prune plan; +/// the guest boot line's token writer and budget). The lane's runner, +/// `scripts/kani.sh`, must prove them with a harness count, so none can go +/// green by never being run. +/// +/// This count is checked only when the lane runs. The workflow's path +/// filter does not name `crates/mlog/**` or `crates/minvmd/**`, so a pull +/// request that changes only one of their harnesses merges without a proof +/// run, and the lane finds the break on the next push to `main`. It runs on +/// a pull request only when `scripts/kani.sh`, `Cargo.toml`, `Cargo.lock` or +/// a listed crate changes. Workflows are frozen for this series: adding the +/// two crates to the filter is a workflow change a maintainer owns, tracked +/// in minimal#2075, and this test then gains a filter check like +/// `kani_workflow_filter_names_bep_crate`. Spec: TEL-001, TEL-002, TEL-004, +/// TEL-017 (their Kani proofs). +#[test] +fn kani_runner_proves_the_telemetry_crates() { + let path = repo_root().join("scripts/kani.sh"); + let runner = + std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())); + let expects: Vec<&str> = runner + .lines() + .map(str::trim) + .filter(|l| l.starts_with("expect ")) + .collect(); + for crate_name in ["mlog", "minvmd"] { + assert!( + expects.iter().any(|l| { + let mut words = l.split_whitespace(); + words.next() == Some("expect") + && words.next() == Some(crate_name) + && words + .next() + .is_some_and(|n| n.parse::().is_ok_and(|n| n > 0)) + }), + "{} does not expect a positive harness count for {crate_name}; found: {expects:?}", + path.display() + ); + } +} + +/// One runaway harness cannot take the runner. +/// With `KANI_MEM_MAX` unset, the CI case, since the workflow sets none, +/// `scripts/kani.sh` caps each crate's `cargo kani` at 14G, and only an +/// explicit `KANI_MEM_MAX=off` runs without a ceiling. The runner's guard +/// block is run as written, under `sh` with no `systemd-run` on `PATH` (the +/// `ulimit` path a GitHub-hosted runner takes), and reports the ceiling it +/// chose. +#[test] +fn kani_runner_caps_memory_when_unset() { + let path = repo_root().join("scripts/kani.sh"); + let runner = + std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())); + let (_, from_guard) = runner + .split_once("# The memory guard, decided once") + .unwrap_or_else(|| panic!("{} has no memory guard block", path.display())); + let (guard, _) = from_guard + .split_once("\n# `cargo kani` on one crate") + .expect("the guard block ends before kani_crate"); + let empty_path = tempfile::TempDir::new().unwrap(); + let decide = |max: Option<&str>| { + let mut sh = std::process::Command::new("/bin/sh"); + sh.env_clear() + .env("PATH", empty_path.path()) + .arg("-c") + .arg(format!( + "set -eu\n# The memory guard{guard}\necho \"guard=$kani_guard kib=$kani_mem_kib\"" + )); + if let Some(max) = max { + sh.env("KANI_MEM_MAX", max); + } + let out = sh.output().unwrap(); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).unwrap().trim().to_owned() + }; + assert_eq!(decide(None), "guard=ulimit kib=14680064", "unset: 14G"); + assert_eq!(decide(Some("8G")), "guard=ulimit kib=8388608"); + assert_eq!(decide(Some("off")), "guard= kib=", "off: no ceiling"); +} diff --git a/crates/minimal/tests/otel_cli.rs b/crates/minimal/tests/otel_cli.rs new file mode 100644 index 0000000000..0e756a71aa --- /dev/null +++ b/crates/minimal/tests/otel_cli.rs @@ -0,0 +1,834 @@ +//! The `min` binary's telemetry against a real (in-process) minimald: the +//! caller's trace crosses into the CLI and on to the daemon (TEL-021), ambient +//! `OTEL_*` alone exports nothing (TEL-001), and a collector that never answers +//! costs a command little and prints nothing (TEL-010, TEL-012). +//! +//! The daemon runs in this test process, so a [`SpanLog`] installed as the +//! process's subscriber sees its spans (process-wide, not `set_default`: a +//! thread-local subscriber in one of several parallel libtest threads can +//! miss callsites another thread registered first); a test picks its own +//! spans out by the trace id it sent. The CLI runs as +//! a child process whose environment starts with every `OTEL_*`, +//! `MINIMAL_*`, `DO_NOT_TRACK`, `TRACEPARENT` and `RUST_LOG` variable +//! removed, and writes its spans to a spool the test reads. +#![cfg(target_os = "linux")] +#![expect( + clippy::unwrap_used, + clippy::panic, + clippy::indexing_slicing, + reason = "test helpers: a failed spawn or a malformed spool must fail the test loudly; \ + serde_json Value indexing returns Null for a missing key" +)] + +mod common; + +use std::path::Path; +use std::time::{Duration, Instant}; + +use minimal::GlobalArgs; +use minimald::test_harness::SpanLog; +use serde_json_lenient::Value; +use tracing_subscriber::layer::SubscriberExt as _; + +// Each test that finds the daemon's spans by trace id in the process-wide +// [`SpanLog`] uses a trace id of its own: under libtest the tests share +// one process, and a shared id would let one test see another's spans. +const PARENT: &str = "b7ad6b7169203331"; + +/// Runs the compiled `min` against the harness daemon with a stripped +/// telemetry environment plus `env`; returns its output and wall time. +async fn run_min_env( + args: &GlobalArgs, + env: &[(&str, &str)], + extra: &[&str], +) -> (std::process::Output, Duration) { + let minimal_dir = args.minimal_dir.as_ref().unwrap(); + let config_dir = tempfile::TempDir::new().unwrap(); + let mut command = tokio::process::Command::new(env!("CARGO_BIN_EXE_min")); + command + .args(["--minimal-dir".as_ref(), minimal_dir.as_os_str()]) + .args(["--config-dir".as_ref(), config_dir.path().as_os_str()]) + .arg("--no-input") + .args(extra); + for (k, _) in std::env::vars_os() { + let name = k.to_string_lossy(); + if name.starts_with("OTEL_") + || name.starts_with("MINIMAL_") + || name == "DO_NOT_TRACK" + || name == "TRACEPARENT" + || name == "RUST_LOG" + { + command.env_remove(&k); + } + } + command.envs(env.iter().copied()); + let t = Instant::now(); + let out = command.output().await.unwrap(); + (out, t.elapsed()) +} + +/// A temporary directory with mode 0700: the spool refuses a directory +/// with group or other bits, and `tempfile` creates its directories with +/// the umask's default (0755). +fn private_tempdir() -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt as _; + tempfile::Builder::new() + .permissions(std::fs::Permissions::from_mode(0o700)) + .tempdir() + .unwrap() +} + +/// Every span in the spool files under `dir`. +fn spool_spans(dir: &Path) -> Vec { + let mut spans = Vec::new(); + for e in std::fs::read_dir(dir).unwrap() { + let p = e.unwrap().path(); + for l in std::fs::read_to_string(&p).unwrap().lines() { + let v: Value = serde_json_lenient::from_str(l).unwrap(); + for rs in v["resourceSpans"].as_array().into_iter().flatten() { + for ss in rs["scopeSpans"].as_array().into_iter().flatten() { + spans.extend(ss["spans"].as_array().into_iter().flatten().cloned()); + } + } + } + } + spans +} + +/// The process-wide [`SpanLog`], installed on first use. +fn span_log() -> SpanLog { + static LOG: std::sync::OnceLock = std::sync::OnceLock::new(); + LOG.get_or_init(|| { + let log = SpanLog::default(); + tracing::subscriber::set_global_default(tracing_subscriber::registry().with(log.clone())) + .unwrap(); + log + }) + .clone() +} + +fn named<'a>(spans: &'a [Value], name: &str) -> &'a Value { + spans + .iter() + .find(|s| s["name"] == name) + .unwrap_or_else(|| panic!("no {name} span in {spans:#?}")) +} + +/// A loopback listener that accepts and never answers: a collector gone +/// silent. Returns its URL and a count of the connections it accepted. +fn tarpit() -> (String, std::sync::Arc) { + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", l.local_addr().unwrap()); + let accepted = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let count = accepted.clone(); + std::thread::spawn(move || { + let mut held = Vec::new(); + for c in l.incoming() { + count.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + held.push(c); // accepted, never read, never answered + } + }); + (url, accepted) +} + +/// TEL-021 across both process boundaries: `min ls` run inside +/// a caller's trace (`TRACEPARENT`) records its `cmd` span as the caller's +/// child and `client.rpc` under it, and the daemon's `rpc` spans for its +/// calls are the `cmd` span's children (the CLI sends one context per +/// process), all in the caller's trace. +#[tokio::test] +async fn a_traced_min_ls_joins_the_callers_trace() { + const TRACE: &str = "0af7651916cd43dd8448eb211c803101"; + let log = span_log(); + let (_daemon, args) = common::setup().await; + let spool = private_tempdir(); + let tp = format!("00-{TRACE}-{PARENT}-01"); + let (out, _) = run_min_env( + &args, + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", spool.path().to_str().unwrap()), + ("TRACEPARENT", &tp), + ], + &["ls"], + ) + .await; + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + + let spans = spool_spans(spool.path()); + let cmd = named(&spans, "cmd"); + assert_eq!(cmd["traceId"], TRACE); + assert_eq!(cmd["parentSpanId"], PARENT); + let rpc = named(&spans, "client.rpc"); + assert_eq!(rpc["traceId"], TRACE); + assert_eq!(rpc["parentSpanId"], cmd["spanId"], "client.rpc under cmd"); + + // `min ls` asks the version, then lists. + let served = log.find("rpc", "trace_id", TRACE); + let names: Vec<_> = served.iter().filter_map(|s| s.field("rpc")).collect(); + assert_eq!( + names, + ["minimald-v1-GetVersion", "minimald-v1-ListSessions"], + "{served:?}" + ); + for s in &served { + assert_eq!( + s.field("parent_span_id"), + cmd["spanId"].as_str(), + "the daemon's rpc span is the CLI's cmd span's child: {s:?}" + ); + } +} + +/// `min` with its telemetry off (`DO_NOT_TRACK=1` here; not +/// opting in, or `OTEL_SDK_DISABLED`, decide the same) tells the daemon so +/// on every channel and sends no `TRACEPARENT`, even with one inherited: +/// the daemon's `rpc` spans for its calls are marked `telemetry=opt-out`, +/// have no parent, and none carries the inherited trace id. That the +/// daemon exports nothing of them is minimald's +/// `an_opted_out_request_leaves_no_daemon_span`. +#[tokio::test] +async fn an_opted_out_min_sends_no_traceparent_and_its_daemon_spans_are_marked() { + const INHERITED: &str = "0af7651916cd43dd8448eb211c80319d"; + let log = span_log(); + let (_daemon, args) = common::setup().await; + let spool = private_tempdir(); + let tp = format!("00-{INHERITED}-{PARENT}-01"); + let (out, _) = run_min_env( + &args, + &[ + ("DO_NOT_TRACK", "1"), + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", spool.path().to_str().unwrap()), + ("TRACEPARENT", &tp), + ], + &["ls"], + ) + .await; + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + assert!( + spool_spans(spool.path()).is_empty(), + "DO_NOT_TRACK: the CLI itself spools nothing" + ); + assert!( + log.find("rpc", "trace_id", INHERITED).is_empty(), + "no daemon span joined the inherited trace: {:?}", + log.find("rpc", "trace_id", INHERITED) + ); + let marked = log.find("rpc", "telemetry", "opt-out"); + assert!( + marked + .iter() + .any(|s| s.field("rpc") == Some("minimald-v1-ListSessions")), + "the listing's rpc span is marked opted out: {marked:?}" + ); + for s in &marked { + assert_eq!( + s.field("parent_span_id"), + None, + "no TRACEPARENT was sent: {s:?}" + ); + } +} + +/// TEL-001: an ambient OTLP endpoint (a developer's shell, a CI +/// runner) without `MINIMAL_TELEMETRY=1` opens no connection to it and +/// spools nothing. +#[tokio::test] +async fn ambient_otel_alone_opens_no_connection() { + let (_daemon, args) = common::setup().await; + let (url, accepted) = tarpit(); + let spool = private_tempdir(); + let dir = spool.path().join("spool"); + let (out, _) = run_min_env( + &args, + &[ + ("OTEL_EXPORTER_OTLP_ENDPOINT", &url), + ("OTEL_TRACES_EXPORTER", "otlp"), + ("MINIMAL_OTEL_SPOOL_DIR", dir.to_str().unwrap()), + ], + &["ls"], + ) + .await; + assert!(out.status.success()); + assert_eq!(accepted.load(std::sync::atomic::Ordering::SeqCst), 0); + assert!(!dir.exists(), "nothing spooled"); +} + +/// TEL-010, TEL-012: against a collector that +/// accepts and never answers, `min ls` takes little longer than with +/// telemetry off (it took 2 s longer before the CLI's exit flush was +/// bounded) and prints none of the SDK's export errors. +/// +/// Each side is the fastest of [`RUNS`] interleaved runs, so one run slowed +/// by a loaded host does not decide the comparison. The allowance is the +/// exit wait's bound (TEL-010, TEL-N01: `CLI_EXIT_FLUSH` plus 250 ms) with +/// as much again for scheduling: a silent collector that blocked the exit +/// past its bound still fails, by the 2 s it once cost. +#[tokio::test] +async fn a_silent_collector_costs_min_ls_little_and_prints_nothing() { + const RUNS: usize = 3; + let allowance = (mlog::otel::CLI_EXIT_FLUSH + Duration::from_millis(250)) * 2; + let (_daemon, args) = common::setup().await; + let (url, _accepted) = tarpit(); + let spool = private_tempdir(); + let (mut off, mut on) = (Duration::MAX, Duration::MAX); + for _ in 0..RUNS { + let (control, took) = run_min_env(&args, &[], &["ls"]).await; + assert!(control.status.success()); + off = off.min(took); + let (out, took) = run_min_env( + &args, + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_EXPORTER_OTLP_ENDPOINT", &url), + ("MINIMAL_OTEL_SPOOL_DIR", spool.path().to_str().unwrap()), + ], + &["ls"], + ) + .await; + assert!(out.status.success()); + on = on.min(took); + let stderr = String::from_utf8_lossy(&out.stderr); + for noise in ["opentelemetry", "OTLP", "BatchSpanProcessor", "ERROR"] { + assert!(!stderr.contains(noise), "{noise} on stderr: {stderr}"); + } + } + assert!( + on < off + allowance, + "min ls took at best {on:?} against a silent collector, {off:?} with \ + telemetry off ({RUNS} runs each; allowance {allowance:?})" + ); +} + +// ---- plan T7 and T11: the join tests ---- + +/// One span as the collector received it, from the OTLP/protobuf body. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct SentSpan { + name: String, + trace_id: String, + span_id: String, + parent_span_id: String, +} + +/// The fields of one protobuf message: `(field number, value)`, where a +/// length-delimited value is its bytes and the other wire types are +/// skipped. Enough to walk an `ExportTraceServiceRequest`. +fn proto_fields(mut b: &[u8]) -> Vec<(u64, &[u8])> { + fn varint(b: &mut &[u8]) -> Option { + let mut v = 0u64; + for shift in (0..64).step_by(7) { + let (&byte, rest) = b.split_first()?; + *b = rest; + v |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return Some(v); + } + } + None + } + let mut out = Vec::new(); + while let Some(key) = varint(&mut b) { + let skip = match key & 7 { + 0 => { + varint(&mut b); + continue; + } + 1 => 8, + 5 => 4, + 2 => { + let len = usize::try_from(varint(&mut b).unwrap()).unwrap(); + let (v, rest) = b.split_at(len); + out.push((key >> 3, v)); + b = rest; + continue; + } + w => panic!("wire type {w} in an OTLP body"), + }; + b = &b[skip..]; + } + out +} + +/// Every span in the trace requests (`/v1/traces`) `col` received. +fn sent_spans(col: &Collector) -> Vec { + let hex = |b: &[u8]| b.iter().map(|x| format!("{x:02x}")).collect::(); + let mut spans = Vec::new(); + for (path, body) in col.requests.lock().unwrap().iter() { + if !path.ends_with("/v1/traces") { + continue; + } + for (_, rs) in proto_fields(body).into_iter().filter(|f| f.0 == 1) { + for (_, ss) in proto_fields(rs).into_iter().filter(|f| f.0 == 2) { + for (_, span) in proto_fields(ss).into_iter().filter(|f| f.0 == 2) { + let mut s = SentSpan { + name: String::new(), + trace_id: String::new(), + span_id: String::new(), + parent_span_id: String::new(), + }; + for (n, v) in proto_fields(span) { + match n { + 1 => s.trace_id = hex(v), + 2 => s.span_id = hex(v), + 4 => s.parent_span_id = hex(v), + 5 => s.name = String::from_utf8_lossy(v).into_owned(), + _ => {} + } + } + spans.push(s); + } + } + } + } + spans +} + +/// The same four fields for every span in the spool files under `dir`. +fn spooled_spans(dir: &Path) -> Vec { + spool_spans(dir) + .iter() + .map(|s| SentSpan { + name: s["name"].as_str().unwrap_or_default().to_owned(), + trace_id: s["traceId"].as_str().unwrap_or_default().to_owned(), + span_id: s["spanId"].as_str().unwrap_or_default().to_owned(), + parent_span_id: s["parentSpanId"].as_str().unwrap_or_default().to_owned(), + }) + .collect() +} + +/// A stub OTLP/HTTP collector on 127.0.0.1: answers every request with an +/// empty 200 and keeps its path and body. +struct Collector { + url: String, + requests: Requests, +} + +/// Each request a [`Collector`] received: its path and body. +type Requests = std::sync::Arc)>>>; + +fn collector() -> Collector { + use std::io::{BufRead as _, Read as _, Write as _}; + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", l.local_addr().unwrap()); + let requests: Requests = std::sync::Arc::default(); + let kept = requests.clone(); + std::thread::spawn(move || { + for conn in l.incoming() { + let Ok(conn) = conn else { continue }; + let kept = kept.clone(); + std::thread::spawn(move || { + let Ok(mut w) = conn.try_clone() else { return }; + let mut r = std::io::BufReader::new(conn); + loop { + let mut request = String::new(); + if r.read_line(&mut request).unwrap_or(0) == 0 { + return; + } + let path = request.split(' ').nth(1).unwrap_or_default().to_owned(); + let mut len = 0usize; + let mut line = String::new(); + while r.read_line(&mut line).is_ok_and(|n| n > 0) { + if line == "\r\n" { + break; + } + if let Some(v) = line.to_ascii_lowercase().strip_prefix("content-length:") { + len = v.trim().parse().unwrap_or(0); + } + line.clear(); + } + let mut body = vec![0; len]; + if r.read_exact(&mut body).is_err() { + return; + } + kept.lock().unwrap().push((path, body)); + let ok = b"HTTP/1.1 200 OK\r\ncontent-type: application/x-protobuf\r\n\ + content-length: 0\r\n\r\n"; + if w.write_all(ok).is_err() { + return; + } + } + }); + } + }); + Collector { url, requests } +} + +/// Plan T7 (spec 25 TEL-002, TEL-005, TEL-021, TEL-022): `MINIMAL_TELEMETRY=1 +/// min ls` against a local collector. The CLI's `cmd` span reaches the +/// collector as a root (no caller's trace here: the CLI starts one), the +/// daemon's `rpc` spans for the version check and the listing are that +/// `cmd` span's children in the same trace, and the spool holds the very +/// records the collector got: the same spans, the same ids. +/// +/// The daemon is the test harness's, in this process, with no exporter of +/// its own: its spans are read from the process's subscriber by the trace +/// id. minimald's own export to a collector is `otel_shutdown_integration`. +#[tokio::test] +async fn a_traced_min_ls_lands_in_one_trace_and_in_the_spool() { + let log = span_log(); + let (_daemon, args) = common::setup().await; + let col = collector(); + let spool = private_tempdir(); + let (out, _) = run_min_env( + &args, + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_EXPORTER_OTLP_ENDPOINT", &col.url), + ("MINIMAL_OTEL_SPOOL_DIR", spool.path().to_str().unwrap()), + ], + &["ls"], + ) + .await; + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + + let mut sent = sent_spans(&col); + let cmd = sent + .iter() + .find(|s| s.name == "cmd") + .unwrap_or_else(|| panic!("no cmd span at the collector: {sent:#?}")) + .clone(); + assert_eq!(cmd.parent_span_id, "", "the CLI started the trace: {cmd:?}"); + assert!( + sent.iter() + .any(|s| s.name == "client.rpc" && s.parent_span_id == cmd.span_id), + "{sent:#?}" + ); + assert!( + sent.iter().all(|s| s.trace_id == cmd.trace_id), + "one trace: {sent:#?}" + ); + + let served = log.find("rpc", "trace_id", &cmd.trace_id); + let names: Vec<_> = served.iter().filter_map(|s| s.field("rpc")).collect(); + assert_eq!( + names, + ["minimald-v1-GetVersion", "minimald-v1-ListSessions"], + "{served:?}" + ); + for s in &served { + assert_eq!( + s.field("parent_span_id"), + Some(cmd.span_id.as_str()), + "the daemon's rpc span is the cmd span's child: {s:?}" + ); + } + + let mut spooled = spooled_spans(spool.path()); + sent.sort(); + spooled.sort(); + assert_eq!(sent, spooled, "the spool holds what the collector got"); +} + +/// Every span the daemon recorded in `trace`. +fn in_trace(log: &SpanLog, trace: &str) -> Vec { + log.spans() + .into_iter() + .filter(|s| s.field("trace_id") == Some(trace)) + .collect() +} + +/// A session of the harness daemon, named `name`, whose project declares +/// the task `tp`, which prints the `TRACEPARENT` its box was given +/// (`TP=none` without one) where a box can start. +async fn session_with_tp_task(daemon: &common::TestDaemon, name: &str) -> String { + use minimald_rpc::{ + ConfigureLoadout, ConfigureLoadoutRequest, CreateSession, FinalizeSession, + FinalizeSessionRequest, + }; + let mut client = daemon.server.connect().await; + let id = client + .call::(&minimald::test_harness::create_session_req(name, "/tmp")) + .await + .unwrap() + .id; + daemon.server.seed_workspace_mfile(id, TP_TASK_MFILE).await; + minimald::test_harness::unwrap_ready( + client + .call::(&ConfigureLoadoutRequest { + session_id: id, + contribution: Default::default(), + }) + .await + .unwrap(), + ); + match client + .call::(&FinalizeSessionRequest { + session_id: id, + report_shared_port_collisions: false, + }) + .await + { + minimald_rpc::Errorable::Ok(_) => {} + minimald_rpc::Errorable::Err { error } => panic!("FinalizeSession failed: {error}"), + } + id.to_string() +} + +const TP_TASK_MFILE: &str = "[tasks.tp]\nbash = \"echo TP=${TRACEPARENT:-none}\"\n"; +const TP_SHELL: &str = "echo TP=${TRACEPARENT:-none}"; + +/// `min` on a pseudo-terminal (`script(1)`; `min session attach` refuses +/// a stdin that is not a TTY), with `stdin` written to it and then closed, +/// as [`run_min_env`] otherwise. +async fn run_min_tty( + args: &GlobalArgs, + env: &[(&str, &str)], + extra: &[&str], + stdin: &[u8], +) -> std::process::Output { + use tokio::io::AsyncWriteExt as _; + let minimal_dir = args.minimal_dir.as_ref().unwrap(); + let config_dir = tempfile::TempDir::new().unwrap(); + let quote = |s: &str| format!("'{}'", s.replace('\'', "'\\''")); + let mut line = vec![ + quote(env!("CARGO_BIN_EXE_min")), + "--minimal-dir".to_owned(), + quote(minimal_dir.to_str().unwrap()), + "--config-dir".to_owned(), + quote(config_dir.path().to_str().unwrap()), + ]; + line.extend(extra.iter().map(|a| quote(a))); + let mut command = tokio::process::Command::new("script"); + command + .args(["-qec", &line.join(" "), "/dev/null"]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + for (k, _) in std::env::vars_os() { + let name = k.to_string_lossy(); + if name.starts_with("OTEL_") + || name.starts_with("MINIMAL_") + || name == "DO_NOT_TRACK" + || name == "TRACEPARENT" + || name == "RUST_LOG" + { + command.env_remove(&k); + } + } + command.envs(env.iter().copied()); + let mut child = command.spawn().unwrap(); + let mut pipe = child.stdin.take().unwrap(); + pipe.write_all(stdin).await.unwrap(); + drop(pipe); + tokio::time::timeout(Duration::from_secs(60), child.wait_with_output()) + .await + .expect("min exits") + .unwrap() +} + +/// One verb's run: its `min`'s output and the `cmd` span from its own spool. +struct VerbRun { + verb: &'static str, + out: std::process::Output, + cmd: Option, +} + +/// The four session verbs, each run as its own `min` with `env` and a +/// spool of its own: `min session exec`, `min session run`, `min session +/// attach` (its shell fed on stdin) and `min task run` (a task session of +/// its own). +async fn run_the_verbs( + daemon: &common::TestDaemon, + args: &GlobalArgs, + env: &[(&str, &str)], +) -> Vec { + static SESSIONS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let n = SESSIONS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let session = session_with_tp_task(daemon, &format!("t11-{n}")).await; + let project = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(project.path().join(".git")).unwrap(); + std::fs::write(project.path().join("minimal.toml"), TP_TASK_MFILE).unwrap(); + let project_dir = project.path().to_str().unwrap().to_owned(); + let mut runs = Vec::new(); + for verb in ["exec", "run", "attach", "task run"] { + let spool = private_tempdir(); + let mut env: Vec<(&str, &str)> = env + .iter() + .copied() + .filter(|(k, _)| *k != "MINIMAL_OTEL_SPOOL_DIR") + .collect(); + env.push(("MINIMAL_OTEL_SPOOL_DIR", spool.path().to_str().unwrap())); + let out = match verb { + "exec" => { + run_min_env(args, &env, &["session", "exec", &session, TP_SHELL]) + .await + .0 + } + "run" => { + run_min_env(args, &env, &["session", "run", &session, "tp"]) + .await + .0 + } + "attach" => { + run_min_tty( + args, + &env, + &["session", "attach", &session], + format!("{TP_SHELL}\nexit\n").as_bytes(), + ) + .await + } + _ => { + run_min_env(args, &env, &["-C", &project_dir, "task", "run", "tp"]) + .await + .0 + } + }; + let cmd = spooled_spans(spool.path()) + .into_iter() + .find(|s| s.name == "cmd"); + runs.push(VerbRun { verb, out, cmd }); + } + runs +} + +/// Plan T11 (spec 25 TEL-021, TEL-022, TEL-025, TEL-045): run `min session +/// exec`, `min session run`, `min session attach` and `min task run` with +/// telemetry on inside a caller's trace. Each `min`'s `cmd` span joins +/// that trace at the collector and in the spool alike, and the daemon's +/// span for each verb's request (`exec` for exec, run and task run, +/// `attach` for attach) is that `cmd` span's child in the same trace. A +/// nested `min` given the `TRACEPARENT` that names the exec's daemon span +/// (what the daemon hands the box once that span is exported, TEL-023) +/// continues the trace as that span's child. With telemetry off +/// (`DO_NOT_TRACK=1`, a caller's `TRACEPARENT` inherited) nothing crosses: +/// nothing is sent or spooled, every verb's daemon span is marked +/// `telemetry=opt-out`, and none joins the inherited trace. +/// +/// What needs more than this in-process harness, and where it is proved: +/// the harness has no packages, so no verb's box process starts (each +/// request is refused at the spawn, after its span); and its daemon has no +/// exporter, so it hands a box no `TRACEPARENT` (`box_traceparent`). That +/// a box gets the value only on opt-in is TEL-023's unit tests +/// (`an_unrecorded_exec_span_gives_the_box_no_traceparent`); a `min` inside +/// a box joining through the in-box helper is minimald's +/// `a_min_in_a_box_joins_the_outer_trace_through_the_helper_prefix`; and +/// the whole path on a real VM is lab scenario 715h. +#[tokio::test] +async fn exec_run_attach_and_task_run_share_the_commands_trace() { + const TRACE: &str = "0af7651916cd43dd8448eb211c803111"; + let log = span_log(); + let (daemon, args) = common::setup().await; + let col = collector(); + let tp = format!("00-{TRACE}-{PARENT}-01"); + let on = [ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_EXPORTER_OTLP_ENDPOINT", col.url.as_str()), + ("TRACEPARENT", tp.as_str()), + ]; + let runs = run_the_verbs(&daemon, &args, &on).await; + let sent = sent_spans(&col); + let daemon_spans = in_trace(&log, TRACE); + let mut exec_span_id = None; + for r in &runs { + let cmd = r.cmd.as_ref().unwrap_or_else(|| { + panic!( + "{}: no cmd span spooled; stderr: {}", + r.verb, + String::from_utf8_lossy(&r.out.stderr) + ) + }); + assert_eq!(cmd.trace_id, TRACE, "{}: {cmd:?}", r.verb); + assert_eq!(cmd.parent_span_id, PARENT, "{}: {cmd:?}", r.verb); + assert!( + sent.contains(cmd), + "{}: the collector has the spooled cmd span: {sent:#?}", + r.verb + ); + let (name, command) = match r.verb { + "exec" => ("exec", Some("min://shell ")), + "run" => ("exec", Some("min://task/run tp")), + "task run" => ("exec", Some("min://task/run --owns-box tp")), + _ => ("attach", None), + }; + let request = daemon_spans + .iter() + .find(|s| { + s.name == name + && s.field("parent_span_id") == Some(cmd.span_id.as_str()) + && command.is_none_or(|c| s.field("command").is_some_and(|v| v.starts_with(c))) + }) + .unwrap_or_else(|| { + panic!( + "{}: no daemon {name} span under cmd {}: {daemon_spans:#?}", + r.verb, cmd.span_id + ) + }); + if r.verb == "exec" { + exec_span_id = request.field("span_id").map(str::to_owned); + } + } + + // A nested `min`, handed what the box would get, continues the trace + // under the exec's daemon span. + let exec_span_id = exec_span_id.unwrap(); + let handed = format!("00-{TRACE}-{exec_span_id}-01"); + let nested_spool = private_tempdir(); + let (out, _) = run_min_env( + &args, + &[ + ("MINIMAL_TELEMETRY", "1"), + ( + "MINIMAL_OTEL_SPOOL_DIR", + nested_spool.path().to_str().unwrap(), + ), + ("TRACEPARENT", handed.as_str()), + ], + &["ls"], + ) + .await; + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + let nested = spooled_spans(nested_spool.path()); + let nested_cmd = nested.iter().find(|s| s.name == "cmd").unwrap(); + assert_eq!(nested_cmd.trace_id, TRACE, "{nested_cmd:?}"); + assert_eq!( + nested_cmd.parent_span_id, exec_span_id, + "the nested min is the exec span's child" + ); + + // Telemetry off: nothing crosses. + const INHERITED: &str = "0af7651916cd43dd8448eb211c80319e"; + let off_col = collector(); + let inherited = format!("00-{INHERITED}-{PARENT}-01"); + let off = [ + ("DO_NOT_TRACK", "1"), + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_EXPORTER_OTLP_ENDPOINT", off_col.url.as_str()), + ("TRACEPARENT", inherited.as_str()), + ]; + let marked_before = log.find("exec", "telemetry", "opt-out").len() + + log.find("attach", "telemetry", "opt-out").len(); + for r in run_the_verbs(&daemon, &args, &off).await { + assert!(r.cmd.is_none(), "{}: spooled with telemetry off", r.verb); + } + assert!(off_col.requests.lock().unwrap().is_empty(), "nothing sent"); + assert!( + in_trace(&log, INHERITED).is_empty(), + "no daemon span joined the inherited trace" + ); + let marked = log.find("exec", "telemetry", "opt-out").len() + + log.find("attach", "telemetry", "opt-out").len(); + assert_eq!( + marked - marked_before, + 4, + "each verb's daemon span is marked opted out" + ); +} diff --git a/crates/minimald/tests/otel_shutdown_integration.rs b/crates/minimald/tests/otel_shutdown_integration.rs new file mode 100644 index 0000000000..7ea68e9c2f --- /dev/null +++ b/crates/minimald/tests/otel_shutdown_integration.rs @@ -0,0 +1,272 @@ +//! minimald's telemetry at shutdown, through the real binary (spec TEL-038, +//! TEL-011): the spans of work still alive at a `Shutdown` are exported, and a +//! collector that never answers costs the stop no more than its bounds +//! (runtime 2 s, then flush 5 s, in `main`). +//! +//! Each test runs `minimald run` against a fresh state dir under `/tmp` (the +//! SSH socket path must fit `sun_path`), with every ambient `OTEL_*`, +//! `MINIMAL_*`, `DO_NOT_TRACK` and `RUST_LOG` variable removed, asks it to +//! shut down over its socket, and reads what a stub collector received. +#![cfg(target_os = "linux")] +#![expect( + clippy::unwrap_used, + clippy::panic, + reason = "test helpers: a failed spawn, connect or RPC must fail the test loudly" +)] + +use std::io::{BufRead as _, Read as _, Write as _}; +use std::net::TcpListener; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +/// A stub OTLP/HTTP collector on 127.0.0.1. `answer`: reply 200 to every +/// request and keep its body; otherwise accept and never read or answer. +struct Collector { + url: String, + bodies: Arc>>>, +} + +fn collector(answer: bool) -> Collector { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", l.local_addr().unwrap()); + let bodies: Arc>>> = Arc::default(); + let kept = bodies.clone(); + std::thread::spawn(move || { + let mut held = Vec::new(); + for conn in l.incoming() { + let Ok(conn) = conn else { continue }; + if !answer { + held.push(conn); // accepted, never read, never answered + continue; + } + let kept = kept.clone(); + std::thread::spawn(move || serve(conn, &kept)); + } + }); + Collector { url, bodies } +} + +/// Answer each request on `conn` with an empty 200, keeping its body. +fn serve(conn: std::net::TcpStream, kept: &Mutex>>) { + let Ok(mut w) = conn.try_clone() else { return }; + let mut r = std::io::BufReader::new(conn); + loop { + let mut len = 0usize; + let mut line = String::new(); + let mut any = false; + while r.read_line(&mut line).is_ok_and(|n| n > 0) { + any = true; + if line == "\r\n" { + break; + } + if let Some(v) = line.to_ascii_lowercase().strip_prefix("content-length:") { + len = v.trim().parse().unwrap_or(0); + } + line.clear(); + } + if !any { + return; + } + let mut body = vec![0; len]; + if r.read_exact(&mut body).is_err() { + return; + } + kept.lock().unwrap().push(body); + let ok = b"HTTP/1.1 200 OK\r\ncontent-type: application/x-protobuf\r\n\ + content-length: 0\r\n\r\n"; + if w.write_all(ok).is_err() { + return; + } + } +} + +/// Accepts the daemon's host key (a fresh one per state dir). +struct Accept; + +impl russh::client::Handler for Accept { + type Error = russh::Error; + async fn check_server_key( + &mut self, + _: &russh::keys::PublicKeyOrCertificate, + ) -> Result { + Ok(true) + } +} + +/// A `minimald run` on a fresh state dir, exporting to `endpoint` with the +/// spool off (so what reaches the collector is the export's alone). +struct Daemon { + child: Child, + sock: PathBuf, + _dir: tempfile::TempDir, +} + +impl Daemon { + fn start(endpoint: &str) -> Self { + let dir = tempfile::Builder::new() + .prefix("mdot") + .tempdir_in("/tmp") + .unwrap(); + let mut c = Command::new(env!("CARGO_BIN_EXE_minimald")); + c.arg("--minimal-state-dir") + .arg(dir.path().join("s")) + .arg("--minimal-cache-dir") + .arg(dir.path().join("c")) + .arg("run"); + for (k, _) in std::env::vars_os() { + let name = k.to_string_lossy(); + if name.starts_with("OTEL_") + || name.starts_with("MINIMAL_") + || name == "DO_NOT_TRACK" + || name == "RUST_LOG" + { + c.env_remove(&k); + } + } + c.env("MINIMAL_TELEMETRY", "1") + .env("MINIMAL_OTEL_SPOOL", "0") + .env("MINIMAL_OTEL_EXPORTER_OTLP_ENDPOINT", endpoint) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let child = c.spawn().unwrap(); + let sock = dir.path().join("s/providers/local-minimald0/ssh.sock"); + let mut d = Self { + child, + sock, + _dir: dir, + }; + let deadline = Instant::now() + Duration::from_secs(20); + while std::os::unix::net::UnixStream::connect(&d.sock).is_err() { + assert!( + d.child.try_wait().unwrap().is_none(), + "minimald exited before listening" + ); + assert!(Instant::now() < deadline, "minimald never listened"); + std::thread::sleep(Duration::from_millis(50)); + } + d + } + + /// An authenticated SSH connection to the daemon. + async fn connect(&self) -> russh::client::Handle { + let stream = tokio::net::UnixStream::connect(&self.sock).await.unwrap(); + let config = Arc::new(russh::client::Config::default()); + let mut handle = russh::client::connect_stream(config, stream, Accept) + .await + .unwrap(); + assert!(handle.authenticate_none("test").await.unwrap().success()); + handle + } + + /// Send `Shutdown` (forced) over a connection of its own and wait for + /// the answer; returns when it came. + async fn shut_down(&self) -> Instant { + use minimald_rpc::OneshotSshRpc as _; + let handle = self.connect().await; + let mut channel = handle.channel_open_session().await.unwrap(); + channel + .request_subsystem(true, minimald_rpc::Shutdown::NAME) + .await + .unwrap(); + channel + .data_bytes(b"{\"force\":true}".to_vec()) + .await + .unwrap(); + channel.eof().await.unwrap(); + let mut answer = Vec::new(); + while let Some(msg) = channel.wait().await { + if let russh::ChannelMsg::Data { data } = msg { + answer.extend_from_slice(&data); + } + } + assert!(!answer.is_empty(), "Shutdown was not answered"); + Instant::now() + } + + /// Wait for the process to exit, at most `bound`; panics past it. + fn exited_within(&mut self, bound: Duration) -> Instant { + let deadline = Instant::now() + bound; + loop { + if self.child.try_wait().unwrap().is_some() { + return Instant::now(); + } + if Instant::now() > deadline { + if let Err(e) = self.child.kill() { + eprintln!("could not kill minimald: {e}"); + } + panic!("minimald did not exit within {bound:?} of its Shutdown"); + } + std::thread::sleep(Duration::from_millis(20)); + } + } +} + +impl Drop for Daemon { + fn drop(&mut self) { + // Already gone, normally: then both fail, and that is fine. + drop(self.child.kill()); + drop(self.child.wait()); + } +} + +/// How many exported spans are named `name`: occurrences of the OTLP +/// protobuf encoding of `Span.name` (field 5, length-delimited) with it. +fn spans_named(bodies: &[Vec], name: &str) -> usize { + let mut field = vec![0x2a, u8::try_from(name.len()).unwrap()]; + field.extend_from_slice(name.as_bytes()); + bodies + .iter() + .map(|b| { + b.windows(field.len()) + .filter(|w| *w == field.as_slice()) + .count() + }) + .sum() +} + +/// TEL-038 (patch 23): a span of a task still alive when `Shutdown` arrives is +/// exported: here the `conn` span of a client connection held open across +/// it, which the server's drain aborts after its grace (`SHUTDOWN_GRACE`). +/// +/// What this does not pin: the order in `main` (runtime shutdown, then the +/// flush). Today the drain ends every connection inside `async_main`, so +/// the spans end before either step and the test passes with the two steps +/// swapped too (checked by hand); the order matters for a task that outlives +/// the drain, which no client can produce here. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_connection_open_at_shutdown_still_exports_its_span() { + let col = collector(true); + let mut d = Daemon::start(&col.url); + let held = d.connect().await; + d.shut_down().await; + d.exited_within(Duration::from_secs(10)); + drop(held); + let bodies = col.bodies.lock().unwrap(); + // `start`'s readiness probe, the `Shutdown` client, the held client. + assert_eq!( + spans_named(&bodies, "conn"), + 3, + "every connection's span is exported, the one still open at the \ + shutdown included" + ); +} + +/// TEL-011: against a collector that accepts and never answers, +/// minimald exits within its shutdown bounds (runtime 2 s, then flush 5 s) +/// of answering `Shutdown`, plus slack, instead of waiting out the export. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_silent_collector_does_not_hold_the_daemons_exit() { + let col = collector(false); + let mut d = Daemon::start(&col.url); + let answered = d.shut_down().await; + let exited = d.exited_within(Duration::from_secs(15)); + let took = exited.duration_since(answered); + assert!( + took < Duration::from_secs(2 + 5 + 1), + "minimald took {took:?} to exit after Shutdown against a silent collector" + ); + drop(col); +} diff --git a/crates/minvmd/tests/otel_integration.rs b/crates/minvmd/tests/otel_integration.rs new file mode 100644 index 0000000000..ae1852b61b --- /dev/null +++ b/crates/minvmd/tests/otel_integration.rs @@ -0,0 +1,240 @@ +//! minvmd's OpenTelemetry through the real binary (spec 25: TEL-001, +//! TEL-009, TEL-032, TEL-036): the +//! trace context crosses the `min` → minvmd process boundary, the resource +//! names the service, and with telemetry off minvmd writes nothing. +//! +//! Like `config_cli_integration`, these run `minvmd status` against an +//! isolated state dir and need no VM. They read the local OTLP-JSON spool +//! (`MINIMAL_OTEL_SPOOL_DIR`), so no collector is involved either. + +use std::path::Path; +use std::process::Command; + +fn minvmd_bin() -> std::ffi::OsString { + std::env::var_os("MINVMD_BIN").unwrap_or_else(|| env!("CARGO_BIN_EXE_minvmd").into()) +} + +const TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c"; +const PARENT_ID: &str = "b7ad6b7169203331"; + +/// Run `minvmd status` with only the given telemetry variables set (every +/// ambient `OTEL_*` / `MINIMAL_*` / `DO_NOT_TRACK` / `RUST_LOG` removed +/// first). +fn status(state_home: &Path, spool: &Path, env: &[(&str, &str)]) { + // A fresh state dir: stopped, exit 1 (the answer, not a failure). + minvmd(&["status", "--json"], 1, state_home, spool, env); +} + +/// Run `minvmd ` as [`status`] does and expect exit `code`. +fn minvmd(args: &[&str], code: i32, state_home: &Path, spool: &Path, env: &[(&str, &str)]) { + let mut cmd = Command::new(minvmd_bin()); + cmd.args(args); + for (k, _) in std::env::vars() { + if k.starts_with("OTEL_") + || k.starts_with("MINIMAL_") + || k == "DO_NOT_TRACK" + || k == "TRACEPARENT" + || k == "RUST_LOG" + { + cmd.env_remove(k); + } + } + cmd.env("XDG_STATE_HOME", state_home) + .env("MINIMAL_OTEL_SPOOL_DIR", spool); + for (k, v) in env { + cmd.env(k, v); + } + let out = cmd.output().expect("spawn minvmd"); + assert_eq!( + out.status.code(), + Some(code), + "minvmd {args:?}: stderr: {}", + String::from_utf8_lossy(&out.stderr) + ); +} + +#[expect( + clippy::unwrap_used, + reason = "a test helper: an unreadable or malformed spool must fail the test loudly" +)] +fn spool_lines(spool: &Path) -> Vec { + let Ok(rd) = std::fs::read_dir(spool) else { + return Vec::new(); + }; + let mut out = Vec::new(); + for e in rd { + let p = e.unwrap().path(); + if p.extension().is_some_and(|x| x == "jsonl") { + for l in std::fs::read_to_string(&p).unwrap().lines() { + out.push(serde_json_lenient::from_str(l).unwrap()); + } + } + } + out +} + +#[expect( + clippy::indexing_slicing, + reason = "serde_json Value indexing returns Null for a missing key; it does not panic" +)] +fn spans(lines: &[serde_json_lenient::Value]) -> Vec<(String, serde_json_lenient::Value)> { + let mut out = Vec::new(); + for l in lines { + for rs in l["resourceSpans"].as_array().into_iter().flatten() { + let service = rs["resource"]["attributes"] + .as_array() + .into_iter() + .flatten() + .find(|a| a["key"] == "service.name") + .and_then(|a| a["value"]["stringValue"].as_str()) + .unwrap_or_default() + .to_string(); + for ss in rs["scopeSpans"].as_array().into_iter().flatten() { + for s in ss["spans"].as_array().into_iter().flatten() { + out.push((service.clone(), s.clone())); + } + } + } + } + out +} + +#[test] +fn a_minvmd_root_span_is_a_child_of_the_callers_traceparent() { + let dir = tempfile::tempdir().unwrap(); + let spool = dir.path().join("spool"); + let tp = format!("00-{TRACE_ID}-{PARENT_ID}-01"); + status( + dir.path(), + &spool, + &[("MINIMAL_TELEMETRY", "1"), ("TRACEPARENT", &tp)], + ); + let spans = spans(&spool_lines(&spool)); + let root = spans + .iter() + .find(|(_, s)| s["name"] == "minvmd") + .unwrap_or_else(|| panic!("no minvmd root span in {spans:?}")); + assert_eq!(root.0, "minvmd", "service.name"); + assert_eq!(root.1["traceId"], TRACE_ID); + assert_eq!(root.1["parentSpanId"], PARENT_ID); + // The subcommand is on the root span. + let cmd = root.1["attributes"] + .as_array() + .into_iter() + .flatten() + .find(|a| a["key"] == "cmd") + .and_then(|a| a["value"]["stringValue"].as_str()); + assert_eq!(cmd, Some("status")); +} + +#[test] +fn without_a_traceparent_minvmd_starts_its_own_trace() { + let dir = tempfile::tempdir().unwrap(); + let spool = dir.path().join("spool"); + status(dir.path(), &spool, &[("MINIMAL_TELEMETRY", "1")]); + let spans = spans(&spool_lines(&spool)); + let root = spans + .iter() + .find(|(_, s)| s["name"] == "minvmd") + .expect("root span"); + assert_ne!(root.1["traceId"], TRACE_ID); + assert!( + root.1["parentSpanId"] + .as_str() + .unwrap_or_default() + .is_empty() + ); +} + +#[test] +fn minvmd_writes_nothing_with_telemetry_off() { + for env in [ + &[][..], + &[("MINIMAL_TELEMETRY", "1"), ("DO_NOT_TRACK", "1")][..], + &[("MINIMAL_TELEMETRY", "1"), ("OTEL_SDK_DISABLED", "true")][..], + // An inbound context alone never turns anything on. + &[( + "TRACEPARENT", + "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", + )][..], + ] { + let dir = tempfile::tempdir().unwrap(); + let spool = dir.path().join("spool"); + status(dir.path(), &spool, env); + assert!( + !spool.exists() || std::fs::read_dir(&spool).unwrap().next().is_none(), + "spool written with {env:?}" + ); + } +} + +#[expect( + clippy::indexing_slicing, + reason = "serde_json Value indexing returns Null for a missing key; it does not panic" +)] +fn attr<'a>(span: &'a serde_json_lenient::Value, key: &str) -> Option<&'a str> { + span["attributes"] + .as_array() + .into_iter() + .flatten() + .find(|a| a["key"] == key) + .and_then(|a| a["value"]["stringValue"].as_str()) +} + +/// TEL-036: `RUST_LOG` filters the +/// console only. With `RUST_LOG=warn` minvmd still spools its info-level +/// root span, in the caller's trace. +#[test] +fn rust_log_warn_does_not_silence_the_spool() { + let dir = tempfile::tempdir().unwrap(); + let spool = dir.path().join("spool"); + let tp = format!("00-{TRACE_ID}-{PARENT_ID}-01"); + status( + dir.path(), + &spool, + &[ + ("MINIMAL_TELEMETRY", "1"), + ("TRACEPARENT", &tp), + ("RUST_LOG", "warn"), + ], + ); + let spans = spans(&spool_lines(&spool)); + let root = spans + .iter() + .find(|(_, s)| s["name"] == "minvmd") + .unwrap_or_else(|| panic!("RUST_LOG=warn silenced the root span: {spans:?}")); + assert_eq!(root.1["traceId"], TRACE_ID); +} + +/// TEL-032: `minvmd stop` exports a `vm.stop` span under the +/// `minvmd` root (`cmd=stop`), in the caller's trace. A fresh state dir +/// takes the idempotent "not running" return, so no VM is needed. +#[test] +fn a_stop_is_a_vm_stop_span_under_the_minvmd_root() { + let dir = tempfile::tempdir().unwrap(); + let spool = dir.path().join("spool"); + let tp = format!("00-{TRACE_ID}-{PARENT_ID}-01"); + minvmd( + &["stop"], + 0, + dir.path(), + &spool, + &[("MINIMAL_TELEMETRY", "1"), ("TRACEPARENT", &tp)], + ); + let spans = spans(&spool_lines(&spool)); + let find = |name: &str| { + spans + .iter() + .map(|(_, s)| s) + .find(|s| s["name"] == name) + .unwrap_or_else(|| panic!("no {name} span in {spans:?}")) + }; + let (root, stop) = (find("minvmd"), find("vm.stop")); + assert_eq!(attr(root, "cmd"), Some("stop")); + assert_eq!(root["parentSpanId"], PARENT_ID); + assert_eq!(stop["traceId"], TRACE_ID); + assert_eq!( + stop["parentSpanId"], root["spanId"], + "vm.stop under the root" + ); +} diff --git a/crates/mlog/src/otel.rs b/crates/mlog/src/otel.rs index 8bbef41930..d2fe39eb06 100644 --- a/crates/mlog/src/otel.rs +++ b/crates/mlog/src/otel.rs @@ -2160,8 +2160,8 @@ mod tests { /// Both kinds of endpoint get a client in an environment with CA roots /// (an `https` one through the platform verifier, as before). The no-roots - /// case needs a process that finds no CA roots on disk, so it is not a - /// unit test here. + /// case needs a process that finds no CA roots on disk, so it is the + /// integration test `tests/otel_without_ca_roots.rs`. #[test] fn http_and_https_endpoints_get_a_client() { if reqwest::blocking::Client::builder().build().is_err() { diff --git a/crates/mlog/tests/otel_requirements.rs b/crates/mlog/tests/otel_requirements.rs new file mode 100644 index 0000000000..c829ae961d --- /dev/null +++ b/crates/mlog/tests/otel_requirements.rs @@ -0,0 +1,1405 @@ +//! Proofs of the telemetry requirements (spec 25, `TEL-*`; the lab's older +//! `OT-*`/`BE-*`/`SP-*`/`TR-*`/`VM-*`/`PV-*` ids map onto them, see +//! [`FORMER_IDS`]) through `mlog::otel`'s public API, the way a binary uses +//! it: `init`, the layers in a subscriber, `align`, `relocate_spool`, the +//! exit flush. +//! +//! `init` is once per process, so each case runs one helper of this binary +//! in a child process whose environment starts with every `OTEL_*`, +//! `MINIMAL_*` and `DO_NOT_TRACK` variable removed. A helper is a test that +//! does nothing unless `MLOG_REQ_CHILD` is set (not `#[ignore]`: `just +//! test-ignored` runs every ignored test, and a helper calls +//! `process::exit`). It reports through `MARK ` lines on +//! stdout; the parent reads those and the spool the child wrote. +//! +//! A requirement the stack does not meet yet (an open review item), or that +//! only the lab exercises, is a test that returns early unless +//! `TEST_KNOWN_GAPS=1`: it fails today, naming the gap and the issue that +//! tracks it ([`KNOWN_GAPS`]), and its fix drops the guard. + +use std::io::{BufRead as _, Read as _, Write as _}; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use serde_json::Value as J; +use tracing_subscriber::prelude::*; + +/// The tests' and helpers' shared code: in a `cfg(test)` module, so the +/// test-code allowances (`unwrap`, `panic`, indexing) apply to it as well. +#[cfg(test)] +mod support { + use super::*; + + // ---- the parent's side ---- + + /// Run the helper test `helper` in a child with a stripped telemetry + /// environment plus `env`; panics unless it succeeds. Returns its stdout. + pub(super) fn child(helper: &str, env: &[(&str, &str)]) -> String { + let mut c = Command::new(std::env::current_exe().unwrap()); + c.args(["--exact", helper, "--nocapture", "--test-threads=1"]); + for (k, _) in std::env::vars_os() { + let name = k.to_string_lossy(); + if name.starts_with("OTEL_") || name.starts_with("MINIMAL_") || name == "DO_NOT_TRACK" { + c.env_remove(&k); + } + } + c.env_remove("RUST_LOG"); + c.env("MLOG_REQ_CHILD", "1"); + c.envs(env.iter().copied()); + let out = c.output().unwrap(); + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + assert!( + out.status.success(), + "{helper} failed\nstdout: {stdout}\nstderr: {}", + String::from_utf8_lossy(&out.stderr) + ); + stdout + } + + /// The value of every `MARK ` line (libtest may print + /// `test ... ` on the same line first). + pub(super) fn marks(out: &str, key: &str) -> Vec { + let tag = format!("MARK {key} "); + out.lines() + .filter_map(|l| Some(l.split_once(&tag)?.1.trim().to_owned())) + .collect() + } + + pub(super) fn mark(out: &str, key: &str) -> String { + marks(out, key) + .pop() + .unwrap_or_else(|| panic!("no MARK {key} in {out}")) + } + + /// Every `*.jsonl` file in `dir`, sorted. + pub(super) fn spool_files(dir: &Path) -> Vec { + let mut v: Vec = std::fs::read_dir(dir) + .map(|rd| { + rd.map(|e| e.unwrap().path()) + .filter(|p| p.extension().is_some_and(|x| x == "jsonl")) + .collect() + }) + .unwrap_or_default(); + v.sort(); + v + } + + /// Every spool line in `dir`, parsed; panics on a line that is not JSON. + pub(super) fn spool(dir: &Path) -> Vec { + spool_files(dir) + .iter() + .flat_map(|p| { + std::fs::read_to_string(p) + .unwrap() + .lines() + .map(|l| { + serde_json::from_str(l) + .unwrap_or_else(|e| panic!("spool line is not JSON ({e}): {l}")) + }) + .collect::>() + }) + .collect() + } + + /// Every span in `lines`, with its resource. + pub(super) fn spans(lines: &[J]) -> Vec<(&J, &J)> { + let mut v = Vec::new(); + for l in lines { + for rs in l["resourceSpans"].as_array().into_iter().flatten() { + for ss in rs["scopeSpans"].as_array().into_iter().flatten() { + for s in ss["spans"].as_array().into_iter().flatten() { + v.push((&rs["resource"], s)); + } + } + } + } + v + } + + /// Every log record in `lines`: its resource, its scope name (the event's + /// target) and the record. + pub(super) fn logs(lines: &[J]) -> Vec<(&J, String, &J)> { + let mut v = Vec::new(); + for l in lines { + for rl in l["resourceLogs"].as_array().into_iter().flatten() { + for sl in rl["scopeLogs"].as_array().into_iter().flatten() { + let scope = sl["scope"]["name"].as_str().unwrap_or_default().to_owned(); + for r in sl["logRecords"].as_array().into_iter().flatten() { + v.push((&rl["resource"], scope.clone(), r)); + } + } + } + } + v + } + + pub(super) fn span_names(lines: &[J]) -> Vec { + spans(lines) + .iter() + .map(|(_, s)| s["name"].as_str().unwrap_or_default().to_owned()) + .collect() + } + + pub(super) fn log_bodies(lines: &[J]) -> Vec { + logs(lines) + .iter() + .map(|(_, _, r)| { + r["body"]["stringValue"] + .as_str() + .unwrap_or_default() + .to_owned() + }) + .collect() + } + + /// A resource attribute's string value, if the resource has it. + pub(super) fn resource_attr(resource: &J, key: &str) -> Option { + resource["attributes"] + .as_array()? + .iter() + .find(|a| a["key"] == key)?["value"]["stringValue"] + .as_str() + .map(str::to_owned) + } + + pub(super) fn to_str(p: &Path) -> &str { + p.to_str().unwrap() + } + + /// A minimal OTLP/HTTP collector on 127.0.0.1: answers every request with + /// an empty 200 and keeps each request's header block (lowercased). + pub(super) struct Collector { + pub(super) url: String, + pub(super) requests: Arc>>, + } + + pub(super) fn collector() -> Collector { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", l.local_addr().unwrap()); + let requests: Arc>> = Arc::default(); + let seen = requests.clone(); + std::thread::spawn(move || { + for conn in l.incoming() { + let Ok(conn) = conn else { continue }; + let seen = seen.clone(); + std::thread::spawn(move || { + let mut w = conn.try_clone().unwrap(); + let mut r = std::io::BufReader::new(conn); + loop { + let mut head = String::new(); + let mut line = String::new(); + while r.read_line(&mut line).is_ok_and(|n| n > 0) { + if line == "\r\n" { + break; + } + head.push_str(&line.to_ascii_lowercase()); + line.clear(); + } + if head.is_empty() { + return; + } + let len: usize = head + .lines() + .find_map(|h| h.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + let mut body = vec![0; len]; + if r.read_exact(&mut body).is_err() { + return; + } + seen.lock().unwrap().push(head); + let ok = b"HTTP/1.1 200 OK\r\ncontent-type: application/x-protobuf\r\n\ + content-length: 0\r\n\r\n"; + if w.write_all(ok).is_err() { + return; + } + } + }); + } + }); + Collector { url, requests } + } + + // ---- the children's side (helpers, run only by the tests above) ---- + + /// Whether this process is a helper's child (see the module docs). + pub(super) fn in_child() -> bool { + std::env::var_os("MLOG_REQ_CHILD").is_some() + } + + /// A layer that records the target of every event and span it sees, and + /// prints each new one as `MARK census ` when `print`. + pub(super) struct Census { + pub(super) seen: Arc>>, + pub(super) print: bool, + } + + impl Census { + fn note(&self, target: &str) { + let mut seen = self.seen.lock().unwrap(); + if self.print && !seen.iter().any(|t| t == target) { + println!("MARK census {target}"); + } + seen.push(target.to_owned()); + } + } + + impl tracing_subscriber::Layer for Census { + fn on_event(&self, e: &tracing::Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) { + self.note(e.metadata().target()); + } + + fn on_new_span( + &self, + a: &tracing::span::Attributes<'_>, + _: &tracing::span::Id, + _: tracing_subscriber::layer::Context<'_, S>, + ) { + self.note(a.metadata().target()); + } + } + + /// `init` plus the export layers, as `min` and the daemons assemble them. + /// A layer that prints the message of every WARN event as + /// `MARK warn `. + pub(super) struct Warnings; + + impl tracing_subscriber::Layer for Warnings { + fn on_event(&self, e: &tracing::Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) { + struct Message(String); + impl tracing::field::Visit for Message { + fn record_debug(&mut self, f: &tracing::field::Field, v: &dyn std::fmt::Debug) { + if f.name() == "message" { + self.0 = format!("{v:?}"); + } + } + } + if *e.metadata().level() == tracing::Level::WARN { + let mut m = Message(String::new()); + e.record(&mut m); + println!("MARK warn {}", m.0); + } + } + } + + pub(super) fn installed() -> impl tracing::Subscriber + Send + Sync { + mlog::otel::init("minimald"); + tracing_subscriber::registry() + .with(mlog::otel::span_layer()) + .with(mlog::otel::log_layer()) + } + + // ---- more helpers ---- + + /// Every `*.jsonl` under `dir`, recursively. + pub(super) fn jsonl_under(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for e in entries { + let p = e.unwrap().path(); + if p.is_dir() { + jsonl_under(&p, out); + } else if p.extension().is_some_and(|x| x == "jsonl") { + out.push(p); + } + } + } + + /// A temporary directory with mode 0700: the spool refuses a directory + /// with group or other bits, and `tempfile` creates its directories with + /// the umask's default (0755). + pub(super) fn private_tempdir() -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt as _; + tempfile::Builder::new() + .permissions(std::fs::Permissions::from_mode(0o700)) + .tempdir() + .unwrap() + } + + /// The guard every known-gap test starts with: the gap `id` is looked + /// up in [`super::KNOWN_GAPS`] (a missing id panics, so a test never + /// names a gap the list no longer holds, and a reordered list cannot + /// move a test onto another gap); then nothing unless + /// `TEST_KNOWN_GAPS=1`, then a failure that names the gap. + pub(super) fn known_gap(id: &str) { + let gap = super::KNOWN_GAPS + .iter() + .find(|g| g.id == id) + .unwrap_or_else(|| panic!("{id} is not in KNOWN_GAPS")); + if std::env::var_os("TEST_KNOWN_GAPS").is_none_or(|v| v.is_empty() || v == "0") { + return; + } + panic!( + "{}: no test in the repository; covered by {}; tracked as {}", + gap.id, gap.covered_by, gap.issue + ); + } + + /// Every `.rs` file under `dir`, recursively (none if it does not exist). + pub(super) fn walk(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for e in entries { + let p = e.unwrap().path(); + if p.is_dir() { + walk(&p, out); + } else if p.extension().is_some_and(|x| x == "rs") { + out.push(p); + } + } + } + + /// Every `` token in `line`: the prefix followed by digits, or + /// by `N` and digits (`TEL-N01`), ended by a non-id byte. + pub(super) fn ids_in(line: &str, prefix: &str) -> Vec { + let mut out = Vec::new(); + let mut rest = line; + while let Some(at) = rest.find(prefix) { + let before_ok = at == 0 + || !rest + .as_bytes() + .get(at - 1) + .is_some_and(|b| b.is_ascii_alphanumeric()); + let tail = rest.get(at + prefix.len()..).unwrap_or_default(); + let n = tail + .bytes() + .take_while(|b| b.is_ascii_digit() || *b == b'N') + .count(); + let id = tail.get(..n).unwrap_or_default(); + let well_formed = n > 0 + && id.bytes().filter(|b| *b == b'N').count() <= 1 + && !id.ends_with('N') + && !tail + .as_bytes() + .get(n) + .is_some_and(|b| b.is_ascii_alphanumeric()); + if before_ok && well_formed { + out.push(format!("{prefix}{id}")); + } + rest = tail; + } + out + } +} + +use support::*; + +// ---- the requirements ---- + +/// TEL-018, TEL-020: an explicit +/// `MINIMAL_OTEL_SPOOL_DIR` survives `relocate_spool`, the call minimald +/// makes once it knows its state directory, so a spool the caller placed +/// (the scenario's small tmpfs) stays where it was put. +#[test] +fn an_explicit_spool_dir_survives_relocation() { + let tmp = private_tempdir(); + let (pinned, state) = (tmp.path().join("pinned"), tmp.path().join("state")); + child( + "helper_record", + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", to_str(&pinned)), + ("MLOG_REQ_RELOCATE", to_str(&state)), + ], + ); + assert!( + span_names(&spool(&pinned)).contains(&"req-span".to_owned()), + "the span is in the pinned directory" + ); + assert!( + !state.exists(), + "nothing was written to the state directory" + ); +} + +/// TEL-006: the resource carries a `service.instance.id` +/// of 32 hex digits, one per process (two processes differ, every record of +/// one signal in a process agrees), never the machine id, and there is no +/// `host.id`. +#[test] +fn one_instance_id_per_process_and_never_the_machine_id() { + let machine_ids: Vec = ["/etc/machine-id", "/var/lib/dbus/machine-id"] + .iter() + .filter_map(|p| std::fs::read_to_string(p).ok()) + .map(|s| s.trim().to_ascii_lowercase()) + .collect(); + let mut per_process = Vec::new(); + for _ in 0..2 { + let tmp = private_tempdir(); + child( + "helper_record", + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", to_str(tmp.path())), + ], + ); + let lines = spool(tmp.path()); + let resources: Vec<&J> = spans(&lines) + .into_iter() + .map(|(r, _)| r) + .chain(logs(&lines).into_iter().map(|(r, _, _)| r)) + .collect(); + assert!(resources.len() >= 2, "a span and a log were spooled"); + let mut ids = Vec::new(); + for r in &resources { + let id = resource_attr(r, "service.instance.id") + .unwrap_or_else(|| panic!("no service.instance.id in {r}")); + assert!( + id.len() == 32 && id.bytes().all(|b| b.is_ascii_hexdigit()), + "not 32 hex digits: {id}" + ); + assert!(!machine_ids.contains(&id), "the machine id was exported"); + assert!(resource_attr(r, "host.id").is_none(), "host.id in {r}"); + ids.push(id); + } + let span_id = resource_attr(spans(&lines)[0].0, "service.instance.id"); + assert!( + spans(&lines) + .iter() + .all(|(r, _)| resource_attr(r, "service.instance.id") == span_id), + "one id across a process's spans" + ); + per_process.push(ids); + } + assert!( + per_process[0].iter().all(|a| !per_process[1].contains(a)), + "two processes share an instance id: {per_process:?}" + ); +} + +/// TEL-006: the traces and the logs of one process carry the same +/// `service.instance.id`, so a backend can join them. +#[test] +fn the_trace_and_log_resources_of_a_process_share_one_instance_id() { + let tmp = private_tempdir(); + child( + "helper_record", + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", to_str(tmp.path())), + ], + ); + let lines = spool(tmp.path()); + let span_id = resource_attr(spans(&lines)[0].0, "service.instance.id"); + let log_id = resource_attr(logs(&lines)[0].0, "service.instance.id"); + assert!(span_id.is_some()); + assert_eq!(span_id, log_id, "traces and logs name different instances"); +} + +/// The exporter's own stack, as targets (the crate's `NEVER_EXPORT`). +const STACK: &[&str] = &[ + "hyper", + "hyper_util::client::legacy", + "h2", + "reqwest::connect", + "tower", + "opentelemetry", + "opentelemetry_sdk", + "opentelemetry_otlp", + "opentelemetry_http", +]; + +/// TEL-013: with the export filter opened all the way (`MINIMAL_OTEL_FILTER= +/// trace`), no span or event from the exporter's HTTP stack or the SDK +/// reaches the spool, at any level, while minimal's own targets do. +#[test] +fn the_export_filter_never_admits_the_exporters_own_stack() { + let tmp = private_tempdir(); + child( + "helper_stack_targets", + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", to_str(tmp.path())), + ("MINIMAL_OTEL_FILTER", "trace"), + ], + ); + let lines = spool(tmp.path()); + let names = span_names(&lines); + let scopes: Vec = logs(&lines).into_iter().map(|(_, s, _)| s).collect(); + assert!(names.contains(&"own-span".to_owned()), "control: {names:?}"); + assert!( + scopes.contains(&"minimald".to_owned()), + "control: {scopes:?}" + ); + for t in STACK { + assert!( + !names.contains(&format!("stack-span {t}")), + "a {t} span was exported" + ); + assert!(!scopes.iter().any(|s| s == t), "a {t} event was exported"); + } +} + +/// TEL-013: a user directive more specific than the never-export one +/// (`hyper_util::client=trace` beside `hyper_util=off`) must not let the +/// exporter's stack back in: an `EnvFilter` lets the more specific +/// directive win (tracing-subscriber 0.3), so the exclusion is +/// a filter of its own after the user's. +#[test] +fn a_specific_directive_cannot_readmit_the_exporters_stack() { + let tmp = private_tempdir(); + child( + "helper_stack_targets", + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", to_str(tmp.path())), + ( + "MINIMAL_OTEL_FILTER", + "trace,hyper_util::client=trace,opentelemetry_sdk::trace=trace", + ), + ], + ); + let lines = spool(tmp.path()); + let scopes: Vec = logs(&lines).into_iter().map(|(_, s, _)| s).collect(); + assert!( + scopes.contains(&"minimald".to_owned()), + "control: {scopes:?}" + ); + for t in STACK { + assert!(!scopes.iter().any(|s| s == t), "a {t} event was exported"); + } +} + +/// TEL-013, end to end: export to a collector that answers, with the filter at +/// `trace` and a census layer beside the export layers. Whatever the HTTP +/// stack and the SDK emit while exporting (the census sees it), none of it +/// is spooled. With today's dependency features the stack emits no tracing +/// events at all (the census sees only this test's own), so this guards a +/// feature flip; the filter itself is proved by +/// [`the_export_filter_never_admits_the_exporters_own_stack`]. +#[test] +fn a_real_export_spools_nothing_from_its_own_stack() { + let tmp = private_tempdir(); + let col = collector(); + let out = child( + "helper_census", + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", to_str(tmp.path())), + ("MINIMAL_OTEL_FILTER", "trace"), + ("MINIMAL_OTEL_EXPORTER_OTLP_ENDPOINT", &col.url), + ], + ); + assert!( + !col.requests.lock().unwrap().is_empty(), + "control: the collector got no export" + ); + let lines = spool(tmp.path()); + assert!(span_names(&lines).contains(&"req-span".to_owned())); + let exported: Vec = logs(&lines).into_iter().map(|(_, s, _)| s).collect(); + for target in marks(&out, "census") { + let stack = STACK + .iter() + .any(|t| target.split("::").next() == t.split("::").next()); + if stack { + assert!( + !exported.contains(&target), + "{target} emitted during export and was spooled" + ); + } + } + for s in &exported { + let root = s.split("::").next().unwrap_or_default(); + assert!( + !STACK.iter().any(|t| t.split("::").next() == Some(root)), + "{s} was spooled" + ); + } +} + +/// TEL-019 (and the code-review refusal): a spool directory the spool +/// creates (and any parent it creates) +/// is 0700, and its files are 0600. A directory that already exists keeps +/// its mode (the spool does not chmod what it did not create); the file in +/// it is still 0600. +#[cfg(unix)] +#[test] +fn spool_files_are_0600_in_a_0700_directory() { + use std::os::unix::fs::{DirBuilderExt as _, PermissionsExt as _}; + let mode = |p: &Path| std::fs::metadata(p).unwrap().permissions().mode() & 0o777; + let tmp = private_tempdir(); + let fresh = tmp.path().join("made").join("spool"); + child( + "helper_record", + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", to_str(&fresh)), + ], + ); + assert_eq!(mode(&fresh), 0o700, "the spool directory"); + assert_eq!(mode(&tmp.path().join("made")), 0o700, "a created parent"); + let files = spool_files(&fresh); + assert_eq!(files.len(), 1, "{files:?}"); + assert_eq!(mode(&files[0]), 0o600, "the spool file"); + + // A directory that already exists keeps its mode. One that grants + // group or other access is refused: nothing is spooled + // there, and the process spools to the state-dir default instead. + let existing = tmp.path().join("existing"); + std::fs::DirBuilder::new() + .mode(0o750) + .create(&existing) + .unwrap(); + std::fs::set_permissions(&existing, std::fs::Permissions::from_mode(0o750)).unwrap(); + let state_home = tmp.path().join("state"); + child( + "helper_record", + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", to_str(&existing)), + ("XDG_STATE_HOME", to_str(&state_home)), + ], + ); + assert_eq!(mode(&existing), 0o750, "a directory it did not create"); + assert_eq!(spool_files(&existing), Vec::::new(), "refused"); + let mut fallback = Vec::new(); + jsonl_under(&state_home, &mut fallback); + assert_eq!( + fallback.len(), + 1, + "one file in the state-dir spool: {fallback:?}" + ); + let spool = fallback[0].parent().unwrap(); + assert!(spool.ends_with("telemetry/spool"), "{}", spool.display()); + assert_eq!(mode(spool), 0o700, "the fallback spool directory"); + assert_eq!(mode(&fallback[0]), 0o600, "its spool file"); +} + +/// TEL-018: when the spool's file +/// cannot grow (the filesystem is full; here `RLIMIT_FSIZE` at the file's +/// size), records are dropped, the spool stays on, and the first record +/// after space returns is written, whole, to the same file, without a +/// restart. Every line in the file is a whole record. +#[cfg(target_os = "linux")] +#[test] +fn a_full_spool_drops_records_then_resumes_when_space_returns() { + let tmp = private_tempdir(); + child( + "helper_full_disk", + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", to_str(tmp.path())), + ], + ); + assert_eq!(spool_files(tmp.path()).len(), 1, "one file, no rotation"); + let names = span_names(&spool(tmp.path())); + assert_eq!(names, ["before-full", "after-full"], "{names:?}"); +} + +/// TEL-024: a span the export filter rejects is not +/// aligned (no ids to propagate, so a box gets no `TRACEPARENT` pointing at +/// a span nobody records); one the filter admits adopts the propagated +/// parent and is spooled under its trace. +#[test] +fn a_span_the_export_filter_rejects_is_not_aligned() { + let tmp = private_tempdir(); + let out = child( + "helper_align", + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", to_str(tmp.path())), + ("MINIMAL_OTEL_FILTER", "warn"), + ("MLOG_REQ_FLAGS", "1"), + ], + ); + assert_eq!(mark(&out, "exec"), "None", "a rejected span is not aligned"); + assert_eq!( + mark(&out, "kept"), + format!("{} 1", "11".repeat(16)), + "an admitted span joins the propagated trace" + ); + let lines = spool(tmp.path()); + let spans = spans(&lines); + assert!(!span_names(&lines).contains(&"exec".to_owned())); + let kept = spans + .iter() + .find(|(_, s)| s["name"] == "kept") + .map(|(_, s)| *s) + .unwrap_or_else(|| panic!("kept is not spooled: {lines:?}")); + assert_eq!(kept["traceId"], "11".repeat(16)); + assert_eq!(kept["parentSpanId"], "22".repeat(8)); +} + +/// TEL-024: under an unsampled propagated parent (flags 00) +/// the span keeps the trace id and the flags 00 it would propagate, nothing +/// of it is spooled, and its events still are. +#[test] +fn an_unsampled_parent_propagates_flags_00_and_spools_nothing() { + let tmp = private_tempdir(); + let out = child( + "helper_align", + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", to_str(tmp.path())), + ("MLOG_REQ_FLAGS", "0"), + ], + ); + assert_eq!( + mark(&out, "exec"), + format!("{} 0", "11".repeat(16)), + "same trace, flags 00" + ); + let lines = spool(tmp.path()); + assert!(span_names(&lines).is_empty(), "{lines:?}"); + assert!( + log_bodies(&lines).contains(&"inside-exec".to_owned()), + "logs still spool: {lines:?}" + ); +} + +/// TEL-005: with traces off (`OTEL_TRACES_EXPORTER=none`) and +/// logs on, no tracer is installed: no span layer, `exporting()` is false, +/// `align` gives nothing to propagate, and events still spool as logs. +#[test] +fn logs_only_installs_no_tracer() { + let tmp = private_tempdir(); + let out = child( + "helper_logs_only", + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", to_str(tmp.path())), + ("OTEL_TRACES_EXPORTER", "none"), + ], + ); + assert_eq!(mark(&out, "layers"), "span=false log=true exporting=false"); + assert_eq!(mark(&out, "align"), "None"); + let lines = spool(tmp.path()); + assert!(span_names(&lines).is_empty(), "{lines:?}"); + assert!(log_bodies(&lines).contains(&"logs-only-event".to_owned())); +} + +/// TEL-010: a CLI that leaves through +/// `process::exit` flushes from the atexit hook within its bound, against a +/// collector that accepts and never answers, and the span is spooled. +#[test] +fn the_atexit_flush_is_bounded_on_process_exit() { + let tarpit = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", tarpit.local_addr().unwrap()); + std::thread::spawn(move || { + let mut held = Vec::new(); + for c in tarpit.incoming() { + held.push(c); // accepted, never read, never answered + } + }); + let tmp = private_tempdir(); + let out = child( + "helper_exit", + &[ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL_DIR", to_str(tmp.path())), + ("OTEL_EXPORTER_OTLP_ENDPOINT", &url), + ], + ); + let exited = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(); + let before: u128 = mark(&out, "exit_at").parse().unwrap(); + let took = exited.saturating_sub(before); + // TEL-010's bound, `CLI_EXIT_FLUSH` plus 250 ms of shutdown slack, and + // a margin for what this measures besides the flush: the child's + // teardown after the hook and this process reading its output. 100 ms + // covers that on a loaded test runner; a regression in the flush + // itself overruns it. + const SHUTDOWN_SLACK_MS: u128 = 250; + const MEASUREMENT_MARGIN_MS: u128 = 100; + let bound = mlog::otel::CLI_EXIT_FLUSH.as_millis() + SHUTDOWN_SLACK_MS + MEASUREMENT_MARGIN_MS; + assert!( + took < bound, + "process::exit took {took} ms against a silent collector (bound {bound} ms)" + ); + assert!(span_names(&spool(tmp.path())).contains(&"exit-span".to_owned())); +} + +/// TEL-008: with a `MINIMAL_`-prefixed endpoint, the ambient +/// `OTEL_EXPORTER_OTLP_HEADERS` (a developer's key for another backend) is +/// never sent to it. Without `MINIMAL_OTEL_EXPORTER_OTLP_HEADERS` the export +/// is refused with one warning per signal and the spool is written as +/// usual; with it, its headers are sent and the plain ones are not. A plain +/// endpoint still gets the plain headers. +#[test] +fn a_prefixed_endpoint_never_carries_plain_headers() { + let ambient = [ + ("MINIMAL_TELEMETRY", "1"), + ("OTEL_EXPORTER_OTLP_HEADERS", "x-api-key=S3CRET,x-plain=1"), + ( + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + "x-api-key=S3CRET,x-traces=1", + ), + ]; + + // Refused: nothing reaches the collector, the spool has the span. + let col = collector(); + let tmp = private_tempdir(); + let mut env = ambient.to_vec(); + env.push(("MINIMAL_OTEL_EXPORTER_OTLP_ENDPOINT", &col.url)); + env.push(("MINIMAL_OTEL_SPOOL_DIR", to_str(tmp.path()))); + let out = child("helper_export", &env); + assert_eq!( + marks(&out, "warn"), + [ + "telemetry: refusing to export traces: OTEL_EXPORTER_OTLP_TRACES_HEADERS is set but \ + the endpoint comes from MINIMAL_OTEL_EXPORTER_OTLP_ENDPOINT; set \ + MINIMAL_OTEL_EXPORTER_OTLP_HEADERS to send headers there", + "telemetry: refusing to export logs: OTEL_EXPORTER_OTLP_HEADERS is set but the \ + endpoint comes from MINIMAL_OTEL_EXPORTER_OTLP_ENDPOINT; set \ + MINIMAL_OTEL_EXPORTER_OTLP_HEADERS to send headers there", + ] + ); + assert!( + col.requests.lock().unwrap().is_empty(), + "a refused export reached the collector" + ); + assert!(span_names(&spool(tmp.path())).contains(&"req-span".to_owned())); + + // MINIMAL_ headers: sent, and only they. + let col = collector(); + let mut env = ambient.to_vec(); + env.push(("MINIMAL_OTEL_EXPORTER_OTLP_ENDPOINT", &col.url)); + env.push(("MINIMAL_OTEL_SPOOL", "0")); + env.push(( + "MINIMAL_OTEL_EXPORTER_OTLP_HEADERS", + "x-api-key=ours,x-minimal=1", + )); + let out = child("helper_export", &env); + assert!(marks(&out, "warn").is_empty(), "{out}"); + let requests = col.requests.lock().unwrap(); + assert!(!requests.is_empty(), "control: the collector got no export"); + for r in requests.iter() { + assert!(!r.contains("s3cret"), "an ambient header was sent: {r}"); + assert!(!r.contains("x-plain") && !r.contains("x-traces"), "{r}"); + assert!( + r.contains("x-api-key: ours") && r.contains("x-minimal: 1"), + "{r}" + ); + } + drop(requests); + + // A plain endpoint: the plain headers, as for any OTel program. + let col = collector(); + let mut env = ambient.to_vec(); + env.push(("OTEL_EXPORTER_OTLP_ENDPOINT", &col.url)); + env.push(("MINIMAL_OTEL_SPOOL", "0")); + child("helper_export", &env); + let requests = col.requests.lock().unwrap(); + assert!(!requests.is_empty(), "control: the collector got no export"); + assert!( + requests.iter().all(|r| r.contains("x-api-key: s3cret")), + "{requests:?}" + ); +} + +/// TEL-012: `quiet` silences the SDK's own events on a console +/// filter (a down collector prints no ERROR lines) and keeps everything the +/// caller's directives show. +#[test] +fn quiet_drops_sdk_errors_and_keeps_the_rest() { + let seen: Arc>> = Arc::default(); + let filter = mlog::otel::quiet(tracing_subscriber::EnvFilter::new("info,minimald=debug")); + let subscriber = tracing_subscriber::registry().with( + Census { + seen: seen.clone(), + print: false, + } + .with_filter(filter), + ); + tracing::subscriber::with_default(subscriber, || { + tracing::error!(target: "opentelemetry", "x"); + tracing::error!(target: "opentelemetry_sdk", "x"); + tracing::error!(target: "opentelemetry_otlp", "x"); + tracing::error!(target: "opentelemetry_http", "x"); + tracing::debug!(target: "minimald", "x"); + tracing::info!(target: "minimal", "x"); + tracing::debug!(target: "minimal", "x"); + }); + assert_eq!(*seen.lock().unwrap(), ["minimald", "minimal"]); +} + +/// A requirement with no test in this repository: what covers it today and +/// the issue that tracks closing the gap. +struct KnownGap { + id: &'static str, + covered_by: &'static str, + issue: &'static str, +} + +/// Every known gap, one test each below. These tests run no product code: +/// each names one gap, what covers it today and the issue that tracks it, +/// and fails on request (`TEST_KNOWN_GAPS=1`) so a lane can list the gaps. +/// An entry leaves when the requirement gets a real test; nothing here +/// notices that by itself, so the change that adds the test removes the +/// entry. The list matches the spec's known-gaps paragraph. +const KNOWN_GAPS: &[KnownGap] = &[ + KnownGap { + id: "TEL-009", + covered_by: "lab scenario 785 only (a daemon keeps its own switches across a client's)", + issue: "minimal#2033", + }, + KnownGap { + id: "TEL-034", + covered_by: "lab scenario 715b only, until the VM test lands (the guest's records \ + reach the host's collector)", + issue: "minimal#2034", + }, + KnownGap { + id: "TEL-044", + covered_by: "each collector's scrub by unit tests \ + (`a_host_bundle_never_carries_a_header_value_or_a_secret_attribute`, \ + `a_daemon_bundle_never_carries_a_header_value_or_a_secret_attribute`), \ + the host's whole `min bug` collection by \ + `a_bundle_collected_with_telemetry_on_carries_no_secret`; \ + the whole path, a nested bundle from a real VM, by lab scenario 782 only", + issue: "minimal#2035", + }, + KnownGap { + id: "TEL-047", + covered_by: "nothing: nothing in the tree reports a box's injected environment, and \ + `min box spec` is spec 24's future work", + issue: "minimal#2036", + }, + KnownGap { + id: "TEL-032 on macOS", + covered_by: "unit coverage only (`a_task_that_outlives_ready_does_not_hold_the_start_phase_open`); \ + the ready()-phase flush was found missing on macOS 2026-10-05 and no macOS \ + lane runs the supervisor", + issue: "minimal#2037", + }, +]; + +#[test] +fn known_gap_tel_009_a_daemon_keeps_its_own_switches() { + known_gap("TEL-009"); +} + +#[test] +fn known_gap_tel_034_the_guest_exports_to_a_reachable_collector() { + known_gap("TEL-034"); +} + +#[test] +fn known_gap_tel_044_the_bug_bundle_carries_the_spools() { + known_gap("TEL-044"); +} + +#[test] +fn known_gap_tel_047_the_box_spec_shows_traceparent() { + known_gap("TEL-047"); +} + +#[test] +fn known_gap_tel_032_on_macos_the_ready_phase_flushes() { + known_gap("TEL-032 on macOS"); +} + +/// Every known gap names a spec id, what covers it today and an issue to +/// close it under; a placeholder issue is spelled `minimal#TBD-` +/// until it is filed. +#[test] +fn known_gaps_name_a_spec_id_a_coverage_and_an_issue() { + for gap in KNOWN_GAPS { + assert!( + gap.id.starts_with("TEL-") && gap.id.len() >= 7, + "{}: not a spec id", + gap.id + ); + assert!( + !gap.covered_by.trim().is_empty(), + "{}: what covers it?", + gap.id + ); + assert!( + gap.issue.starts_with("minimal#") && gap.issue.len() > "minimal#".len(), + "{}: issue {} is neither minimal# nor minimal#TBD-", + gap.id, + gap.issue + ); + if let Some(slug) = gap.issue.strip_prefix("minimal#TBD-") { + assert!( + !slug.is_empty() + && slug + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'), + "{}: placeholder slug {slug:?} is not kebab-case", + gap.id + ); + } + } +} + +/// Gaps are found by id: each id is listed once, and each +/// listed gap has the test that names it, so a gap cannot sit in the list +/// with no test failing for it under `TEST_KNOWN_GAPS=1`. +#[test] +fn every_known_gap_is_listed_once_and_has_its_test() { + let src = include_str!("otel_requirements.rs"); + for (i, gap) in KNOWN_GAPS.iter().enumerate() { + assert!( + KNOWN_GAPS.iter().skip(i + 1).all(|g| g.id != gap.id), + "{} is listed twice", + gap.id + ); + assert!( + src.contains(&format!(" known_gap(\"{}\");", gap.id)), + "{} has no known_gap test", + gap.id + ); + } +} + +/// The requirement ids the lab's notes, scenario headers and proof tables +/// used before 2026-10-05, and the spec 25 id each became +/// (`patches/lab-scenarios-spec-ids-20261005/idmap.json`, the rename's +/// own map). One to one. +const FORMER_IDS: &[(&str, &str)] = &[ + ("OT-1", "TEL-001"), + ("OT-2", "TEL-002"), + ("OT-3", "TEL-003"), + ("OT-4", "TEL-004"), + ("OT-5", "TEL-005"), + ("OT-6", "TEL-006"), + ("OT-7", "TEL-007"), + ("OT-8", "TEL-008"), + ("OT-9", "TEL-009"), + ("BE-1", "TEL-010"), + ("BE-2", "TEL-011"), + ("BE-3", "TEL-012"), + ("BE-4", "TEL-013"), + ("SP-1", "TEL-014"), + ("SP-2", "TEL-015"), + ("SP-3", "TEL-016"), + ("SP-4", "TEL-017"), + ("SP-5", "TEL-018"), + ("SP-6", "TEL-019"), + ("SP-7", "TEL-020"), + ("TR-1", "TEL-021"), + ("TR-2", "TEL-022"), + ("TR-3", "TEL-023"), + ("TR-4", "TEL-024"), + ("TR-5", "TEL-025"), + ("TR-6", "TEL-026"), + ("TR-7", "TEL-027"), + ("TR-8", "TEL-028"), + ("TR-9", "TEL-029"), + ("TR-10", "TEL-030"), + ("TR-11", "TEL-031"), + ("VM-1", "TEL-032"), + ("VM-2", "TEL-033"), + ("VM-3", "TEL-034"), + ("VM-4", "TEL-035"), + ("VM-5", "TEL-036"), + ("VM-6", "TEL-037"), + ("SD-1", "TEL-038"), + ("SD-2", "TEL-039"), + ("PV-1", "TEL-040"), + ("PV-2", "TEL-041"), + ("PV-3", "TEL-042"), + ("PV-4", "TEL-043"), + ("DG-1", "TEL-044"), + ("BE-N01", "TEL-N01"), + ("SP-N01", "TEL-N02"), +]; + +/// Spec 25 ids with no former id (the spec's table +/// maps them from `(none)`): a source that cites one cites a requirement +/// that exists, so they count as known below. +const NEW_IDS: &[&str] = &["TEL-045", "TEL-046", "TEL-047", "TEL-048", "TEL-051"]; + +/// The former ids map one to one onto spec 25's, and every `TEL-*` id a +/// test or a source comment in this workspace cites is one the map +/// produces, so a lab note in the old ids and a test in the new ones name +/// the same requirement; no former id survives in the sources. +#[test] +fn former_ids_map_one_to_one_onto_the_ids_the_tests_use() { + use std::collections::BTreeSet; + + let former: BTreeSet<&str> = FORMER_IDS.iter().map(|(f, _)| *f).collect(); + let now: BTreeSet<&str> = FORMER_IDS.iter().map(|(_, n)| *n).collect(); + assert_eq!(former.len(), FORMER_IDS.len(), "a former id maps twice"); + assert_eq!(now.len(), FORMER_IDS.len(), "two former ids map to one"); + for (f, n) in FORMER_IDS { + let (group, num) = f.split_once('-').unwrap_or_else(|| panic!("{f}: no group")); + assert!( + ["OT", "BE", "SP", "TR", "VM", "SD", "PV", "DG"].contains(&group), + "{f}: unknown former group" + ); + assert!( + num.trim_start_matches('N') + .bytes() + .all(|b| b.is_ascii_digit()), + "{f}: not a former id" + ); + let tail = n + .strip_prefix("TEL-") + .unwrap_or_else(|| panic!("{n}: not a spec 25 id")); + assert!( + tail.len() == 3 + && tail + .trim_start_matches('N') + .bytes() + .all(|b| b.is_ascii_digit()), + "{n}: not TEL-NNN or TEL-NNN with an N" + ); + } + + // What the sources cite. + let crates = Path::new(env!("CARGO_MANIFEST_DIR")).join(".."); + let mut files = Vec::new(); + for entry in std::fs::read_dir(&crates).unwrap() { + let krate = entry.unwrap().path(); + for sub in ["src", "tests"] { + walk(&krate.join(sub), &mut files); + } + } + assert!( + files.len() > 50, + "found only {} source files under {}", + files.len(), + crates.display() + ); + let mut cited = BTreeSet::new(); + let mut stale = Vec::new(); + for f in &files { + let text = std::fs::read_to_string(f).unwrap_or_default(); + for (i, line) in text.lines().enumerate() { + for id in ids_in(line, "TEL-") { + cited.insert(id); + } + for group in ["OT-", "BE-", "SP-", "TR-", "VM-", "SD-", "PV-", "DG-"] { + for id in ids_in(line, group) { + // This file carries the map itself. + if f.ends_with("otel_requirements.rs") { + continue; + } + if former.contains(id.as_str()) { + stale.push(format!("{}:{}: {id}", f.display(), i + 1)); + } + } + } + } + } + assert!( + stale.is_empty(), + "former ids still in the sources:\n{}", + stale.join("\n") + ); + let known: BTreeSet<&str> = now.iter().copied().chain(NEW_IDS.iter().copied()).collect(); + let unknown: Vec<_> = cited + .iter() + .filter(|id| !known.contains(id.as_str())) + .collect(); + assert!( + unknown.is_empty(), + "ids cited by the sources that no former id maps to: {unknown:?}" + ); + let core: BTreeSet<&str> = FORMER_IDS + .iter() + .map(|(_, n)| *n) + .filter(|n| !n.contains('N')) + .collect(); + let uncited: Vec<_> = core.iter().filter(|id| !cited.contains(**id)).collect(); + assert!( + uncited.len() < core.len() / 2, + "most spec ids are cited by a test or a source comment; not these: {uncited:?}" + ); +} + +/// One span with one event inside, after an optional `relocate_spool` to +/// `MLOG_REQ_RELOCATE`. +#[test] +fn helper_record() { + if !in_child() { + return; + } + let subscriber = installed(); + if let Some(d) = std::env::var_os("MLOG_REQ_RELOCATE") { + mlog::otel::relocate_spool(d.into()); + } + tracing::subscriber::with_default(subscriber, || { + tracing::info_span!("req-span").in_scope(|| tracing::info!("req-event")); + }); +} + +/// A span and an event at TRACE and at ERROR under each [`STACK`] target, +/// and one of each under `minimald` as the control. +#[test] +fn helper_stack_targets() { + if !in_child() { + return; + } + macro_rules! emit { + ($($t:literal),*) => {$( + drop(tracing::trace_span!(target: $t, concat!("stack-span ", $t))); + drop(tracing::error_span!(target: $t, concat!("stack-span ", $t))); + tracing::trace!(target: $t, "stack-event"); + tracing::error!(target: $t, "stack-event"); + )*}; + } + tracing::subscriber::with_default(installed(), || { + emit!( + "hyper", + "hyper_util::client::legacy", + "h2", + "reqwest::connect", + "tower", + "opentelemetry", + "opentelemetry_sdk", + "opentelemetry_otlp", + "opentelemetry_http" + ); + drop(tracing::trace_span!(target: "minimald", "own-span")); + tracing::trace!(target: "minimald", "own-event"); + }); +} + +/// One span, exported through a real OTLP/HTTP client under a global +/// subscriber (the SDK's threads see it too) with a printing [`Census`]. +#[test] +fn helper_census() { + if !in_child() { + return; + } + let census = Census { + seen: Arc::default(), + print: true, + }; + tracing::subscriber::set_global_default(installed().with(census)).unwrap(); + tracing::info_span!("req-span").in_scope(|| tracing::info!("req-event")); + mlog::otel::flush(Duration::from_secs(5)); + mlog::otel::shutdown(Duration::from_secs(5)); +} + +/// `report_init` (its warnings printed by [`Warnings`]), one span, then a +/// bounded shutdown (the export happens there). +#[test] +fn helper_export() { + if !in_child() { + return; + } + tracing::subscriber::with_default(installed().with(Warnings), || { + mlog::otel::report_init(); + drop(tracing::info_span!("req-span")); + }); + mlog::otel::shutdown(Duration::from_secs(5)); +} + +/// `before-full`; then the file is held at its size (`RLIMIT_FSIZE`, with +/// `SIGXFSZ` ignored so the write fails with `EFBIG` as on a full disk) +/// while three spans end; then the limit is lifted and `after-full` ends. +#[cfg(target_os = "linux")] +#[test] +fn helper_full_disk() { + if !in_child() { + return; + } + let dir = PathBuf::from(std::env::var_os("MINIMAL_OTEL_SPOOL_DIR").unwrap()); + tracing::subscriber::with_default(installed(), || { + drop(tracing::info_span!("before-full")); + let files = spool_files(&dir); + let len = std::fs::metadata(&files[0]).unwrap().len(); + let mut old = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + // SAFETY: sets this process's own disposition of SIGXFSZ to ignore. + unsafe { libc::signal(libc::SIGXFSZ, libc::SIG_IGN) }; + // SAFETY: a valid pointer to a local rlimit the call fills in. + let got = unsafe { libc::getrlimit(libc::RLIMIT_FSIZE, &raw mut old) }; + assert_eq!(got, 0, "getrlimit"); + let full = libc::rlimit { + rlim_cur: len, + rlim_max: old.rlim_max, + }; + // SAFETY: a valid pointer to a local rlimit; lowers only the soft limit. + let set = unsafe { libc::setrlimit(libc::RLIMIT_FSIZE, &raw const full) }; + assert_eq!(set, 0, "setrlimit"); + for _ in 0..3 { + drop(tracing::info_span!("while-full")); + } + assert_eq!(std::fs::metadata(&files[0]).unwrap().len(), len, "full"); + // SAFETY: a valid pointer to the limit read before, restored. + let restored = unsafe { libc::setrlimit(libc::RLIMIT_FSIZE, &raw const old) }; + assert_eq!(restored, 0, "setrlimit back"); + drop(tracing::info_span!("after-full")); + }); +} + +/// `align` for an `info` span `exec` given the parent `11..`/`22..` with +/// flags `MLOG_REQ_FLAGS` (an event inside it), and for a `warn` span `kept` +/// given the same parent. Prints `MARK ` or +/// `MARK None`. +#[test] +fn helper_align() { + if !in_child() { + return; + } + let flags: u8 = std::env::var("MLOG_REQ_FLAGS").unwrap().parse().unwrap(); + let parent = Some(([0x11; 16], [0x22; 8], flags)); + let show = |a: Option<([u8; 16], [u8; 8], u8)>| { + a.map_or_else( + || "None".to_owned(), + |(t, _, f)| { + let hex: String = t.iter().map(|b| format!("{b:02x}")).collect(); + format!("{hex} {f}") + }, + ) + }; + tracing::subscriber::with_default(installed(), || { + let exec = tracing::info_span!("exec"); + println!("MARK exec {}", show(mlog::otel::align(&exec, parent))); + exec.in_scope(|| tracing::warn!("inside-exec")); + drop(exec); + let kept = tracing::warn_span!("kept"); + println!("MARK kept {}", show(mlog::otel::align(&kept, parent))); + drop(kept); + }); +} + +/// The layers `init` leaves with traces off, `align`'s answer, one event. +#[test] +fn helper_logs_only() { + if !in_child() { + return; + } + mlog::otel::init("minimald"); + let span_layer = mlog::otel::span_layer::(); + let log_layer = mlog::otel::log_layer::(); + println!( + "MARK layers span={} log={} exporting={}", + span_layer.is_some(), + log_layer.is_some(), + mlog::otel::exporting() + ); + let subscriber = tracing_subscriber::registry().with(log_layer); + tracing::subscriber::with_default(subscriber, || { + let span = tracing::info_span!("exec"); + let a = mlog::otel::align(&span, Some(([0x11; 16], [0x22; 8], 1))); + println!("MARK align {}", if a.is_some() { "Some" } else { "None" }); + span.in_scope(|| tracing::info!("logs-only-event")); + }); +} + +/// The CLI's exit path: one span, the CLI exit bound, then `process::exit` +/// (no explicit shutdown: the atexit hook flushes). Prints the time just +/// before the exit. +#[test] +fn helper_exit() { + if !in_child() { + return; + } + let subscriber = installed(); + mlog::otel::set_exit_flush(mlog::otel::CLI_EXIT_FLUSH); + tracing::subscriber::with_default(subscriber, || { + drop(tracing::info_span!("exit-span")); + }); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(); + println!("MARK exit_at {now}"); + std::process::exit(0); +} diff --git a/crates/mlog/tests/otel_without_ca_roots.rs b/crates/mlog/tests/otel_without_ca_roots.rs new file mode 100644 index 0000000000..bb73a7eebf --- /dev/null +++ b/crates/mlog/tests/otel_without_ca_roots.rs @@ -0,0 +1,62 @@ +//! The microVM guest's `/init` initialises telemetry from an initramfs that +//! holds the daemon binary and nothing else, so there are no CA roots on +//! disk. An `http://` collector must still get an exporter (spec 25, +//! TEL-007). This is its own test binary, so the environment it sets is +//! this process's alone. + +use std::time::Duration; + +/// Linux only, like the guest. On macOS rustls-native-certs reads the +/// keychain and ignores `SSL_CERT_FILE` and `SSL_CERT_DIR`, so the +/// environment cannot take the roots away and the test would prove +/// nothing; it is reported as ignored there rather than passing. +#[test] +#[cfg_attr( + not(target_os = "linux"), + ignore = "needs Linux: elsewhere the platform finds CA roots without SSL_CERT_FILE/SSL_CERT_DIR, so the guest's no-roots case cannot be simulated" +)] +fn an_http_exporter_builds_with_no_ca_roots_on_disk() { + let empty = tempfile::tempdir().unwrap(); + // SAFETY: the only test in this binary, so no other thread reads the + // environment while it is set, and `init` starts its threads afterwards. + // When set, rustls-native-certs reads only these: a missing file and + // an empty directory are the initramfs's "no roots at all". + unsafe { std::env::set_var("SSL_CERT_FILE", empty.path().join("none.pem")) }; + // SAFETY: as above. + unsafe { std::env::set_var("SSL_CERT_DIR", empty.path()) }; + // Premise: the client `opentelemetry-otlp` would build fails here, as it + // does in the guest. If it builds, this host finds roots some other way + // and the test would prove nothing, so that fails rather than passes. + assert!( + reqwest::blocking::Client::builder().build().is_err(), + "premise: the default reqwest client still finds CA roots with SSL_CERT_FILE and \ + SSL_CERT_DIR pointing at nothing, so this host cannot simulate the guest" + ); + for k in [ + "OTEL_SDK_DISABLED", + "DO_NOT_TRACK", + "OTEL_TRACES_EXPORTER", + "MINIMAL_OTEL_TRACES_EXPORTER", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "MINIMAL_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_ENDPOINT", + ] { + // SAFETY: as above. + unsafe { std::env::remove_var(k) }; + } + for (k, v) in [ + ("MINIMAL_TELEMETRY", "1"), + ("MINIMAL_OTEL_SPOOL", "0"), + ("MINIMAL_OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:9"), + ] { + // SAFETY: as above. + unsafe { std::env::set_var(k, v) }; + } + mlog::otel::init("minimald"); + // With the spool off, a tracer is installed only if the exporter built. + assert!( + mlog::otel::exporting(), + "no trace exporter for an http endpoint without CA roots" + ); + mlog::otel::shutdown(Duration::from_millis(100)); +} diff --git a/scripts/kani.sh b/scripts/kani.sh index 638f994884..ddaa39cc24 100755 --- a/scripts/kani.sh +++ b/scripts/kani.sh @@ -3,7 +3,10 @@ # crates: rcache (index_file untrusted-bytes parse path) and sessions # (PathDecision combination lattice, the egress frame verdict's # admit-iff-declared property, and the publish-verb decision's -# applied-iff-table-admits property). +# applied-iff-table-admits property), mlog (the telemetry switches and +# the spool's clean-up plan), and minvmd (the guest boot line: the token +# alphabet both its readers agree on and the budget that leaves libkrun +# its reserve). # # Install: cargo install --locked kani-verifier --version 0.68.0 # && cargo kani setup @@ -24,6 +27,55 @@ # # Sequential on purpose: -j once OOMed CBMC running four byte-level # harnesses at once; the whole suite solves in seconds sequentially. +# +# Measured cost (2026-10-07, Kani 0.68.0 / CBMC 6.11.0, on the lab box: a +# Dell R730, 2x Xeon E5-2680 v3, 48 threads, 377 GB). Each run sat in a +# systemd scope with swap off (MemorySwapMax=0); peak is the scope's +# cgroup memory.peak, page cache included. Per crate, run alone with a +# warm Kani target dir: +# sessions 10/10 harnesses 1060 s 11.87 GB peak +# rcache 3/3 31 s 0.99 GB peak +# mlog 9/9 47 s 0.56 GB peak +# minvmd 5/5 42 s 1.67 GB peak (122 s, 5.55 GB cold) +# The whole script verifies 27/27 in 1157 s at a 12.33 GB peak under the +# default ceiling below (the ulimit path), and in 1260 s at 12.32 GB in a +# 16 GB scope; no OOM event in any run. sessions sets the peak. On +# GitHub-hosted CI (run 37364930742, "CI (Kani proofs)", 2026-10-05, +# ubuntu-24.04) all 27 harnesses passed, minvmd in about 66 s. +# +# Memory guard: a box that runs the lane as one job has nothing else +# between one harness's CBMC and the machine. On 2026-10-05 an uncapped +# run on the lab box, with swap on, reached 302 GB of RAM and 365 GB of +# swap in the minvmd lane before systemd stopped the runner under memory +# pressure, and its log went with it. Its cause, re-measured on that tree +# (series v10 before patch 0092), each harness alone in a 16 GB scope +# with swap off and a 15-minute limit: the minvmd harnesses then built +# Strings of symbolic length under #[kani::unwind(32)] (a token of +# symbolic length push_str'd onto the 1779-byte base line; a value pushed +# char by char from symbolic bytes), and CBMC models every byte of such a +# String. a_filter_at_the_budget_is_forwarded_or_skipped_whole reached +# 16 GB in 14 minutes, every_rejected_value_has_a_reason_and_no_token and +# every_accepted_token_is_in_the_alphabet in 11.5 minutes each, and +# guest_env_tokens_cross_all_or_none ran 15 minutes at 3.9 GB without a +# verdict. Kani checks a crate's harnesses in reverse source order, so the +# uncapped run met guest_env_tokens_cross_all_or_none first and +# a_filter_at_the_budget_is_forwarded_or_skipped_whole next; which one held +# the 302 GB is not recorded. The fifth, the_budget_leaves_libkrun_its_reserve, +# had no unwind bound at all but verifies in 8 s at 0.62 GB. KANI_MEM_MAX +# did not exist then. The same day those proofs were rewritten over +# lengths and bytes, the five measured above. +# +# KANI_MEM_MAX is the ceiling (systemd's size syntax: bytes, or K/M/G/T +# for 1024-based units, e.g. KANI_MEM_MAX=64G). Unset, it is 14G, which +# leaves a 16 GB runner its OS: CI sets nothing, so CI runs under it. +# KANI_MEM_MAX=off runs with no ceiling. Each crate's `cargo kani` runs +# under the ceiling, so a harness that outgrows it dies alone and the log +# says which: in a transient systemd scope with MemoryMax and swap denied +# to the scope (MemorySwapMax=0; a cgroup at its MemoryMax swaps rather +# than dies, which is the incident's shape) when systemd-run can reach a +# user manager; else under `ulimit -v` in a subshell, which caps each +# process's address space (CBMC is one process). The 1157 s run above +# took the ulimit path, with no user manager reachable. set -eu # Build artifacts land in the REAL workspace's target dir (not the @@ -123,6 +175,69 @@ elif [ -n "$btver" ]; then fi cd "$ws" + +# The memory guard, decided once (see the header). Unset means the +# default ceiling, 14G, which leaves a 16 GB runner its OS; `off` means +# no guard. `kani_guard` is "scope", "ulimit" or empty; `kani_mem_kib` is +# the ulimit figure. +KANI_MEM_MAX="${KANI_MEM_MAX:-14G}" +kani_guard="" +kani_mem_kib="" +if [ "$KANI_MEM_MAX" = off ]; then + echo "kani.sh: memory guard off (KANI_MEM_MAX=off)" >&2 +else + digits="${KANI_MEM_MAX%[KMGT]}" + case "$digits" in + "" | *[!0-9]*) + echo "FATAL: KANI_MEM_MAX='$KANI_MEM_MAX' is not a size: digits, optionally K, M, G or T" >&2 + exit 1 + ;; + # Plain decimal only: a leading zero would make the arithmetic + # below read the figure as octal (08G is an error, 010G is 8G). + 0?*) + echo "FATAL: KANI_MEM_MAX='$KANI_MEM_MAX' has a leading zero; write the size in plain decimal" >&2 + exit 1 + ;; + esac + # Probe the scope path on `true`: no user manager (no session bus, a + # box without systemd) fails here, not on the real run, and the lane + # falls back to ulimit rather than on its face. + if command -v systemd-run >/dev/null 2>&1 && + systemd-run --user --scope -q -p MemoryMax="$KANI_MEM_MAX" -p MemorySwapMax=0 \ + true >/dev/null 2>&1; then + kani_guard="scope" + else + case "$KANI_MEM_MAX" in + *K) kani_mem_kib="$digits" ;; + *M) kani_mem_kib=$((digits * 1024)) ;; + *G) kani_mem_kib=$((digits * 1024 * 1024)) ;; + *T) kani_mem_kib=$((digits * 1024 * 1024 * 1024)) ;; + *) kani_mem_kib=$((digits / 1024)) ;; + esac + kani_guard="ulimit" + fi + echo "kani.sh: memory guard '$kani_guard' at KANI_MEM_MAX=$KANI_MEM_MAX" >&2 +fi + +# `cargo kani` on one crate, under the memory guard when one is set. +# `--scope` runs the command in the caller's foreground and returns its +# status, so no --wait (systemd-run refuses --wait with --scope). Not -q: +# the "Running scope as unit: run-….scope" line names the unit to watch +# (`systemctl --user status run-….scope` shows its memory while it runs). +kani_crate() { # crate + case "$kani_guard" in + scope) + systemd-run --user --scope -p MemoryMax="$KANI_MEM_MAX" -p MemorySwapMax=0 \ + cargo kani -p "$1" --output-format=terse + ;; + ulimit) + # shellcheck disable=SC3045 # ulimit -v is not POSIX; dash, bash and busybox sh have it + (ulimit -v "$kani_mem_kib" && exec cargo kani -p "$1" --output-format=terse) + ;; + *) cargo kani -p "$1" --output-format=terse ;; + esac +} + # Assert the harness COUNT, not just exit status: `cargo kani` exits 0 # on a crate with zero harnesses, so if the #[cfg(kani)] modules ever # stop compiling in, the lane would go green having proved nothing. @@ -131,7 +246,7 @@ cd "$ws" # in POSIX sh. The log file lives in the scratch ws, cleaned by trap. expect() { # crate expected_count log="$ws/kani-$1.log" - cargo kani -p "$1" --output-format=terse 2>&1 | tee "$log" + kani_crate "$1" 2>&1 | tee "$log" # tee masks cargo's status (no pipefail in sh): the count grep below # is the gate, and a failed run cannot print the success line. grep -q "Complete - $2 successfully verified harnesses, 0 failures" "$log" || { @@ -155,3 +270,5 @@ expect() { # crate expected_count } expect sessions 10 expect rcache 3 +expect mlog 9 +expect minvmd 5