From edf167d05408a56dbe5a15864a1e2c62c277f4df Mon Sep 17 00:00:00 2001 From: Norrie Taylor Date: Thu, 9 Jul 2026 21:43:28 -0700 Subject: [PATCH 1/5] ci: adopt .config/nextest.toml with a ci profile Add a nextest config with two profiles: profile.default for local runs (surface slow tests, never kill), and profile.ci for the core-tests composite (fail-fast off for full failure reports, hard-kill at 5 minutes via slow-timeout terminate-after, leak detection at 1s). No retries anywhere: the workspace suite has no recorded flakes, and a same-invocation retry of a VM boot test would run against the leaked __krun-vmm/gvproxy processes of the failed attempt (#588) - the lane reap steps plus terminate-after remain the mitigation. core-tests also gains --no-tests=fail: an empty selection is a broken filter or a dropped target, not a pass. Refs: #687 Co-Authored-By: Claude Fable 5 --- .config/nextest.toml | 22 ++++++++++++++++++++++ .github/actions/core-tests/action.yml | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 .config/nextest.toml diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 0000000000..84a17473c1 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,22 @@ +# Nextest configuration. CI lanes select a profile explicitly (--profile ci); +# local `cargo nextest run` uses profile.default. +# +# No retries anywhere: there is no recorded flakiness in the workspace suite, +# and for VM boot tests a same-invocation retry runs against the leaked +# __krun-vmm/gvproxy processes of the failed attempt (#588) — the lane reap +# steps plus slow-timeout terminate-after are the mitigation. Revisit once the +# harness reaps its own process group. + +[profile.default] +# Local runs: surface slow tests, never kill them. +slow-timeout = "60s" + +[profile.ci] +# Workspace suite (core-tests composite). A per-test override with a larger +# slow-timeout is the escape hatch for a legitimately slow test — not raising +# the cap. +fail-fast = false +retries = 0 +slow-timeout = { period = "60s", terminate-after = 5 } +leak-timeout = "1s" +final-status-level = "slow" diff --git a/.github/actions/core-tests/action.yml b/.github/actions/core-tests/action.yml index 909d91afb1..e3f7890782 100644 --- a/.github/actions/core-tests/action.yml +++ b/.github/actions/core-tests/action.yml @@ -19,7 +19,9 @@ runs: run: cargo fetch --locked - name: Run tests shell: bash - run: cargo nextest run --workspace + # --no-tests=fail: an empty selection is a broken filter or a dropped + # target, not a pass. + run: cargo nextest run --workspace --profile ci --no-tests=fail - name: Run doctests shell: bash run: cargo test --workspace --doc From 841445314a5ff623eb23ed4383eb4569ea9d728f Mon Sep 17 00:00:00 2001 From: Norrie Taylor Date: Thu, 9 Jul 2026 22:14:08 -0700 Subject: [PATCH 2/5] ci(linux-native): trigger the lane on nextest config changes A PR touching only .config/nextest.toml (e.g. a future profile edit) must re-run the tests that consume it; this PR's own run only triggered because it also edited the core-tests composite. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci-linux-native.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-linux-native.yml b/.github/workflows/ci-linux-native.yml index 94e13d2e81..4fb68d32b8 100644 --- a/.github/workflows/ci-linux-native.yml +++ b/.github/workflows/ci-linux-native.yml @@ -57,6 +57,8 @@ jobs: - "crates/**" - "Cargo.toml" - "Cargo.lock" + # Nextest profiles change how this lane's tests run. + - ".config/nextest.toml" - "scripts/**" - ".minimal/**" - ".github/actions/setup-rust/**" From 3633d0a348fdf873d9304db362e0c098174cc747 Mon Sep 17 00:00:00 2001 From: Norrie Taylor Date: Thu, 9 Jul 2026 22:25:02 -0700 Subject: [PATCH 3/5] ci(linux-kvm): reshape the lane to its end state Four changes, one review story (nextest archive, unified session e2e, scripts over YAML, composite verify): - nextest archive replaces the hand-rolled testbins.json + run-testbin.sh: build-linux ships testbed/nextest-archive.tar.zst and the test job selects harnesses with filtersets under the new profile.vm (one VM at a time, hung boots hard-killed at 6 min, no retries - a same-invocation retry runs against the leaked vmm/gvproxy of the failed attempt, #588). - the unified session e2e (scripts/session-e2e.sh) joins the lane, covering Deployment Model 3 (native Linux + VM): the minimal CLI is built in build-linux (separate invocation - a combined build unifies libkrun into the CLI, the #694 regression class), ships in the testbed, and drives activate/exec/destroy from PATH, with the #588 reap (sudo: relay leftovers are root-owned) before it and cli-e2e-boot.log uploaded. - the daemon-lifecycle shell blob becomes scripts/lifecycle-e2e.sh: PATH-resolved minvmd (no cargo, macOS-reusable), temp workdir, and an EXIT-trap teardown so a failed assert cannot strand a daemon. - the krun_add_disk3 export assert moves into setup-libkrun-linux, mirroring the macOS composite and covering release.yml (which uses the composite but had no check); an early actionable failure - minvmd's compile-time link is the real backstop. Also per owner direction: workflow comments drop PR/issue and R-numbers (history belongs in commits, not YAML), and the DM labels are corrected against docs/specs/03-spec-networking (DM1 is the macOS model; this lane is DM3, session-e2e.sh header fixed accordingly). Refs: #687 Co-Authored-By: Claude Fable 5 --- .config/nextest.toml | 12 ++ .../actions/setup-libkrun-linux/action.yml | 13 ++ .github/workflows/ci-linux-kvm.yml | 182 +++++++++--------- scripts/ci/run-testbin.sh | 24 --- scripts/lifecycle-e2e.sh | 65 +++++++ scripts/session-e2e.sh | 5 +- 6 files changed, 183 insertions(+), 118 deletions(-) delete mode 100755 scripts/ci/run-testbin.sh create mode 100755 scripts/lifecycle-e2e.sh diff --git a/.config/nextest.toml b/.config/nextest.toml index 84a17473c1..b434b819ed 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -20,3 +20,15 @@ retries = 0 slow-timeout = { period = "60s", terminate-after = 5 } leak-timeout = "1s" final-status-level = "slow" + +[profile.vm] +# Prebuilt-harness VM runs (the KVM lane's archived e2e tests). One VM at a +# time on a 2-vCPU runner; boots are legitimately slow, so the slow period is +# generous — but a hung boot dies at 6 minutes instead of burning the job +# timeout. The vmm/gvproxy children linger briefly at teardown by design, +# hence the longer leak window. +test-threads = 1 +retries = 0 +fail-fast = true +slow-timeout = { period = "120s", terminate-after = 3 } +leak-timeout = "5s" diff --git a/.github/actions/setup-libkrun-linux/action.yml b/.github/actions/setup-libkrun-linux/action.yml index 39142f46e7..46e70c2365 100644 --- a/.github/actions/setup-libkrun-linux/action.yml +++ b/.github/actions/setup-libkrun-linux/action.yml @@ -37,3 +37,16 @@ runs: echo "LIBKRUN_PREFIX=$prefix" echo "LD_LIBRARY_PATH=$prefix${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" } >> "$GITHUB_ENV" + - name: Verify libkrun exports krun_add_disk3 (>= 1.19.0) + # Early, actionable failure if the materialized prefix is too old for + # krun_add_disk3 (the /dev/vdb attach) — at the point every consumer of + # this composite links it, instead of a cryptic runtime error at VM + # boot. minvmd's compile-time link is the real backstop; this mirrors + # the export check the macOS composite performs on its built dylib. + shell: bash + run: | + so="$(ls "$HOME/.krun"/libkrun.so* 2>/dev/null | head -1)" + test -n "$so" || { echo "::error::no libkrun.so under $HOME/.krun"; exit 1; } + nm -D "$so" 2>/dev/null | grep -q krun_add_disk3 \ + || { echo "::error::materialized libkrun ($so) does not export krun_add_disk3; need >= 1.19.0"; exit 1; } + echo "libkrun OK: $so exports krun_add_disk3" diff --git a/.github/workflows/ci-linux-kvm.yml b/.github/workflows/ci-linux-kvm.yml index 901040a50a..86450ec329 100644 --- a/.github/workflows/ci-linux-kvm.yml +++ b/.github/workflows/ci-linux-kvm.yml @@ -1,6 +1,6 @@ name: CI (Linux KVM) -# minvmd Linux/KVM end-to-end on a GitHub-hosted runner (#410, #411). +# minvmd Linux/KVM end-to-end on a GitHub-hosted runner. # # It needs no preprovisioned runner — KVM is available on GitHub-hosted # x86_64 Linux runners, and libkrun is materialized from the upstream package @@ -11,7 +11,7 @@ name: CI (Linux KVM) # /dev/kvm, so the Linux/KVM e2e can only run on x86_64. aarch64 + libkrun boot # is already covered by the macOS lane (ci-macos.yml, Apple Silicon / HVF). # -# GATE-READY (#687): ALWAYS-TRIGGERED, path-scoped INSIDE the workflow — the +# GATE-READY: ALWAYS-TRIGGERED, path-scoped INSIDE the workflow — the # `changes` job gates the e2e, and the `if: always()` aggregator # (ci-linux-kvm-success) reports success even when it was path-skipped, so # the context can join the required checks after this lane soaks green. A @@ -108,10 +108,10 @@ jobs: 'apt-get update && apt-get install -y --no-install-recommends \ protobuf-compiler libprotobuf-dev cpio git' - - name: Install cross + - name: Install cross + nextest uses: taiki-e/install-action@v2 with: - tool: cross + tool: cross,nextest - name: Cache cargo + build artifacts uses: actions/cache@v6 @@ -141,24 +141,28 @@ jobs: - name: Build minvmd run: cargo build -p minvmd --bin minvmd - - name: Build e2e harnesses + stage the testbed - # The manifest records exactly which hash-suffixed binaries cargo - # produced; the test job resolves them via scripts/ci/run-testbin.sh - # (never `ls | head`). The harnesses locate minvmd via the MINVMD_BIN - # override (their compile-time CARGO_BIN_EXE path does not exist on - # the test runner). + - name: Build the minimal CLI + # SEPARATE invocation from the minvmd build: a combined `cargo build + # -p minvmd -p minimal` unifies minvmd's default `libkrun` feature + # into the CLI's default-features=false opt-out, linking libkrun into + # `minimal` (the regression the macOS lane hit and now guards). The + # CLI drives the unified session e2e in the test job. + run: cargo build -p minimal --bin minimal + + - name: Archive e2e harnesses + stage the testbed + # `cargo nextest archive` replaces the hand-rolled testbins.json + # manifest: the archive carries every minvmd test binary with exec + # bits intact, and the test job selects harnesses with filtersets. + # The sidecar files stay explicit — libkrun lives outside target/ (an + # archive cannot carry it), and the harnesses/CLI locate minvmd via + # MINVMD_BIN / PATH (their compile-time CARGO_BIN_EXE path does not + # exist on the test runner). run: | set -euo pipefail mkdir -p testbed - cargo test -p minvmd --test boot_e2e --test minimald_session_e2e --test vsock_relay_e2e \ - --no-run --message-format=json > "$RUNNER_TEMP/build-plan.json" - jq -rs '[.[] | select(.reason=="compiler-artifact" and .executable != null and (.target.kind | index("test"))) - | {key: .target.name, value: (.executable | split("/") | last)}] | from_entries' \ - "$RUNNER_TEMP/build-plan.json" > testbed/testbins.json - jq -r 'select(.reason=="compiler-artifact" and .executable != null and (.target.kind | index("test"))) | .executable' \ - "$RUNNER_TEMP/build-plan.json" | while read -r exe; do cp "$exe" testbed/; done - cat testbed/testbins.json + cargo nextest archive -p minvmd --archive-file testbed/nextest-archive.tar.zst cp target/debug/minvmd testbed/minvmd + cp target/debug/minimal testbed/minimal cp "$RUNNER_TEMP/initramfs.cpio" testbed/initramfs.cpio # Ship the exact libkrun prefix the binaries were linked against. tar czf testbed/libkrun-prefix.tgz -C "$LIBKRUN_PREFIX" . @@ -172,10 +176,10 @@ jobs: retention-days: 3 test-kvm: - # TEST phase: no Rust toolchain, no cargo — download the testbed, pull - # the guest images, and run the prebuilt harnesses directly. A flake - # rerun of this job alone costs ~15 minutes instead of the old - # 60-minute build+test monolith. + # TEST phase: no Rust toolchain, no cargo build — download the testbed, + # pull the guest images, and run the archived harnesses with the + # standalone cargo-nextest binary. A flake rerun of this job alone costs + # ~15 minutes instead of the old 60-minute build+test monolith. needs: [changes, build-linux] if: github.event_name == 'workflow_dispatch' || needs.changes.outputs.kvm == 'true' runs-on: ubuntu-latest # x86_64; KVM-capable @@ -185,6 +189,11 @@ jobs: with: persist-credentials: false + - name: Install nextest + uses: taiki-e/install-action@v2 + with: + tool: nextest + - name: Enable /dev/kvm access # GitHub-hosted x86_64 Linux runners expose /dev/kvm but the runner user # is not in the `kvm` group; widen the node so libkrun can open it. Fail @@ -207,18 +216,23 @@ jobs: path: ${{ runner.temp }}/testbed - name: Unpack testbed + resolve paths - # Artifact transfer strips exec bits; restore them from the manifest. - # MINVMD_BIN points the harnesses at the shipped minvmd (their baked - # CARGO_BIN_EXE path only exists on the build runner). The libkrun - # prefix is unpacked to the same $HOME/.krun path the build linked - # (rpath) and exported for the dynamic loader. + # Artifact transfer strips exec bits on the sidecar binaries; restore + # them (the nextest archive preserves its own). MINVMD_BIN points the + # harnesses at the shipped minvmd (their baked CARGO_BIN_EXE path only + # exists on the build runner); the testbed joins PATH so the CLI's + # autospawn and the lifecycle script resolve minvmd/minimal by name. + # The libkrun prefix is unpacked to the same $HOME/.krun path the + # build linked (rpath) and exported for the dynamic loader; its + # krun_add_disk3 export was verified at build time by the + # setup-libkrun-linux composite — minvmd loading it here is the + # runtime backstop. run: | set -euo pipefail TESTBED="$RUNNER_TEMP/testbed" - chmod +x "$TESTBED/minvmd" - jq -r '.[]' "$TESTBED/testbins.json" | while read -r b; do chmod +x "$TESTBED/$b"; done + chmod +x "$TESTBED/minvmd" "$TESTBED/minimal" mkdir -p "$HOME/.krun" tar xzf "$TESTBED/libkrun-prefix.tgz" -C "$HOME/.krun" + echo "$TESTBED" >> "$GITHUB_PATH" { echo "TESTBED=$TESTBED" echo "MINVMD_BIN=$TESTBED/minvmd" @@ -230,19 +244,8 @@ jobs: echo "MINVMD_INITRAMFS=$TESTBED/initramfs.cpio" } >> "$GITHUB_ENV" - - name: Assert libkrun exports krun_add_disk3 (>= 1.19.0) - # krun_add_disk3 (the /dev/vdb attach) needs libkrun >= 1.19.0. Fail - # loudly here rather than with a cryptic runtime error at VM boot if - # the shipped upstream libkrun is too old. - run: | - so="$(ls "$LIBKRUN_PREFIX"/libkrun.so* 2>/dev/null | head -1)" - test -n "$so" || { echo "::error::no libkrun.so under $LIBKRUN_PREFIX"; exit 1; } - nm -D "$so" 2>/dev/null | grep -q krun_add_disk3 \ - || { echo "::error::shipped libkrun ($so) does not export krun_add_disk3; need >= 1.19.0"; exit 1; } - echo "libkrun OK: $so exports krun_add_disk3" - - - name: Materialize gvproxy switch binary (DM1 relay) - # The DM1 gvproxy relay e2e (vsock_relay_e2e) spawns this per-host switch + - name: Materialize gvproxy switch binary + # The gvproxy relay e2e (vsock_relay_e2e) spawns this per-host switch # binary; pin-verified, no Go toolchain. Publish its path for the gated # step below. run: | @@ -265,15 +268,22 @@ jobs: arch: x86_64 channel: unstable - - name: Boot E2E (READY round-trip, R3.1) + - name: Boot E2E (READY round-trip) env: MINVMD_BOOT_LOG: ${{ runner.temp }}/boot.log - run: ./scripts/ci/run-testbin.sh "$TESTBED" boot_e2e --include-ignored --nocapture + run: | + cargo-nextest nextest run --archive-file "$TESTBED/nextest-archive.tar.zst" \ + --workspace-remap "$GITHUB_WORKSPACE" --profile vm \ + --run-ignored all --no-tests=fail -E 'binary(boot_e2e)' - name: Session E2E (full minimald session over the bridge) - run: ./scripts/ci/run-testbin.sh "$TESTBED" minimald_session_e2e --include-ignored --nocapture --exact minimald_exec_over_bridge + run: | + cargo-nextest nextest run --archive-file "$TESTBED/nextest-archive.tar.zst" \ + --workspace-remap "$GITHUB_WORKSPACE" --profile vm \ + --run-ignored all --no-tests=fail \ + -E 'binary(minimald_session_e2e) and test(=minimald_exec_over_bridge)' - - name: DM1 gvproxy relay E2E (OwnIp PTask gets a switch IP, R1.5) + - name: Gvproxy relay E2E (OwnIp PTask gets a switch IP) # Boots an OwnIp VM and asserts its netns receives a 100.64.0.0/16 switch # IP via the async TAP↔gvproxy relay, with frames traversing the switch. # Needs /dev/kvm, the gvproxy binary, and CAP_NET_ADMIN (tap + netns), so @@ -284,7 +294,9 @@ jobs: MINVMD_INTEGRATION_TEST: "1" run: | sudo -E env "PATH=$PATH" "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" \ - ./scripts/ci/run-testbin.sh "$TESTBED" vsock_relay_e2e --include-ignored --nocapture + cargo-nextest nextest run --archive-file "$TESTBED/nextest-archive.tar.zst" \ + --workspace-remap "$GITHUB_WORKSPACE" --profile vm \ + --run-ignored all --no-tests=fail -E 'binary(vsock_relay_e2e)' # NOTE: bridge_e2e is intentionally NOT run here. It targets the Stage-1 # guest "vsock stub" (a socat/cat echo on port 2222) that minimald-as-pid1 @@ -293,53 +305,38 @@ jobs: # minimald_session_e2e; session coverage of the bridge is provided by # Session E2E above plus the daemon lifecycle step below. - - name: Daemon lifecycle (run --detach → status → stop, R4.2/R4.3/R4.4) - # Exercises the supervised daemon path that manages the lifecycle state, - # which the raw `boot` e2e tests above do not cover. + - name: Daemon lifecycle (run --detach → status → stop) + # Exercises the supervised daemon path that manages the lifecycle + # state, which the harness e2e tests above do not cover. The script + # resolves minvmd from PATH (the testbed dir, added at unpack). env: MINVMD_BOOT_LOG: ${{ runner.temp }}/boot-daemon.log + run: ./scripts/lifecycle-e2e.sh + + - name: Reap leftover VM processes + # The harness runs above can leak __krun-vmm / gvproxy children on + # failure (Guest::drop kills only its own minvmd), and a leftover + # wedges the vsock bridge for the CLI e2e's fresh daemon. Root-owned + # leftovers are possible here (the relay e2e runs under sudo), hence + # sudo. Ported from the macOS lane; the proper fix is for the harness + # to reap its own process group, tracked separately. run: | - set -euo pipefail - BIN="$MINVMD_BIN" - - echo "::group::run --detach" - "$BIN" run --detach --timeout 30 - echo "::endgroup::" - - echo "::group::status (expect running)" - # `run --detach` returns once the host UDS accepts connections, which - # libkrun opens early in VM setup — slightly ahead of the supervisor's - # Starting->Running transition (set after the guest READY marker). Poll - # status (up to ~15s) for Running rather than asserting immediately. - # jq parses structurally (preinstalled on GitHub runners) so the check - # doesn't break on harmless JSON formatting changes. - for _ in $(seq 1 75); do - "$BIN" status --json > status.json || true - if jq -e '.state == "running" and (.vmm_pid | type == "number")' status.json >/dev/null; then - break - fi - sleep 0.2 - done - cat status.json - "$BIN" status # exit 0 == Running - jq -e '.state == "running" and (.vmm_pid | type == "number")' status.json - echo "::endgroup::" - - echo "::group::stop" - "$BIN" stop - echo "::endgroup::" - - echo "::group::status (expect stopped)" - # `status`/`status --json` exit 1 when stopped (documented), so capture - # without tripping set -e / pipefail, then assert structurally. - "$BIN" status --json > status-after.json || true - cat status-after.json - jq -e '.state == "stopped"' status-after.json - # Also confirm the documented non-zero exit for the stopped state. - if "$BIN" status; then echo "::error::expected non-zero status exit after stop"; exit 1; fi - echo "::endgroup::" - - echo "daemon lifecycle OK" + sudo pkill -x minvmd || true + sudo pkill -f __krun-vmm || true + sudo pkill -f gvproxy || true + + - name: Session e2e via the CLI (activate/exec/destroy; autospawns minvmd) + # The SAME unified proof every target lane runs (scripts/session-e2e.sh) + # against Deployment Model 3 (native Linux + VM — the target this + # lane owns; see docs/specs/03-spec-networking): from a clean state + # `minimal --minvmd activate` must auto-spawn minvmd, boot the microVM, + # and round-trip a session exec over the vsock bridge — the Linux/KVM + # counterpart of the macOS lane's CLI gate. E2E_PROJECT_DIR=/tmp: no + # project sync into the guest yet, so activate a path that exists in + # the guest image. minimal/minvmd resolve from the testbed on PATH. + env: + MINVMD_BOOT_LOG: ${{ runner.temp }}/cli-e2e-boot.log + run: E2E_VM=1 E2E_MINIMAL_ARGS=--minvmd E2E_PROJECT_DIR=/tmp ./scripts/session-e2e.sh - name: Upload guest boot console logs if: always() @@ -349,10 +346,11 @@ jobs: path: | ${{ runner.temp }}/boot.log ${{ runner.temp }}/boot-daemon.log + ${{ runner.temp }}/cli-e2e-boot.log if-no-files-found: ignore # Single required status check for this lane (branch-protection context: - # ci-linux-kvm-success, joins the ruleset after the soak — #687). Green when + # ci-linux-kvm-success, joins the ruleset after the soak). Green when # build + tests succeeded OR were path-skipped; red only on failure/cancellation. ci-linux-kvm-success: if: always() diff --git a/scripts/ci/run-testbin.sh b/scripts/ci/run-testbin.sh deleted file mode 100755 index d75d85ba55..0000000000 --- a/scripts/ci/run-testbin.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# Run a prebuilt cargo test harness by target name, resolved through the -# testbins.json manifest the build job wrote next to the binaries. Replaces -# the fragile `ls -1t target/debug/deps/-* | grep -v .d | head -1` -# pattern: the manifest records exactly which hash-suffixed binary cargo -# produced, so a stale sibling from an older build can never be picked up. -# -# Usage: run-testbin.sh [harness args...] -# testbins.json: {"": "", ...} -set -euo pipefail - -TESTBED="${1:?usage: run-testbin.sh [args...]}" -NAME="${2:?usage: run-testbin.sh [args...]}" -shift 2 - -MANIFEST="$TESTBED/testbins.json" -test -f "$MANIFEST" || { echo "::error::no manifest at $MANIFEST" >&2; exit 1; } - -bin="$(jq -er --arg name "$NAME" '.[$name]' "$MANIFEST")" \ - || { echo "::error::no '$NAME' entry in $MANIFEST" >&2; exit 1; } -exe="$TESTBED/$bin" -test -x "$exe" || { echo "::error::test binary $exe missing or not executable" >&2; exit 1; } - -exec "$exe" "$@" diff --git a/scripts/lifecycle-e2e.sh b/scripts/lifecycle-e2e.sh new file mode 100755 index 0000000000..b9f2948a70 --- /dev/null +++ b/scripts/lifecycle-e2e.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Daemon lifecycle proof: `minvmd run --detach` → `status` (Running) → +# `stop` → `status` (Stopped, documented non-zero exit). Exercises the +# supervised daemon path that the boot/session e2e +# harnesses do not cover. +# +# Target-agnostic: resolves `minvmd` from PATH (no cargo), so the same proof +# runs on the Linux/KVM lane today and a macOS testbed later. Requires jq, +# plus the usual VM env from the caller: +# MINVMD_KERNEL_PATH / MINVMD_ROOTFS_PATH / MINVMD_INITRAMFS +# MINVMD_BOOT_LOG (optional) guest console capture +# +# Usage: scripts/lifecycle-e2e.sh +set -euo pipefail + +command -v minvmd >/dev/null || { echo "::error::minvmd not on PATH"; exit 1; } +command -v jq >/dev/null || { echo "::error::jq is required"; exit 1; } + +# Scratch dir for status captures; EXIT-trap teardown so a failed assert +# can't strand a running daemon on the runner. +WORK="$(mktemp -d /tmp/mnl-lifecycle.XXXXXX)" +teardown() { + minvmd stop >/dev/null 2>&1 || true + rm -rf "$WORK" +} +trap teardown EXIT + +echo "::group::run --detach" +minvmd run --detach --timeout 30 +echo "::endgroup::" + +echo "::group::status (expect running)" +# `run --detach` returns once the host UDS accepts connections, which +# libkrun opens early in VM setup — slightly ahead of the supervisor's +# Starting->Running transition (set after the guest READY marker). Poll +# status (up to ~15s) for Running rather than asserting immediately. jq +# parses structurally so the check doesn't break on harmless JSON +# formatting changes. +for _ in $(seq 1 75); do + minvmd status --json > "$WORK/status.json" || true + if jq -e '.state == "running" and (.vmm_pid | type == "number")' "$WORK/status.json" >/dev/null; then + break + fi + sleep 0.2 +done +cat "$WORK/status.json" +minvmd status # exit 0 == Running +jq -e '.state == "running" and (.vmm_pid | type == "number")' "$WORK/status.json" +echo "::endgroup::" + +echo "::group::stop" +minvmd stop +echo "::endgroup::" + +echo "::group::status (expect stopped)" +# `status`/`status --json` exit 1 when stopped (documented), so capture +# without tripping set -e / pipefail, then assert structurally. +minvmd status --json > "$WORK/status-after.json" || true +cat "$WORK/status-after.json" +jq -e '.state == "stopped"' "$WORK/status-after.json" +# Also confirm the documented non-zero exit for the stopped state. +if minvmd status; then echo "::error::expected non-zero status exit after stop"; exit 1; fi +echo "::endgroup::" + +echo "daemon lifecycle OK" diff --git a/scripts/session-e2e.sh b/scripts/session-e2e.sh index b3ae1c162a..800eb569ee 100755 --- a/scripts/session-e2e.sh +++ b/scripts/session-e2e.sh @@ -4,8 +4,9 @@ # so the identical proof runs against all three deployment targets: # # Linux native (DM2) minimald on the host (no extra env) -# Linux KVM (DM1) minimald in a minvmd microVM E2E_VM=1 E2E_MINIMAL_ARGS=--minvmd -# macOS HVF minimald in a minvmd microVM E2E_VM=1 (macOS is always VM-backed) +# Linux KVM (DM3) minimald in a minvmd microVM E2E_VM=1 E2E_MINIMAL_ARGS=--minvmd +# macOS HVF (DM1) minimald in a minvmd microVM E2E_VM=1 (macOS is always VM-backed) +# (DM numbers are the deployment models in docs/specs/03-spec-networking.) # # Flow: from a guaranteed-clean state, `minimal activate` must auto-spawn the # target's daemon and create a session; `minimal attach --command` execs in From b557f46527476d1e042902d30067e58a8b07845a Mon Sep 17 00:00:00 2001 From: Norrie Taylor Date: Thu, 9 Jul 2026 22:52:22 -0700 Subject: [PATCH 4/5] ci(linux-kvm): reap harness leftovers before the daemon lifecycle step The lifecycle step spawns a fresh daemon just like the CLI session e2e, so it was equally exposed to leaked __krun-vmm/gvproxy children from a failed harness run. Move the reap to run immediately after the harness e2es, ahead of both fresh-daemon steps. Addresses CodeRabbit review on #702. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci-linux-kvm.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-linux-kvm.yml b/.github/workflows/ci-linux-kvm.yml index 86450ec329..a608670176 100644 --- a/.github/workflows/ci-linux-kvm.yml +++ b/.github/workflows/ci-linux-kvm.yml @@ -305,26 +305,27 @@ jobs: # minimald_session_e2e; session coverage of the bridge is provided by # Session E2E above plus the daemon lifecycle step below. - - name: Daemon lifecycle (run --detach → status → stop) - # Exercises the supervised daemon path that manages the lifecycle - # state, which the harness e2e tests above do not cover. The script - # resolves minvmd from PATH (the testbed dir, added at unpack). - env: - MINVMD_BOOT_LOG: ${{ runner.temp }}/boot-daemon.log - run: ./scripts/lifecycle-e2e.sh - - name: Reap leftover VM processes # The harness runs above can leak __krun-vmm / gvproxy children on # failure (Guest::drop kills only its own minvmd), and a leftover - # wedges the vsock bridge for the CLI e2e's fresh daemon. Root-owned - # leftovers are possible here (the relay e2e runs under sudo), hence - # sudo. Ported from the macOS lane; the proper fix is for the harness - # to reap its own process group, tracked separately. + # wedges the vsock bridge for the fresh daemons the next two steps + # spawn (lifecycle, then the CLI session e2e). Root-owned leftovers + # are possible here (the relay e2e runs under sudo), hence sudo. + # Ported from the macOS lane; the proper fix is for the harness to + # reap its own process group, tracked separately. run: | sudo pkill -x minvmd || true sudo pkill -f __krun-vmm || true sudo pkill -f gvproxy || true + - name: Daemon lifecycle (run --detach → status → stop) + # Exercises the supervised daemon path that manages the lifecycle + # state, which the harness e2e tests above do not cover. The script + # resolves minvmd from PATH (the testbed dir, added at unpack). + env: + MINVMD_BOOT_LOG: ${{ runner.temp }}/boot-daemon.log + run: ./scripts/lifecycle-e2e.sh + - name: Session e2e via the CLI (activate/exec/destroy; autospawns minvmd) # The SAME unified proof every target lane runs (scripts/session-e2e.sh) # against Deployment Model 3 (native Linux + VM — the target this From cb3dcbeb80d778dbfc05fd62e2ac17b9cda1dd72 Mon Sep 17 00:00:00 2001 From: Norrie Taylor Date: Thu, 9 Jul 2026 22:58:39 -0700 Subject: [PATCH 5/5] ci: cache discipline, slim CI debug info, and a --locked sweep Three cross-lane changes that must fork every cache exactly once: - rust-cache gains shared-key (one per job class that compiles the same artifacts: workspace/tests/clippy/native-e2e/mac-unit) and save-if restricted to main, via new setup-rust inputs forwarded from core-tests. PR branches restore but never write, so PR churn stops LRU-evicting the main caches every PR restores from (the 10 GB pool is shared repo-wide). - the two multi-GB raw actions/cache users (KVM build-linux, macOS artifacts stage2) split into cache/restore + main-only cache/save guarded on cache-hit. - CARGO_PROFILE_DEV_DEBUG=line-tables-only in CI (setup-rust env step; explicit job env on the mac and KVM jobs that bypass the composite): usable backtraces, smaller target trees and caches, faster links - without a named Cargo profile, which would move output out of target/debug and break codesign/testbed/justfile paths. - --locked on every workflow cargo invocation (build/test/clippy/ archive), so a stale Cargo.lock fails loudly instead of silently re-resolving; previously only core-tests' cargo fetch enforced it. Refs: #687 Co-Authored-By: Claude Fable 5 --- .github/actions/core-tests/action.yml | 12 +++++-- .github/actions/setup-rust/action.yml | 31 ++++++++++++++++-- .github/workflows/ci-linux-kvm.yml | 29 ++++++++++++++--- .github/workflows/ci-linux-native.yml | 6 +++- .github/workflows/ci-macos.yml | 46 ++++++++++++++++++++++----- .github/workflows/ci.yml | 9 ++++-- 6 files changed, 113 insertions(+), 20 deletions(-) diff --git a/.github/actions/core-tests/action.yml b/.github/actions/core-tests/action.yml index e3f7890782..71f48eb45f 100644 --- a/.github/actions/core-tests/action.yml +++ b/.github/actions/core-tests/action.yml @@ -5,11 +5,19 @@ description: > proofs behind non-default features (e.g. networking-proxy, networking-wg) are NOT run here — invoke those explicitly in the calling workflow. +inputs: + shared-key: + description: rust-cache shared-key, forwarded to setup-rust. + required: false + default: tests + runs: using: composite steps: - name: Setup Rust uses: ./.github/actions/setup-rust + with: + shared-key: ${{ inputs.shared-key }} - name: Install nextest uses: taiki-e/install-action@v2 with: @@ -21,7 +29,7 @@ runs: shell: bash # --no-tests=fail: an empty selection is a broken filter or a dropped # target, not a pass. - run: cargo nextest run --workspace --profile ci --no-tests=fail + run: cargo nextest run --workspace --locked --profile ci --no-tests=fail - name: Run doctests shell: bash - run: cargo test --workspace --doc + run: cargo test --workspace --doc --locked diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index d023a72032..0c5b00495f 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -1,8 +1,24 @@ name: Setup Rust description: > Common Rust CI preamble for GitHub-hosted Ubuntu runners: free disk space, - install protoc, and restore the cargo registry/build cache. Assumes the - repository is already checked out. + install protoc, slim CI debug info, and restore the cargo registry/build + cache. Assumes the repository is already checked out. + +inputs: + shared-key: + description: > + rust-cache shared-key. One key per job CLASS that compiles the same + artifacts (same packages, features, profile); jobs that compile + differently must not share a key, or they thrash each other's cache. + required: false + default: workspace + save-if: + description: > + Whether this job may WRITE the cache. Defaults to main only: PR + branches restore but never save, so PR churn cannot LRU-evict the + main-branch caches every PR wants to restore from. + required: false + default: ${{ github.ref == 'refs/heads/main' }} runs: using: composite @@ -17,5 +33,16 @@ runs: - name: Install protoc shell: bash run: sudo sh -c 'apt-get update && apt-get install -y protobuf-compiler' + - name: Slim debug info for CI builds + # line-tables-only keeps usable backtraces while shrinking target/ (and + # every cache built from it) and speeding linking. An env var rather + # than a named Cargo profile: a profile would move output to + # target// and break every hardcoded target/debug path (codesign + # steps, testbeds, PATH exports, justfile). + shell: bash + run: echo "CARGO_PROFILE_DEV_DEBUG=line-tables-only" >> "$GITHUB_ENV" - name: Cache cargo registry and build artifacts uses: Swatinem/rust-cache@v2 + with: + shared-key: ${{ inputs.shared-key }} + save-if: ${{ inputs.save-if }} diff --git a/.github/workflows/ci-linux-kvm.yml b/.github/workflows/ci-linux-kvm.yml index a608670176..e1d9e53d24 100644 --- a/.github/workflows/ci-linux-kvm.yml +++ b/.github/workflows/ci-linux-kvm.yml @@ -77,6 +77,11 @@ jobs: if: github.event_name == 'workflow_dispatch' || needs.changes.outputs.kvm == 'true' runs-on: ubuntu-latest timeout-minutes: 45 + env: + # Slim debug info (usable backtraces, smaller caches, faster links); + # this job doesn't run the setup-rust composite that sets this on the + # other Linux lanes. + CARGO_PROFILE_DEV_DEBUG: line-tables-only steps: - name: Free Disk Space # The workspace build + libkrun prefix routinely exceed the ~14 GB @@ -114,7 +119,9 @@ jobs: tool: cross,nextest - name: Cache cargo + build artifacts - uses: actions/cache@v6 + # Restore-only; the save half (main only) runs at the end of the job. + id: cargo-cache + uses: actions/cache/restore@v6 with: path: | ~/.cargo/bin/ @@ -139,7 +146,7 @@ jobs: run: ./scripts/build-initramfs.sh "$RUNNER_TEMP/initramfs.cpio" x86_64-unknown-linux-musl - name: Build minvmd - run: cargo build -p minvmd --bin minvmd + run: cargo build -p minvmd --bin minvmd --locked - name: Build the minimal CLI # SEPARATE invocation from the minvmd build: a combined `cargo build @@ -147,7 +154,7 @@ jobs: # into the CLI's default-features=false opt-out, linking libkrun into # `minimal` (the regression the macOS lane hit and now guards). The # CLI drives the unified session e2e in the test job. - run: cargo build -p minimal --bin minimal + run: cargo build -p minimal --bin minimal --locked - name: Archive e2e harnesses + stage the testbed # `cargo nextest archive` replaces the hand-rolled testbins.json @@ -160,7 +167,7 @@ jobs: run: | set -euo pipefail mkdir -p testbed - cargo nextest archive -p minvmd --archive-file testbed/nextest-archive.tar.zst + cargo nextest archive -p minvmd --locked --archive-file testbed/nextest-archive.tar.zst cp target/debug/minvmd testbed/minvmd cp target/debug/minimal testbed/minimal cp "$RUNNER_TEMP/initramfs.cpio" testbed/initramfs.cpio @@ -175,6 +182,20 @@ jobs: if-no-files-found: error retention-days: 3 + - name: Save cargo + build artifacts cache (main only) + # PR branches restore but never write: multi-GB caches written by + # every branch LRU-evict the main entries all PRs restore from. + if: github.ref == 'refs/heads/main' && steps.cargo-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ steps.cargo-cache.outputs.cache-primary-key }} + test-kvm: # TEST phase: no Rust toolchain, no cargo build — download the testbed, # pull the guest images, and run the archived harnesses with the diff --git a/.github/workflows/ci-linux-native.yml b/.github/workflows/ci-linux-native.yml index 4fb68d32b8..f6c338e834 100644 --- a/.github/workflows/ci-linux-native.yml +++ b/.github/workflows/ci-linux-native.yml @@ -104,6 +104,10 @@ jobs: with: persist-credentials: false - uses: ./.github/actions/setup-rust + with: + # `-p` builds unify features differently than the workspace test + # suite — own cache class. + shared-key: native-e2e - name: Enable unprivileged user namespaces # The session sandbox (hakoniwa) needs userns; Ubuntu 24.04 restricts # them via AppArmor by default. @@ -120,7 +124,7 @@ jobs: key: native-e2e-minimal-cache-${{ hashFiles('.minimal/minimal.toml') }} restore-keys: native-e2e-minimal-cache- - name: Build minimal + minimald - run: cargo build -p minimald --bin minimald -p minimal --bin minimal + run: cargo build -p minimald --bin minimald -p minimal --bin minimal --locked - name: Session e2e (native minimald) run: | export PATH="$PWD/target/debug:$PATH" diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 05e69d32ba..bf2fdbb6d9 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -77,8 +77,10 @@ jobs: - name: Cache cargo + build artifacts (incl. the aarch64 cross target) # Raw actions/cache (not Swatinem) — it preserves the full # target/aarch64-unknown-linux-musl tree, so the initramfs cross-compile - # is incremental after the first run. - uses: actions/cache@v6 + # is incremental after the first run. Restore-only here; the save + # half (main only) runs at the end of the job. + id: stage2-cache + uses: actions/cache/restore@v6 with: path: | ~/.cargo/bin/ @@ -129,6 +131,19 @@ jobs: name: minimald-initramfs-aarch64 path: ${{ runner.temp }}/initramfs.cpio if-no-files-found: error + - name: Save cargo + build artifacts cache (main only) + # PR branches restore but never write: multi-GB caches written by + # every branch LRU-evict the main entries all PRs restore from. + if: github.ref == 'refs/heads/main' && steps.stage2-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ steps.stage2-cache.outputs.cache-primary-key }} e2e: # Hypervisor E2E on real hardware, using the cache-pulled kernel + rootfs @@ -154,6 +169,10 @@ jobs: needs: [artifacts] runs-on: [self-hosted, macOS, ARM64] timeout-minutes: 30 + env: + # Slim debug info; see the unit job. First run after this change + # rebuilds the mini's persistent target dir once (new fingerprints). + CARGO_PROFILE_DEV_DEBUG: line-tables-only steps: - uses: actions/checkout@v7 with: @@ -187,8 +206,8 @@ jobs: # package bins for integration tests) and bakes its absolute # CARGO_BIN_EXE path — same-machine build+run, so it resolves. run: | - cargo test -p minvmd --test krun_smoke --test minimald_session_e2e --no-run - cargo build -p minvmd --bin minvmd + cargo test -p minvmd --test krun_smoke --test minimald_session_e2e --no-run --locked + cargo build -p minvmd --bin minvmd --locked # SEPARATE invocation for the CLI (mirrors release.yml): built # together, cargo unifies minvmd's default `libkrun` feature # into minimal's default-features=false opt-out, and the CLI @@ -196,7 +215,7 @@ jobs: # name loaded from anywhere); our @rpath dylib turns it into # dyld "Library not loaded" at autospawn, since minimal bakes # no rpaths. - cargo build -p minimal --bin minimal + cargo build -p minimal --bin minimal --locked - name: Verify minimal links only system libraries # The CLI spawns minvmd as a subprocess and calls no libkrun; a # krun load command means the feature unification above regressed. @@ -314,6 +333,11 @@ jobs: # mac-buildable. Kept lean (private repo: macOS minutes bill at 10x). runs-on: macos-latest timeout-minutes: 45 + env: + # Slim debug info (usable backtraces, smaller caches, faster + # links); the mac jobs don't run the setup-rust composite that + # sets this on the Linux lanes. + CARGO_PROFILE_DEV_DEBUG: line-tables-only steps: - uses: actions/checkout@v7 with: @@ -324,12 +348,18 @@ jobs: components: clippy - name: Cache cargo registry and build artifacts uses: Swatinem/rust-cache@v2 + with: + shared-key: mac-unit + # PR branches restore but never save, so PR churn cannot + # LRU-evict the main-branch caches (same policy as the + # setup-rust composite). + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Provision libkrun (pinned own build) uses: ./.github/actions/setup-libkrun-macos - name: Clippy (minvmd) - run: cargo clippy -p minvmd --all-targets -- -D warnings + run: cargo clippy -p minvmd --all-targets --locked -- -D warnings - name: Test (minvmd) - run: cargo test -p minvmd + run: cargo test -p minvmd --locked - name: Test (sessions) # Platform-sensitive surface: std::fs::canonicalize behavior on # prefix-symlinked roots (/tmp → /private/tmp on macOS) and the @@ -341,4 +371,4 @@ jobs: # (`cargo build -p minimal` is fine: no dev-deps.) The CLI's macOS # coverage stays the autospawn e2e until minimald's sandbox deps # are cfg(linux)-gated. - run: cargo test -p sessions + run: cargo test -p sessions --locked diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5c0a785a8..4b40b95230 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,8 +42,11 @@ jobs: steps: - uses: actions/checkout@v7 - uses: ./.github/actions/setup-rust + with: + # check artifacts, not build artifacts — own cache class + shared-key: clippy - name: Run Clippy - run: cargo clippy --workspace --all-targets -- -D warnings + run: cargo clippy --workspace --all-targets --locked -- -D warnings test: runs-on: ubuntu-latest @@ -57,14 +60,14 @@ jobs: # feature, so the core-tests `--workspace` run does not compile or run # its proofs (R4.5 mtls_missing_cert_returns_401_with_no_topology, UC2b # mtls_valid_cert_routes_to_backend). Exercise them explicitly. - run: cargo nextest run -p minimald --features networking-proxy + run: cargo nextest run -p minimald --features networking-proxy --locked - name: Run WireGuard mesh tests # The WireGuard mesh peer is behind the non-default `networking-wg` # feature, so the core-tests `--workspace` run does not compile or run # its proofs. Exercise them explicitly (R-WG: # two_meshes_handshake_and_relay_a_packet, rpc get_mesh_status). The # `#[ignore]` two-namespace mesh_uc7 proof runs in the netns lane. - run: cargo test -p minimald --features networking-wg + run: cargo test -p minimald --features networking-wg --locked dogfood: runs-on: ubuntu-latest