Repository navigation
Expand file tree
/
Copy pathmain.rs
More file actions
1878 lines (1764 loc) · 83.1 KB
/
Copy pathmain.rs
File metadata and controls
1878 lines (1764 loc) · 83.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
//! The minimal daemon, an SSH server which hosts sessions and
//! task/sandbox executions within them.
#![recursion_limit = "256"]
use clap::{Args, CommandFactory, Parser, Subcommand};
use clap_complete::Shell;
use paths::{CwdRelative, Daemon, DaemonAbsPath, sub_path};
use std::io::Write as _;
use tokio::{net::UnixListener, runtime::Builder};
use minimald::server::{Config, HostKey, Server};
mod logging;
use logging::{DaemonLogger, LogMode};
#[cfg(target_os = "linux")]
use tokio_vsock::{VMADDR_CID_ANY, VsockAddr, VsockListener};
/// Base AF_VSOCK port the guest relay listens on when in vsock mode.
///
/// The actual port is `DEFAULT_VSOCK_PORT_BASE` + `instance_num`.
#[cfg(target_os = "linux")]
const DEFAULT_VSOCK_PORT_BASE: u32 = 2222;
/// Receive window installed on the guest's vsock listener, in bytes.
///
/// `virtio_transport_inc_rx_pkt` refuses an incoming packet — resetting the
/// connection and setting `sk_err` to `ENOBUFS` — when either the peer overruns
/// its credit (`buf_used + len > buf_alloc`) or the queued socket-buffer
/// overhead does (`(queue_len + 1) * SKB_TRUESIZE(0) > buf_alloc`). At the
/// 256 KiB default window the second ceiling lands at 455 queued skbs on arm64,
/// where `SKB_TRUESIZE(0)` is 576 bytes. libkrun clamps each read to the credit
/// still outstanding, so under receiver backpressure its packets shrink to
/// roughly 540 bytes — below that per-skb overhead — and a workspace upload
/// trips the overhead ceiling with kilobytes of window still unused.
///
/// 8 MiB is empirical headroom, not immunity. It does not change the ratio
/// between the two ceilings; it keeps a normal upload out of the credit-starved
/// tail where packets degenerate, and a larger upload or a slower reader can
/// still reach it. The real fix belongs in libkrun, which should not shrink a
/// packet below the per-skb overhead it costs to queue; this is the
/// consumer-side mitigation.
#[cfg(target_os = "linux")]
const VSOCK_RX_WINDOW_BYTES: u64 = 8 * 1024 * 1024;
/// Env var `spawn_detached` sets on the child so `async_main` knows
/// its stdio has been redirected to `/dev/null` and needs to swap
/// the tracing writer over to a rolling log file.
const DETACHED_ENV: &str = "MINIMALD_DETACHED";
#[derive(Parser)]
#[command(name = "minimald", version = version::VERSION, long_version = version::LONG_VERSION)]
#[command(about = "The Minimal daemon")]
struct Cli {
#[command(subcommand)]
command: Command,
#[command(flatten)]
global_args: GlobalArgs,
}
impl Cli {
/// Returns the path to the minimal-dir (base directory for state)
/// based on command-line arguments.
pub fn minimal_state_dir(&self) -> DaemonAbsPath {
match &self.global_args.minimal_state_dir {
Some(p) => p
.resolve()
.expect("could not resolve --minimal-state-dir against the current directory"),
None => paths::minimal_state_dir(),
}
}
/// Returns the path to base directory for caching
/// based on command-line arguments.
pub fn minimal_cache_dir(&self) -> DaemonAbsPath {
match &self.global_args.minimal_cache_dir {
Some(p) => p
.resolve()
.expect("could not resolve --minimal-cache-dir against the current directory"),
None => paths::minimal_cache_dir(),
}
}
fn listen_args(&self) -> Option<&ListenArgs> {
match &self.command {
Command::Run(a) => Some(a),
_ => None,
}
}
fn instance_num(&self) -> u32 {
self.listen_args().map(|a| a.instance_num).unwrap_or(0)
}
/// Returns the path to the directory containing sockets/info about this daemon for clients.
pub fn client_instance_dir(&self) -> DaemonAbsPath {
paths::host_dir(
&self.minimal_state_dir(),
paths::HostKind::Minimald,
self.instance_num(),
)
}
/// Returns fragments of the command-line arguments which should be passed to an ssh invocation in
/// order to connect to the UDS socket.
///
/// The first argument is a list of SSH options and their values, the second is the name of
/// the ssh server.
pub fn ssh_args(&self) -> (Vec<(&'static str, String)>, String) {
(
vec![
(
"ProxyCommand",
if cfg!(target_os = "macos") {
format!("nc -U {}", self.listen_on())
} else {
format!("socat - UNIX-CONNECT:{}", self.listen_on())
},
),
(
"UserKnownHostsFile",
sub_path!(self.client_instance_dir(), "known_hosts")
.as_utf8_path()
.to_string(),
),
],
paths::host_name(paths::HostKind::Minimald, self.instance_num()),
)
}
/// Returns the path to the UDS socket we should listen on.
pub fn listen_on(&self) -> DaemonAbsPath {
self.client_instance_dir()
.sub_path_unchecked(paths::SSH_SOCK_FILE)
}
}
#[derive(Subcommand)]
enum Command {
/// Generate shell completion script
#[command(
long_about = "Generate a shell tab-completion script for the minimal daemon.\nSupported shells include bash, zsh, elvish and fish.\n\n source <(minimald completions bash)"
)]
Completions(CompletionsArgs),
/// Runs the minimald server in the foreground.
Run(ListenArgs),
/// Internal: join a session's namespaces and run a program there.
///
/// Not a supported command line — this is the daemon re-execing itself to
/// power launching a process in existing namespaces.
#[command(name = minimald::nsenter::SUBCOMMAND, hide = true)]
Nsenter(minimald::nsenter::ShimArgs),
}
/// The arguments for the completions subcommand.
#[derive(Debug, clap::Args)]
struct CompletionsArgs {
/// The shell type for a CLI completion script should be printed
#[arg(value_parser)]
shell: Shell,
}
/// Shared arguments for all subcommands.
#[derive(Debug, Args)]
pub struct GlobalArgs {
/// Override the directory where state is stored (default: $XDG_STATE_DIR/minimal)
#[arg(long, alias = "minimal_dir")]
minimal_state_dir: Option<CwdRelative<Daemon>>,
/// Override the directory where artifacts are cached (default: $XDG_CACHE_DIR/minimal)
#[arg(long)]
minimal_cache_dir: Option<CwdRelative<Daemon>>,
/// Load the minimal standard library from the given path instead
#[arg(long)]
#[clap(hide = true)]
stdlib_dir: Option<CwdRelative<Daemon>>,
/// Configure the number of parallel builds
#[arg(short, long, global = true)]
#[clap(hide = true)]
num_parallel_builds: Option<usize>,
}
/// Arguments describing where minimald should listen for connections.
#[derive(Debug, Args)]
pub struct ListenArgs {
/// Instance number for this minimald; determines client-relevant paths under
/// `<minimal_state_dir>/providers/local-minimald<instance-num>`.
///
/// The SSH socket is accessible as `ssh.sock`.
#[arg(long, default_value_t = 0)]
instance_num: u32,
/// Host the SSH socket over vsock instead of UDS.
///
/// The vsock port will be `DEFAULT_VSOCK_PORT_BASE` + `instance_num`.
#[arg(long, default_value_t = false)]
vsock: bool,
/// Mount `/dev`. Only useful if minimald is a VM's init process.
#[arg(long, default_value_t = false)]
#[clap(hide = true)]
mount_dev: bool,
/// Mounts the given device as the rootfs and pivot to it. This also mounts
/// standard puesdo-filesystems in the / including proc, sys, dev, and run.
#[arg(long)]
#[clap(hide = true)]
mount_rootfs: Option<String>,
/// Device to format-on-first-boot + mount as the writable state volume at
/// `/var/lib/minimal` when running as a microVM init (R1.5/R1.6). When set
/// and the mount succeeds, cache + state are relocated onto it. Only useful
/// as a VM init process; `None` leaves state on the tmpfs default.
#[arg(long)]
#[clap(hide = true)]
mk_mount_state_volume: Option<String>,
/// Raise `RLIMIT_NOFILE`, soft and hard, to this many descriptors, which
/// everything the daemon forks inherits. Only useful as a VM init process,
/// where nothing else widens the kernel's 1024-fd default; `None` keeps the
/// limits we were started with.
#[arg(long)]
#[clap(hide = true)]
rlimit_nofile: Option<u64>,
/// Vsock port to listen on for host time updates: 8-byte little-endian
/// nanoseconds-since-epoch stamps, dialed in by the host half in `minvmd`
/// (see `guest::run_timekeep_listener`). Only useful as a VM init process;
/// `None` leaves the guest clock free-running.
#[arg(long)]
#[clap(hide = true)]
timekeep_listener_port: Option<u32>,
/// Port the host-side hostname proxy must listen on, when this deployment
/// pins one — the port clients point `HTTP(S)_PROXY` at, whose documented
/// default is 7654. Unset (the default) tries that default first and only
/// when it is busy asks the OS for a free port, which the daemon reports
/// wherever a client needs it: `min ls` prints it, and a second daemon on
/// the same machine gets its own port instead of silently losing hostname
/// routing. A pinned port that is busy stays a hard failure — the
/// operator named it, and moving the listener would hide the loss.
#[arg(long)]
hostname_proxy_port: Option<u16>,
/// Port the machine's box-zone answerer serves on (UDP), when this
/// deployment pins one — the port the host's resolver is pointed at to
/// answer `*.min.internal`, whose documented default is 7656. On a
/// native host the daemon is first a client of the installed
/// `minzoned` service (its rows publish over the machine-global
/// channel and it hosts nothing); the port is the hook port it hosts
/// the single-operator interim on while no service serves, and
/// unlike the hostname proxy there is no select-when-busy for it: the
/// address is the one the host's resolver is routed to, so a held
/// hook port is a surfaced error, never a move.
#[arg(long)]
zone_answerer_port: Option<u16>,
/// Daemonize: spawn minimald in a new session (setsid) and return once the
/// SSH socket accepts connections, or an 8s timeout elapses. Used by the
/// `min` CLI to auto-start a native daemon on Linux.
#[arg(long, default_value_t = false)]
detach: bool,
/// Path to the gvproxy ("gvisor-tap-vsock") binary backing the per-host
/// `OwnIp` switch. Defaults to the installed location when unset: the
/// user-local `bin/gvproxy-min` the installer stamps, else the system
/// install path. Point it at a local build to run own-IP without an
/// install.
#[arg(long)]
gvproxy_bin: Option<std::path::PathBuf>,
/// Keep the earlier allow-all egress default for a box that declares no
/// `egress` section (NET-077). With the deny-all default in force (see
/// [`boxes::EGRESS_DEFAULT_PHASE`]), an own-address box created with
/// no egress declaration reaches nothing outside itself (NET-074) and
/// shows `deny-all (default)` in `min session policy` (NET-075). Opt out
/// to keep the prior default — a deployment that cannot carry the change
/// yet — and retire the flag once yours declares its boxes' egress. A
/// box that declares its own egress section is unaffected either way.
/// `MINIMALD_EGRESS_DENY_ALL_OPT_OUT=1` in the environment opts out too,
/// and survives the restarts `min` makes without the flag.
#[arg(long, default_value_t = false)]
egress_deny_all_opt_out: bool,
}
/// The environment fallback for `run --egress-deny-all-opt-out` (NET-077).
/// `min` auto-starts the native daemon with no flags of its own, and the
/// spawned daemon inherits the environment it was started from, so an
/// operator who exports this keeps the opt-out across every restart `min`
/// makes — the native counterpart of `MINVMD_EGRESS_DENY_ALL_OPT_OUT`. The
/// spelling is the boot token a VM host hands its guest daemon
/// ([`minimald::guest::HANDED_EGRESS_DENY_ALL_OPT_OUT_TOKEN`]), so the one name means
/// the one thing on either side.
const EGRESS_DENY_ALL_OPT_OUT_ENV: &str = minimald::guest::HANDED_EGRESS_DENY_ALL_OPT_OUT_TOKEN;
/// The daemon's deny-all opt-out (NET-077): the `run` flag, or the
/// [`EGRESS_DENY_ALL_OPT_OUT_ENV`] value `env` read through the one parse
/// every opt-out reader shares — `1`, `true`, `yes` or `on`; anything else,
/// or unset, keeps the deny-all default.
fn egress_deny_all_opt_out(flag: bool, env: Option<&str>) -> bool {
flag || boxes::egress_deny_all_opt_out_from_raw(env)
}
/// An error at the top level of minimald.
#[derive(Debug)]
pub enum MainError {
IO(std::io::Error, &'static str),
Other(String),
}
impl From<russh::keys::ssh_key::Error> for MainError {
fn from(value: russh::keys::ssh_key::Error) -> Self {
Self::Other(format!("ssh key: {value}"))
}
}
impl From<russh::keys::Error> for MainError {
fn from(value: russh::keys::Error) -> Self {
Self::Other(format!("ssh key: {value}"))
}
}
/// Flattens an error and its `source()` chain into one line, so a typed error
/// reaching [`MainError::Other`] keeps the context its variants carry.
fn error_chain(err: &dyn std::error::Error) -> String {
std::iter::successors(err.source(), |e| e.source()).fold(err.to_string(), |mut acc, e| {
acc.push_str(&format!(": {e}"));
acc
})
}
/// The `guest.ready` span: the microVM guest's READY beacon, a child of the
/// `TRACEPARENT` minvmd put on the kernel command line (which the kernel
/// hands `/init` as its environment) when telemetry is on. Its ids are
/// recorded for the file log the way every sibling span's are
/// (`exec::adopt_trace`): the exported span's when the export records it,
/// otherwise a child of the host's context, otherwise fresh ones.
fn guest_ready_span() -> tracing::Span {
use minimald_rpc::trace::{TRACEPARENT_ENV, TraceContext};
let span = tracing::info_span!(
"guest.ready",
trace_id = tracing::field::Empty,
span_id = tracing::field::Empty,
parent_span_id = tracing::field::Empty,
);
let parent = mlog::otel::exporting()
.then(|| std::env::var(TRACEPARENT_ENV).ok())
.flatten()
.and_then(|v| TraceContext::parse_traceparent(&v));
let ctx = mlog::otel::align(&span, parent.as_ref().map(TraceContext::parts))
.map(|(t, s, f)| TraceContext::from_parts(t, s, f))
.unwrap_or_else(|| {
parent
.as_ref()
.map(TraceContext::child)
.unwrap_or_else(TraceContext::mint)
});
span.record("trace_id", tracing::field::display(ctx.trace_id_hex()));
span.record("span_id", tracing::field::display(ctx.span_id_hex()));
if let Some(parent) = &parent {
span.record(
"parent_span_id",
tracing::field::display(parent.span_id_hex()),
);
}
span
}
/// Open (creating if absent) a lock file; only its fd matters, for flock.
fn open_lock_file(path: impl AsRef<std::path::Path>) -> std::io::Result<std::fs::File> {
std::fs::OpenOptions::new()
.create(true)
.read(true)
.write(true)
.truncate(false)
.open(path)
}
fn main() -> Result<(), MainError> {
// We re-exec to power some machinery for joining namespaces, do that before async/other-threads
// are setup.
if std::env::args_os()
.nth(1)
.is_some_and(|arg| arg == minimald::nsenter::SUBCOMMAND)
{
let Command::Nsenter(args) = Cli::parse().command else {
unreachable!(
"argv[1] is {}, so clap parsed that subcommand",
minimald::nsenter::SUBCOMMAND
)
};
let code =
minimald::nsenter::shim_main(args).map_err(|e| MainError::Other(error_chain(&e)))?;
std::process::exit(code);
}
// As the microVM's init, fork the daemon off before anything else, while
// this process is still single-threaded: pid 1 stays behind and reaps
// the orphans every process in the guest is reparented to, which the
// daemon cannot do without stealing its own children's exit statuses
// (see `minimald::reaper`). Returns in the daemon only.
#[cfg(target_os = "linux")]
if is_minimal_microvm() {
minimald::reaper::split_init();
}
let runtime = Builder::new_multi_thread()
.thread_name("minimald-worker")
.thread_stack_size(8 * 1024 * 1024)
.enable_all()
.build()
.unwrap();
// The microVM's own egress lives outside the server runtime, so the
// final telemetry flush below still has a network; see [`GuestEgress`].
let mut egress = GuestEgress::new(is_minimal_microvm());
let result = runtime.block_on(async_main(&mut egress));
// Drop the tasks still running when `async_main` returns (the connection
// that sent `Shutdown`, a session reaper) before the final telemetry
// flush: their spans end as they are dropped, and must reach the exporter,
// or every span they parented is an orphan in its trace (the last stray
// `sessions.manager.message` roots: `Shutdown` and the reaper's
// `GetRecord`). Bounded, so a stuck task cannot hold the exit.
runtime.shutdown_timeout(std::time::Duration::from_secs(2));
mlog::otel::shutdown(std::time::Duration::from_secs(5));
// The guest's forwarded records (TEL-034): give the sender a moment to
// write the spans that just ended to the host before the VM goes down.
#[cfg(target_os = "linux")]
if is_minimal_microvm()
&& !minimald::telemetry_forward::drain(std::time::Duration::from_millis(500))
{
tracing::warn!(
"telemetry forward: records still queued at shutdown; the host spool keeps what arrived"
);
}
// Only now, with the flush done (or given up on), take the egress down.
egress.tear_down();
// As the microVM's daemon we must not return: exiting init panics the
// guest kernel and wedges the VM (#730), and when the daemon runs as pid
// 1's fork, pid 1 would only take the VM down after it, without this
// log line. Take the VM down here instead — a clean
// shutdown (the `Shutdown` RPC drained the server) and a failed one alike,
// since either way there is no init left to run. Diverges on success.
#[cfg(target_os = "linux")]
if is_minimal_microvm() {
match &result {
Ok(()) => tracing::info!("microVM init finished; shutting the VM down"),
Err(e) => tracing::error!(error = ?e, "microVM init failed; shutting the VM down"),
}
let error = minimald::guest::shut_down_vm();
// Unreachable in practice — `reboot(2)` only fails for a caller without
// CAP_SYS_BOOT, and the microVM's pid-1 has it. But falling through to
// `return result` would exit init and panic the guest kernel, which is
// the wedge #730 is about; never trade one wedge for another. Park
// instead, as the boot path's degraded arms do: the kernel stays alive
// and idle (no panic-handler spin), the console keeps working, and
// `minvmd stop`'s SIGTERM can still reap the VMM.
tracing::error!(%error, "shutting the VM down failed; parking pid-1 (exiting it would panic the guest kernel)");
loop {
std::thread::sleep(std::time::Duration::from_secs(3600));
}
}
result
}
/// Re-exec this binary in a new session (`setsid`) with `--detach` stripped, so
/// the child runs the foreground server fully detached from the caller's
/// controlling terminal, then poll the SSH socket until it accepts connections.
/// Mirrors `minvmd run --detach`.
fn spawn_detached(cli: &Cli) -> Result<(), MainError> {
use std::os::unix::process::CommandExt as _;
use std::time::{Duration, Instant};
// The child may first wait out a departing daemon's lock
// (`SHUTDOWN_DEPARTURE_WAIT`) before its own startup.
const DETACH_TIMEOUT_SECS: u64 = 8 + minimald_rpc::SHUTDOWN_DEPARTURE_WAIT.as_secs();
let exe = std::env::current_exe().map_err(|e| MainError::IO(e, "resolving current exe"))?;
let args: Vec<String> = std::env::args()
.skip(1)
.filter(|a| a != "--detach")
.collect();
let mut cmd = std::process::Command::new(&exe);
cmd.args(&args)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
// Mark the child so `async_main` knows its stdio has been
// null'd and can route tracing output to a log file instead.
.env(DETACHED_ENV, "1");
// SAFETY: setsid() is async-signal-safe. In the child it starts a new
// session so the daemon outlives the CLI and is unaffected by SIGHUP when
// the invoking shell exits.
unsafe {
cmd.pre_exec(|| {
if libc::setsid() == -1 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
let mut child = cmd
.spawn()
.map_err(|e| MainError::IO(e, "spawning detached minimald"))?;
// Ready = socket connectable AND a native minimald holds the instance
// lock. ssh.sock is shared with the minvmd bridge, so a bare connect can
// be satisfied by a live peer backend while our child's bail goes to
// /dev/null. A child exit surfaces as an error instead of a timeout.
let sock = cli.listen_on();
let sock_path = std::path::Path::new(sock.as_utf8_path().as_str());
let lock_path = cli
.client_instance_dir()
.sub_path_unchecked(paths::MINIMALD_LOCK_FILE);
let deadline = Instant::now() + Duration::from_secs(DETACH_TIMEOUT_SECS);
loop {
if let Some(status) = child
.try_wait()
.map_err(|e| MainError::IO(e, "polling detached minimald"))?
{
return Err(MainError::Other(format!(
"detached minimald exited during startup ({status}); \
run without --detach to see the error"
)));
}
if std::os::unix::net::UnixStream::connect(sock_path).is_ok()
&& lock_held(lock_path.as_utf8_path().as_std_path())
.map_err(|e| MainError::IO(e, "probing instance lock"))?
{
return Ok(());
}
if Instant::now() >= deadline {
return Err(MainError::Other(format!(
"detached minimald did not start listening on {sock} within {DETACH_TIMEOUT_SECS}s"
)));
}
std::thread::sleep(Duration::from_millis(100));
}
}
/// Wait, bounded by `wait`, while the instance lock at `lock_path` is held by
/// a daemon on its way out: one whose socket at `sock_path` refuses a
/// connection (or is gone), because a departing daemon closes its listener
/// before it drains. Returns at once when nothing holds the lock, and when
/// the socket accepts: that holder is a live daemon, and the caller's own
/// lock attempt refuses to start beside it. Returns when the wait runs out
/// too; the caller's lock attempt then reports the holder.
async fn wait_for_departing_daemon(
lock_path: &std::path::Path,
sock_path: &std::path::Path,
wait: std::time::Duration,
) -> std::io::Result<()> {
let deadline = tokio::time::Instant::now() + wait;
let mut announced = false;
while lock_held(lock_path)? {
match tokio::net::UnixStream::connect(sock_path).await {
Err(e)
if matches!(
e.kind(),
std::io::ErrorKind::ConnectionRefused | std::io::ErrorKind::NotFound
) => {}
// Accepted, or an error that proves nothing: not a departing daemon.
_ => return Ok(()),
}
if tokio::time::Instant::now() >= deadline {
return Ok(());
}
if !announced {
tracing::info!(
wait = ?wait,
"the instance lock is held by a daemon that has stopped listening; \
waiting for it to exit"
);
announced = true;
}
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
}
Ok(())
}
/// Whether some process holds an exclusive advisory lock on `path`.
/// Read-only probe: a missing file means no holder.
fn lock_held(path: &std::path::Path) -> std::io::Result<bool> {
let file = match std::fs::File::open(path) {
Ok(f) => f,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(e) => return Err(e),
};
match fd_lock::RwLock::new(file).try_read() {
Ok(_guard) => Ok(false),
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => Ok(true),
Err(e) => Err(e),
}
}
/// The microVM daemon's own egress: the root-netns tap relayed to the host
/// gvproxy over the vsock shuttle ([`minimald::guest::bring_up_root_egress`]).
///
/// `main` owns it, not the server future, and its relay runs on a small
/// runtime of its own. `main` shuts the server runtime down before the
/// bounded telemetry flush (so the spans of the tasks it drops end and are
/// exported), and a relay held by the server future, or running on that
/// runtime, would be torn down with it: the guest's final flush would then
/// have no network. [`GuestEgress::tear_down`] runs after the flush.
struct GuestEgress {
/// The relay's runtime: `None` off the microVM, and the build error when
/// it could not be built. Either way the relay then runs on the server
/// runtime, as it did before it had one of its own.
runtime: Option<std::io::Result<tokio::runtime::Runtime>>,
/// The relay, once [`GuestEgress::bring_up`] succeeded. Dropping it
/// aborts the relay's tasks.
relay: Option<minimald::net::switch::SwitchRelay>,
}
impl GuestEgress {
/// Prepares the egress for the microVM's init (`in_microvm`) by building
/// the relay's runtime. Brings nothing up yet: `async_main` does, once
/// the guest is ready.
fn new(in_microvm: bool) -> Self {
let runtime = in_microvm.then(|| {
Builder::new_multi_thread()
.worker_threads(1)
.thread_name("minimald-egress")
.enable_all()
.build()
});
Self {
runtime,
relay: None,
}
}
/// Brings the egress up and holds it until [`GuestEgress::tear_down`].
/// Best effort: if the host gvproxy is absent the daemon serves without
/// network, the prior behaviour.
async fn bring_up(&mut self) {
use minimald::guest::bring_up_root_egress;
let relay = match &self.runtime {
Some(Ok(runtime)) => match runtime
.spawn(tracing::Instrument::in_current_span(bring_up_root_egress()))
.await
{
Ok(relay) => relay,
Err(join) => Err(std::io::Error::other(join)),
},
Some(Err(e)) => {
tracing::warn!(
error = %e,
"no runtime of its own for the guest root egress; \
the final telemetry flush will go without network"
);
bring_up_root_egress().await
}
None => bring_up_root_egress().await,
};
match relay {
Ok(relay) => self.relay = Some(relay),
Err(e) => {
tracing::warn!(error = %e, "guest root egress unavailable; serving without network");
}
}
}
/// Takes the egress down: aborts the relay's tasks, then drops its
/// runtime without waiting (the relay keeps no state worth draining).
/// Call outside any runtime, after the final telemetry flush.
fn tear_down(self) {
let Self { runtime, relay } = self;
drop(relay);
if let Some(Ok(runtime)) = runtime {
runtime.shutdown_background();
}
}
}
async fn async_main(egress: &mut GuestEgress) -> Result<(), MainError> {
// Use hardcoded configuration if we are the init process (`argv[0] == "/init"`), which
// would indicate we are operating in a single-purpose micro-vm.
//
// If we are not the init process, we load our config from CLI args.
let cli = if is_minimal_microvm() {
Cli {
command: Command::Run(ListenArgs {
instance_num: 0,
vsock: true,
mount_dev: true,
mount_rootfs: Some("/dev/vda".to_string()),
mk_mount_state_volume: Some("/dev/vdb".to_string()),
detach: false,
// No service manager above pid-1 to widen the kernel default.
rlimit_nofile: Some(guest::DEFAULT_MICROVM_NOFILE_LIMIT),
// Host time updates arrive on a vsock stream we listen on, from
// minvmd. Always listen: the guest cannot see what the
// host runs.
timekeep_listener_port: Some(guest::TIMEKEEP_PORT),
// In-VM (DM1/3/4) the PTask attaches to the host gvproxy over the
// vsock shuttle, so no in-guest gvproxy binary path is needed.
gvproxy_bin: None,
// The node ports are the host's, not the daemon's: minvmd
// assigns them before the VM boots (NET-025) and hands them
// on the kernel command line, so the guest binds exactly
// what the host's box table already admits (NET-138) and the
// host gvproxy publishes it at the same number. Binds as
// handed and selects none; a boot that carries no tokens — an
// older minvmd, a native run — falls back to default-then-
// select, the pre-handoff behaviour. The tokens are read
// once the log sink is live, below, because a token present
// but unusable is a surfaced boot error, which `Cli`
// construction has nowhere to raise.
hostname_proxy_port: None,
zone_answerer_port: None,
// The microVM's pid-1 has no flags to read, but the host
// hands it the operator's opt-out on the boot line (NET-077):
// the guest runs the egress default its host was started with
// — the rollout phase [`boxes::EGRESS_DEFAULT_PHASE`]
// carries — unless the operator opted out.
egress_deny_all_opt_out: guest::handed_egress_deny_all_opt_out(),
}),
global_args: GlobalArgs {
minimal_state_dir: Some(DaemonAbsPath::try_new("/run/minimal").unwrap().into()),
minimal_cache_dir: Some(
DaemonAbsPath::try_new("/run/minimal/cache").unwrap().into(),
),
num_parallel_builds: None,
stdlib_dir: None,
},
}
} else {
Cli::parse()
};
// Handle non-{launch,run} commands.
let mut cli = match cli.command {
Command::Nsenter(_) => unreachable!("nsenter is intercepted before CLI args are parsed"),
Command::Completions(CompletionsArgs { shell }) => {
let mut cmd = Cli::command();
let name = cmd.get_name().to_string();
clap_complete::generate(shell, &mut cmd, name, &mut std::io::stdout());
return Ok(());
}
_ => cli,
};
// Install tracing. A foreground run logs to stdout only. A detached
// native daemon (stdio null'd, marked by `MINIMALD_DETACHED`) and the
// microVM pid-1 both log to a daily-rotated file, wired up by
// `logger.activate` once the log directory is final (below): immediately
// for the native daemon, after the state volume mounts for the microVM.
// The activation yields a release the server state runs at shutdown.
let log_mode = if is_minimal_microvm() || std::env::var_os(DETACHED_ENV).is_some() {
LogMode::File
} else {
LogMode::Console
};
let logger = DaemonLogger::install(log_mode)?;
// The guest's telemetry sender (TEL-034): started once telemetry is
// initialised (in the logger), a no-op unless the boot line named a
// forward destination.
#[cfg(target_os = "linux")]
if is_minimal_microvm() {
minimald::telemetry_forward::start();
}
let listen_args = cli.listen_args().unwrap();
// The node ports this daemon listens on, resolved once: the hostname
// proxy's port from the token the VM host handed on the boot line in a
// microVM, the CLI's otherwise. The read lives here, once the log sink
// is live, because a token present but unusable is a surfaced boot
// failure, not a fallback: the daemon would otherwise publish a
// listener the host's box table does not name (NET-025, NET-138) and
// strand every client pointed at the handed one. A handed port that
// cannot bind — something in the guest already holds it — fails the
// boot here too, probed in the bind base the daemon's own listeners
// use, rather than surfacing after READY, when the host already
// believes the VM healthy.
//
// The answerer's half is `None` on a VM boot by construction, not by
// read: a VM-hosted daemon starts no zone answerer — the VM host
// daemon's host answerer owns the zone (NET-138) — so the boot line
// carries no answerer token and no port exists to report.
let (hostname_proxy_port, zone_answerer_port) = if is_minimal_microvm() {
let proxy = guest::handed_proxy_port().map_err(|e| MainError::Other(e.to_string()))?;
guest::probe_handed_node_port(proxy)
.map_err(|e| MainError::IO(e, "binding the handed node port"))?;
// The egress opt-out was read off the boot line once, at `Cli`
// construction; say what the guest runs with, so a diag bundle shows
// whether the deny-all default was opted out (NET-077).
tracing::info!(
egress_deny_all_opt_out = listen_args.egress_deny_all_opt_out,
"guest egress deny-all opt-out from the boot line"
);
(proxy, None)
} else {
(
listen_args.hostname_proxy_port,
listen_args.zone_answerer_port,
)
};
// Daemonize before doing any work: re-exec ourselves in a new session and
// wait until the SSH socket is accepting connections, then return so the
// caller (the `minimal` CLI autospawn) gets a clean ready/timeout result.
if listen_args.detach {
// `spawn_detached` polls `cli.listen_on()` (a UDS) for readiness, but a
// `--vsock` child binds the vsock listener instead, so the UDS never
// appears and the parent would always hit the 8s timeout while leaving a
// detached child running. Reject the combination up front.
if listen_args.vsock {
return Err(MainError::Other(
"--detach is only supported for Unix-socket listeners (not --vsock)".to_string(),
));
}
return spawn_detached(&cli);
}
// Pin the path this daemon re-execs itself from as the `__nsenter` shim,
// now, while the file is certainly still there. Resolved per-spawn instead,
// `current_exe()` reads `/proc/self/exe`, which the kernel reports as a
// dangling `<path> (deleted)` once the binary has been replaced — so every
// lifecycle hook and in-session exec fails with `ENOENT` after any rebuild
// of a running daemon. `just min` rebuilds `-p minimald` on every
// invocation, so a dev daemon meets that the first time its source changes.
// The path resolved here keeps naming whatever now lives at it.
//
// Not in the microVM: pid-1's own path is the initramfs `/init`, which is
// unreachable after the rootfs switch, so the guest stages a runnable copy
// and registers *that* (#1175). Registration is first-wins, so claiming it
// here would lock the guest's out.
if !is_minimal_microvm() {
match std::env::current_exe() {
Ok(exe) => minimald::nsenter::set_shim_exe(exe),
// Non-fatal: without a registration each injection falls back to
// resolving `current_exe()` itself, which is the prior behaviour.
Err(e) => tracing::warn!(
error = %e,
"could not resolve this daemon's executable to register as the nsenter shim; \
in-session exec will re-resolve it per spawn",
),
}
}
// Handle setup specific to operating in a micro-vm.
use minimald::guest;
// Before anything forks from us and inherits the limits. Best effort: the
// kernel default is the prior behaviour, not a reason to refuse to boot.
if let Some(limit) = listen_args.rlimit_nofile {
match guest::raise_nofile_limit(limit) {
Ok(effective) if effective >= limit => {
tracing::debug!(nofile = effective, "raised the open-file limit");
}
Ok(effective) => tracing::warn!(
requested = limit,
effective,
"the open-file limit could not be raised as far as requested; fd-hungry builds \
may fail with EMFILE"
),
Err(e) => tracing::warn!(
error = %e,
requested = limit,
"could not raise the open-file limit; fd-hungry builds may fail with EMFILE"
),
}
}
if listen_args.mount_dev {
guest::mount_dev();
}
if let Some(root_dev) = &listen_args.mount_rootfs
&& let Err(e) = guest::enter_rootfs(root_dev)
{
tracing::warn!(error = %e, "no rootfs disk; initramfs READY-only");
guest::mount_pseudo_filesystems();
let _ = guest::emit_simple_ready_marker().await;
loop {
tokio::time::sleep(std::time::Duration::from_secs(3600)).await;
}
}
// VM-wide kernel settings every session on the VM shares, before any box
// starts. Never on a native daemon: those are the host's to set.
if is_minimal_microvm() {
guest::apply_microvm_sysctls();
}
// NET-079: the daemon's own classifier leaf, entered at start so its own
// traffic is decided as the daemon's (NET-080), never classed with a
// box's, and so the daemon sits in a *sibling* of every box leaf — never
// above them, where a controller it enabled could make the box leaves
// domain invalid (spike finding F). Each box then joins its own leaf in
// the sandbox's pre-exec closure and unshares its cgroup namespace onto
// it, which is what keeps a box out of every other leaf on the tree (see
// `sandbox::classifier`).
//
// In the guest this is pid 1, which mounts cgroup2 itself
// (`guest::enter_rootfs`) and so builds the tree here; a tree it could
// not build is a broken image, not a deployment state, and
// `box_runtime` refuses a host-address box on it (design §7.1).
// Natively the installer installs the tree and delegates it to this
// account, and entering it is a migration whose common ancestor this
// daemon can write only from *inside* the tree: the installer's `--pid`
// step or a `Delegate=yes` unit places it. A daemon left outside keeps
// running and places no box — `create_session_leaf` decides that per
// launch, by migrating a throwaway child, so no box is ever spawned into
// a join it dies making.
let tree_root = std::path::Path::new(sandbox::classifier::TREE_ROOT);
if let Err(e) = sandbox::classifier::enter_daemon_leaf(tree_root) {
if guest::is_microvm_daemon() {
tracing::error!(
error = %e,
tree = sandbox::classifier::TREE_ROOT,
"entering the daemon's own classifier leaf: this guest image \
cannot decide per box, and host-address boxes will be refused"
);
} else {
tracing::warn!(
error = %e,
tree = sandbox::classifier::TREE_ROOT,
daemon_cgroup = ?sandbox::classifier::own_cgroup_path(),
install = %sandbox::classifier::install_hint(),
"entering the daemon's own classifier leaf failed, so every \
box launch now decides its placement by whether it can \
migrate into the tree: enter it with the installer's --pid \
step, or start the daemon from a Delegate=yes unit"
);
}
}
// NET-079: the empty leaves a daemon death left behind are swept here, at
// the next start, so a leaf named by a session's id is never mistaken for
// a leftover its next launch could take: a fresh launch finding one
// reports the collision. A leaf that still holds a session refuses its
// own removal and stays, which is the emptiness the sweep tests by.
match sandbox::classifier::sweep_box_leaves(tree_root) {
Ok(swept) if swept.is_empty() => {}
Ok(swept) => tracing::info!(
count = swept.len(),
leaves = ?swept
.iter()
.map(|leaf| leaf.display().to_string())
.collect::<Vec<_>>(),
"swept the empty classifier leaves a previous daemon left behind"
),
Err(e) => tracing::debug!(
error = %e,
"sweeping the box cohort at start"
),
}
// NET-079: the guest's own classifier boot, after the tree above exists
// and before the start-up decision reads it — the one host whose table this
// daemon loads itself, because its kernel is its image's alone and no
// person can run an installer inside a microVM. [`boot_guest_classifier`]
// runs it in the one order its halves can keep: loopback up first —
// nothing else in the boot has brought `lo` up by this point, and the
// probe's listener on 127.0.0.1 is a bind on an address a guest without
// its loopback up does not carry — then the listeners the effect probe
// connects to, held for the daemon's life because the probe's evidence
// is a port that is refused whenever a launch looks, not one the probe
// brings with it, then the load itself: the installer's table rendered
// for the tree this daemon just entered its own leaf of, checked with
// `nft -c`, loaded in one transaction, the presence marker written only
// once it loaded. The load logs its own outcome — check line and load
// line with the digest of the exact bytes it piped — and nothing here
// branches on it: the start-up decision after the bind
// (`read_host_ip_enforcement_fact`) is the fact's own reader, and a
// boot whose load failed reports the guest as unable to decide per box
// exactly as a native host without the step does.
if guest::is_microvm_daemon() {
// NET-078's two source identities are the guest's own address on a
// guest — the cohort and the node plane share it — but the render is
// told them as the two inputs they are, never left to assume.
let identity = std::net::IpAddr::V4(minimald::net::DEFAULT_SUBNET.daemon_ip());
let render = minimald::net::classifier::GuestRender {
tree_root,
// The listener the deny subtree's DNS-port lookups are dstnat'd
// onto (NET-079): the guest's loopback answerer, which refuses
// every name outside the box zone and forwards the zone's A
// questions to the switch gateway alone.
answerer_address: minimald::net::guest_dns::LISTENER_ADDRESS,
answerer_port: minimald::net::guest_dns::LISTENER_PORT,
cohort_address: identity,
node_plane_address: identity,
ct_mark_mask: minimald::net::classifier::GUEST_CT_MARK_MASK,
// BEP-083: the proxy's listener, whose address the table refuses
// to every box whose own leaf is not in the lane set.
proxy_listener: std::net::SocketAddrV4::new(
minimald::net::DEFAULT_SUBNET.box_egress_proxy_address(),
boxes::core::egress::PROXY_LISTENER_PORT,
),
mountinfo_override: None,
};
minimald::net::classifier::boot_guest_classifier(
guest::bring_up_loopback,
minimald::net::classifier::hold_probe_listeners,
|| {
minimald::net::classifier::load_guest_table(
std::path::Path::new(minimald::net::classifier::GUEST_BASH),