Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 117 additions & 22 deletions crates/minimald/src/box_manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -686,8 +686,14 @@ impl BoxManagerHandle {
/// in the same step, so a reap or a resume that lands between the two
/// cannot leave the name with no row while a box with it is retained.
/// A failure here does not fail the reap: the daemon log names it.
/// Answers with the box that took the row, or `None` when no box did.
#[cfg(target_os = "linux")]
async fn hand_over_name(&self, reaped: SessionId, reaped_box: SessionId, name: &str) {
async fn hand_over_name(
&self,
reaped: SessionId,
reaped_box: SessionId,
name: &str,
) -> Option<SessionId> {
let (send, recv) = Responder::channel();
// A send error means the manager is gone: the recv fails as well.
let _ = self
Expand All @@ -701,7 +707,7 @@ impl BoxManagerHandle {
send,
))
.await;
let _ = recv.await;
recv.await.ok().and_then(Result::ok).flatten()
}

/// Reaps every `stopped` or `exited` box that matches all the given
Expand Down Expand Up @@ -1012,7 +1018,7 @@ enum ManagerMessage {
/// removed. A no-op for the successor when another box took the row
/// since, or when a resume of a box with that name is in flight.
#[cfg(target_os = "linux")]
HandOverName(NameHandOver, Responder<()>),
HandOverName(NameHandOver, Responder<Option<SessionId>>),
Shutdown(bool, Responder<Result<(), ()>>),
/// Bring up the actor of every `Active` record whose box owns a name,
/// so a restarted daemon routes the names of the sessions it restored
Expand Down Expand Up @@ -3659,7 +3665,7 @@ impl BoxManager {
}
// Deletes a session: see `delete_box`.
ManagerMessage::DeleteSession(id, only_if_unfinalized, r) => {
self.delete_box(id, only_if_unfinalized, r).await;
self.destroy_box(id, only_if_unfinalized, r).await;
}
// Reaps a stopped or exited box (BOX-019): the state check and
// the delete's claim run in this one message, so no resume
Expand All @@ -3671,8 +3677,9 @@ impl BoxManager {
},
#[cfg(target_os = "linux")]
ManagerMessage::HandOverName(hand_over, r) => {
self.hand_over_reaped_name(hand_over).await;
r.handle(async { Ok::<_, BoxManagerError>(()) }).await;
let successor = self.hand_over_reaped_name(hand_over).await;
r.handle(async move { Ok::<_, BoxManagerError>(successor) })
.await;
}
ManagerMessage::Shutdown(force, r) => {
r.handle(async {
Expand Down Expand Up @@ -4162,6 +4169,91 @@ impl BoxManager {
}
}

/// The destroy that [`ManagerMessage::DeleteSession`] asks for: as
/// [`Self::delete_box`], and then, when the destroyed box held its
/// name's row, the row goes to the next retained box with that name, as
/// a reap hands it over (BOX-019, NET-012). The destroy answers after
/// both. The `destroy` span records the box that took the row as
/// `successor`, or `none`.
async fn destroy_box(
&mut self,
id: SessionId,
only_if_unfinalized: bool,
r: Responder<Option<Vec<String>>>,
) {
let span = tracing::info_span!(
"destroy",
session_id = %id,
successor = tracing::field::Empty,
);
#[cfg(target_os = "linux")]
let held = self.held_name_of_box(id).await;
#[cfg(not(target_os = "linux"))]
let held: Option<(SessionId, String)> = None;
let Some((destroyed_box, name)) = held else {
span.record("successor", tracing::field::display("none"));
tracing::Instrument::instrument(self.delete_box(id, only_if_unfinalized, r), span)
.await;
return;
};
let (send, recv) = Responder::channel();
tracing::Instrument::instrument(
self.delete_box(id, only_if_unfinalized, send),
span.clone(),
)
.await;
let manager = self.weak_self.clone();
// Not on `teardowns`: the handover is a message to this manager, and
// a shutdown that waits on `teardowns` reads no message meanwhile.
crate::traced::spawn(tracing::Instrument::instrument(
async move {
let answer = recv.await.unwrap_or_else(|_| {
Err(std::io::Error::new(
std::io::ErrorKind::ConnectionRefused,
minimald_rpc::SHUTDOWN_REFUSAL,
))
});
// Only a destroyed box gives up its row: `Ok(None)` is a box
// that was kept, and an error left the box as it is.
let destroyed = matches!(answer, Ok(Some(_)));
let successor = match manager.upgrade() {
#[cfg(target_os = "linux")]
Some(manager) if destroyed => {
manager.hand_over_name(id, destroyed_box, &name).await
}
_ => None,
};
let shown = successor.map_or_else(|| "none".to_string(), |s| s.to_string());
tracing::Span::current().record("successor", tracing::field::display(&shown));
if destroyed {
tracing::info!(
box_id = %destroyed_box,
session_name = &name,
successor = %shown,
"destroy: the destroyed box held the name's row; \
the successor holds it now (none: no box does)"
);
}
r.handle(async { answer }).await;
},
span,
));
}

/// The `box_id` and name of box `id`, when the box holds that name's
/// row in the hostname registry.
#[cfg(target_os = "linux")]
async fn held_name_of_box(&self, id: SessionId) -> Option<(SessionId, String)> {
let handle = self.store.find(RecordPredicate::Id(id)).await.ok()??;
let record = handle.record().await.ok()?;
let name = crate::box_actor::registry_name(&record);
self.hostnames
.read()
.expect("hostname registry lock poisoned")
.name_held_by(id, &name)
.then(|| (record.box_id.unwrap_or(id), name))
}

/// The name of box `id`, when the box holds that name's row in the
/// hostname registry.
#[cfg(target_os = "linux")]
Expand All @@ -4181,9 +4273,10 @@ impl BoxManager {
/// without an actor leaves it, so it is withdrawn here. Then the
/// successor is picked among the retained boxes that are not being
/// removed, and takes the row. Both steps run in one message, so no
/// other message of the manager lands between them.
/// other message of the manager lands between them. Answers with the
/// box that took the row, or `None` when no box did.
#[cfg(target_os = "linux")]
async fn hand_over_reaped_name(&mut self, hand_over: NameHandOver) {
async fn hand_over_reaped_name(&mut self, hand_over: NameHandOver) -> Option<SessionId> {
let NameHandOver {
reaped,
reaped_box,
Expand Down Expand Up @@ -4229,26 +4322,27 @@ impl BoxManager {
%error,
"reap: the records cannot be read; the name is not handed over"
);
return;
return None;
}
let Some(successor) = name_successor(&records, reaped, &name) else {
tracing::info!(
box_id = %reaped_box,
session_name = &name,
"reap: no retained box has the name; it answers nothing"
);
return;
return None;
};
self.hand_over_name(successor).await;
self.hand_over_name(successor).await.then_some(successor)
}

/// Registers the name row of the retained box `successor`, after a
/// reap removed the box that held it (NET-012). A box with a live actor
/// is registered here, and a stopped or exited one is marked stopped
/// (NET-128). A box with no actor gets one, and its bring-up registers
/// the name as a resume's bring-up does.
/// the name as a resume's bring-up does. Answers whether `successor`
/// holds the row afterwards.
#[cfg(target_os = "linux")]
async fn hand_over_name(&mut self, successor: SessionId) {
async fn hand_over_name(&mut self, successor: SessionId) -> bool {
use boxes::core::record::RecordState;
let removing = self
.tearing_down
Expand All @@ -4260,18 +4354,18 @@ impl BoxManager {
session_id = %successor,
"reap: the next holder of the name is being removed; the name is not handed over"
);
return;
return false;
}
let handle = match self.store.find(RecordPredicate::Id(successor)).await {
Ok(Some(handle)) => handle,
Ok(None) => return,
Ok(None) => return false,
Err(error) => {
tracing::warn!(
session_id = %successor,
%error,
"reap: the next holder of the name cannot be found; the name is not handed over"
);
return;
return false;
}
};
let record = match handle.record().await {
Expand All @@ -4283,7 +4377,7 @@ impl BoxManager {
"reap: the record of the next holder of the name cannot be read; \
the name is not handed over"
);
return;
return false;
}
};
let name = crate::box_actor::registry_name(&record);
Expand All @@ -4297,15 +4391,15 @@ impl BoxManager {
session_name = &name,
"reap: a resume of a box with the name is in flight; it registers the name"
);
return;
return false;
}
{
let registry = self
.hostnames
.read()
.expect("hostname registry lock poisoned");
if registry.name_held_by(successor, &name) {
return;
return true;
}
if let Some(owner) = registry.hostname_held_by_another(successor, &name) {
tracing::info!(
Expand All @@ -4314,7 +4408,7 @@ impl BoxManager {
owner,
"reap: another box took the name since the reap; the name is not handed over"
);
return;
return false;
}
}
let ended = matches!(
Expand Down Expand Up @@ -4347,7 +4441,7 @@ impl BoxManager {
"reap: the next holder of the name has no published address; \
the name is not handed over"
);
return;
return false;
}
if ended {
registry.mark_stopped(successor);
Expand All @@ -4364,7 +4458,7 @@ impl BoxManager {
"reap: the next holder of the name did not come up; \
the name is not handed over"
);
return;
return false;
}
}
}
Expand All @@ -4376,6 +4470,7 @@ impl BoxManager {
action = "name-handed-over",
"reap: the name now answers for the next box that holds it"
);
true
}
}

Expand Down
63 changes: 63 additions & 0 deletions crates/minimald/src/box_manager/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4757,6 +4757,69 @@ async fn reap_hands_name_to_retained_holder() {
);
}

/// BOX-019, NET-012, NET-128: a destroy of the box that holds a name's row
/// hands the row to the next retained box with that name, as a reap does.
/// The destroy returns after the handover. A destroy of a box that does not
/// hold the row leaves the row where it is. Once the last holder is
/// destroyed, the name answers nothing. The daemon log names the destroyed
/// box and the successor.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn destroy_hands_name_to_retained_holder() {
use boxes::core::record::RecordState;

let log = crate::test_harness::captured_log();
let (_state, _cache, mngr) = manager().await;

// `first` stops. A running `dev` beside it holds the row.
let first = host_net_box(&mngr, "dev").await;
stop(&mngr, first).await;
let second = host_net_box(&mngr, "dev").await;
assert_eq!(row_of(&mngr, "dev", &[first, second]), Some(second));

// The destroy of the running holder hands the row to the stopped box.
mngr.delete_session(second).await.unwrap();
assert!(
mngr.get_record(BoxKeyPredicate::Id(second))
.await
.unwrap()
.is_none(),
"the destroyed record is gone when the destroy returns"
);
assert_eq!(row_of(&mngr, "dev", &[first, second]), Some(first));
assert_eq!(answers_to(&mngr, "dev").await, Some(first));
assert!(matches!(
state_of(&mngr, first).await,
RecordState::Stopped(_)
));

// A stopped `third` takes the row. The destroy of `first`, which does
// not hold it, leaves the row with `third`.
let third = host_net_box(&mngr, "dev").await;
stop(&mngr, third).await;
assert_eq!(row_of(&mngr, "dev", &[first, third]), Some(third));
mngr.delete_session(first).await.unwrap();
assert_eq!(row_of(&mngr, "dev", &[first, third]), Some(third));

// The last holder goes: the name answers nothing.
mngr.delete_session(third).await.unwrap();
assert_eq!(row_of(&mngr, "dev", &[first, second, third]), None);
assert_eq!(answers_to(&mngr, "dev").await, None);

let logged = log.contents();
assert!(
logged.lines().any(|line| line.contains("destroy:")
&& line.contains(&second.to_string())
&& line.contains(&format!("successor={first}"))),
"the daemon log names the destroyed box and its successor:\n{logged}"
);
assert!(
logged.lines().any(|line| line.contains("destroy:")
&& line.contains(&third.to_string())
&& line.contains("successor=none")),
"the daemon log says no box took the row:\n{logged}"
);
}

/// BOX-019, BOX-021, NET-012: a reap returns only once the reaped box's
/// teardown and the handover of its name are both done, under contention.
/// The removal of another box with the name is held in its teardown while
Expand Down
Loading