diff --git a/crates/minimald/src/box_manager.rs b/crates/minimald/src/box_manager.rs index fe89f335e..0c95d26f3 100644 --- a/crates/minimald/src/box_manager.rs +++ b/crates/minimald/src/box_manager.rs @@ -686,8 +686,14 @@ impl BoxManagerHandle { /// in the same step, so a reap or a resume that lands between the two /// cannot leave the name with no row while a box with it is retained. /// A failure here does not fail the reap: the daemon log names it. + /// Answers with the box that took the row, or `None` when no box did. #[cfg(target_os = "linux")] - async fn hand_over_name(&self, reaped: SessionId, reaped_box: SessionId, name: &str) { + async fn hand_over_name( + &self, + reaped: SessionId, + reaped_box: SessionId, + name: &str, + ) -> Option { let (send, recv) = Responder::channel(); // A send error means the manager is gone: the recv fails as well. let _ = self @@ -701,7 +707,7 @@ impl BoxManagerHandle { send, )) .await; - let _ = recv.await; + recv.await.ok().and_then(Result::ok).flatten() } /// Reaps every `stopped` or `exited` box that matches all the given @@ -1012,7 +1018,7 @@ enum ManagerMessage { /// removed. A no-op for the successor when another box took the row /// since, or when a resume of a box with that name is in flight. #[cfg(target_os = "linux")] - HandOverName(NameHandOver, Responder<()>), + HandOverName(NameHandOver, Responder>), Shutdown(bool, Responder>), /// Bring up the actor of every `Active` record whose box owns a name, /// so a restarted daemon routes the names of the sessions it restored @@ -3659,7 +3665,7 @@ impl BoxManager { } // Deletes a session: see `delete_box`. ManagerMessage::DeleteSession(id, only_if_unfinalized, r) => { - self.delete_box(id, only_if_unfinalized, r).await; + self.destroy_box(id, only_if_unfinalized, r).await; } // Reaps a stopped or exited box (BOX-019): the state check and // the delete's claim run in this one message, so no resume @@ -3671,8 +3677,9 @@ impl BoxManager { }, #[cfg(target_os = "linux")] ManagerMessage::HandOverName(hand_over, r) => { - self.hand_over_reaped_name(hand_over).await; - r.handle(async { Ok::<_, BoxManagerError>(()) }).await; + let successor = self.hand_over_reaped_name(hand_over).await; + r.handle(async move { Ok::<_, BoxManagerError>(successor) }) + .await; } ManagerMessage::Shutdown(force, r) => { r.handle(async { @@ -4162,6 +4169,91 @@ impl BoxManager { } } + /// The destroy that [`ManagerMessage::DeleteSession`] asks for: as + /// [`Self::delete_box`], and then, when the destroyed box held its + /// name's row, the row goes to the next retained box with that name, as + /// a reap hands it over (BOX-019, NET-012). The destroy answers after + /// both. The `destroy` span records the box that took the row as + /// `successor`, or `none`. + async fn destroy_box( + &mut self, + id: SessionId, + only_if_unfinalized: bool, + r: Responder>>, + ) { + let span = tracing::info_span!( + "destroy", + session_id = %id, + successor = tracing::field::Empty, + ); + #[cfg(target_os = "linux")] + let held = self.held_name_of_box(id).await; + #[cfg(not(target_os = "linux"))] + let held: Option<(SessionId, String)> = None; + let Some((destroyed_box, name)) = held else { + span.record("successor", tracing::field::display("none")); + tracing::Instrument::instrument(self.delete_box(id, only_if_unfinalized, r), span) + .await; + return; + }; + let (send, recv) = Responder::channel(); + tracing::Instrument::instrument( + self.delete_box(id, only_if_unfinalized, send), + span.clone(), + ) + .await; + let manager = self.weak_self.clone(); + // Not on `teardowns`: the handover is a message to this manager, and + // a shutdown that waits on `teardowns` reads no message meanwhile. + crate::traced::spawn(tracing::Instrument::instrument( + async move { + let answer = recv.await.unwrap_or_else(|_| { + Err(std::io::Error::new( + std::io::ErrorKind::ConnectionRefused, + minimald_rpc::SHUTDOWN_REFUSAL, + )) + }); + // Only a destroyed box gives up its row: `Ok(None)` is a box + // that was kept, and an error left the box as it is. + let destroyed = matches!(answer, Ok(Some(_))); + let successor = match manager.upgrade() { + #[cfg(target_os = "linux")] + Some(manager) if destroyed => { + manager.hand_over_name(id, destroyed_box, &name).await + } + _ => None, + }; + let shown = successor.map_or_else(|| "none".to_string(), |s| s.to_string()); + tracing::Span::current().record("successor", tracing::field::display(&shown)); + if destroyed { + tracing::info!( + box_id = %destroyed_box, + session_name = &name, + successor = %shown, + "destroy: the destroyed box held the name's row; \ + the successor holds it now (none: no box does)" + ); + } + r.handle(async { answer }).await; + }, + span, + )); + } + + /// The `box_id` and name of box `id`, when the box holds that name's + /// row in the hostname registry. + #[cfg(target_os = "linux")] + async fn held_name_of_box(&self, id: SessionId) -> Option<(SessionId, String)> { + let handle = self.store.find(RecordPredicate::Id(id)).await.ok()??; + let record = handle.record().await.ok()?; + let name = crate::box_actor::registry_name(&record); + self.hostnames + .read() + .expect("hostname registry lock poisoned") + .name_held_by(id, &name) + .then(|| (record.box_id.unwrap_or(id), name)) + } + /// The name of box `id`, when the box holds that name's row in the /// hostname registry. #[cfg(target_os = "linux")] @@ -4181,9 +4273,10 @@ impl BoxManager { /// without an actor leaves it, so it is withdrawn here. Then the /// successor is picked among the retained boxes that are not being /// removed, and takes the row. Both steps run in one message, so no - /// other message of the manager lands between them. + /// other message of the manager lands between them. Answers with the + /// box that took the row, or `None` when no box did. #[cfg(target_os = "linux")] - async fn hand_over_reaped_name(&mut self, hand_over: NameHandOver) { + async fn hand_over_reaped_name(&mut self, hand_over: NameHandOver) -> Option { let NameHandOver { reaped, reaped_box, @@ -4229,7 +4322,7 @@ impl BoxManager { %error, "reap: the records cannot be read; the name is not handed over" ); - return; + return None; } let Some(successor) = name_successor(&records, reaped, &name) else { tracing::info!( @@ -4237,18 +4330,19 @@ impl BoxManager { session_name = &name, "reap: no retained box has the name; it answers nothing" ); - return; + return None; }; - self.hand_over_name(successor).await; + self.hand_over_name(successor).await.then_some(successor) } /// Registers the name row of the retained box `successor`, after a /// reap removed the box that held it (NET-012). A box with a live actor /// is registered here, and a stopped or exited one is marked stopped /// (NET-128). A box with no actor gets one, and its bring-up registers - /// the name as a resume's bring-up does. + /// the name as a resume's bring-up does. Answers whether `successor` + /// holds the row afterwards. #[cfg(target_os = "linux")] - async fn hand_over_name(&mut self, successor: SessionId) { + async fn hand_over_name(&mut self, successor: SessionId) -> bool { use boxes::core::record::RecordState; let removing = self .tearing_down @@ -4260,18 +4354,18 @@ impl BoxManager { session_id = %successor, "reap: the next holder of the name is being removed; the name is not handed over" ); - return; + return false; } let handle = match self.store.find(RecordPredicate::Id(successor)).await { Ok(Some(handle)) => handle, - Ok(None) => return, + Ok(None) => return false, Err(error) => { tracing::warn!( session_id = %successor, %error, "reap: the next holder of the name cannot be found; the name is not handed over" ); - return; + return false; } }; let record = match handle.record().await { @@ -4283,7 +4377,7 @@ impl BoxManager { "reap: the record of the next holder of the name cannot be read; \ the name is not handed over" ); - return; + return false; } }; let name = crate::box_actor::registry_name(&record); @@ -4297,7 +4391,7 @@ impl BoxManager { session_name = &name, "reap: a resume of a box with the name is in flight; it registers the name" ); - return; + return false; } { let registry = self @@ -4305,7 +4399,7 @@ impl BoxManager { .read() .expect("hostname registry lock poisoned"); if registry.name_held_by(successor, &name) { - return; + return true; } if let Some(owner) = registry.hostname_held_by_another(successor, &name) { tracing::info!( @@ -4314,7 +4408,7 @@ impl BoxManager { owner, "reap: another box took the name since the reap; the name is not handed over" ); - return; + return false; } } let ended = matches!( @@ -4347,7 +4441,7 @@ impl BoxManager { "reap: the next holder of the name has no published address; \ the name is not handed over" ); - return; + return false; } if ended { registry.mark_stopped(successor); @@ -4364,7 +4458,7 @@ impl BoxManager { "reap: the next holder of the name did not come up; \ the name is not handed over" ); - return; + return false; } } } @@ -4376,6 +4470,7 @@ impl BoxManager { action = "name-handed-over", "reap: the name now answers for the next box that holds it" ); + true } } diff --git a/crates/minimald/src/box_manager/tests.rs b/crates/minimald/src/box_manager/tests.rs index bb104a327..4e353c2b0 100644 --- a/crates/minimald/src/box_manager/tests.rs +++ b/crates/minimald/src/box_manager/tests.rs @@ -4757,6 +4757,69 @@ async fn reap_hands_name_to_retained_holder() { ); } +/// BOX-019, NET-012, NET-128: a destroy of the box that holds a name's row +/// hands the row to the next retained box with that name, as a reap does. +/// The destroy returns after the handover. A destroy of a box that does not +/// hold the row leaves the row where it is. Once the last holder is +/// destroyed, the name answers nothing. The daemon log names the destroyed +/// box and the successor. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn destroy_hands_name_to_retained_holder() { + use boxes::core::record::RecordState; + + let log = crate::test_harness::captured_log(); + let (_state, _cache, mngr) = manager().await; + + // `first` stops. A running `dev` beside it holds the row. + let first = host_net_box(&mngr, "dev").await; + stop(&mngr, first).await; + let second = host_net_box(&mngr, "dev").await; + assert_eq!(row_of(&mngr, "dev", &[first, second]), Some(second)); + + // The destroy of the running holder hands the row to the stopped box. + mngr.delete_session(second).await.unwrap(); + assert!( + mngr.get_record(BoxKeyPredicate::Id(second)) + .await + .unwrap() + .is_none(), + "the destroyed record is gone when the destroy returns" + ); + assert_eq!(row_of(&mngr, "dev", &[first, second]), Some(first)); + assert_eq!(answers_to(&mngr, "dev").await, Some(first)); + assert!(matches!( + state_of(&mngr, first).await, + RecordState::Stopped(_) + )); + + // A stopped `third` takes the row. The destroy of `first`, which does + // not hold it, leaves the row with `third`. + let third = host_net_box(&mngr, "dev").await; + stop(&mngr, third).await; + assert_eq!(row_of(&mngr, "dev", &[first, third]), Some(third)); + mngr.delete_session(first).await.unwrap(); + assert_eq!(row_of(&mngr, "dev", &[first, third]), Some(third)); + + // The last holder goes: the name answers nothing. + mngr.delete_session(third).await.unwrap(); + assert_eq!(row_of(&mngr, "dev", &[first, second, third]), None); + assert_eq!(answers_to(&mngr, "dev").await, None); + + let logged = log.contents(); + assert!( + logged.lines().any(|line| line.contains("destroy:") + && line.contains(&second.to_string()) + && line.contains(&format!("successor={first}"))), + "the daemon log names the destroyed box and its successor:\n{logged}" + ); + assert!( + logged.lines().any(|line| line.contains("destroy:") + && line.contains(&third.to_string()) + && line.contains("successor=none")), + "the daemon log says no box took the row:\n{logged}" + ); +} + /// BOX-019, BOX-021, NET-012: a reap returns only once the reaped box's /// teardown and the handover of its name are both done, under contention. /// The removal of another box with the name is held in its teardown while