From deca8d2eaf82a93b665953282ff9b273bf2f4d18 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Tue, 6 Oct 2026 21:40:09 -0400 Subject: [PATCH 1/2] fix: recover exited and unlaunched bridges with dead owners --- docs/guides/native-candidate.md | 15 + packages/runtime-core/README.md | 13 + .../src/provider/graph/bridges.rs | 25 +- .../src/provider/graph/bridges/cleanup.rs | 5 +- .../graph/bridges/cleanup/normalization.rs | 390 ++++++++++++++++++ .../native_test/same_boot_absent_relay.rs | 10 +- .../bridge_normalization.rs | 273 ++++++++++++ .../src/provider/graph/live_owner_cleanup.rs | 10 + .../provider/graph/relay-normalize-observe.sh | 126 ++++++ .../runtime-core/src/provider/graph/relay.rs | 152 +++++++ .../tests/relay_fence_contract.rs | 69 ++++ 11 files changed, 1084 insertions(+), 4 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/bridges/cleanup/normalization.rs create mode 100644 packages/runtime-core/src/provider/graph/foreground/native_test/same_boot_absent_relay/bridge_normalization.rs create mode 100644 packages/runtime-core/src/provider/graph/relay-normalize-observe.sh diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 294c42a0a..585c4b3a6 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -1365,6 +1365,21 @@ the live relay acknowledgement protocol. When a restored publisher is admitted, older absence-retirement records remain validated history; the exact current cleanup proof supplies retention authority. +Before the main recovery intent, the operation journals and normalizes owned +bridge reservations that never launched (`reserved` with no relay intent) or whose +exact reservation-v1 helper has exited. The journal binds the original receipt, +boot, owner witnesses and complete same-run bridge inventory. It resumes confirmed +stop/removal interruptions, including canonical pending fence writes, without +removing a replacement socket, helper, allocation or reservation. A helper may +remove its own socket on exit; its exact process identity and retained private +socket receipt remain required. An originally live, unchanged bridge that exits +before main intent enrollment can be added to the journal with fresh exit proof. +Starting reservations, unknown or partial metadata, added assignments and replaced +identities refuse. Live helpers are left for the existing main recovery path; +other graphs are not selected. The normalizer does not run on an existing main +intent retry. Completed normalization journals are generation-bound history after +an independently verified retained restore. + After completion, retained restore can create a fresh owner; `graph retire-recovered-publisher --run-id RUN --expect-owner OWNER` remains an idempotent compatibility operation. Interrupted cleanup retains its journal for an diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index c1d3b85cf..0d30531e1 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -221,6 +221,19 @@ idempotent retry, and never restarts the pool. On failure, inspect its retained graph evidence before another attempt. This fixture does not qualify application authentication, dependency listeners, browser routing, or publication. +The two ignored tests under +`foreground::native_test::same_boot_absent_relay::bridge_normalization` use the same +300-second external watchdog and a caller-owned capacity-two development pool. +Their pinned image additionally supplies BusyBox `httpd`; the target has two +HTTP-probed services on an internal network. They qualify an exited bridge plus a +never-launched reservation, and separately a second originally live bridge that +exits after normalization selection. Exact pending-fence writes, socket unlink, +owner unlink, registry removal and completion are interrupted and retried. A +replaced reservation refuses. Named markers and a live **unbridged** sibling must +survive; both owned graphs are removed through guarded cleanup on success. These +fixtures do not qualify simultaneous bridged siblings or application/browser +acceptance. Inspect failed-run resources before allocating another fixture. + The ignored foreground-owner fixture `foreground::native_test::stop14::fourteen_services_retain_named_data_across_cleanup_and_restore` qualifies normal retaining cleanup of thirteen running services and one completed diff --git a/packages/runtime-core/src/provider/graph/bridges.rs b/packages/runtime-core/src/provider/graph/bridges.rs index 13da3ab07..9eceb198a 100644 --- a/packages/runtime-core/src/provider/graph/bridges.rs +++ b/packages/runtime-core/src/provider/graph/bridges.rs @@ -628,12 +628,24 @@ fn stop_slot( store: &mut Store, slot: u8, ) -> Result<(), CandidateError> { + stop_slot_fenced(candidate, engine, store, slot, &|| Ok(()), &|| Ok(())) +} +fn stop_slot_fenced( + candidate: &Candidate, + engine: &Engine<'_>, + store: &mut Store, + slot: u8, + fence: &dyn Fn() -> Result<(), CandidateError>, + finish_partial: &dyn Fn() -> Result<(), CandidateError>, +) -> Result<(), CandidateError> { + fence()?; let a = store.slots.get(&slot).expect("selected slot"); super::super::publication::release( candidate, engine.guest().incarnation(), Some((&a.run, &a.reservation)), )?; + fence()?; if a.relay.is_none() { return Ok(()); } @@ -645,12 +657,23 @@ fn stop_slot( if a.phase != "stopped" { store.slots.get_mut(&slot).expect("selected slot").phase = "stopping".into(); save(candidate, store)?; + fence()?; relay::operate(engine, slot, &store.slots[&slot], "stop", None)?; + #[cfg(test)] + fault_pause( + &directory(candidate, &store.slots[&slot].run)?, + &store.slots[&slot].run, + "bridge-normalization-after-guest-stop", + )?; + fence()?; store.slots.get_mut(&slot).expect("selected slot").phase = "stopped".into(); save(candidate, store)?; } + fence()?; + finish_partial()?; + fence()?; relay::operate(engine, slot, &store.slots[&slot], "remove", None)?; - Ok(()) + fence() } pub fn inspect_bridges(candidate: &Candidate, run: &str) -> Result { let engine = Engine::connect_cleanup(candidate)?; diff --git a/packages/runtime-core/src/provider/graph/bridges/cleanup.rs b/packages/runtime-core/src/provider/graph/bridges/cleanup.rs index 939fd1911..5a445a588 100644 --- a/packages/runtime-core/src/provider/graph/bridges/cleanup.rs +++ b/packages/runtime-core/src/provider/graph/bridges/cleanup.rs @@ -1,5 +1,6 @@ //! Retained bridge selection survives registry release and graph archival. use super::*; +pub(crate) mod normalization; #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -964,7 +965,7 @@ mod tests { selected: BTreeMap::new(), } } - fn live_selection() -> (Selection, Store) { + pub(super) fn live_selection() -> (Selection, Store) { let mut selected = selection(1); selected.boot = "11111111-1111-1111-1111-111111111111".into(); selected.capacity = 3; @@ -1009,7 +1010,7 @@ mod tests { (selected, store) } - fn receipt_for_assignment(selected: &Selection, assignment: &Assignment) -> Receipt { + pub(super) fn receipt_for_assignment(selected: &Selection, assignment: &Assignment) -> Receipt { serde_json::from_value(json!({ "version":1, "run":selected.run, diff --git a/packages/runtime-core/src/provider/graph/bridges/cleanup/normalization.rs b/packages/runtime-core/src/provider/graph/bridges/cleanup/normalization.rs new file mode 100644 index 000000000..66dc4236b --- /dev/null +++ b/packages/runtime-core/src/provider/graph/bridges/cleanup/normalization.rs @@ -0,0 +1,390 @@ +//! Bounded pre-admission journal for dead-owner bridge normalization. The caller +//! holds provider, foreground and relay witnesses; the journal never replaces them. +use super::*; +use sha2::{Digest, Sha256}; +use std::path::Path; +const FILE: &str = "live-owner-bridge-normalization.json"; + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Journal { + version: u8, + owner: String, + boot: String, + authority: String, + original: Receipt, + original_sha256: String, + inventory: BTreeMap, + targets: BTreeMap>, + complete: bool, +} +fn hash(value: &impl Serialize) -> Result { + Ok(format!( + "{:x}", + Sha256::digest(serde_json::to_vec_pretty(value).map_err(|_| invalid())?) + )) +} +fn exists(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(state::io(e)), + } +} +fn write(root: &Path, value: &Journal) -> Result<(), CandidateError> { + journal::retain_file( + root, + "live-owner-bridge-normalization.pending", + "bridge-normalization-interrupted", + 2 * 1024 * 1024, + )?; + state::write(&root.join(FILE), value) +} +fn validate( + journal: &Journal, + receipt: &Receipt, + owner: &str, + boot: &str, + authority: &str, +) -> Result<(), CandidateError> { + if journal.version != 1 + || journal.owner != owner + || journal.boot != boot + || journal.authority != authority + || !hex(authority, 64) + || journal.original_sha256 != hash(receipt)? + || hash(&journal.original)? != journal.original_sha256 + || journal.inventory.len() > 32 + || journal + .targets + .keys() + .any(|slot| !journal.inventory.contains_key(slot)) + { + return Err(invalid()); + } + for (slot, a) in &journal.inventory { + binding(receipt, boot, a)?; + match (a.phase.as_str(), &a.relay, journal.targets.get(slot)) { + ("reserved", None, Some(None)) => {} + ("running", Some(relay), Some(Some(proof))) + if relay.transport == relay::Transport::ReservationV1 && proof.valid() => {} + ("running", Some(_), None) => {} + _ => return Err(invalid()), + } + } + Ok(()) +} +fn binding(receipt: &Receipt, boot: &str, a: &Assignment) -> Result<(), CandidateError> { + if a.run != receipt.run + || a.boot_id != boot + || !receipt + .resources + .get(&format!("container:{}", a.service)) + .is_some_and(|r| { + r.kind == Kind::Container + && r.key == a.service + && r.id.as_deref() == Some(a.container_id.as_str()) + }) + || !receipt + .resources + .values() + .any(|r| r.kind == Kind::Network && r.id.as_deref() == Some(a.network_id.as_str())) + { + return Err(invalid()); + } + Ok(()) +} +/// An interrupted normalization can only advance its exact selected assignments +/// toward absence. New/reused slots and changes to surviving live reservations refuse. +fn remaining(journal: &Journal, store: &Store) -> Result<(), CandidateError> { + if store.owner != journal.owner + || store + .slots + .iter() + .any(|(slot, a)| a.run == journal.original.run && !journal.inventory.contains_key(slot)) + { + return Err(invalid()); + } + for (slot, expected) in &journal.inventory { + match store.slots.get(slot) { + None if journal.targets.contains_key(slot) => {} + Some(current) => { + let mut comparable = current.clone(); + if journal.targets.get(slot).is_some_and(Option::is_some) + && ["running", "stopping", "stopped"].contains(¤t.phase.as_str()) + { + comparable.phase = expected.phase.clone(); + } + if comparable != *expected + || (journal.complete && journal.targets.contains_key(slot)) + { + return Err(invalid()); + } + } + _ => return Err(invalid()), + } + } + Ok(()) +} +fn archive_previous(root: &Path, receipt: &Receipt, prior: &Journal) -> Result<(), CandidateError> { + if !prior.complete + || prior.original.run != receipt.run + || prior.original.owner != receipt.owner + || prior.original.namespace != receipt.namespace + || prior.original.plan_id != receipt.plan_id + || !super::super::super::restore_history::confirms_prior_generation(root, receipt)? + || !prior.original.resources.iter().any(|(key, r)| { + r.kind == Kind::Container + && r.id != receipt.resources.get(key).and_then(|v| v.id.clone()) + }) + { + return Err(invalid()); + } + validate( + prior, + &prior.original, + &prior.owner, + &prior.boot, + &prior.authority, + )?; + let archived = root.join("live-owner-bridge-normalization-previous.json"); + if exists(&archived)? { + let old: Journal = state::read(&archived)?; + validate(&old, &old.original, &old.owner, &old.boot, &old.authority)?; + if !old.complete + || old.original.run != receipt.run + || old.original.owner != receipt.owner + || old.original.plan_id != receipt.plan_id + { + return Err(invalid()); + } + } + fs::rename(root.join(FILE), archived).map_err(state::io)?; + fs::File::open(root) + .and_then(|f| f.sync_all()) + .map_err(state::io) +} + +pub(crate) fn normalize( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: &Receipt, + root: &Path, + authority: &str, + verify: &dyn Fn() -> Result<(), CandidateError>, +) -> Result<(), CandidateError> { + verify()?; + let store = strict_store(candidate, engine)?; + let path = root.join(FILE); + if exists(&path)? { + let previous: Journal = state::read(&path)?; + if previous.original_sha256 != hash(receipt)? { + archive_previous(root, receipt, &previous)?; + } + } + let mut journal = if exists(&path)? { + state::read::(&path)? + } else { + let inventory: BTreeMap<_, _> = store + .slots + .iter() + .filter(|(_, a)| a.run == receipt.run) + .map(|(s, a)| (*s, a.clone())) + .collect(); + let mut targets = BTreeMap::new(); + for (slot, a) in &inventory { + binding(receipt, engine.guest().boot_id(), a)?; + match (a.phase.as_str(), &a.relay) { + ("reserved", None) => { + targets.insert(*slot, None); + } + ("running", Some(_)) => { + if let Some(exited) = relay::normalization_selection(engine, *slot, a)? { + targets.insert(*slot, Some(exited)); + } + } + _ => return Err(invalid()), + } + } + Journal { + version: 1, + owner: engine.guest().incarnation().into(), + boot: engine.guest().boot_id().into(), + authority: authority.into(), + original: receipt.clone(), + original_sha256: hash(receipt)?, + inventory, + targets, + complete: false, + } + }; + validate( + &journal, + receipt, + engine.guest().incarnation(), + engine.guest().boot_id(), + authority, + )?; + remaining(&journal, &store)?; + // A previously live member of this exact inventory can exit before main + // recovery enrollment. Promote only that unchanged member, never replace a + // selected proof or add a reservation from another generation. + for (slot, original) in &journal.inventory { + if !journal.targets.contains_key(slot) + && let Some(exited) = relay::normalization_selection(engine, *slot, original)? + { + journal.targets.insert(*slot, Some(exited)); + journal.complete = false; + } + } + // Save selection before any publication, helper or reservation is changed. + write(root, &journal)?; + #[cfg(test)] + super::super::super::fault_pause(root, &receipt.run, "bridge-normalization-after-intent")?; + for (slot, proof) in &journal.targets { + let expected = &journal.inventory[slot]; + let fence = || { + verify()?; + remaining(&journal, &strict_store(candidate, engine)?)?; + if let Some(proof) = proof { + relay::verify_normalization(engine, *slot, expected, proof)?; + } + Ok(()) + }; + fence()?; + let mut current = strict_store(candidate, engine)?; + if current.slots.contains_key(slot) { + let finish_partial = || match proof { + Some(p) => relay::finish_normalization_removal(engine, *slot, expected, p), + None => Ok(()), + }; + stop_slot_fenced( + candidate, + engine, + &mut current, + *slot, + &fence, + &finish_partial, + )?; + fence()?; + current.slots.remove(slot); + save(candidate, ¤t)?; + #[cfg(test)] + super::super::super::fault_pause( + root, + &receipt.run, + "bridge-normalization-after-removal", + )?; + } + fence()?; + } + verify()?; + journal.complete = true; + remaining(&journal, &strict_store(candidate, engine)?)?; + write(root, &journal)?; + #[cfg(test)] + super::super::super::fault_pause(root, &receipt.run, "bridge-normalization-complete")?; + verify() +} + +#[cfg(test)] +mod tests { + use super::*; + fn fixture() -> (Journal, Store) { + let (selection, store) = super::super::tests::live_selection(); + let original = super::super::tests::receipt_for_assignment(&selection, &store.slots[&0]); + let journal = Journal { + version: 1, + owner: selection.owner, + boot: selection.boot, + authority: "a".repeat(64), + original_sha256: hash(&original).unwrap(), + original, + inventory: store.slots.clone(), + targets: BTreeMap::from([( + 0, + Some( + serde_json::from_value(json!( + "relay-normalization-exited-v1 20 20 1:2 1:3 1:4 1:5 1:6" + )) + .unwrap(), + ), + )]), + complete: false, + }; + (journal, store) + } + #[test] + fn normalization_binds_generation_boot_authority_and_never_launched_targets() { + let (mut j, _) = fixture(); + validate(&j, &j.original, &j.owner, &j.boot, &j.authority).unwrap(); + for change in 0..5 { + let (mut bad, _) = fixture(); + match change { + 0 => bad.authority = "b".repeat(64), + 1 => bad.boot = "other".into(), + 2 => bad.owner = "b".repeat(32), + 3 => bad.original_sha256 = "b".repeat(64), + 4 => bad.inventory.get_mut(&0).unwrap().container_id = "b".repeat(64), + _ => unreachable!(), + } + assert!(validate(&bad, &j.original, &j.owner, &j.boot, &j.authority).is_err()); + } + j.inventory.get_mut(&0).unwrap().phase = "reserved".into(); + j.inventory.get_mut(&0).unwrap().relay = None; + j.targets.insert(0, None); + validate(&j, &j.original, &j.owner, &j.boot, &j.authority).unwrap(); + j.inventory.get_mut(&0).unwrap().phase = "starting".into(); + assert!(validate(&j, &j.original, &j.owner, &j.boot, &j.authority).is_err()); + } + #[test] + fn normalization_retry_preserves_siblings_and_refuses_replacements_or_added_inventory() { + let (j, mut store) = fixture(); + let mut sibling = store.slots[&0].clone(); + sibling.run = "9".repeat(32); + sibling.reservation = "8".repeat(32); + store.slots.insert(1, sibling.clone()); + for phase in ["running", "stopping", "stopped"] { + store.slots.get_mut(&0).unwrap().phase = phase.into(); + remaining(&j, &store).unwrap(); + } + for change in 0..5 { + let (_, mut bad) = fixture(); + let a = bad.slots.get_mut(&0).unwrap(); + match change { + 0 => a.reservation = "b".repeat(32), + 1 => a.run = "9".repeat(32), + 2 => a.generation = "b".repeat(64), + 3 => a.relay.as_mut().unwrap().launch_serial += 1, + 4 => a.phase = "starting".into(), + _ => unreachable!(), + } + assert!(remaining(&j, &bad).is_err()); + } + store.slots.remove(&0); + remaining(&j, &store).unwrap(); + assert_eq!(store.slots[&1], sibling); + store.slots.insert(0, sibling); + assert!(remaining(&j, &store).is_err()); + store.slots.remove(&0); + let mut added = j.inventory[&0].clone(); + added.service = "added".into(); + store.slots.insert(2, added); + assert!(remaining(&j, &store).is_err()); + } + #[test] + fn surviving_live_reservation_cannot_disappear_or_become_a_target_on_retry() { + let (mut j, mut store) = fixture(); + j.targets.clear(); + remaining(&j, &store).unwrap(); + store.slots.get_mut(&0).unwrap().phase = "stopping".into(); + assert!(remaining(&j, &store).is_err()); + store.slots.clear(); + assert!(remaining(&j, &store).is_err()); + let (mut j, mut store) = fixture(); + j.complete = true; + assert!(remaining(&j, &store).is_err()); + store.slots.clear(); + remaining(&j, &store).unwrap(); + } +} diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/same_boot_absent_relay.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/same_boot_absent_relay.rs index 3c9478cfa..f0321a270 100644 --- a/packages/runtime-core/src/provider/graph/foreground/native_test/same_boot_absent_relay.rs +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/same_boot_absent_relay.rs @@ -2,6 +2,7 @@ //! drop only their managed relay runtime before exiting. No receipt is fabricated //! to manufacture the absent endpoint. Fault controls affect only owned fixtures. use super::*; +mod bridge_normalization; use crate::provider::{identity::ProcessIdentity, relay_owner::publication::dead}; use std::collections::{BTreeMap, BTreeSet}; @@ -44,7 +45,14 @@ fn owner_child() { ) .unwrap(); let _publication = Publication::bind(&candidate, &run).unwrap(); - let readiness = BTreeMap::from([("app".into(), graph::Condition::Started)]); + let readiness = if std::env::var_os("HACK_NORMALIZATION_FIXTURE").is_some() { + BTreeMap::from([ + ("app".into(), graph::Condition::Healthy), + ("reserved".into(), graph::Condition::Healthy), + ]) + } else { + BTreeMap::from([("app".into(), graph::Condition::Started)]) + }; let values = BTreeMap::new(); let receipt = graph::run_with_host_dependencies_until( &candidate, diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/same_boot_absent_relay/bridge_normalization.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/same_boot_absent_relay/bridge_normalization.rs new file mode 100644 index 000000000..4f4648c5e --- /dev/null +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/same_boot_absent_relay/bridge_normalization.rs @@ -0,0 +1,273 @@ +//! Real exited/unlaunched bridge recovery; the live sibling uses no bridge slot. +use super::*; +#[test] +#[ignore = "Caller-owned running capacity-two disposable pool, pinned BusyBox HTTP image and 300s watchdog required"] +fn exited_and_reserved_bridges_recover_with_crash_retry_and_live_sibling() { + exercise(false); +} +#[test] +#[ignore = "Caller-owned capacity-two pool and pinned HTTP image; 300s external watchdog"] +fn late_exited_original_bridge_is_promoted_without_replacing_selection() { + exercise(true); +} +fn exercise(late_exit: bool) { + let deadline = Instant::now() + Duration::from_secs(270); + let candidate = candidate(); + let image = std::env::var("HACK_LOCAL_TEST_IMAGE").unwrap(); + assert!( + image + .strip_prefix("sha256:") + .is_some_and(|v| graph::hex(v, 64)) + ); + let fixtures = [graph::tests::Fixture::new(), graph::tests::Fixture::new()]; + let runs = [ + graph::probes::token().unwrap(), + graph::probes::token().unwrap(), + ]; + let service = json!({"image":image,"read_only":true,"init":true,"user":"0:0", + "entrypoint":["/bin/sh","-c","printf ok > /data/index.html; exec httpd -f -p 3000 -h /data"],"command":[],"volumes":["data:/data"], + "healthcheck":{"x-hack-http":{"port":3000,"path":"/","interval_ms":100,"timeout_ms":1000,"retries":20,"start_period_ms":500}}}); + state::write( + &fixtures[0].0.join("compose.yaml"), + &json!({"services":{"app":service,"reserved":service},"volumes":{"data":{}},"networks":{"default":{"internal":true}}}), + ) + .unwrap(); + state::write(&fixtures[1].0.join("compose.yaml"),&json!({"services":{"app":{"image":image,"read_only":true,"network_mode":"none","init":true,"user":"0:0","entrypoint":["/bin/sh","-c","exec sleep 300"],"command":[],"volumes":["data:/data"]}},"volumes":{"data":{}},"networks":{"default":{"internal":true}}})).unwrap(); + let mut owners = Vec::new(); + for index in 0..2 { + let armed = fixtures[index].0.join("armed.json"); + let mut env = vec![ + ( + "HACK_ABSENT_RELAY_PROJECT", + fixtures[index].0.to_str().unwrap(), + ), + ("HACK_ABSENT_RELAY_ARMED", armed.to_str().unwrap()), + ]; + if index == 0 { + env.push(("HACK_NORMALIZATION_FIXTURE", "1")); + } + let mut owner = child(&candidate, &runs[index], OWNER, &env); + while !armed.exists() { + if let Some(status) = owner.poll() { + panic!( + "synthetic normalization owner exited {status}: {}", + String::from_utf8_lossy(&owner.out) + ); + } + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(20)); + } + owners.push(owner); + } + let markers = [ + marker(&candidate, &runs[0], true), + marker(&candidate, &runs[1], true), + ]; + let sibling = running_container(&candidate, &runs[1]); + let before = graph::inspect(&candidate, &runs[0]).unwrap(); + let mut assignments = Vec::new(); + for (slot, service) in [(0, "app"), (1, "reserved")] { + assignments.push( + graph::reserve_bridge( + &candidate, + graph::ReserveBridgeOptions { + run: &runs[0], + service, + slot, + expected_generation: &before.guest_endpoints[service].generation, + }, + ) + .unwrap(), + ); + } + assignments[0] = + graph::start_bridge(&candidate, &runs[0], 0, &assignments[0].reservation).unwrap(); + if late_exit { + assignments[1] = + graph::start_bridge(&candidate, &runs[0], 1, &assignments[1].reservation).unwrap(); + } + let bridge_path = candidate + .state_root + .join("run/bridge-assignments/state.json"); + let initial_bridges = fs::read(&bridge_path).unwrap(); + let root = graph::directory(&candidate, &runs[0]).unwrap(); + let expected = sha(&fs::read(root.join("state.json")).unwrap()); + refused(&candidate, &runs[0], &expected); + assert_eq!(fs::read(&bridge_path).unwrap(), initial_bridges); + stop_owned_helper(&candidate, 0, &assignments[0]); + drop(owners[0].child.stdin.take()); + assert!(owners[0].wait(deadline).success()); + pause_recovery( + &candidate, + &runs[0], + &expected, + "bridge-normalization-after-intent", + deadline, + ); + if late_exit { + stop_owned_helper(&candidate, 1, &assignments[1]); + } + let journal = root.join("live-owner-bridge-normalization.json"); + assert_eq!(state::read::(&journal).unwrap()["complete"], false); + // Replacing a selected reservation cannot lend the original journal authority. + let original_registry = fs::read(&bridge_path).unwrap(); + let mut changed: Value = serde_json::from_slice(&original_registry).unwrap(); + changed["slots"]["0"]["reservation"] = json!("f".repeat(32)); + state::write(&bridge_path, &changed).unwrap(); + let refused_registry = fs::read(&bridge_path).unwrap(); + refused(&candidate, &runs[0], &expected); + assert_eq!(fs::read(&bridge_path).unwrap(), refused_registry); + fs::write(&bridge_path, &original_registry).unwrap(); + for fault in [ + "bridge-normalization-closing-pending", + "bridge-normalization-stopped-pending", + ] { + let mut interrupted = child( + &candidate, + &runs[0], + RECOVERY, + &[ + ("HACK_ABSENT_RELAY_RECEIPT", &expected), + ("HACK_LOCAL_GRAPH_FAULT", fault), + ], + ); + assert!(!interrupted.wait(deadline).success()); + let engine = graph::Engine::connect_cleanup(&candidate).unwrap(); + let proof = + serde_json::from_value(state::read::(&journal).unwrap()["targets"]["0"].clone()) + .unwrap(); + graph::relay::verify_normalization(&engine, 0, &assignments[0], &proof).unwrap(); + let serial = assignments[0] + .relay + .as_ref() + .unwrap() + .launch_serial + .to_string(); + let phase = if fault.ends_with("closing-pending") { + "closing" + } else { + "stopped" + }; + assert_eq!(engine.guest().execute_cleanup("set -eu; printf '%s %s %s\\n' \"$1\" \"$2\" \"$3\" | cmp -s - /run/hack-local/relay-slots/slot-0/pending; printf verified", &[&serial,&assignments[0].reservation,phase]).unwrap(), "verified"); + } + let mut interrupted = child( + &candidate, + &runs[0], + RECOVERY, + &[ + ("HACK_ABSENT_RELAY_RECEIPT", &expected), + ( + "HACK_LOCAL_GRAPH_FAULT", + "bridge-normalization-socket-unlinked", + ), + ], + ); + assert!(!interrupted.wait(deadline).success()); + { + let engine = graph::Engine::connect_cleanup(&candidate).unwrap(); + let proof = + serde_json::from_value(state::read::(&journal).unwrap()["targets"]["0"].clone()) + .unwrap(); + graph::relay::verify_normalization(&engine, 0, &assignments[0], &proof).unwrap(); + } + pause_recovery( + &candidate, + &runs[0], + &expected, + "bridge-normalization-after-guest-stop", + deadline, + ); + let mut interrupted = child( + &candidate, + &runs[0], + RECOVERY, + &[ + ("HACK_ABSENT_RELAY_RECEIPT", &expected), + ( + "HACK_LOCAL_GRAPH_FAULT", + "bridge-normalization-owner-unlinked", + ), + ], + ); + assert!(!interrupted.wait(deadline).success()); + { + let engine = graph::Engine::connect_cleanup(&candidate).unwrap(); + let proof = + serde_json::from_value(state::read::(&journal).unwrap()["targets"]["0"].clone()) + .unwrap(); + graph::relay::verify_normalization(&engine, 0, &assignments[0], &proof).unwrap(); + } + for point in [ + "bridge-normalization-after-removal", + "bridge-normalization-complete", + ] { + pause_recovery(&candidate, &runs[0], &expected, point, deadline); + assert_eq!(marker(&candidate, &runs[0], false), markers[0]); + assert_eq!(marker(&candidate, &runs[1], false), markers[1]); + assert!(owners[1].poll().is_none()); + assert_eq!(running_container(&candidate, &runs[1]), sibling); + } + let normalized = fs::read(&journal).unwrap(); + assert_eq!(state::read::(&journal).unwrap()["complete"], true); + graph::recover_live_owner(&candidate, &runs[0], &expected).unwrap(); + assert_eq!(fs::read(&journal).unwrap(), normalized); + assert_eq!( + graph::inspect(&candidate, &runs[0]).unwrap().receipt.phase, + "stopped-data-retained" + ); + assert_eq!(marker(&candidate, &runs[0], false), markers[0]); + assert_eq!(running_container(&candidate, &runs[1]), sibling); + assert!( + graph::inspect_bridges(&candidate, &runs[0]).unwrap()["slots"] + .as_object() + .unwrap() + .is_empty() + ); + drop(owners[1].child.stdin.take()); + assert!(owners[1].wait(deadline).success()); + let sibling_expected = sha(&fs::read( + graph::directory(&candidate, &runs[1]) + .unwrap() + .join("state.json"), + ) + .unwrap()); + graph::recover_live_owner(&candidate, &runs[1], &sibling_expected).unwrap(); + for run in &runs { + graph::foreground::cleanup_request(&candidate, run, true).unwrap(); + assert_eq!( + graph::inspect(&candidate, run).unwrap().receipt.phase, + "removed" + ); + } +} + +fn stop_owned_helper(candidate: &Candidate, slot: u8, assignment: &graph::bridges::Assignment) { + { + let engine = graph::Engine::connect_cleanup(&candidate).unwrap(); + assert!( + graph::relay::normalization_selection(&engine, slot, assignment) + .unwrap() + .is_none() + ); + let evidence = + serde_json::to_value(graph::relay::capture_cleanup(&engine, slot, assignment).unwrap()) + .unwrap(); + let relay = format!( + "/run/hack-local/graph-relays/{}/relay", + assignment.reservation + ); + let pid = evidence["pid"].as_u64().unwrap().to_string(); + let start = evidence["start"].as_u64().unwrap().to_string(); + let identity = format!( + "{}:{}", + evidence["executable_device"].as_u64().unwrap(), + evidence["executable_inode"].as_u64().unwrap() + ); + engine.guest().execute_cleanup("set -eu; test ! -L \"$1\"; test \"$(stat -c %d:%i \"$1\")\" = \"$4\"; \"$1\" --stop \"$2\" \"$3\"",&[&relay,&pid,&start,&identity]).unwrap(); + assert!( + graph::relay::normalization_selection(&engine, slot, assignment) + .unwrap() + .is_some() + ); + } +} diff --git a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs index 01596ee10..a6b620b37 100644 --- a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs +++ b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs @@ -329,6 +329,16 @@ pub fn recover_live_owner( inspect_resource(&engine, &receipt, resource)?; } let selected = relay::Selection::capture(&relay, foreground.process())?; + let authority = digest(&(expected, foreground.fingerprint(), &selected))?; + let verify = || { + engine.guest().verify()?; + foreground.verify_retirement_ready()?; + relay.verify()?; + relay::verify_receipt(&root, &receipt) + }; + bridges::cleanup::normalization::normalize( + candidate, &engine, &receipt, &root, &authority, &verify, + )?; Intent { version: selected.version(), boot: engine.guest().boot_id().into(), diff --git a/packages/runtime-core/src/provider/graph/relay-normalize-observe.sh b/packages/runtime-core/src/provider/graph/relay-normalize-observe.sh new file mode 100644 index 000000000..8766f4a49 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/relay-normalize-observe.sh @@ -0,0 +1,126 @@ +# Read-only selection/retry proof for an exited, launch-fenced bridge helper. +# Capture/verify never mutate. finish-empty removes only a pinned empty allocation +# after its committed stopped fence, under the same exclusive guest slot lock. +set -efu +stage=1 +trap 'status=$?; if test "$status" -ne 0; then printf "relay-normalization-refused %s\n" "$stage"; exit 0; fi' 0 +exec 2>/dev/null +action=$1; allocation=$2; marker=$3; socket=$4; digest=$5; serial=$6; slot=$7 +expected=${8:-} +base=/run/hack-local/graph-relays +root="$base/$allocation" +control=/run/hack-local/relay-slots/slot-$slot +private_dir() { test ! -L "$1" && test -d "$1" && test "$(stat -c %u:%g:%a "$1")" = 0:0:700; } +private_file() { test ! -L "$1" && test -f "$1" && test "$(stat -c %u:%g:%a:%h "$1")" = 0:0:600:1 && test "$(stat -c %s "$1")" -le 256; } +absent() { test ! -e "$1" && test ! -L "$1"; } +number() { case "$1" in ''|*[!0-9]*) return 1;; esac; test "$1" -gt 0; } +identity() { stat -c %d:%i "$1"; } +stage=2 +private_dir /run/hack-local +test "$(findmnt -n -o FSTYPE --target /run/hack-local)" = tmpfs +private_dir /run/hack-local/relay-slots +private_dir "$control"; private_file "$control/lock" +control_id=$(identity "$control"); lock_id=$(identity "$control/lock") +exec 9< "$control/lock" +if test "$action" = finish-empty; then flock -x -w 5 9; else flock -s -w 5 9; fi +test "$(identity "$control")" = "$control_id" +test "$(stat -Lc %d:%i /proc/$$/fd/9)" = "$lock_id" +test "$(identity "$control/lock")" = "$lock_id" +stage=3 +private_file "$control/state" +set -- $(cat "$control/state") +test "$#" = 3; test "$1" = "$serial"; test "$2" = "$allocation" +phase=$3 +printf '%s %s %s\n' "$serial" "$allocation" "$phase" | cmp -s - "$control/state" +case "$phase" in launching|closing|stopped) :;; *) exit 1;; esac +# NORMALIZATION_PENDING_BEGIN +if ! absent "$control/pending"; then + test "$action" != capture + private_file "$control/pending" + set -- $(cat "$control/pending") + test "$#" = 3; test "$1" = "$serial"; test "$2" = "$allocation" + next=$3 + printf '%s %s %s\n' "$serial" "$allocation" "$next" | cmp -s - "$control/pending" + case "$phase:$next" in launching:closing|closing:closing|closing:stopped|stopped:closing) :;; *) exit 1;; esac + # Read-only admission. Ordinary stop promotes this under its exclusive lock. +fi +# NORMALIZATION_PENDING_END +if test "$action" = capture; then + test "$phase" = launching + stage=4 + private_dir "$base"; private_dir "$root"; private_file "$root/owner" + test "$(cat "$root/owner")" = "$marker" + root_id=$(identity "$root") + stage=5 + test ! -L "$root/relay"; test -f "$root/relay" + test "$(stat -c %u:%g:%a:%h "$root/relay")" = 0:0:500:1 + test "$(sha256sum "$root/relay" | cut -d' ' -f1)" = "$digest" + binary_id=$(identity "$root/relay") + stage=6 + private_file "$root/process" + set -- $(cat "$root/process") + test "$#" = 2; pid=$1; born=$2; number "$pid"; number "$born"; test "$pid" -gt 1 + printf '%s %s\n' "$pid" "$born" | cmp -s - "$root/process" + stage=7 + if test -e "/proc/$pid"; then + test -r "/proc/$pid/stat" + test "$(sed 's/.*) //' "/proc/$pid/stat" | awk '{print $20}')" = "$born" + if test "$(sed 's/.*) //' "/proc/$pid/stat" | cut -d' ' -f1)" != Z; then printf 'relay-normalization-running\n'; exit; fi + fi + stage=8 + private_file "$root/socket" + socket_id=$(cat "$root/socket") + # The exact helper unlinks its own listener during a graceful exit. + if ! absent "$socket"; then + test ! -L "$socket"; test -S "$socket" + test "$(stat -c %u:%g:%a:%h "$socket")" = 0:0:700:1 + test "$(identity "$socket")" = "$socket_id" + fi + printf 'relay-normalization-exited-v1 %s %s %s %s %s %s %s\n' "$pid" "$born" "$root_id" "$binary_id" "$socket_id" "$control_id" "$lock_id" + exit +fi +stage=9 +case "$action" in verify|finish-empty) :;; *) exit 1;; esac +set -- $expected +test "$#" = 8; test "$1" = relay-normalization-exited-v1 +pid=$2; born=$3; root_id=$4; binary_id=$5; socket_id=$6 +number "$pid"; number "$born"; test "$pid" -gt 1 +test "$7" = "$control_id"; test "$8" = "$lock_id" +stage=10 +if test -e "/proc/$pid"; then + test -r "/proc/$pid/stat" + test "$(sed 's/.*) //' "/proc/$pid/stat" | awk '{print $20}')" = "$born" + test "$(sed 's/.*) //' "/proc/$pid/stat" | cut -d' ' -f1)" = Z +fi +stage=11 +if absent "$root"; then + test "$phase" = stopped; absent "$socket" +else + private_dir "$base"; private_dir "$root"; test "$(identity "$root")" = "$root_id" + if absent "$root/owner"; then + test "$phase" = stopped; absent "$socket" + test -z "$(find "$root" -mindepth 1 -maxdepth 1 -print -quit)" + if test "$action" = finish-empty; then rmdir "$root"; fi + printf 'relay-normalization-verified\n'; exit + fi + private_file "$root/owner"; test "$(cat "$root/owner")" = "$marker" + if ! absent "$root/relay"; then + test ! -L "$root/relay"; test -f "$root/relay" + test "$(stat -c %u:%g:%a:%h "$root/relay")" = 0:0:500:1 + test "$(identity "$root/relay")" = "$binary_id" + test "$(sha256sum "$root/relay" | cut -d' ' -f1)" = "$digest" + else test "$phase" = stopped; fi + if ! absent "$root/process"; then + private_file "$root/process" + printf '%s %s\n' "$pid" "$born" | cmp -s - "$root/process" + else test "$phase" = stopped; fi + if ! absent "$root/socket"; then + private_file "$root/socket"; test "$(cat "$root/socket")" = "$socket_id" + else test "$phase" = stopped; fi + if ! absent "$socket"; then + test ! -L "$socket"; test -S "$socket" + test "$(stat -c %u:%g:%a:%h "$socket")" = 0:0:700:1 + test "$(identity "$socket")" = "$socket_id" + else case "$phase" in launching|closing|stopped) :;; *) exit 1;; esac; fi +fi +printf 'relay-normalization-verified\n' diff --git a/packages/runtime-core/src/provider/graph/relay.rs b/packages/runtime-core/src/provider/graph/relay.rs index d9a3fb69e..5f31c5bae 100644 --- a/packages/runtime-core/src/provider/graph/relay.rs +++ b/packages/runtime-core/src/provider/graph/relay.rs @@ -110,6 +110,35 @@ pub(super) fn operate( ]; let args = args.iter().map(String::as_str).collect::>(); let script = include_str!("relay.sh").replace("# RELAY_FENCE", include_str!("relay-fence.sh")); + #[cfg(test)] + let script = { + if std::env::var("HACK_LOCAL_GRAPH_RUN").ok().as_deref() == Some(assignment.run.as_str()) { + match ( + action, + std::env::var("HACK_LOCAL_GRAPH_FAULT").ok().as_deref(), + ) { + ("stop", Some("bridge-normalization-closing-pending")) => script.replace( + " mv \"$control/pending\" \"$control/state\"", + " if test \"$1\" = closing; then exit 97; fi\n mv \"$control/pending\" \"$control/state\"", + ), + ("stop", Some("bridge-normalization-stopped-pending")) => script.replace( + " mv \"$control/pending\" \"$control/state\"", + " if test \"$1\" = stopped; then exit 97; fi\n mv \"$control/pending\" \"$control/state\"", + ), + ("stop", Some("bridge-normalization-socket-unlinked")) => script.replace( + "if test \"$serial\" -ne 0; then fence_write stopped; fi", + "exit 97", + ), + ("remove", Some("bridge-normalization-owner-unlinked")) => script.replace( + "rm \"$root/owner\"\n rmdir \"$root\"", + "rm \"$root/owner\"\n exit 97", + ), + _ => script, + } + } else { + script + } + }; let result = if action == "start" { engine.guest().execute(&script, &args, input)? } else { @@ -335,6 +364,129 @@ pub(super) fn capture_cleanup( ) -> Result { CleanupEvidence::parse(&observe_cleanup(engine, slot, assignment, None)?) } +/// Canonical numeric-only metadata pins an already exited helper and the guest +/// allocation/fence identities throughout interrupted stop and removal. +#[cfg(target_os = "macos")] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub(super) struct ExitedEvidence(String); +#[cfg(target_os = "macos")] +impl ExitedEvidence { + pub(super) fn valid(&self) -> bool { + let fields: Vec<_> = self.0.split(' ').collect(); + let canonical = |v: &str| { + v.parse::() + .is_ok_and(|n| n > 0 && n <= i64::MAX as u64 && n.to_string() == v) + }; + fields.len() == 8 + && fields[0] == "relay-normalization-exited-v1" + && canonical(fields[1]) + && canonical(fields[2]) + && fields[3..].iter().all(|v| { + v.split_once(':').is_some_and(|(a, b)| { + a.parse::().is_ok_and(|n| n.to_string() == a) && canonical(b) + }) + }) + } +} +#[cfg(target_os = "macos")] +fn observe_normalization( + engine: &Engine<'_>, + slot: u8, + assignment: &bridges::Assignment, + expected: Option<&ExitedEvidence>, + finish_empty: bool, +) -> Result { + let relay = assignment.relay.as_ref().ok_or_else(observation_refused)?; + if !relay.valid() + || relay.transport != Transport::ReservationV1 + || relay.launch_serial == 0 + || assignment.boot_id != engine.guest().boot_id() + || !hex(&assignment.reservation, 32) + || !hex(&assignment.run, 32) + || expected.is_some_and(|v| !v.valid()) + { + return Err(observation_refused()); + } + let args = [ + if finish_empty { + "finish-empty".into() + } else if expected.is_some() { + "verify".into() + } else { + "capture".into() + }, + assignment.reservation.clone(), + format!( + "{}:{}:{}:{}", + engine.guest().incarnation(), + assignment.run, + assignment.reservation, + assignment.boot_id + ), + format!("/run/hack-local/bridge-{slot:02}.sock"), + relay.binary_sha256.clone(), + relay.launch_serial.to_string(), + slot.to_string(), + expected.map_or_else(String::new, |v| v.0.clone()), + ]; + engine.guest().execute_cleanup( + include_str!("relay-normalize-observe.sh"), + &args.iter().map(String::as_str).collect::>(), + ) +} +#[cfg(target_os = "macos")] +pub(super) fn normalization_selection( + engine: &Engine<'_>, + slot: u8, + assignment: &bridges::Assignment, +) -> Result, CandidateError> { + let output = observe_normalization(engine, slot, assignment, None, false)?; + if output == "relay-normalization-running\n" { + capture_cleanup(engine, slot, assignment)?; + return Ok(None); + } + let value = ExitedEvidence( + output + .strip_suffix('\n') + .ok_or_else(observation_refused)? + .into(), + ); + if !value.valid() { + return Err(observation_refused()); + } + Ok(Some(value)) +} +#[cfg(target_os = "macos")] +pub(super) fn verify_normalization( + engine: &Engine<'_>, + slot: u8, + assignment: &bridges::Assignment, + expected: &ExitedEvidence, +) -> Result<(), CandidateError> { + if observe_normalization(engine, slot, assignment, Some(expected), false)? + != "relay-normalization-verified\n" + { + return Err(observation_refused()); + } + Ok(()) +} + +/// Resume only the final empty-directory window of this exact selected allocation. +#[cfg(target_os = "macos")] +pub(super) fn finish_normalization_removal( + engine: &Engine<'_>, + slot: u8, + assignment: &bridges::Assignment, + expected: &ExitedEvidence, +) -> Result<(), CandidateError> { + if observe_normalization(engine, slot, assignment, Some(expected), true)? + != "relay-normalization-verified\n" + { + return Err(observation_refused()); + } + Ok(()) +} /// Independently verify captured generation retirement without invoking the helper, /// editing its fence, or deleting allocations. This proves the actual helper /// generation exited and its allocation/socket path vanished, not global descriptor diff --git a/packages/runtime-core/tests/relay_fence_contract.rs b/packages/runtime-core/tests/relay_fence_contract.rs index fe8253c8c..23775b8ef 100644 --- a/packages/runtime-core/tests/relay_fence_contract.rs +++ b/packages/runtime-core/tests/relay_fence_contract.rs @@ -134,3 +134,72 @@ fn interrupted_fence_publication_requires_canonical_valid_transition() { } fs::remove_dir_all(root).unwrap(); } + +#[test] +fn normalization_pending_probe_is_readonly_and_requires_exact_complete_transition() { + let source = include_str!("../src/provider/graph/relay-normalize-observe.sh"); + let block = source + .split("# NORMALIZATION_PENDING_BEGIN\n") + .nth(1) + .unwrap() + .split("# NORMALIZATION_PENDING_END") + .next() + .unwrap(); + let script = format!( + "set -efu\ncontrol=$1; action=$2; phase=$3; serial=7; allocation=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\nabsent() {{ test ! -e \"$1\" && test ! -L \"$1\"; }}\nprivate_file() {{ test ! -L \"$1\" && test -f \"$1\"; }}\n{block}" + ); + let root = + std::env::temp_dir().join(format!("hack-normalization-fence-{}", std::process::id())); + fs::create_dir(&root).unwrap(); + let allocation = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + for phase in ["launching", "closing", "stopped"] { + for next in ["launching", "closing", "stopped"] { + for action in ["capture", "verify"] { + let before = format!("7 {allocation} {phase}\n"); + let pending = format!("7 {allocation} {next}\n"); + fs::write(root.join("state"), &before).unwrap(); + fs::write(root.join("pending"), &pending).unwrap(); + let allowed = action == "verify" + && matches!( + (phase, next), + ("launching", "closing") + | ("closing", "closing") + | ("closing", "stopped") + | ("stopped", "closing") + ); + let status = Command::new("/bin/sh") + .args(["-c", &script, "test"]) + .arg(&root) + .args([action, phase]) + .output() + .unwrap() + .status; + assert_eq!(status.success(), allowed, "{action}: {phase} -> {next}"); + assert_eq!(fs::read_to_string(root.join("state")).unwrap(), before); + assert_eq!(fs::read_to_string(root.join("pending")).unwrap(), pending); + } + } + } + for pending in [ + String::new(), + format!("7 {allocation} closing"), + format!("7 {allocation} closing\n\n"), + format!("8 {allocation} closing\n"), + "7 bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb closing\n".into(), + format!("7 {allocation} closing extra\n"), + ] { + fs::write(root.join("pending"), &pending).unwrap(); + assert!( + !Command::new("/bin/sh") + .args(["-c", &script, "test"]) + .arg(&root) + .args(["verify", "launching"]) + .output() + .unwrap() + .status + .success() + ); + assert_eq!(fs::read_to_string(root.join("pending")).unwrap(), pending); + } + fs::remove_dir_all(root).unwrap(); +} From 18659324d20d0f01dde56eb2920d2a16660c74fe Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Tue, 6 Oct 2026 21:47:41 -0400 Subject: [PATCH 2/2] test: qualify recovery CLI and audit removed bridge fixtures --- packages/runtime-core/README.md | 5 ++ .../bridge_normalization.rs | 83 ++++++++++++++++++- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 0d30531e1..cfb307043 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -224,6 +224,11 @@ application authentication, dependency listeners, browser routing, or publicatio The two ignored tests under `foreground::native_test::same_boot_absent_relay::bridge_normalization` use the same 300-second external watchdog and a caller-owned capacity-two development pool. +Set `HACK_LOCAL_TEST_NATIVE` to the absolute current `hack-native` binary to run +the final positive recovery through its public CLI. The read-only ignored +`removed_fixture_has_no_engine_resources_or_selected_guest_helpers` child audits +an exact `HACK_LOCAL_GRAPH_RUN` afterward, including engine absence and the +selected guest process identities. Their pinned image additionally supplies BusyBox `httpd`; the target has two HTTP-probed services on an internal network. They qualify an exited bridge plus a never-launched reservation, and separately a second originally live bridge that diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/same_boot_absent_relay/bridge_normalization.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/same_boot_absent_relay/bridge_normalization.rs index 4f4648c5e..00eb30c72 100644 --- a/packages/runtime-core/src/provider/graph/foreground/native_test/same_boot_absent_relay/bridge_normalization.rs +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/same_boot_absent_relay/bridge_normalization.rs @@ -209,7 +209,37 @@ fn exercise(late_exit: bool) { } let normalized = fs::read(&journal).unwrap(); assert_eq!(state::read::(&journal).unwrap()["complete"], true); - graph::recover_live_owner(&candidate, &runs[0], &expected).unwrap(); + if let Some(binary) = std::env::var_os("HACK_LOCAL_TEST_NATIVE") { + assert!(Path::new(&binary).is_absolute()); + let output = Command::new(binary) + .env_clear() + .env("PATH", "/usr/bin:/bin:/usr/sbin:/sbin") + .env("HOME", std::env::var_os("HOME").unwrap()) + .arg("--candidate-root") + .arg(&candidate.checkout) + .args([ + "graph", + "recover-live-owner", + "--run-id", + &runs[0], + "--expect-receipt", + &expected, + "--json", + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "candidate CLI refused: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + serde_json::from_slice::(&output.stdout).unwrap()["phase"], + "stopped-data-retained" + ); + } else { + graph::recover_live_owner(&candidate, &runs[0], &expected).unwrap(); + } assert_eq!(fs::read(&journal).unwrap(), normalized); assert_eq!( graph::inspect(&candidate, &runs[0]).unwrap().receipt.phase, @@ -238,12 +268,13 @@ fn exercise(late_exit: bool) { graph::inspect(&candidate, run).unwrap().receipt.phase, "removed" ); + audit_removed(&candidate, run); } } fn stop_owned_helper(candidate: &Candidate, slot: u8, assignment: &graph::bridges::Assignment) { { - let engine = graph::Engine::connect_cleanup(&candidate).unwrap(); + let engine = graph::Engine::connect_cleanup(candidate).unwrap(); assert!( graph::relay::normalization_selection(&engine, slot, assignment) .unwrap() @@ -271,3 +302,51 @@ fn stop_owned_helper(candidate: &Candidate, slot: u8, assignment: &graph::bridge ); } } + +/// Read-only post-test audit; caller selects one exact removed synthetic run. +#[test] +#[ignore = "Explicit owned synthetic graph run/home; no allocation or cleanup effects"] +fn removed_fixture_has_no_engine_resources_or_selected_guest_helpers() { + audit_removed(&candidate(), &run()); +} +fn audit_removed(candidate: &Candidate, run: &str) { + let snapshot = graph::inspect(candidate, run).unwrap(); + assert_eq!(snapshot.receipt.phase, "removed"); + assert!(!snapshot.observations.is_empty()); + assert!( + snapshot + .observations + .values() + .all(|v| v["state"] == "absent") + ); + assert!( + graph::inspect_bridges(candidate, run).unwrap()["slots"] + .as_object() + .unwrap() + .is_empty() + ); + let root = graph::directory(candidate, run).unwrap(); + let journal: Value = state::read(&root.join("live-owner-bridge-normalization.json")).unwrap(); + assert_eq!(journal["complete"], true); + assert_eq!(journal["original"]["run"], run); + let engine = graph::Engine::connect_cleanup(candidate).unwrap(); + for (slot, assignment) in journal["inventory"].as_object().unwrap() { + let allocation = assignment["reservation"].as_str().unwrap(); + assert!(graph::hex(allocation, 32)); + let proof = &journal["targets"][slot]; + let (pid, start) = if let Some(proof) = proof.as_str() { + let fields: Vec<_> = proof.split(' ').collect(); + assert_eq!(fields.len(), 8); + assert_eq!(fields[0], "relay-normalization-exited-v1"); + assert!(fields[1].parse::().unwrap() > 1); + assert!(fields[2].parse::().unwrap() > 0); + (fields[1], fields[2]) + } else { + assert!(assignment["relay"].is_null()); + ("0", "0") + }; + assert_eq!(engine.guest().execute_cleanup( + r#"set -efu; root=/run/hack-local/graph-relays/$1; test ! -e "$root"; test ! -L "$root"; if test "$2" != 0 && test -e "/proc/$2/stat"; then born=$(sed 's/.*) //' "/proc/$2/stat" | awk '{print $20}'); phase=$(sed 's/.*) //' "/proc/$2/stat" | cut -d' ' -f1); test "$born" != "$3" || test "$phase" = Z; fi; printf verified"#, + &[allocation,pid,start]).unwrap(),"verified"); + } +}