Runner: retain the provider across hosted park and reclaim (#19140)

This commit is contained in:
Nicky Leach authored and GitHub committed 2026-11-05 18:40:44 -08:00
1 parent 7c04adb2d6
commit 5ab7b78f2b
6 files changed
+903 -7

No files matched your search

@@ -162,3 +162,32 @@ after that drain, so a late active-goal notification cannot be hidden by the
older executor journal. This hook does not detach a controller, stop a process,
or advertise hosted parking support. The host-control consumer must separately
validate the exact Runner identity and acknowledged sequence cut.
### Hosted park and reclaim loop
A trusted Launcher may pass a connected Unix stream at fd 4 and start durable
listener mode with `--host-control-fd 4`. The descriptor is close-on-exec before
provider startup. The private channel accepts the `park`, `claim`, and `retire`
requests defined in the host-control codec; it cannot execute PRP commands.
`park` requires the exact six-field authority, acknowledged command/event cut,
no unfinished transition or cleanup, and the provider eligibility proof above.
The Runner drops its controller connection and credentials, retains its provider
and journals, and blocks on host control without provider polling. Ordinary
reconnect grace does not expire a parked Runner. The configured process lifetime
still applies. Mount ownership stays with Launcher.
`claim` names the accepted park request and supplies a fresh bootstrap ticket.
It resumes authenticated PRP under the retained authority; ordinary `run.attach`
must still establish the next authority before new work. Pending reconnect and
live commands both reject execution before that attachment; snapshots, drain,
and cleanup remain available. Replaying the same claim returns its acknowledgement without consuming another ticket. A parked
retirement names the park request. Cancelling a claim additionally requires its
claim ID, the unchanged authority and sequence cut, and provider quiescence.
Retirement, parked host-channel loss, or process-lifetime expiry uses the existing
suspended lifecycle and provider shutdown path. There is no new checkpoint state.
This native loop does not advertise hosted parking support: Launcher retention,
Core orchestration, provider-specific qualification, and live VM sleep/wake
verification are required before rollout. Providers without an eligibility
implementation remain ineligible.
@@ -9,8 +9,8 @@ use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt};
use paperclip_runner_core::acpx_candidate_admission::AcpxCandidateAdmission;
use paperclip_runner_core::durable::{
capture_bootstrap_ticket, redact_diagnostic_text, run_durable_runner, AcpxLaunchProfile,
CursorCloudLaunchProfile, DurableRunnerConfig, OpenCodeLaunchProfile, QualifiedLaunchArtifact,
capture_bootstrap_ticket, redact_diagnostic_text, AcpxLaunchProfile, CursorCloudLaunchProfile,
DurableRunnerConfig, OpenCodeLaunchProfile, QualifiedLaunchArtifact,
};
use paperclip_runner_core::local_runner::{run_local_runner, LocalRunnerError, RunnerConfig};
use paperclip_runner_core::native_provider_backend::NativeProviderCommandExecutor;
@@ -417,6 +417,38 @@ fn durable_connect_url(args: &[String], run_id: &str) -> Result<String, LocalRun
}
fn run_durable(args: &[String]) -> Result<(), LocalRunnerError> {
#[cfg(unix)]
let host_control = if let Some(fd) = optional_u64(args, "--host-control-fd")? {
use std::os::fd::{FromRawFd, OwnedFd};
use std::os::unix::fs::FileTypeExt;
if fd != 4
|| !fs::metadata("/dev/fd/4")
.map(|metadata| metadata.file_type().is_socket())
.unwrap_or(false)
{
return Err(LocalRunnerError::invalid(
"host control requires an inherited socket at fd 4",
));
}
// SAFETY: fd 4 was inherited from the trusted host and verified open.
// No other Runner owner takes or closes it; HostControl sets CLOEXEC
// before any provider can spawn and validates connected AF_UNIX STREAM.
let socket = std::os::unix::net::UnixStream::from(unsafe { OwnedFd::from_raw_fd(4) });
Some(
paperclip_runner_core::host_control::HostControl::new(socket)
.map_err(|error| LocalRunnerError::invalid(error.to_string()))?,
)
} else {
None
};
#[cfg(not(unix))]
let host_control = if args.iter().any(|arg| arg == "--host-control-fd") {
return Err(LocalRunnerError::invalid(
"host control is unavailable on this platform",
));
} else {
None
};
let ticket = capture_bootstrap_ticket()
.map_err(|error| LocalRunnerError::invalid(error.to_string()))?
.ok_or_else(|| {
@@ -462,6 +494,11 @@ fn run_durable(args: &[String]) -> Result<(), LocalRunnerError> {
reconnect_grace: optional_u64(args, "--reconnect-grace-ms")?.map(Duration::from_millis),
max_runtime: duration("--max-runtime-ms", 0)?,
};
if host_control.is_some() && !config.connect_url.starts_with("listen://") {
return Err(LocalRunnerError::invalid(
"host control requires trusted Runner listener mode",
));
}
#[cfg(unix)]
std::env::set_var(
paperclip_runner_core::runtime_notifications::CONTEXT_ENV,
@@ -479,8 +516,13 @@ fn run_durable(args: &[String]) -> Result<(), LocalRunnerError> {
}
}
let executor = NativeProviderCommandExecutor::with_runner_config(state_dir, &config);
run_durable_runner(config, ticket, executor)
.map_err(|error| LocalRunnerError::invalid(error.to_string()))
paperclip_runner_core::durable::run_durable_runner_with_host_control(
config,
ticket,
executor,
host_control,
)
.map_err(|error| LocalRunnerError::invalid(error.to_string()))
}
fn apply_oom_score_adj(args: &[String]) -> Result<(), LocalRunnerError> {
@@ -545,6 +587,11 @@ fn run(args: &[String]) -> Result<(), LocalRunnerError> {
"ACPX candidate admission requires trusted durable runner startup",
));
}
if args.iter().any(|arg| arg == "--host-control-fd") {
return Err(LocalRunnerError::invalid(
"host control requires trusted durable runner startup",
));
}
run_local_runner(RunnerConfig {
run_id: value(args, "--run-id")?,
normalized_session_id: value(args, "--session-id")?,
@@ -678,6 +725,7 @@ mod tests {
assert!(metadata.get("codexSourceRecoveryVersion").is_none());
assert_eq!(metadata["acpxRecoveryIntegrityVersion"], 1);
assert!(metadata.get("acpxSourceRecoveryVersion").is_none());
assert!(metadata.get("hostedParkingVersion").is_none());
assert_eq!(metadata["schema"], RUNNERD_BUILD_METADATA_SCHEMA);
assert_eq!(metadata["binaryContractVersion"], 2);
assert_eq!(
@@ -694,6 +742,20 @@ mod tests {
);
}
#[test]
#[cfg(unix)]
fn host_control_rejects_other_fds_before_bootstrap_or_provider_startup() {
let args = vec!["--host-control-fd".to_owned(), "3".to_owned()];
assert_eq!(
run_durable(&args).unwrap_err().to_string(),
"host control requires an inherited socket at fd 4"
);
assert_eq!(
run(&args).unwrap_err().to_string(),
"host control requires trusted durable runner startup"
);
}
#[test]
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
fn candidate_startup_requires_a_complete_trusted_launch_profile() {
@@ -12,8 +12,8 @@ use sha2::{Digest, Sha256};
use crate::stable_identity::{is_stable_id, DURABLE_STABLE_ID_CHARS, SHORT_STABLE_ID_CHARS};
pub use runner::{
run_durable_runner, CommandExecution, CommandExecutor, PolledEvent,
TerminalDeliveryReconciliation,
run_durable_runner, run_durable_runner_with_host_control, CommandExecution, CommandExecutor,
PolledEvent, TerminalDeliveryReconciliation,
};
pub(crate) use state::{
create_private_temporary_file, open_private_regular_file, redact_sensitive_text_values,
@@ -447,10 +447,44 @@ impl CumulativeAckPersistence {
}
pub fn run_durable_runner<E: CommandExecutor>(
config: DurableRunnerConfig,
bootstrap_ticket: BootstrapTicket,
executor: E,
) -> Result<(), DurableRunnerError> {
run_durable_runner_with_host_control(config, bootstrap_ticket, executor, None)
}
#[cfg(not(unix))]
pub struct HostControl;
#[cfg(unix)]
use crate::host_control::{HostControl, HostIdentity, HostRequest};
#[cfg(unix)]
struct HostedRunnerControl {
channel: HostControl,
park_id: Option<String>,
claiming: Option<(String, String)>,
}
#[cfg(unix)]
impl HostedRunnerControl {
fn new(channel: HostControl) -> Self {
Self {
channel,
park_id: None,
claiming: None,
}
}
}
pub fn run_durable_runner_with_host_control<E: CommandExecutor>(
mut config: DurableRunnerConfig,
bootstrap_ticket: BootstrapTicket,
mut executor: E,
#[allow(unused_variables)] host_control: Option<HostControl>,
) -> Result<(), DurableRunnerError> {
#[cfg(unix)]
let mut host_control = host_control.map(HostedRunnerControl::new);
let started = Instant::now();
config.validate()?;
if let Some(admission) = config
@@ -508,7 +542,50 @@ pub fn run_durable_runner<E: CommandExecutor>(
let mut reconnect_attempt = 0_u32;
let mut next_connection_attempt = Instant::now();
loop {
'session: loop {
#[cfg(unix)]
if let Some(control) = host_control.as_mut() {
if control.park_id.is_some() {
let remaining = (!config.max_runtime.is_zero())
.then(|| config.max_runtime.saturating_sub(started.elapsed()));
if remaining.is_some_and(|duration| duration.is_zero()) {
persist_lifecycle_before_shutdown(
&mut state,
&store,
&mut executor,
"suspended",
)?;
return Ok(());
}
match process_host_control(
control,
&mut state,
&store,
&mut executor,
true,
remaining,
) {
Ok(HostAction::Claim(ticket)) => {
bootstrap_ticket = Some(ticket);
lease = None;
authenticated_once = false;
disconnected_since = None;
reconnect_attempt = 0;
}
Ok(HostAction::Retire) => return Ok(()),
Ok(_) => continue,
Err(error) => {
let _ = persist_lifecycle_before_shutdown(
&mut state,
&store,
&mut executor,
"suspended",
);
return Err(error);
}
}
}
}
if authenticated_once {
let disconnected_at = disconnected_since.get_or_insert_with(Instant::now);
if config
@@ -584,6 +661,22 @@ pub fn run_durable_runner<E: CommandExecutor>(
continue;
}
let using_bootstrap = lease.is_none();
#[cfg(unix)]
if let Some(control) = host_control.as_mut() {
match process_host_control(control, &mut state, &store, &mut executor, false, None) {
Ok(HostAction::Park) => {
lease = None;
bootstrap_ticket = None;
continue 'session;
}
Ok(HostAction::Retire) => return Ok(()),
Err(error) => {
let _ = shutdown_preserving_cleanup(&state, &mut executor);
return Err(error);
}
_ => {}
}
}
let connect_deadline = connection_attempt_deadline(
&config,
started,
@@ -743,6 +836,10 @@ pub fn run_durable_runner<E: CommandExecutor>(
&mut endpoint,
next,
)?;
#[cfg(unix)]
if let Some(control) = host_control.as_mut() {
control.claiming = None;
}
}
Err(error) => {
state.record_diagnostic(format!(
@@ -786,6 +883,8 @@ pub fn run_durable_runner<E: CommandExecutor>(
welcome.warm_transition_version,
connection.protocol_version,
)?;
#[cfg(unix)]
admit_host_claim(host_control.as_ref(), &command, next_authority.as_ref())?;
let (result, lifecycle) =
process_command(&mut state, &store, &config, &mut executor, &command)?;
let next_authority = next_authority.filter(|_| completed_attachment(&result));
@@ -861,6 +960,10 @@ pub fn run_durable_runner<E: CommandExecutor>(
}
if let Some(next) = authority_rotation {
apply_authority_rotation(&mut state, &store, &mut config, &mut endpoint, next)?;
#[cfg(unix)]
if let Some(control) = host_control.as_mut() {
control.claiming = None;
}
disconnected_since = Some(Instant::now());
continue;
}
@@ -994,6 +1097,34 @@ pub fn run_durable_runner<E: CommandExecutor>(
// Read control before starting another fsynced provider batch.
// Consuming the last cumulative ACK must not let a new output
// suffix overtake the stop/suspend already queued behind it.
#[cfg(unix)]
{
if let Some(control) = host_control.as_mut() {
match process_host_control(
control,
&mut state,
&store,
&mut executor,
false,
None,
) {
Ok(HostAction::Park) => {
// The private owner now holds the exact idle cut. No
// old PRP frame, lease renewal, or provider poll may follow.
lease = None;
bootstrap_ticket = None;
drop(transport);
continue 'session;
}
Ok(HostAction::Retire) => return Ok(()),
Err(error) => {
let _ = shutdown_preserving_cleanup(&state, &mut executor);
return Err(error);
}
_ => {}
}
}
}
let control_message = transport.receive_json();
poll_executor_events_when_control_idle(
&mut state,
@@ -1058,6 +1189,8 @@ pub fn run_durable_runner<E: CommandExecutor>(
welcome.warm_transition_version,
connection.protocol_version,
)?;
#[cfg(unix)]
admit_host_claim(host_control.as_ref(), &command, next_authority.as_ref())?;
let (result, lifecycle) =
process_command(&mut state, &store, &config, &mut executor, &command)?;
let next_authority = next_authority.filter(|_| completed_attachment(&result));
@@ -1135,6 +1268,10 @@ pub fn run_durable_runner<E: CommandExecutor>(
&mut endpoint,
next,
)?;
#[cfg(unix)]
if let Some(control) = host_control.as_mut() {
control.claiming = None;
}
disconnected_since = Some(Instant::now());
break;
}
@@ -1238,6 +1375,148 @@ pub fn run_durable_runner<E: CommandExecutor>(
}
}
#[cfg(unix)]
enum HostAction {
None,
Park,
Claim(BootstrapTicket),
Retire,
}
#[cfg(unix)]
fn admit_host_claim(
control: Option<&HostedRunnerControl>,
command: &Command,
next_authority: Option<&DurableRunnerConfig>,
) -> Result<(), DurableRunnerError> {
if control.is_some_and(|control| control.claiming.is_some())
&& !(matches!(
command.command_type.as_str(),
"session.snapshot"
| "runner.drain"
| "runner.suspend"
| "runner.shutdown"
| "turn.stop"
) || (command.command_type == "run.attach" && next_authority.is_some()))
{
return Err(DurableRunnerError::invalid(
"claimed Runner requires authority attachment before execution",
));
}
Ok(())
}
#[cfg(unix)]
fn process_host_control<E: CommandExecutor>(
control: &mut HostedRunnerControl,
state: &mut DurableState,
store: &DurableStateStore,
executor: &mut E,
parked: bool,
remaining: Option<Duration>,
) -> Result<HostAction, DurableRunnerError> {
let Some(request) = control.channel.receive(parked, remaining)? else {
return Ok(HostAction::None);
};
if control.channel.replay(&request)? {
return Ok(HostAction::None);
}
match &request {
HostRequest::Park {
id,
identity,
last_controller_command_seq,
acked_source_seq,
} if !parked
&& control.park_id.is_none()
&& control.claiming.is_none()
&& host_park_cut_matches(
state,
identity,
*last_controller_command_seq,
*acked_source_seq,
)
&& executor.host_park_ready()? =>
{
control.park_id = Some(id.clone());
control.channel.reply(&request, "parked")?;
Ok(HostAction::Park)
}
HostRequest::Claim {
id,
park_id,
bootstrap_ticket,
..
} if parked && control.park_id.as_ref() == Some(park_id) => {
let ticket = BootstrapTicket::new(bootstrap_ticket.clone())?;
control.claiming = Some((park_id.clone(), id.clone()));
control.park_id = None;
control.channel.reply(&request, "claimed")?;
Ok(HostAction::Claim(ticket))
}
HostRequest::Retire {
park_id,
claim_id,
identity,
last_controller_command_seq,
acked_source_seq,
..
} if match (
claim_id,
identity,
last_controller_command_seq,
acked_source_seq,
) {
(None, None, None, None) => parked && control.park_id.as_ref() == Some(park_id),
(Some(claim_id), Some(identity), Some(command_seq), Some(acked_seq)) => {
!parked
&& control
.claiming
.as_ref()
.is_some_and(|(park, claim)| park == park_id && claim == claim_id)
&& host_park_cut_matches(state, identity, *command_seq, *acked_seq)
&& executor.host_park_ready()?
}
_ => false,
} =>
{
persist_lifecycle_before_shutdown(state, store, executor, "suspended")?;
control.channel.reply(&request, "retired")?;
Ok(HostAction::Retire)
}
_ => {
control.channel.reply(&request, "conflict")?;
Ok(HostAction::None)
}
}
}
#[cfg(unix)]
fn host_park_cut_matches(
state: &DurableState,
identity: &HostIdentity,
command_seq: u64,
acked_seq: u64,
) -> bool {
identity.runner_instance_id == state.runner_instance_id
&& identity.environment_lease_id == state.environment_lease_id
&& identity.run_id == state.run_id
&& identity.normalized_session_id == state.normalized_session_id
&& identity.turn_id == state.turn_id
&& identity.item_id == state.item_id
&& command_seq == state.last_controller_command_seq
&& acked_seq == state.acked_source_seq
&& state.outbox.is_empty()
&& state.acked_source_seq == state.highest_source_seq()
&& state.warm_transition.is_none()
&& state.pending_terminal_delivery.is_none()
&& state.pending_provider_cleanup.is_none()
&& state
.processed_commands
.values()
.all(|command| !matches!(command.status.as_str(), "pending" | "indeterminate"))
}
fn lease_renewal_deadline(now: u64, expires_at: u64) -> u64 {
now.saturating_add(expires_at.saturating_sub(now) / 2)
}
@@ -2721,6 +3000,271 @@ mod tests {
}
}
#[test]
#[cfg(unix)]
fn hosted_parking_requires_exact_cut_and_replays_claim_without_reusing_ticket() {
use std::io::{BufRead, BufReader, Write};
use std::os::unix::net::UnixStream;
struct IdleExecutor {
proofs: usize,
shutdowns: usize,
ready: bool,
}
impl CommandExecutor for IdleExecutor {
fn execute(&mut self, _: &Command) -> Result<CommandExecution, DurableRunnerError> {
panic!("private control cannot execute PRP commands");
}
fn host_park_ready(&mut self) -> Result<bool, DurableRunnerError> {
self.proofs += 1;
Ok(self.ready)
}
fn shutdown(&mut self) -> Result<(), DurableRunnerError> {
self.shutdowns += 1;
Ok(())
}
}
let directory = std::env::temp_dir().join(format!(
"paperclip-host-retirement-{}",
uuid::Uuid::new_v4()
));
let config = config(directory.clone());
let store = DurableStateStore::new(&config.state_dir).unwrap();
let mut state = DurableState::new(&config);
let (socket, mut host) = UnixStream::pair().unwrap();
let mut reader = BufReader::new(host.try_clone().unwrap());
let mut control = HostedRunnerControl::new(HostControl::new(socket).unwrap());
let mut executor = IdleExecutor {
proofs: 0,
shutdowns: 0,
ready: true,
};
let park = json!({"op":"park","id":"park-1","identity":{
"runnerInstanceId":state.runner_instance_id,"environmentLeaseId":state.environment_lease_id,
"runId":state.run_id,"normalizedSessionId":state.normalized_session_id,"turnId":state.turn_id,"itemId":state.item_id,
},"lastControllerCommandSeq":0,"ackedSourceSeq":0});
let mut exchange = |request: &Value,
state: &mut DurableState,
control: &mut HostedRunnerControl,
executor: &mut IdleExecutor,
parked: bool| {
writeln!(host, "{request}").unwrap();
let action = process_host_control(
control,
state,
&store,
executor,
parked,
Some(Duration::from_secs(1)),
)
.unwrap();
let mut line = String::new();
reader.read_line(&mut line).unwrap();
(action, serde_json::from_str::<Value>(&line).unwrap())
};
state.last_controller_command_seq = 1;
assert_eq!(
exchange(&park, &mut state, &mut control, &mut executor, false).1["status"],
"conflict"
);
state.last_controller_command_seq = 0;
assert!(matches!(
exchange(&park, &mut state, &mut control, &mut executor, false).0,
HostAction::Park
));
assert!(matches!(
exchange(&park, &mut state, &mut control, &mut executor, true).0,
HostAction::None
));
assert_eq!(executor.proofs, 1);
let claim = json!({"op":"claim","id":"claim-1","parkId":"park-1","bootstrapTicket":"fresh-private-ticket"});
assert!(matches!(
exchange(&claim, &mut state, &mut control, &mut executor, true).0,
HostAction::Claim(_)
));
assert!(matches!(
exchange(&claim, &mut state, &mut control, &mut executor, false).0,
HostAction::None
));
assert!(admit_host_claim(Some(&control), &command("turn.start"), None).is_err());
assert!(admit_host_claim(Some(&control), &command("session.snapshot"), None).is_ok());
assert!(admit_host_claim(Some(&control), &command("run.attach"), None).is_err());
let mut next_authority = config.clone();
next_authority.run_id = "next-run".to_owned();
assert!(admit_host_claim(
Some(&control),
&command("run.attach"),
Some(&next_authority)
)
.is_ok());
let mut changed = claim;
changed["bootstrapTicket"] = json!("different-private-ticket");
assert_eq!(
exchange(&changed, &mut state, &mut control, &mut executor, false).1["status"],
"conflict"
);
assert_eq!(executor.shutdowns, 0);
let mut premature_park = park.clone();
premature_park["id"] = json!("park-before-attach");
assert_eq!(
exchange(
&premature_park,
&mut state,
&mut control,
&mut executor,
false
)
.1["status"],
"conflict"
);
let retire = json!({"op":"retire","id":"retire-park","parkId":"park-1"});
assert_eq!(
exchange(&retire, &mut state, &mut control, &mut executor, false).1["status"],
"conflict"
);
let conditional_retire = json!({"op":"retire","id":"retire-claim","parkId":"park-1",
"claimId":"claim-1","identity":park["identity"],
"lastControllerCommandSeq":0,"ackedSourceSeq":0});
for (field, value) in [
("parkId", json!("wrong-park")),
("claimId", json!("wrong-claim")),
("lastControllerCommandSeq", json!(1)),
("ackedSourceSeq", json!(1)),
] {
let mut stale = conditional_retire.clone();
stale["id"] = json!(format!("stale-{field}"));
stale[field] = value;
assert_eq!(
exchange(&stale, &mut state, &mut control, &mut executor, false).1["status"],
"conflict"
);
}
let mut wrong_identity = conditional_retire.clone();
wrong_identity["id"] = json!("stale-identity");
wrong_identity["identity"]["runId"] = json!("another-run");
assert_eq!(
exchange(
&wrong_identity,
&mut state,
&mut control,
&mut executor,
false
)
.1["status"],
"conflict"
);
executor.ready = false;
assert_eq!(
exchange(
&conditional_retire,
&mut state,
&mut control,
&mut executor,
false
)
.1["status"],
"conflict"
);
assert_eq!(executor.shutdowns, 0);
executor.ready = true;
// Simulate completed run.attach before a later exact parked cut.
control.claiming = None;
assert!(admit_host_claim(Some(&control), &command("turn.start"), None).is_ok());
let mut next_park = park.clone();
next_park["id"] = json!("park-2");
assert!(matches!(
exchange(&next_park, &mut state, &mut control, &mut executor, false).0,
HostAction::Park
));
let retire = json!({"op":"retire","id":"retire-park","parkId":"park-2"});
assert!(matches!(
exchange(&retire, &mut state, &mut control, &mut executor, true).0,
HostAction::Retire
));
assert_eq!(executor.shutdowns, 1);
fs::remove_dir_all(directory).unwrap();
}
#[test]
#[cfg(unix)]
fn hosted_park_blocks_without_provider_polls_and_retires_on_eof_or_lifetime() {
use std::io::{BufRead, BufReader, Write};
use std::os::unix::net::UnixStream;
use std::sync::{
atomic::{AtomicUsize, Ordering},
Arc,
};
struct ParkedExecutor(Arc<AtomicUsize>);
impl CommandExecutor for ParkedExecutor {
fn execute(&mut self, _: &Command) -> Result<CommandExecution, DurableRunnerError> {
panic!("parked provider cannot execute");
}
fn host_park_ready(&mut self) -> Result<bool, DurableRunnerError> {
Ok(true)
}
fn poll_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {
panic!("parked provider cannot poll");
}
fn shutdown(&mut self) -> Result<(), DurableRunnerError> {
self.0.fetch_add(1, Ordering::SeqCst);
Ok(())
}
}
for eof in [false, true] {
let directory = std::env::temp_dir().join(format!(
"paperclip-hosted-park-stop-{}-{eof}",
std::process::id()
));
let _ = fs::remove_dir_all(&directory);
let mut config = config(directory.clone());
config.max_runtime = if eof {
Duration::ZERO
} else {
// Include durable startup under parallel-suite filesystem load.
Duration::from_secs(2)
};
let (socket, mut host) = UnixStream::pair().unwrap();
let control = HostControl::new(socket).unwrap();
writeln!(host,"{}",json!({"op":"park","id":"park-stop","identity":{
"runnerInstanceId":config.runner_instance_id,"environmentLeaseId":config.environment_lease_id,
"runId":config.run_id,"normalizedSessionId":config.normalized_session_id,"turnId":config.turn_id,"itemId":config.item_id,
},"lastControllerCommandSeq":0,"ackedSourceSeq":0})).unwrap();
let shutdowns = Arc::new(AtomicUsize::new(0));
let executor = ParkedExecutor(shutdowns.clone());
let saved_config = config.clone();
let task = std::thread::spawn(move || {
run_durable_runner_with_host_control(
config,
BootstrapTicket::new("unused-bootstrap".to_owned()).unwrap(),
executor,
Some(control),
)
});
let mut reply = String::new();
BufReader::new(&host).read_line(&mut reply).unwrap();
assert_eq!(
serde_json::from_str::<Value>(&reply).unwrap()["status"],
"parked"
);
if eof {
drop(host);
assert!(task.join().unwrap().is_err());
} else {
assert!(task.join().unwrap().is_ok());
}
assert_eq!(shutdowns.load(Ordering::SeqCst), 1);
assert_eq!(
DurableStateStore::new(&directory)
.unwrap()
.load_or_create(&saved_config)
.unwrap()
.0
.lifecycle,
"suspended"
);
fs::remove_dir_all(directory).unwrap();
}
}
#[test]
fn indeterminate_recovery_result_is_a_failed_wire_result() {
let state = DurableState::new(&config(PathBuf::from("unused")));
@@ -0,0 +1,258 @@
// Shares this module's authenticated PRP server helpers.
use crate::durable::{
run_durable_runner_with_host_control, CommandExecution, CommandExecutor, DurableStateStore,
PolledEvent,
};
use crate::host_control::HostControl;
use std::io::BufRead;
use std::os::unix::net::UnixStream;
struct RetentionExecutor {
polls: Arc<AtomicUsize>,
shutdowns: Arc<AtomicUsize>,
}
impl CommandExecutor for RetentionExecutor {
fn execute(&mut self, _: &Command) -> Result<CommandExecution, DurableRunnerError> {
panic!("host control must not execute business commands");
}
fn host_park_ready(&mut self) -> Result<bool, DurableRunnerError> {
Ok(true)
}
fn poll_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {
self.polls.fetch_add(1, Ordering::SeqCst);
Ok(vec![])
}
fn shutdown(&mut self) -> Result<(), DurableRunnerError> {
self.shutdowns.fetch_add(1, Ordering::SeqCst);
Ok(())
}
}
fn host_exchange(host: &mut UnixStream, request: Value) -> Value {
writeln!(host, "{request}").unwrap();
let mut line = String::new();
BufReader::new(host).read_line(&mut line).unwrap();
serde_json::from_str(&line).unwrap()
}
#[test]
fn hosted_loop_parks_and_reclaims_with_fresh_bootstrap_during_busy_prp() {
exercise_hosted_loop(None);
}
#[test]
fn hosted_claim_blocks_pending_and_live_work_before_attachment() {
for pending in [true, false] {
exercise_hosted_loop(Some(pending));
}
}
fn exercise_hosted_loop(blocked_work: Option<bool>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let mut config = config(listener.local_addr().unwrap().port());
let directory =
std::env::temp_dir().join(format!("paperclip-host-loop-{}", uuid::Uuid::new_v4()));
config.state_dir = directory.clone();
config.max_runtime = Duration::from_secs(15);
config.reconnect_grace = Some(Duration::from_millis(20));
let state = test_state(&config);
let server_config = config.clone();
let server_state = state.clone();
let (busy_tx, busy_rx) = mpsc::channel();
let (claimed_tx, claimed_rx) = mpsc::channel();
let (finish_tx, finish_rx) = mpsc::channel();
let server = thread::spawn(move || {
let expires = current_unix_ms().unwrap() + 60_000;
let (stream, _) = listener.accept().unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
stream
.set_write_timeout(Some(Duration::from_secs(5)))
.unwrap();
let mut socket = accept(stream).unwrap();
let mut secure = server_authenticate(
&mut socket,
&server_config,
&server_state,
ServerCredential {
token: "initial-private-bootstrap",
kind: "bootstrap",
lease_id: None,
expires_at_unix_ms: expires,
revocation_epoch: 0,
},
);
send_secure(
&mut socket,
&mut secure,
&server_config,
&welcome(
&server_state,
"connection-1",
Some("old-private-lease"),
expires,
0,
vec![],
),
);
let mut announced = false;
// Keep PRP readable. Host control must not depend on an idle socket read.
loop {
let ping = control(&server_state, "connection-1", "ping", json!({}));
let encrypted = secure
.encrypt(&serde_json::to_vec(&ping).unwrap(), true)
.unwrap();
if send_plain(&mut socket, &encrypted, server_config.max_frame_bytes).is_err() {
break;
}
let Ok(frame) = receive_plain(&mut socket, server_config.max_frame_bytes) else {
break;
};
assert_eq!(secure.decrypt(&frame, true).unwrap()["kind"], "pong");
if !announced {
busy_tx.send(()).unwrap();
announced = true;
}
}
drop(socket);
// A parked Runner must accept the new bootstrap, never the old lease.
let (stream, _) = listener.accept().unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
let mut socket = accept(stream).unwrap();
let mut secure = server_authenticate(
&mut socket,
&server_config,
&server_state,
ServerCredential {
token: "fresh-private-bootstrap",
kind: "bootstrap",
lease_id: None,
expires_at_unix_ms: expires,
revocation_epoch: 0,
},
);
let forbidden = json!({
"schema":"paperclip.prp.command.v1", "commandId":"premature-turn",
"controllerSeq":1, "type":"turn.start", "issuedAt":"2026-01-01T00:00:00Z", "payload":{}
});
send_secure(
&mut socket,
&mut secure,
&server_config,
&welcome(
&server_state,
"connection-2",
Some("new-private-lease"),
expires,
0,
if blocked_work == Some(true) { vec![forbidden.clone()] } else { vec![] },
),
);
if let Some(pending) = blocked_work {
if !pending {
send_secure(&mut socket, &mut secure, &server_config,
&control(&server_state, "connection-2", "command", forbidden));
}
finish_rx.recv_timeout(Duration::from_secs(5)).unwrap();
return;
}
send_secure(
&mut socket,
&mut secure,
&server_config,
&control(&server_state, "connection-2", "ping", json!({})),
);
assert_eq!(
receive_secure(&mut socket, &mut secure, &server_config)["kind"],
"pong"
);
claimed_tx.send(()).unwrap();
finish_rx.recv_timeout(Duration::from_secs(5)).unwrap();
});
let polls = Arc::new(AtomicUsize::new(0));
let shutdowns = Arc::new(AtomicUsize::new(0));
let executor = RetentionExecutor {
polls: polls.clone(),
shutdowns: shutdowns.clone(),
};
let (socket, mut host) = UnixStream::pair().unwrap();
host.set_read_timeout(Some(Duration::from_secs(5))).unwrap();
let runner_config = config.clone();
let runner = thread::spawn(move || {
run_durable_runner_with_host_control(
runner_config,
BootstrapTicket::new("initial-private-bootstrap".to_owned()).unwrap(),
executor,
Some(HostControl::new(socket).unwrap()),
)
});
busy_rx.recv_timeout(Duration::from_secs(5)).unwrap();
let identity = json!({"runnerInstanceId":state.runner_instance_id,"environmentLeaseId":state.environment_lease_id,
"runId":state.run_id,"normalizedSessionId":state.normalized_session_id,"turnId":state.turn_id,"itemId":state.item_id});
assert_eq!(
host_exchange(
&mut host,
json!({"op":"park","id":"park-1","identity":identity,
"lastControllerCommandSeq":0,"ackedSourceSeq":0})
)["status"],
"parked"
);
let parked_polls = polls.load(Ordering::SeqCst);
// Outlast the ordinary reconnect grace without any polling or shutdown.
thread::sleep(Duration::from_millis(80));
assert_eq!(polls.load(Ordering::SeqCst), parked_polls);
assert_eq!(shutdowns.load(Ordering::SeqCst), 0);
assert_eq!(
host_exchange(
&mut host,
json!({"op":"claim","id":"claim-1","parkId":"park-1",
"bootstrapTicket":"fresh-private-bootstrap"})
)["status"],
"claimed"
);
if blocked_work.is_some() {
let error = runner.join().unwrap().unwrap_err();
assert_eq!(error.to_string(), "claimed Runner requires authority attachment before execution");
finish_tx.send(()).unwrap();
server.join().unwrap();
let retained = DurableStateStore::new(&directory).unwrap().load_or_create(&config).unwrap().0;
assert_eq!(retained.last_controller_command_seq, 0);
assert!(retained.processed_commands.is_empty());
std::fs::remove_dir_all(directory).unwrap();
return;
}
claimed_rx.recv_timeout(Duration::from_secs(5)).unwrap();
// A stale parked-only retirement cannot kill a newly claimed process.
assert_eq!(
host_exchange(
&mut host,
json!({"op":"retire","id":"stale-retire","parkId":"park-1"})
)["status"],
"conflict"
);
assert_eq!(
host_exchange(
&mut host,
json!({"op":"retire","id":"retire-claim","parkId":"park-1",
"claimId":"claim-1","identity":identity,"lastControllerCommandSeq":0,"ackedSourceSeq":0})
)["status"],
"retired"
);
assert!(runner.join().unwrap().is_ok());
finish_tx.send(()).unwrap();
server.join().unwrap();
assert_eq!(shutdowns.load(Ordering::SeqCst), 1);
assert_eq!(
DurableStateStore::new(&directory)
.unwrap()
.load_or_create(&config)
.unwrap()
.0
.lifecycle,
"suspended"
);
std::fs::remove_dir_all(directory).unwrap();
}
@@ -1747,6 +1747,9 @@ fn hex_nibble(byte: u8) -> Result<u8, DurableRunnerError> {
#[cfg(test)]
mod tests {
#[cfg(unix)]
include!("tests/hosted_retention.rs");
use std::net::{IpAddr, Ipv4Addr, TcpListener};
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};