diff --git a/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md b/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md index 24d651c0d37..ff9a14d0891 100644 --- a/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md +++ b/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md @@ -313,6 +313,14 @@ claim ID, the unchanged authority and sequence cut, and provider quiescence. Retirement, parked host-channel loss, or process-lifetime expiry uses the existing suspended lifecycle and provider shutdown path. There is no new checkpoint state. +When a trusted host relaunches an absent Runner from a verified ACPX park, it can +pass `--acpx-park-relaunch` with the private host-control descriptor. For +park-eligible ACPX providers, Runner validates the settled checkpoint and keeps +the claim fence without starting the provider. The authenticated `run.attach` +opens the provider with the incoming run's bindings and checks live session +identity and dormancy before committing the attachment. An existing warm +transition receipt follows the ordinary replay path. + This native loop does not advertise hosted parking support: Launcher retention, Core orchestration, provider-specific qualification, and live VM sleep/wake verification are required before rollout. Providers without an eligibility diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index 43d40c98580..8f9bb560e22 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -325,6 +325,13 @@ impl AcpxProviderDescriptor { }) } + fn supports_host_parking(&self) -> bool { + matches!(self.agent.as_str(), "codex" | "claude") + && self.names_qualified_release() + && acpx_release_profile(&self.agent) + .is_some_and(|profile| !profile.qualification_pending) + } + fn validate_session( &self, context: &AcpxEventProjectionContext, @@ -1035,6 +1042,7 @@ pub struct AcpxCommandExecutor { fail_after_state_rename: bool, launch_profile: Option, recovery_pending_effects: Option, + parked_relaunch: bool, /// The failed turn-end restart's cause, reported by the next turn start. failed_turn_end_restart: Option, /// The failed turn-end restart's cause, until the Runner records it. @@ -1079,6 +1087,7 @@ impl AcpxCommandExecutor { session: None, restore_checked: false, recovery_pending_effects: None, + parked_relaunch: false, failed_turn_end_restart: None, unreported_turn_end_restart_failure: None, turn_accepts_provider_turns: false, @@ -1098,6 +1107,33 @@ impl AcpxCommandExecutor { self.recovery_pending_effects = Some(state.recovery_pending_effects()); } + pub(crate) fn defer_parked_acpx_restore(&mut self) -> Result<(), DurableRunnerError> { + self.ensure_persistence_healthy()?; + self.load_state_without_session()?; + let settled = self.state.as_ref().is_some_and(|state| { + state.descriptor.supports_host_parking() + && state.descriptor.run_id == self.context.run_id + && matches!(state.lifecycle.as_str(), "session_open" | "suspended") + && settled_checkpoint_blockers(state).is_empty() + && state.pending_events.is_empty() + && state.pending_runtime_requests.is_empty() + && state.provider_turn_wait.is_none() + && (state.goal_projection.is_null() || goal_is_dormant(&state.goal_projection)) + }); + if !settled + || self.recovery_pending_effects != Some(0) + || self.candidate_admission().is_some() + { + return Err(DurableRunnerError::invalid( + "parked ACPX recovery requires a settled provider checkpoint qualified for host parking", + ) + .with_classification(crate::durable::PROVIDER_ATTACH_REFUSED)); + } + self.restore_checked = true; + self.parked_relaunch = true; + Ok(()) + } + fn candidate_admission( &self, ) -> Option<&crate::acpx_candidate_admission::AcpxCandidateAdmission> { @@ -1556,6 +1592,14 @@ impl AcpxCommandExecutor { .state .as_ref() .ok_or_else(|| DurableRunnerError::invalid("ACPX provider has not been prepared"))?; + self.start_session_from(state, recovering) + } + + fn start_session_from( + &self, + state: &AcpxDurableState, + recovering: bool, + ) -> Result { let expected = recovering.then(|| state.identity.clone()).flatten(); let config = state.descriptor.session_config( state.tool_set.clone(), @@ -1933,7 +1977,12 @@ impl AcpxCommandExecutor { if clears_turn_end_restart_failure { state.turn_end_restart_failed_run = None; } - let saved = if cold_candidate_attachment { + let saved = if self.parked_relaunch { + let session = self.start_session_from(&state, true)?; + self.commit_staged_state(state)?; + self.session = Some(session); + Ok(()) + } else if cold_candidate_attachment { self.commit_staged_state(state) } else { self.state = Some(state); @@ -3135,7 +3184,7 @@ impl CommandExecutor for AcpxCommandExecutor { let Some(state) = self.state.as_ref() else { return Ok(false); }; - if !matches!(state.descriptor.agent.as_str(), "codex" | "claude") + if !state.descriptor.supports_host_parking() || self.session.is_none() || state.lifecycle != "session_open" || state.provider_exit_unconfirmed @@ -3145,11 +3194,8 @@ impl CommandExecutor for AcpxCommandExecutor { || !state.pending_events.is_empty() || !state.pending_runtime_requests.is_empty() || (!state.goal_projection.is_null() && !goal_is_dormant(&state.goal_projection)) - || !state.descriptor.names_qualified_release() || state.provider_turn_wait.is_some() || self.candidate_admission().is_some() - || !acpx_release_profile(&state.descriptor.agent) - .is_some_and(|profile| !profile.qualification_pending) { return Ok(false); } @@ -3277,6 +3323,7 @@ impl CommandExecutor for AcpxCommandExecutor { } fn rotate_authority(&mut self, config: &DurableRunnerConfig) { + self.parked_relaunch = false; self.recovery_pending_effects = None; self.context.run_id = config.run_id.clone(); self.context.normalized_session_id = config.normalized_session_id.clone(); @@ -3512,6 +3559,10 @@ pub(crate) fn secure_directory(path: &Path, label: &str) -> Result<(), DurableRu }) } +#[cfg(all(test, unix))] +#[path = "tests/acpx_park_relaunch.rs"] +pub(crate) mod park_relaunch_tests; + #[cfg(test)] mod tests { use super::*; @@ -3532,7 +3583,7 @@ mod tests { directory } - fn write_artifact(path: &Path, contents: &[u8], executable: bool) { + pub(super) fn write_artifact(path: &Path, contents: &[u8], executable: bool) { fs::write(path, contents).unwrap(); #[cfg(unix)] fs::set_permissions( @@ -3542,7 +3593,7 @@ mod tests { .unwrap(); } - fn artifact(path: &Path) -> QualifiedLaunchArtifact { + pub(super) fn artifact(path: &Path) -> QualifiedLaunchArtifact { QualifiedLaunchArtifact { path: path.to_owned(), sha256: format!("sha256:{:x}", Sha256::digest(fs::read(path).unwrap())), @@ -6327,7 +6378,7 @@ mod tests { ) } - fn cold_attach_command(payload: Value) -> Command { + pub(super) fn cold_attach_command(payload: Value) -> Command { Command { schema: "paperclip.prp.command.v1".into(), command_id: "cold-attach".into(), @@ -6629,12 +6680,13 @@ mod tests { fs::remove_dir_all(directory).unwrap(); } - // A settled Claude session saved by one runner build. The next deploy + // A settled ACPX session saved by one runner build. The next deploy // ships a new sidecar build and provider-pack authority at the same // qualified launch paths, and the next run cold-attaches in a new runner. #[cfg(unix)] - fn settled_session_after_runner_upgrade( + pub(super) fn settled_session_after_runner_upgrade( label: &str, + agent: &str, ) -> (PathBuf, DurableRunnerConfig, Value, Vec) { let directory = temporary_directory(label); let command = directory.join("node"); @@ -6654,7 +6706,7 @@ mod tests { fs::create_dir_all(&workspace).unwrap(); fs::set_permissions(&runtime, fs::Permissions::from_mode(0o700)).unwrap(); fs::set_permissions(&workspace, fs::Permissions::from_mode(0o700)).unwrap(); - let mut value = descriptor("claude"); + let mut value = descriptor(agent); value["runtimeDirectory"] = json!(runtime); value["cwd"] = json!(workspace); value["sidecarCommand"] = json!(command); @@ -6717,7 +6769,7 @@ mod tests { #[test] fn settled_session_rebinds_to_the_upgraded_runner_launch_profile() { let (directory, config, payload, saved) = - settled_session_after_runner_upgrade("upgrade-rebind"); + settled_session_after_runner_upgrade("upgrade-rebind", "claude"); let prior: AcpxDurableState = serde_json::from_slice(&saved).unwrap(); let current_digest = config .acpx_launch_profile @@ -6781,7 +6833,7 @@ mod tests { ]; for (label, seed) in settled { let (directory, config, payload, saved) = - settled_session_after_runner_upgrade("upgrade-retired-settled"); + settled_session_after_runner_upgrade("upgrade-retired-settled", "claude"); let state_path = directory.join(ACPX_PROVIDER_STATE_FILE); let mut prior: AcpxDurableState = serde_json::from_slice(&saved).unwrap(); seed(&mut prior); @@ -6825,7 +6877,8 @@ mod tests { fn interrupted_turn_after_runner_upgrade( label: &str, ) -> (PathBuf, DurableRunnerConfig, PathBuf) { - let (directory, mut config, _, saved) = settled_session_after_runner_upgrade(label); + let (directory, mut config, _, saved) = + settled_session_after_runner_upgrade(label, "claude"); config.run_id = "run-1".to_owned(); config.turn_id = "turn-1".to_owned(); config.item_id = "item-1".to_owned(); @@ -6955,7 +7008,7 @@ mod tests { fs::remove_dir_all(directory).unwrap(); let (directory, mut config, payload, _) = - settled_session_after_runner_upgrade("upgrade-relocated-settled"); + settled_session_after_runner_upgrade("upgrade-relocated-settled", "claude"); let (command, args) = relocate_upgraded_install(&directory, &mut config); // The controller attaches with the payload it saved, which still names // the old install's paths. @@ -7037,7 +7090,7 @@ mod tests { ("acpx_release", ""), ] { let (directory, mut config, _, saved) = - settled_session_after_runner_upgrade("upgrade-strict"); + settled_session_after_runner_upgrade("upgrade-strict", "claude"); let state_path = directory.join(ACPX_PROVIDER_STATE_FILE); let mut state: AcpxDurableState = serde_json::from_slice(&saved).unwrap(); let profile = config.acpx_launch_profile.as_mut().unwrap(); @@ -7119,7 +7172,7 @@ mod tests { #[test] fn a_settled_session_saved_by_the_previous_acpx_release_continues() { let (directory, config, payload, saved) = - settled_session_after_runner_upgrade("previous-acpx-settled"); + settled_session_after_runner_upgrade("previous-acpx-settled", "claude"); let state_path = directory.join(ACPX_PROVIDER_STATE_FILE); let mut prior: AcpxDurableState = serde_json::from_slice(&saved).unwrap(); prior.descriptor.acpx_version = "0.13.1".into(); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs index 6d5ffa68fbf..2b9984486d5 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs @@ -460,6 +460,28 @@ fn run_durable(args: &[String]) -> Result<(), LocalRunnerError> { } else { None }; + let host_control = if args.iter().any(|arg| arg == "--acpx-park-relaunch") { + #[cfg(unix)] + { + Some( + host_control + .ok_or_else(|| { + LocalRunnerError::invalid( + "ACPX park relaunch requires private host control", + ) + })? + .with_acpx_park_relaunch(), + ) + } + #[cfg(not(unix))] + { + return Err(LocalRunnerError::invalid( + "ACPX park relaunch requires private host control", + )); + } + } else { + host_control + }; let ticket = capture_bootstrap_ticket() .map_err(|error| LocalRunnerError::invalid(error.to_string()))? .ok_or_else(|| { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs index e9c2eafc585..53326fb8a64 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs @@ -339,6 +339,12 @@ pub trait CommandExecutor { Ok(()) } + fn defer_parked_acpx_restore(&mut self) -> Result<(), DurableRunnerError> { + Err(DurableRunnerError::invalid( + "parked ACPX recovery requires an ACPX checkpoint", + )) + } + /// Opt in only when semantic result delivery validates an exact durable /// call receipt and cannot dispatch the underlying business operation. /// Ordinary commands and providers without this proof remain indeterminate. @@ -494,17 +500,28 @@ struct HostedRunnerControl { channel: HostControl, park_id: Option, claiming: Option<(String, String)>, + acpx_park_relaunch: bool, } #[cfg(unix)] impl HostedRunnerControl { fn new(channel: HostControl) -> Self { Self { + acpx_park_relaunch: channel.acpx_park_relaunch, channel, park_id: None, claiming: None, } } + + fn awaiting_attachment(&self) -> bool { + self.claiming.is_some() || self.acpx_park_relaunch + } + + fn finish_attachment(&mut self) { + self.claiming = None; + self.acpx_park_relaunch = false; + } } pub fn run_durable_runner_with_host_control( @@ -536,6 +553,22 @@ pub fn run_durable_runner_with_host_control( } let store = DurableStateStore::new(&config.state_dir)?; let (mut state, recovered) = store.load_or_create(&config)?; + #[cfg(unix)] + if let Some(control) = host_control + .as_mut() + .filter(|control| control.acpx_park_relaunch) + { + if state.warm_transition.is_some() { + control.finish_attachment(); + } else if !recovered + || !matches!(state.lifecycle.as_str(), "ready" | "suspended") + || !host_park_checkpoint_is_settled(&state) + { + return Err(DurableRunnerError::invalid( + "parked ACPX recovery requires a settled Runner checkpoint", + )); + } + } if state.lifecycle == "revoked" || (state.lifecycle == "stopped" && state.pending_terminal_delivery.is_none() @@ -547,6 +580,13 @@ pub fn run_durable_runner_with_host_control( // A restarted runner process emits a new event generation (section G). state.begin_recovered_generation(); executor.observe_recovery_state(&state)?; + #[cfg(unix)] + if host_control + .as_ref() + .is_some_and(|control| control.acpx_park_relaunch) + { + executor.defer_parked_acpx_restore()?; + } } if recovered && state.warm_transition.is_none() { state.reconnect_count = state.reconnect_count.saturating_add(1); @@ -871,7 +911,7 @@ pub fn run_durable_runner_with_host_control( )?; #[cfg(unix)] if let Some(control) = host_control.as_mut() { - control.claiming = None; + control.finish_attachment(); } } Err(error) => { @@ -996,7 +1036,7 @@ pub fn run_durable_runner_with_host_control( apply_authority_rotation(&mut state, &store, &mut config, &mut endpoint, next)?; #[cfg(unix)] if let Some(control) = host_control.as_mut() { - control.claiming = None; + control.finish_attachment(); } disconnected_since = Some(Instant::now()); continue; @@ -1321,7 +1361,7 @@ pub fn run_durable_runner_with_host_control( )?; #[cfg(unix)] if let Some(control) = host_control.as_mut() { - control.claiming = None; + control.finish_attachment(); } disconnected_since = Some(Instant::now()); break; @@ -1440,7 +1480,7 @@ fn admit_host_claim( command: &Command, next_authority: Option<&DurableRunnerConfig>, ) -> Result<(), DurableRunnerError> { - if control.is_some_and(|control| control.claiming.is_some()) + if control.is_some_and(HostedRunnerControl::awaiting_attachment) && !(matches!( command.command_type.as_str(), "session.snapshot" @@ -1480,7 +1520,7 @@ fn process_host_control( acked_source_seq, } if !parked && control.park_id.is_none() - && control.claiming.is_none() + && !control.awaiting_attachment() && host_park_cut_matches( state, identity, @@ -1571,7 +1611,12 @@ fn host_park_cut_matches( && identity.item_id == state.item_id && command_seq == state.last_controller_command_seq && acked_seq == state.acked_source_seq - && state.outbox.is_empty() + && host_park_checkpoint_is_settled(state) +} + +#[cfg(unix)] +fn host_park_checkpoint_is_settled(state: &DurableState) -> bool { + state.outbox.is_empty() && state.acked_source_seq == state.highest_source_seq() && state.warm_transition.is_none() && state.pending_terminal_delivery.is_none() @@ -3271,7 +3316,12 @@ mod tests { assert_eq!(executor.shutdowns, 0); executor.ready = true; // Simulate completed run.attach before a later exact parked cut. - control.claiming = None; + control.finish_attachment(); + control.acpx_park_relaunch = true; + assert!(admit_host_claim(Some(&control), &command("turn.start"), None).is_err()); + assert!(admit_host_claim(Some(&control), &command("run.attach"), None).is_err()); + assert!(admit_host_claim(Some(&control), &command("session.snapshot"), None).is_ok()); + control.finish_attachment(); assert!(admit_host_claim(Some(&control), &command("turn.start"), None).is_ok()); let mut next_park = park.clone(); next_park["id"] = json!("park-2"); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/tests/acpx_park_relaunch.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/tests/acpx_park_relaunch.rs new file mode 100644 index 00000000000..c6374a88369 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/tests/acpx_park_relaunch.rs @@ -0,0 +1,154 @@ +#[test] +fn parked_acpx_relaunch_replays_an_interrupted_attachment_before_admitting_work() { + use crate::acpx_provider_backend::{ + park_relaunch_tests::parked_acpx_fixture, ACPX_PROVIDER_STATE_FILE, + }; + use crate::durable::state::{ + PendingWarmRunTransition, WarmRunIdentity, WarmRunTransition, TRANSITION_STATE_SCHEMA, + }; + use crate::native_provider_backend::NativeProviderCommandExecutor; + + struct ResumedExecutor(bool); + impl CommandExecutor for ResumedExecutor { + fn execute(&mut self, command: &Command) -> Result { + assert!(self.0, "commands are fenced until activation"); + assert_eq!(command.command_type, "turn.start", "attachment must only replay"); + Ok(CommandExecution { + result: json!({"runId":"run-2"}), + events: vec![], + }) + } + + fn rotate_authority(&mut self, config: &DurableRunnerConfig) { + assert_eq!(config.run_id, "run-2"); + self.0 = true; + } + } + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let (directory, mut old, mut attach, _, _) = parked_acpx_fixture("claude", false); + old.connect_url = format!("ws://{}/api/runner/v1/connect/run-1", listener.local_addr().unwrap()); + old.max_runtime = Duration::from_secs(10); + old.runner_digest = format!("sha256:{}", "a".repeat(64)); + let mut next = old.clone(); + next.run_id = "run-2".into(); + next.turn_id = "turn-2".into(); + next.item_id = "item-2".into(); + attach.payload["paperclipNextAuthority"] = json!({ + "identity": WarmRunIdentity::from_config(&next), + "connection": {"mode":"external"}, + }); + let store = DurableStateStore::new(&directory).unwrap(); + let mut state = DurableState::new(&old); + state.lifecycle = "ready".into(); + let mut prior = NativeProviderCommandExecutor::with_runner_config(&directory, &old); + prior.observe_recovery_state(&state).unwrap(); + prior.defer_parked_acpx_restore().unwrap(); + state.begin_command(&attach).unwrap(); + let execution = prior.execute(&attach).unwrap(); + assert_eq!(execution.result["status"], "resumed"); + let result = state.complete_command(&attach, execution.result).unwrap(); + let expires = current_unix_ms().unwrap() + 60_000; + let receipt = WarmRunTransition::new( + &old, &next, &attach, &result, state.acked_source_seq, "lease_1".into(), expires, 1, + ).unwrap(); + state.schema = TRANSITION_STATE_SCHEMA.into(); + state.warm_transition = Some(PendingWarmRunTransition { + receipt: receipt.clone(), + phase: "prepared".into(), + command: attach, + result, + }); + store.save(&state).unwrap(); + drop(prior); + + let provider_path = directory.join(ACPX_PROVIDER_STATE_FILE); + let provider = std::fs::read(&provider_path).unwrap(); + let provider_state: Value = serde_json::from_slice(&provider).unwrap(); + assert_eq!(provider_state["descriptor"]["runId"], next.run_id); + assert!(store.load_or_create(&next).is_err(), "incoming authority is still fenced"); + + let (socket, host) = UnixStream::pair().unwrap(); + let runner_config = old.clone(); + let runner = thread::spawn(move || { + run_durable_runner_with_host_control( + runner_config, + BootstrapTicket::new("recovery-bootstrap".into()).unwrap(), + ResumedExecutor(false), + Some(HostControl::new(socket).unwrap().with_acpx_park_relaunch()), + ) + }); + let receipt_value = serde_json::to_value(&receipt).unwrap(); + let turn = json!({ + "schema":"paperclip.prp.command.v1", "commandId":"next-turn", + "controllerSeq":1, "type":"turn.start", "issuedAt":"2026-01-01T00:00:00Z", "payload":{} + }); + let mut prior_turn = turn.clone(); + prior_turn["controllerSeq"] = json!(2); + let mut new_state = DurableState::new(&next); + new_state.warm_transition = state.warm_transition.clone(); + listener.set_nonblocking(true).unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + for (expected, config, activating) in [(&state, &old, false), (&new_state, &next, true)] { + let stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock && Instant::now() < deadline => { + thread::sleep(Duration::from_millis(5)); + } + Err(e) => panic!("accept recovery connection: {e}"), + } + }; + stream.set_nonblocking(false).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let mut socket = accept(stream).unwrap(); + let mut secure = server_authenticate(&mut socket, config, expected, ServerCredential { + token: if activating { "recovery-lease" } else { "recovery-bootstrap" }, + kind: if activating { "lease" } else { "bootstrap" }, + lease_id: activating.then_some("lease_1"), + expires_at_unix_ms: expires, + revocation_epoch: 1, + }); + let mut welcome = welcome( + expected, "recovery", (!activating).then_some("recovery-lease"), expires, 0, + if activating { vec![] } else { vec![prior_turn.clone()] }, + ); + welcome["payload"]["warmTransitionVersion"] = json!(1); + if activating { + welcome["payload"]["warmTransitionPhase"] = json!("activated"); + welcome["payload"]["warmTransition"] = receipt_value.clone(); + } + send_secure(&mut socket, &mut secure, config, &welcome); + let replay = receive_secure(&mut socket, &mut secure, config); + assert_eq!(replay["runId"], expected.run_id); + let ack = if activating { + assert_eq!(replay["kind"], "warm_transition_activated"); + assert_eq!(replay["payload"]["transitionId"], receipt.transition_id); + control(expected, "recovery", "warm_transition_activated_ack", + json!({"transitionId": receipt.transition_id})) + } else { + assert_eq!(replay["kind"], "command_result"); + assert_eq!(replay["payload"]["commandId"], receipt.command_id); + control(expected, "recovery", "command_result_ack", json!({ + "commandId":receipt.command_id, "controllerSeq":receipt.controller_seq, + "commandType":"run.attach", "status":"completed", "warmTransition":receipt_value, + })) + }; + send_secure(&mut socket, &mut secure, config, &ack); + if activating { + send_secure(&mut socket, &mut secure, config, + &control(expected, "recovery", "command", turn.clone())); + let result = receive_secure(&mut socket, &mut secure, config); + assert_eq!(result["payload"]["commandId"], "next-turn"); + assert_eq!(result["payload"]["result"]["runId"], "run-2"); + } + } + drop(host); + assert!(runner.join().unwrap().is_err()); + assert!(store.load_or_create(&old).is_err()); + let (active, _) = store.load_or_create(&next).unwrap(); + assert!(active.warm_transition.is_none()); + assert_eq!(std::fs::read(provider_path).unwrap(), provider); + assert_eq!(std::fs::read_to_string(directory.join("starts")).unwrap(), "x"); + std::fs::remove_dir_all(directory).unwrap(); +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs index 578419e8bb4..fe648064c29 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs @@ -1771,6 +1771,8 @@ fn hex_nibble(byte: u8) -> Result { mod tests { #[cfg(unix)] include!("tests/hosted_retention.rs"); + #[cfg(unix)] + include!("tests/acpx_park_relaunch.rs"); include!("tests/ack_release_poll.rs"); include!("tests/indeterminate_terminal_ack.rs"); @@ -1834,6 +1836,13 @@ mod tests { let credential = CredentialMaterial::from_token(server_credential.token); assert_eq!(payload["credentialId"], credential.credential_id); assert_eq!(payload["credentialKind"], server_credential.kind); + let warm_transition_id = state + .warm_transition + .as_ref() + .map(|transition| transition.receipt.transition_id.clone()); + if let Some(id) = &warm_transition_id { + assert_eq!(payload["warmTransitionId"], *id); + } let mut challenge = AuthChallenge { credential_id: credential.credential_id.clone(), credential_kind: server_credential.kind.to_owned(), @@ -1853,8 +1862,8 @@ mod tests { credential_lease_id: server_credential.lease_id.map(str::to_owned), revocation_epoch: server_credential.revocation_epoch, server_proof: String::new(), - warm_transition_version: None, - warm_transition_id: None, + warm_transition_version: warm_transition_id.as_ref().map(|_| 1), + warm_transition_id, recovery_cursor_version: Some(1), recovery_acked_source_seq: payload["resume"]["ackedSourceSeq"].as_u64(), }; @@ -1889,6 +1898,8 @@ mod tests { "credentialLeaseId": &challenge.credential_lease_id, "revocationEpoch": challenge.revocation_epoch, "serverProof": &challenge.server_proof, + "warmTransitionVersion": challenge.warm_transition_version, + "warmTransitionId": &challenge.warm_transition_id, "recoveryCursorVersion": challenge.recovery_cursor_version, "recoveryAckedSourceSeq": challenge.recovery_acked_source_seq, }, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/host_control.rs b/packages/paperclip-runner/runner/crates/runner-core/src/host_control.rs index 12886ac43a2..fbb8ecb4f55 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/host_control.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/host_control.rs @@ -99,6 +99,7 @@ impl HostRequest { } pub struct HostControl { + pub(crate) acpx_park_relaunch: bool, socket: UnixStream, input: Vec, receipts: BTreeMap, @@ -130,12 +131,18 @@ impl HostControl { .set_write_timeout(Some(Duration::from_secs(1))) .map_err(|_| failure())?; Ok(Self { + acpx_park_relaunch: false, socket, input: Vec::new(), receipts: BTreeMap::new(), }) } + pub fn with_acpx_park_relaunch(mut self) -> Self { + self.acpx_park_relaunch = true; + self + } + pub(crate) fn receive( &mut self, parked: bool, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs index ce58aa81193..1bb19a6c0af 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs @@ -403,6 +403,15 @@ impl CommandExecutor for NativeProviderCommandExecutor { Ok(()) } + fn defer_parked_acpx_restore(&mut self) -> Result<(), DurableRunnerError> { + match self.selected.as_mut() { + Some(SelectedExecutor::Acpx(executor)) => executor.defer_parked_acpx_restore(), + _ => Err(DurableRunnerError::invalid( + "parked ACPX recovery requires an ACPX checkpoint", + )), + } + } + fn can_reconcile_result_delivery(&mut self) -> Result { self.select_recovery()?; self.selected diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/tests/acpx_park_relaunch.rs b/packages/paperclip-runner/runner/crates/runner-core/src/tests/acpx_park_relaunch.rs new file mode 100644 index 00000000000..f79ac9e6d3a --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/tests/acpx_park_relaunch.rs @@ -0,0 +1,108 @@ +use super::tests::{ + artifact, cold_attach_command, settled_session_after_runner_upgrade, write_artifact, +}; +use super::*; + +pub(crate) fn parked_acpx_fixture( + agent: &str, + active: bool, +) -> ( + PathBuf, + DurableRunnerConfig, + Command, + Vec, + AcpxProviderSessionIdentity, +) { + let (directory, mut config, mut payload, before) = + settled_session_after_runner_upgrade("acpx-park-relaunch", agent); + config.run_id = "run-1".into(); + config.turn_id = "turn-1".into(); + config.item_id = "item-1".into(); + let profile = config.acpx_launch_profile.as_mut().unwrap(); + let checkpoint: AcpxDurableState = serde_json::from_slice(&before).unwrap(); + let identity = checkpoint.identity.unwrap(); + let marker = directory.join("starts"); + let goal = json!({ + "schema":"paperclip.session_goal.snapshot.v1", + "goal": if active { json!({"status":"active"}) } else { Value::Null }, + "workingNow":false, + }); + let mut script = format!("#!/bin/sh\nprintf x >> '{}'\n", marker.display()); + for (index, (command, result)) in [ + ("initialize", json!({"protocolVersion":2,"sidecarPid":900001,"profile":{}, + "capabilities":{"persistentSessions":true,"exactModelVerification":true, + "permissions":"runner_policy","semanticTools":"runner_bridge","structuredInput":"paperclip.question_set.v1"}})), + ("session.open", json!({"sidecarPid":900001,"identity":identity,"status":{}})), + ("run.attach", json!({"runId":"run-2","catalogRevision":1})), + ("session.goal.get", goal.clone()), + ("session.snapshot", json!({"identity":identity,"runId":"run-2","turnId":null, + "pendingToolCount":0,"pendingInputCount":0,"pendingRuntimeRequests":[]})), + ("session.goal.get", goal), + ].into_iter().enumerate() { + let frame = json!({"protocolVersion":2,"id":index+1,"ok":true,"result":result}); + script.push_str(&format!("IFS= read -r line || exit 0\ncase \"$line\" in *'\"command\":\"{command}\"'*) ;; *) exit 1;; esac\nprintf '%s\\n' '{}'\n", + frame.to_string().replace("900001", "'\"$$\"'"))); + } + script.push_str("while IFS= read -r line; do :; done\n"); + write_artifact(&profile.command, script.as_bytes(), true); + profile.artifacts[0] = artifact(&profile.command); + + payload["paperclipNextAuthority"] = + json!({"identity":{"runId":"run-2","normalizedSessionId":"session-1"}}); + ( + directory, + config, + cold_attach_command(payload), + before, + identity, + ) +} + +#[test] +fn parked_acpx_relaunch_checks_dormancy_on_the_only_sidecar() { + for (agent, active) in [("claude", false), ("claude", true), ("codex", false)] { + let (directory, config, attach, before, identity) = parked_acpx_fixture(agent, active); + let marker = directory.join("starts"); + let mut unsettled = AcpxCommandExecutor::with_runner_config(&directory, &config); + unsettled.recovery_pending_effects = Some(1); + assert!(unsettled.defer_parked_acpx_restore().is_err()); + assert!(!marker.exists()); + + let mut recovered = AcpxCommandExecutor::with_runner_config(&directory, &config); + recovered.recovery_pending_effects = Some(0); + recovered.defer_parked_acpx_restore().unwrap(); + let mut snapshot = cold_attach_command(json!({"quiesceForWarmAttach":true})); + snapshot.command_type = "session.snapshot".into(); + assert_eq!( + recovered.execute(&snapshot).unwrap().result["warmAttachReady"], + true + ); + assert!(recovered.poll_events().unwrap().is_empty()); + assert!(!marker.exists()); + let result = recovered.execute(&attach); + + assert_eq!(fs::read_to_string(marker).unwrap(), "x"); + if active { + assert!(result + .unwrap_err() + .to_string() + .contains("active or unknown goal")); + assert!(recovered.session.is_none()); + assert_eq!(fs::read(recovered.state_path()).unwrap(), before); + } else { + let result = result.unwrap(); + assert_eq!(result.result["status"], "resumed"); + assert!(result + .events + .iter() + .any(|(name, _, _)| name == "run.attached")); + assert_eq!(recovered.state.as_ref().unwrap().descriptor.run_id, "run-2"); + assert_eq!( + recovered.state.as_ref().unwrap().identity.as_ref(), + Some(&identity) + ); + } + drop(recovered); + fs::remove_dir_all(directory).unwrap(); + } +}