diff --git a/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md b/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md index 1c1242934d0..3b61ad38db9 100644 --- a/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md +++ b/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md @@ -216,6 +216,55 @@ identities. The report's shape is `$defs.backgroundTasks` in generated from it and tested against `protocol/fixtures/provider-boundary/acpx-background-tasks.json`. +The Claude sidecar reports `{"live": N}` with N above zero only while N shell +tasks are running and nothing has made the count unknown; finished tasks do +not count. It reports `{"live": 0}` only while its provider process has never +had a background task and Claude has not acted between prompts. Every other +case is unknown and the field is omitted: once a background task has finished +or a provider-started cycle has run and no shell is running, and permanently +after a dropped, declined, failed or cancelled cycle, a task that is not a +shell, a malformed task update, a scheduling tool, a full ledger, or idle +activity that was not adopted. The ACP stream does not say which task a Claude +cycle reports on, so the sidecar does not guess: after a provider-started turn, +Runner keeps waiting only while a background command is still running, and a +command that finishes while Claude is already reporting on an earlier one gets +no turn of its own. An origin-tagged usage update that directly follows the +previous one with the same context size and cost and zero usage is an empty +result, not a cycle, and is ignored. + +A controller opts in per turn with `providerStartedTurns: true` on +`turn.start`. Only then, when that completed Claude turn reported no accepted +`paperclip_finish` or `paperclip_block` result for the run, its terminal reports +`backgroundTasks: {"live": N}` with N above zero, and the same restart +conditions hold, Runner sends the sidecar `session.provider_turn` with +`armed: true` for that terminal. Only if the sidecar replies `armed: true` does +Runner keep it and await a provider-started turn: the PRP `turn.completed` +carries `awaitingProviderTurn: true` and no `run.terminal` follows. Without the +opt-in, with a result, or when arming is refused, every event and the turn-end +restart are exactly as before. + +The sidecar adopts Claude's own cycle only while armed. A cycle that starts +after the terminal and before the arming command is held, with nothing lost, +for at most 5 s; arming adopts it and anything else drops it. When armed, the +sidecar emits `runtime.turn_started` with `{"origin": "provider"}` and a +sidecar-minted turn id. Runner admits it only while awaiting and only for +Claude, with the same settled-identity and receipt rules as a controller turn, +and publishes PRP `turn.started` with `origin: "provider"`. Any other +provider-started turn fails the session closed. A controller `turn.start` while +awaiting disarms and replaces the sidecar first. If the provider exits while armed, the +sidecar reports `runtime.diagnostic` with code `ACPX_PROVIDER_PROCESS_LOST`; that, +or a failed sidecar transport, ends the wait and Runner publishes the +`run.terminal` it held back for the completed turn. Park is refused while awaiting. +Runner waits at most 5 s for the sidecar to answer an arm or a disarm. No answer +is handled as a lost sidecar, and Runner still publishes the held `run.terminal`. +When a `run.attach` from a different run succeeds, the wait ends and Runner +discards the held `run.terminal`. If that attach fails, the wait and its held +`run.terminal` stay with the old run. A `run.attach` from the same run publishes +the held `run.terminal`. +Runner saves the wait with its held `run.terminal` in the provider state; a +Runner restart during the wait publishes that `run.terminal` once and resumes +in a fresh process. + The check also inspects the live provider after that drain, so a late active-goal notification cannot be hidden by the older executor journal. This hook does not detach a controller, stop a process, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs index c32d37d89de..d56ade1a9b5 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs @@ -26,10 +26,11 @@ impl AcpxCommandExecutor { pub(super) fn restart_session_after_turn(&mut self) -> Result<(), DurableRunnerError> { self.ensure_persistence_healthy()?; - if self - .session - .as_ref() - .is_some_and(AcpxProviderSession::continues_in_process) + if self.awaiting_provider_turn() + || self + .session + .as_ref() + .is_some_and(AcpxProviderSession::continues_in_process) || !self.generation_restartable(AcpxProviderSession::can_restart_after_turn) { return Ok(()); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index d29cd3ed265..43d40c98580 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -43,6 +43,8 @@ use crate::qualified_launch::{verify_executable_launch_artifact, verify_launch_a #[path = "acpx_park_preparation.rs"] mod park_preparation; +#[path = "acpx_provider_turn_wait.rs"] +mod provider_turn_wait; fn bounded_turn_timing(response: &Value, elapsed: Duration) -> Value { let mut receipt = serde_json::Map::new(); @@ -639,6 +641,8 @@ struct AcpxDurableState { /// clears it and resumes the saved session. Written only while set. #[serde(default, skip_serializing_if = "Option::is_none")] turn_end_restart_failed_run: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + provider_turn_wait: Option, } /// The saved launch names the same executable and sidecar files as this @@ -719,6 +723,7 @@ impl AcpxDurableState { open_semantic_calls: BTreeSet::new(), retired_semantic_calls: BTreeSet::new(), turn_end_restart_failed_run: None, + provider_turn_wait: None, } } @@ -806,6 +811,10 @@ impl AcpxDurableState { .turn_end_restart_failed_run .as_ref() .is_some_and(|run_id| *run_id != self.descriptor.run_id) + || self + .provider_turn_wait + .as_ref() + .is_some_and(|wait| !wait.is_consistent(self)) { return Err(DurableRunnerError::invalid( "ACPX durable provider state is malformed or inconsistent", @@ -1030,6 +1039,7 @@ pub struct AcpxCommandExecutor { failed_turn_end_restart: Option, /// The failed turn-end restart's cause, until the Runner records it. unreported_turn_end_restart_failure: Option, + turn_accepts_provider_turns: bool, } // The durable command's turn and the live provider callback have separate IDs. @@ -1071,6 +1081,7 @@ impl AcpxCommandExecutor { recovery_pending_effects: None, failed_turn_end_restart: None, unreported_turn_end_restart_failure: None, + turn_accepts_provider_turns: false, restore_error: None, persistence_error: None, #[cfg(test)] @@ -1284,6 +1295,14 @@ impl AcpxCommandExecutor { if self.session.is_some() { return Ok(()); } + if self.awaiting_provider_turn() { + let same_run = self + .state + .as_ref() + .is_some_and(|state| state.descriptor.run_id == self.context.run_id); + self.end_provider_turn_wait(same_run)?; + self.save_state()?; + } let Some(state) = self.state.as_ref() else { return Ok(()); }; @@ -1594,6 +1613,9 @@ impl AcpxCommandExecutor { fn save_state(&mut self) -> Result<(), DurableRunnerError> { self.ensure_persistence_healthy()?; + if let Some(state) = self.state.as_mut() { + state.leave_finished_provider_turn_wait()?; + } let result = self .state .as_ref() @@ -1865,6 +1887,7 @@ impl AcpxCommandExecutor { "run.attach requires the same settled ACPX provider profile and session", )); } + let different_run = state.descriptor.run_id != attachment_context.run_id; if !self.rebind_prepared_session(&descriptor, &tool_set, next_candidate.is_some()) { if let Some(session) = self.session.as_mut() { session @@ -1895,6 +1918,11 @@ impl AcpxCommandExecutor { state.semantic_result = None; state.pending_events.clear(); state.lifecycle = "suspended".to_owned(); + let ended_wait = if different_run { + state.provider_turn_wait.take() + } else { + None + }; // A new run starts a fresh provider generation; the previous run's // failed turn-end restart no longer applies to it. A same-run attach // keeps the restriction. @@ -1914,6 +1942,11 @@ impl AcpxCommandExecutor { if saved.is_ok() && clears_turn_end_restart_failure { self.failed_turn_end_restart = None; } + if let (true, Some(wait), Some(session)) = + (saved.is_ok(), ended_wait.as_ref(), self.session.as_mut()) + { + session.disarm_provider_turn(&wait.turn_id); + } saved } @@ -2239,6 +2272,8 @@ impl AcpxCommandExecutor { .map(|state| PathBuf::from(&state.descriptor.cwd)) .expect("ACPX state exists before turn start"); let measure = payload.get("performance").and_then(Value::as_bool) == Some(true); + self.turn_accepts_provider_turns = + payload.get("providerStartedTurns").and_then(Value::as_bool) == Some(true); let timing_started = std::time::Instant::now(); let start_result = self .session @@ -2475,10 +2510,7 @@ impl AcpxCommandExecutor { .as_ref() .and_then(|state| state.active_turn_id.clone()); let Some(turn_id) = turn_id else { - return Ok(CommandExecution::result(json!({ - "status": "already_settled", - "reason": reason, - }))); + return self.settled_without_turn(reason); }; self.session .as_mut() @@ -2509,10 +2541,7 @@ impl AcpxCommandExecutor { .as_ref() .is_some_and(|state| state.descriptor.agent == "pi"); if turn_id.is_none() && !stop_idle_pi { - return Ok(CommandExecution::result(json!({ - "status": "already_settled", - "reason": reason, - }))); + return self.settled_without_turn(reason); } let provider_lifetime_fence_candidates = { let session = self @@ -2879,6 +2908,9 @@ impl AcpxCommandExecutor { self.state .as_ref() .and_then(|state| state.goal_source_revision), + self.state + .as_ref() + .is_some_and(|state| state.provider_turn_wait.is_some()), ); let events = match events { Ok(events) => events, @@ -2913,6 +2945,25 @@ impl AcpxCommandExecutor { }; let mut provider_turn_settled = false; for event in events { + if let crate::acpx_provider_state::AcpxProviderStateEvent::ProviderTurnStarted { + turn_id, + } = &event + { + let state = self + .state + .as_mut() + .expect("ACPX state remains available while polling"); + if state.lifecycle != "session_open" || state.active_turn_id.is_some() { + return Err(DurableRunnerError::invalid( + "ACPX provider turn started outside an open idle session", + )); + } + state.active_turn_id = Some(turn_id.clone()); + state.attempted_turn_controls.clear(); + state.semantic_result = None; + state.lifecycle = "turn_active".to_owned(); + self.context.provider_turn_id = Some(turn_id.clone()); + } let provider_process_lost = matches!( &event, crate::acpx_provider_state::AcpxProviderStateEvent::TurnTerminal { @@ -2947,10 +2998,16 @@ impl AcpxCommandExecutor { })?; } } + let provider_lost = matches!( + &event, + crate::acpx_provider_state::AcpxProviderStateEvent::Diagnostic { code, .. } + if code == crate::generated_acpx_sidecar_contract::GENERATED_ACPX_PROVIDER_TURN_LOST_CODE + ); let state = self .state .as_mut() .expect("ACPX state remains available while polling"); + let mut accepted = Vec::with_capacity(normalized.len()); for mut event in normalized { if matches!( event.event_type.as_str(), @@ -2973,8 +3030,10 @@ impl AcpxCommandExecutor { if event.event_type == "run.result.proposed" { state.semantic_result = Some(event.payload.clone()); } - state.push(event)?; + accepted.push(event); } + let turn_ended = terminal.is_some(); + let mut run_terminal = None; if let Some(event_type) = terminal { state.active_turn_id = None; state.provider_exit_unconfirmed |= provider_process_lost; @@ -2992,15 +3051,12 @@ impl AcpxCommandExecutor { provider_turn_settled = true; // An ACP goal has session lifetime, not prompt lifetime. // Out-of-prompt goal updates remain observable after quiescence. - if !provider_process_lost + let goal_continues = !provider_process_lost && state .goal_projection .pointer("/goal/status") .and_then(Value::as_str) - == Some("active") - { - continue; - } + == Some("active"); let (turn_terminal_state, status) = match event_type.as_str() { "turn.completed" => ("completed", "succeeded"), "turn.cancelled" => ("cancelled", "cancelled"), @@ -3019,18 +3075,20 @@ impl AcpxCommandExecutor { .and_then(Value::as_str), status == "succeeded", ); - state.push(NormalizedProviderEvent { - event_type: "run.terminal".to_owned(), - priority: EventPriority::P0, - payload: json!({ + run_terminal = (!goal_continues).then(|| { + json!({ "schema": "paperclip.prp.terminal.v1", "status": status, "turnTerminalState": turn_terminal_state, "runTerminalState": status, "reportedWorkDisposition": disposition, "provider": "acpx", - }), - })?; + }) + }); + } + self.settle_turn_events(accepted, turn_ended, run_terminal)?; + if provider_lost { + self.end_provider_turn_wait(true)?; } } if provider_turn_settled { @@ -3088,6 +3146,7 @@ impl CommandExecutor for AcpxCommandExecutor { || !state.pending_runtime_requests.is_empty() || (!state.goal_projection.is_null() && !goal_is_dormant(&state.goal_projection)) || !state.descriptor.names_qualified_release() + || state.provider_turn_wait.is_some() || self.candidate_admission().is_some() || !acpx_release_profile(&state.descriptor.agent) .is_some_and(|profile| !profile.qualification_pending) @@ -3529,7 +3588,7 @@ mod tests { } } - fn descriptor(agent: &str) -> Value { + pub(super) fn descriptor(agent: &str) -> Value { let profile = acpx_release_profile(agent).unwrap(); let mut value = json!({ "kind": "acpx", diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs index 3c79aaa114e..6da5c8d45f3 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs @@ -24,6 +24,8 @@ use crate::stable_identity::{is_stable_id, DURABLE_STABLE_ID_CHARS, SHORT_STABLE #[path = "acpx_provider_generation.rs"] mod generation; +#[path = "acpx_provider_turn.rs"] +mod provider_turn; const MAX_ID_CHARS: usize = 240; const MAX_MODEL_CHARS: usize = 240; @@ -364,6 +366,8 @@ pub struct AcpxProviderSession { quiet_turn_observed: bool, continues_in_process: bool, turn_start_replaced_provider: bool, + provider_turn_offer: Option, + run_result_reported: Option, generation_has_run_turn: bool, restarted_after_turn_from: Option, prepared_generation: Option, @@ -429,6 +433,8 @@ impl AcpxProviderSession { quiet_turn_observed: false, continues_in_process: false, turn_start_replaced_provider: false, + provider_turn_offer: None, + run_result_reported: None, generation_has_run_turn: false, restarted_after_turn_from: None, prepared_generation: None, @@ -684,6 +690,7 @@ impl AcpxProviderSession { && self.background_work_is_quiet(); self.continues_in_process = false; self.quiet_turn_observed = false; + self.provider_turn_offer = None; let restart_now = self.generation_has_run_turn && !continue_in_process; if restart_now { if let Err(error) = self.restart_idle_provider(None) { @@ -857,7 +864,16 @@ impl AcpxProviderSession { timeout: Duration, ) -> Result>, LocalRunnerError> { Ok(self - .poll_output(timeout, false, None)? + .poll_output(timeout, false, None, false)? + .map(|(events, _)| events)) + } + + pub fn poll_awaited_event( + &mut self, + timeout: Duration, + ) -> Result>, LocalRunnerError> { + Ok(self + .poll_output(timeout, false, None, true)? .map(|(events, _)| events)) } @@ -866,6 +882,7 @@ impl AcpxProviderSession { timeout: Duration, disconnected: bool, last_goal_revision: Option, + awaited: bool, ) -> Result, Option)>, LocalRunnerError> { self.ensure_open()?; let event = match self.transport.poll_event(timeout) { @@ -873,6 +890,11 @@ impl AcpxProviderSession { Err(error) => return Err(self.fail_closed(error)), }; let Some(mut event) = event else { + if awaited { + if let Some(error) = self.transport.exited_error("provider turn wait") { + return Err(self.fail_closed(error)); + } + } return Ok(None); }; let mut terminal_goal = None; @@ -921,8 +943,18 @@ impl AcpxProviderSession { payload.insert("origin".to_owned(), origin); } } + if event.event_type == GeneratedAcpxSidecarEventType::RuntimeTurnStarted { + return match self.admit_provider_turn(&event, awaited) { + Ok(started) => Ok(Some((vec![started], terminal_goal))), + Err(error) => Err(self.fail_closed(error)), + }; + } let terminal_quiet = event.event_type == GeneratedAcpxSidecarEventType::RuntimeTurnTerminal && generation::reports_no_live_background_tasks(&event.payload); + let terminal_live = (event.event_type + == GeneratedAcpxSidecarEventType::RuntimeTurnTerminal) + .then(|| provider_turn::reported_live_background_tasks(&event.payload)) + .flatten(); let mut next_state = self.state.clone(); let events = match next_state.accept_event(&event) { Ok(events) => events, @@ -976,6 +1008,9 @@ impl AcpxProviderSession { if let Err(error) = validate_reserved_terminal_result(&next_state, result) { return Err(self.fail_closed(error)); } + if result.ok { + self.record_run_result(); + } if let Err(error) = next_reserved_bridge.apply_result(crate::provider_bridge::ToolResult { call_id: result.call_id.clone(), @@ -1082,9 +1117,17 @@ impl AcpxProviderSession { && self.cancelled_turn_id.as_deref() != Some(turn_id.as_str()); } } + let terminal_completed = self.completed_turn_observed; + let terminal_turn = reconciled_events.iter().find_map(|event| match event { + AcpxProviderStateEvent::TurnTerminal { turn_id, .. } => Some(turn_id.clone()), + _ => None, + }); self.state = next_state; self.tool_bridge = next_bridge; self.reserved_tool_bridge = next_reserved_bridge; + if let Some(turn_id) = terminal_turn { + self.offer_provider_turn(&turn_id, terminal_completed, terminal_live); + } Ok(Some((reconciled_events, terminal_goal))) } @@ -1103,12 +1146,17 @@ impl AcpxProviderSession { }; let duplicate = bridge.has_completed_call(&result.call_id); let detached = bridge.turn_closed(); + let reported_run_result = + is_reserved_terminal_operation(&result.operation_id) && !result.is_error; bridge.apply_result(result.clone()).map_err(|error| { LocalRunnerError::invalid(format!( "ACPX tool result for call {} operation {} is invalid: {error}", result.call_id, result.operation_id )) })?; + if reported_run_result { + self.record_run_result(); + } if duplicate { return Ok(()); } @@ -1406,6 +1454,7 @@ impl AcpxProviderSession { self.generation_has_run_turn = false; self.quiet_turn_observed = false; self.continues_in_process = false; + self.provider_turn_offer = None; Ok(()) } @@ -2314,6 +2363,8 @@ mod mode_tests { quiet_turn_observed: false, continues_in_process: false, turn_start_replaced_provider: false, + provider_turn_offer: None, + run_result_reported: None, generation_has_run_turn: false, restarted_after_turn_from: None, prepared_generation: None, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs index 58f537c07a8..91608c307d6 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs @@ -80,6 +80,9 @@ pub enum AcpxProviderStateEvent { status: AcpxTurnStatus, error: Option, }, + ProviderTurnStarted { + turn_id: String, + }, Process(Value), Goal(Value), Diagnostic { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_turn.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_turn.rs new file mode 100644 index 00000000000..a1f67c12004 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_turn.rs @@ -0,0 +1,131 @@ +use std::time::Instant; + +use serde_json::{json, Value}; + +use super::AcpxProviderSession; +use crate::acpx_provider_state::AcpxProviderStateEvent; +use crate::acpx_sidecar_transport::AcpxSidecarEvent; +use crate::generated_acpx_sidecar_contract::{ + read_generated_acpx_background_tasks, GeneratedAcpxSidecarCommand, + GENERATED_ACPX_PROVIDER_TURN_ORIGIN, +}; +use crate::local_runner::LocalRunnerError; + +pub(super) fn reported_live_background_tasks(payload: &Value) -> Option { + payload + .get("backgroundTasks") + .and_then(read_generated_acpx_background_tasks) +} + +impl AcpxProviderSession { + pub fn run_result_reported(&self) -> bool { + self.run_result_reported.as_deref() == Some(self.state.run_id()) + } + + pub(super) fn record_run_result(&mut self) { + self.run_result_reported = Some(self.state.run_id().to_owned()); + } + + pub fn take_provider_turn_offer(&mut self) -> Option { + self.provider_turn_offer.take() + } + + pub fn arm_provider_turn(&mut self, turn_id: &str) -> bool { + self.request_provider_turn(turn_id, true) + .is_some_and(|response| response.get("armed").and_then(Value::as_bool) == Some(true)) + } + + pub fn disarm_provider_turn(&mut self, turn_id: &str) { + if !self.transport_terminated { + self.request_provider_turn(turn_id, false); + } + } + + fn request_provider_turn(&mut self, turn_id: &str, armed: bool) -> Option { + let deadline = Instant::now() + + super::ACPX_TURN_CANCEL_ACK_TIMEOUT.min(self.config.transport.request_timeout); + match self.transport.request_by( + GeneratedAcpxSidecarCommand::SessionProviderTurn, + json!({"turnId": turn_id, "armed": armed}), + deadline, + ) { + Ok(Ok(response)) => Some(response), + Ok(Err(_)) => None, + Err(error) => { + self.fail_closed(error); + None + } + } + } + + pub(super) fn offer_provider_turn( + &mut self, + turn_id: &str, + completed: bool, + live: Option, + ) { + let eligible = completed + && live.is_some_and(|live| live > 0) + && !self.run_result_reported() + && self.config.agent == "claude" + && self.config.candidate_admission.is_none() + && !self.runtime_retired + && self.failure.is_none() + && !self.state.has_pending_requests() + && !self.state.has_pending_tools() + && !self.state.settled_turn_identity_capacity_reached() + && !self.tool_bridge.replay_history_blocks_admission() + && !self.reserved_tool_bridge.replay_history_blocks_admission(); + self.provider_turn_offer = eligible.then(|| turn_id.to_owned()); + } + + pub(super) fn admit_provider_turn( + &mut self, + event: &AcpxSidecarEvent, + awaited: bool, + ) -> Result { + if !awaited + || self.config.agent != "claude" + || self.state.active_turn_id().is_some() + || self.state.has_pending_tools() + { + return Err(LocalRunnerError::invalid( + "ACPX sidecar started a provider turn outside an awaited continuation", + )); + } + if event.payload != json!({"origin": GENERATED_ACPX_PROVIDER_TURN_ORIGIN}) + || event.run_id.as_deref() != Some(self.state.run_id()) + { + return Err(LocalRunnerError::invalid( + "ACPX provider turn start violated its bounded contract", + )); + } + let turn_id = event.turn_id.clone().ok_or_else(|| { + LocalRunnerError::invalid("ACPX provider turn start omitted its turn binding") + })?; + self.state.validate_new_turn_identity(&turn_id)?; + let mut next_tool_bridge = self.tool_bridge.clone(); + next_tool_bridge.prepare_turn().map_err(|error| { + LocalRunnerError::invalid(format!( + "ACPX dynamic tool receipt rotation failed: {error}" + )) + })?; + let mut next_reserved_tool_bridge = self.reserved_tool_bridge.clone(); + next_reserved_tool_bridge.prepare_turn().map_err(|error| { + LocalRunnerError::invalid(format!( + "ACPX reserved tool receipt rotation failed: {error}" + )) + })?; + let mut next_state = self.state.clone(); + next_state.begin_turn(turn_id.clone())?; + self.state = next_state; + self.tool_bridge = next_tool_bridge; + self.reserved_tool_bridge = next_reserved_tool_bridge; + self.provider_turn_offer = None; + self.completed_turn_observed = false; + self.quiet_turn_observed = false; + self.continues_in_process = false; + self.cancelled_turn_id = None; + Ok(AcpxProviderStateEvent::ProviderTurnStarted { turn_id }) + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_turn_wait.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_turn_wait.rs new file mode 100644 index 00000000000..e9d35b1ebd4 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_turn_wait.rs @@ -0,0 +1,224 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +use super::{AcpxCommandExecutor, AcpxDurableState}; +use crate::durable::{CommandExecution, DurableRunnerError, EventPriority}; +use crate::provider_events::NormalizedProviderEvent; +use crate::stable_identity::{is_stable_id, DURABLE_STABLE_ID_CHARS}; + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct AcpxProviderTurnWait { + pub(super) turn_id: String, + pub(super) held_run_terminal: Value, +} + +impl AcpxProviderTurnWait { + pub(super) fn is_consistent(&self, state: &AcpxDurableState) -> bool { + is_stable_id(&self.turn_id, DURABLE_STABLE_ID_CHARS) + && self.held_run_terminal.is_object() + && state.lifecycle == "session_open" + && state.active_turn_id.is_none() + && state.identity.is_some() + } + + pub(super) fn into_run_terminal(self) -> NormalizedProviderEvent { + run_terminal_event(self.held_run_terminal) + } +} + +fn run_terminal_event(payload: Value) -> NormalizedProviderEvent { + NormalizedProviderEvent { + event_type: "run.terminal".to_owned(), + priority: EventPriority::P0, + payload, + } +} + +impl AcpxDurableState { + pub(super) fn leave_finished_provider_turn_wait(&mut self) -> Result<(), DurableRunnerError> { + if self + .provider_turn_wait + .as_ref() + .is_none_or(|wait| wait.is_consistent(self)) + { + return Ok(()); + } + let wait = self + .provider_turn_wait + .take() + .expect("checked provider turn wait"); + if matches!(self.lifecycle.as_str(), "turn_starting" | "turn_active") { + return Ok(()); + } + self.push(wait.into_run_terminal()) + } +} + +impl AcpxCommandExecutor { + pub(super) fn awaiting_provider_turn(&self) -> bool { + self.state + .as_ref() + .is_some_and(|state| state.provider_turn_wait.is_some()) + } + + pub(super) fn settle_turn_events( + &mut self, + events: Vec, + turn_ended: bool, + run_terminal: Option, + ) -> Result<(), DurableRunnerError> { + let offer = turn_ended + .then(|| { + self.session + .as_mut() + .and_then(|session| session.take_provider_turn_offer()) + }) + .flatten(); + let wait = match (offer, run_terminal) { + (Some(turn_id), Some(held_run_terminal)) + if self.turn_accepts_provider_turns + && self + .session + .as_mut() + .is_some_and(|session| session.arm_provider_turn(&turn_id)) => + { + Ok(AcpxProviderTurnWait { + turn_id, + held_run_terminal, + }) + } + (_, run_terminal) => Err(run_terminal), + }; + let state = self + .state + .as_mut() + .expect("ACPX state remains available while polling"); + for mut event in events { + if wait.is_ok() && event.event_type == "turn.completed" { + event.payload["awaitingProviderTurn"] = json!(true); + } + state.push(event)?; + } + match wait { + Ok(wait) => state.provider_turn_wait = Some(wait), + Err(Some(payload)) => state.push(run_terminal_event(payload))?, + Err(None) => {} + } + Ok(()) + } + + pub(super) fn end_provider_turn_wait( + &mut self, + publish: bool, + ) -> Result<(), DurableRunnerError> { + let Some(wait) = self + .state + .as_mut() + .and_then(|state| state.provider_turn_wait.take()) + else { + return Ok(()); + }; + if let Some(session) = self.session.as_mut() { + session.disarm_provider_turn(&wait.turn_id); + } + if publish { + self.state + .as_mut() + .expect("ACPX state exists with a provider turn wait") + .push(wait.into_run_terminal())?; + } + Ok(()) + } + + pub(super) fn settled_without_turn( + &mut self, + reason: &str, + ) -> Result { + if self.awaiting_provider_turn() { + self.end_provider_turn_wait(true)?; + self.save_state()?; + } + Ok(CommandExecution { + result: json!({"status": "already_settled", "reason": reason}), + events: Vec::new(), + }) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::super::tests::descriptor; + use super::super::AcpxDurableState; + use super::AcpxProviderTurnWait; + use crate::provider_bridge::{authorized_tool_catalog_digest, AuthorizedToolSet}; + + fn state() -> AcpxDurableState { + let operations = Vec::new(); + AcpxDurableState::new( + serde_json::from_value(descriptor("claude")).unwrap(), + AuthorizedToolSet { + schema: "paperclip.runner.authorized-tools.v1".into(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(), + operations, + }, + "test".into(), + ) + } + + fn wait() -> AcpxProviderTurnWait { + AcpxProviderTurnWait { + turn_id: "turn-1".into(), + held_run_terminal: json!({"schema":"paperclip.prp.terminal.v1","turnTerminalState":"completed"}), + } + } + + #[test] + fn state_outside_a_wait_keeps_the_previous_release_shape_in_both_directions() { + let saved = serde_json::to_value(state()).unwrap(); + assert!(saved.get("providerTurnWait").is_none()); + let resumed: AcpxDurableState = serde_json::from_value(saved).unwrap(); + assert_eq!(resumed.provider_turn_wait, None); + } + + #[test] + fn a_wait_round_trips_and_an_unknown_field_is_refused_like_a_previous_release_refuses_it() { + let mut waiting = state(); + waiting.provider_turn_wait = Some(wait()); + let saved = serde_json::to_value(&waiting).unwrap(); + assert_eq!(saved["providerTurnWait"]["turnId"], json!("turn-1")); + let resumed: AcpxDurableState = serde_json::from_value(saved.clone()).unwrap(); + assert_eq!(resumed.provider_turn_wait, Some(wait())); + let mut future = saved; + future["providerTurnWaitFromAFutureRelease"] = json!({}); + let refused = serde_json::from_value::(future).unwrap_err(); + assert!(refused.to_string().contains("unknown field"), "{refused}"); + } + + #[test] + fn leaving_the_open_idle_session_ends_the_wait_by_the_next_lifecycle() { + for (lifecycle, active, published) in [ + ("closed", None, true), + ("suspended", None, true), + ("prepared", None, true), + ("turn_starting", Some("turn-2"), false), + ("turn_active", Some("provider-1"), false), + ] { + let mut left = state(); + left.provider_turn_wait = Some(wait()); + left.lifecycle = lifecycle.into(); + left.active_turn_id = active.map(str::to_owned); + left.leave_finished_provider_turn_wait().unwrap(); + assert_eq!(left.provider_turn_wait, None, "{lifecycle}"); + let terminals = left + .pending_events + .iter() + .filter(|event| event.event_type == "run.terminal") + .count(); + assert_eq!(terminals, usize::from(published), "{lifecycle}"); + } + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs index 350cece37b6..7efa6d3e2f6 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs @@ -484,6 +484,22 @@ impl AcpxSidecarTransport { } } + pub(crate) fn request_by( + &mut self, + command: GeneratedAcpxSidecarCommand, + params: Value, + deadline: Instant, + ) -> Result, LocalRunnerError> { + let bound = self.request_deadline; + self.request_deadline = Some(bound.map_or(deadline, |bound| bound.min(deadline))); + let outcome = self.request_outcome(command, params); + self.request_deadline = bound; + match outcome? { + CommandOutcome::Success(value) => Ok(Ok(value)), + CommandOutcome::Rejected { error, .. } => Ok(Err(error)), + } + } + pub(crate) fn request_turn_start( &mut self, params: Value, @@ -941,6 +957,17 @@ impl AcpxSidecarTransport { )) } + pub(crate) fn exited_error(&mut self, stage: &str) -> Option { + match self.process.try_wait() { + Ok(None) => None, + Ok(Some(_)) | Err(_) => { + let error = self.closed_error(stage); + self.poison(); + Some(error) + } + } + } + fn closed_error(&mut self, stage: &str) -> LocalRunnerError { self.drain_diagnostics(Duration::from_millis(20)); let suffix = self.diagnostic_suffix(); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs index f9b9e78563b..c66ae154349 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs @@ -123,6 +123,8 @@ fn run() -> Result<(), Box> { let mut provider_lost = false; let mut session_identity = Value::Null; let mut source_run = String::new(); + let mut pending_provider_run: Option = None; + let mut provider_rearmable = false; let mut source_pending_completion: Option = None; let mut source_goal_revision = 0_u64; let mut runtime_directory: Option = None; @@ -217,6 +219,94 @@ fn run() -> Result<(), Box> { continue; } } + if command == "turn.cancel" + && provider_turn_tokens() + .iter() + .any(|token| token == "provider-turn-open") + && request["params"]["turnId"] + .as_str() + .is_some_and(|turn| turn.starts_with("provider-")) + { + let turn = request["params"]["turnId"].as_str().unwrap().to_owned(); + write_json( + &mut stdout, + &json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"id":id,"ok":true, + "result":{"cancelled":true,"sessionClosed":true}}), + )?; + write_json( + &mut stdout, + &json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"sequence":next_sequence, + "eventType":"runtime.turn_terminal","runId":"run-1","turnId":turn, + "payload":{"status":"cancelled","stopReason":"cancelled"}}), + )?; + next_sequence += 1; + continue; + } + if command == "session.provider_turn" { + if let Some(journal) = source_journal.as_mut() { + write_json(journal, &request)?; + } + let tokens = provider_turn_tokens(); + let lost = if request["params"]["armed"] == json!(true) { + "provider-arm-reply-lost" + } else { + "provider-disarm-reply-lost" + }; + if tokens.iter().any(|token| token == lost) { + continue; + } + let armed = request["params"]["armed"] == json!(true) + && (pending_provider_run.is_some() || provider_rearmable) + && !tokens.iter().any(|token| token == "provider-arm-reject"); + write_json( + &mut stdout, + &json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"id":id,"ok":true, + "result":{"armed":armed}}), + )?; + if request["params"]["armed"] == json!(false) + && tokens + .iter() + .any(|token| token == "provider-turn-after-disarm") + { + if let Some(run) = pending_provider_run.take() { + next_sequence = write_provider_turn(&mut stdout, next_sequence, &run)?; + } + } + if armed + && tokens + .iter() + .any(|token| token == "provider-sidecar-exit-after-arm") + { + std::thread::spawn(|| { + std::thread::sleep(Duration::from_millis(300)); + std::process::exit(0); + }); + } + if armed + && tokens + .iter() + .any(|token| token == "provider-lost-after-arm") + { + write_json( + &mut stdout, + &json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"sequence":next_sequence, + "eventType":"runtime.diagnostic","runId":null,"turnId":null, + "payload":{"code":"ACPX_PROVIDER_PROCESS_LOST","message":"The provider exited."}}), + )?; + next_sequence += 1; + } + if armed && tokens.iter().any(|token| token == "provider-turn-hold") { + continue; + } + if let (true, Some(run)) = (armed, pending_provider_run.take()) { + next_sequence = write_provider_turn(&mut stdout, next_sequence, &run)?; + provider_rearmable = tokens.iter().any(|token| token == "provider-terminal-live"); + } else { + provider_rearmable = false; + } + pending_provider_run = None; + continue; + } if let Some(journal) = source_journal.as_mut() { write_json(journal, &request)?; if command == "run.attach" { @@ -452,6 +542,12 @@ fn run() -> Result<(), Box> { json!({"status":status}), )?; next_sequence += 1; + next_sequence = offer_provider_turn( + &mut stdout, + next_sequence, + &source_run, + &mut pending_provider_run, + )?; } if command == "tool.resolve" { let turn = source_pending_completion @@ -974,6 +1070,22 @@ fn run() -> Result<(), Box> { )?; next_sequence += 1; } + if command == "turn.start" + && matches!( + mode, + "turns-tool-terminal" + | "turns-reused-tool-id-terminal" + | "turns-tool-result-terminal" + | "turns-reserved-result-terminal" + ) + { + next_sequence = offer_provider_turn( + &mut stdout, + next_sequence, + "run-1", + &mut pending_provider_run, + )?; + } if command == "turn.start" && mode == "turns-multiple-tool-results-terminal" { for index in 1..=2 { write_turn_event( @@ -1448,6 +1560,134 @@ fn bootstrap_success( }) } +fn provider_turn_tokens() -> Vec { + BACKGROUND_TASKS + .get() + .and_then(Option::as_ref) + .and_then(|path| std::fs::read_to_string(path).ok()) + .map(|modes| modes.split_whitespace().map(str::to_owned).collect()) + .unwrap_or_default() +} + +fn offer_provider_turn( + output: &mut impl Write, + sequence: u64, + default_run: &str, + pending: &mut Option, +) -> io::Result { + let tokens = provider_turn_tokens(); + if !tokens.iter().any(|token| token == "provider-turn") { + return Ok(sequence); + } + if tokens.iter().any(|token| token == "provider-turn-unarmed") { + return write_provider_turn(output, sequence, default_run); + } + *pending = Some(default_run.to_owned()); + Ok(sequence) +} + +fn write_provider_turn( + output: &mut impl Write, + mut sequence: u64, + default_run: &str, +) -> io::Result { + let tokens = provider_turn_tokens(); + if !tokens.iter().any(|token| token == "provider-turn") { + return Ok(sequence); + } + let value = |prefix: &str, fallback: &str| { + tokens + .iter() + .find_map(|token| token.strip_prefix(prefix).map(str::to_owned)) + .unwrap_or_else(|| fallback.to_owned()) + }; + let turn_id = value("provider-turn-id=", "provider-1"); + let run_id = value("provider-run=", default_run); + let frame = |sequence: u64, event_type: &str, payload: Value| { + json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"sequence":sequence, + "eventType":event_type,"runId":run_id,"turnId":turn_id,"payload":payload}) + }; + write_json( + output, + &frame( + sequence, + "runtime.turn_started", + json!({"origin":"provider"}), + ), + )?; + sequence += 1; + if let Some(call_id) = tokens + .iter() + .find_map(|token| token.strip_prefix("provider-reuse-call=")) + { + write_json( + output, + &frame( + sequence, + "runtime.tool_called", + json!({"callId":call_id,"operationId":"issues.read","input":{"id":"issue-1"}}), + ), + )?; + sequence += 1; + } + if tokens.iter().any(|token| token == "provider-result") { + let result = valid_reserved_completion_result(); + write_json( + output, + &frame( + sequence, + "runtime.tool_called", + json!({"callId":"call-provider-finish","operationId":"paperclip_finish","input":result.clone()}), + ), + )?; + write_json( + output, + &frame( + sequence + 1, + "runtime.event", + json!({"type":"semantic_result","callId":"call-provider-finish","operationId":"paperclip_finish","ok":true,"result":result}), + ), + )?; + sequence += 2; + } + write_json( + output, + &frame( + sequence, + "runtime.event", + json!({"type":"text_delta","text":"Claude finished its own work."}), + ), + )?; + sequence += 1; + if tokens.iter().any(|token| token == "provider-turn-open") { + return Ok(sequence); + } + write_json( + output, + &frame( + sequence, + "runtime.event", + json!({"type":"status","tag":"usage_update","text":"terminal prompt usage", + "breakdown":{"inputTokens":6,"outputTokens":4,"cachedReadTokens":0,"cachedWriteTokens":0,"thoughtTokens":0}}), + ), + )?; + sequence += 1; + let live = if tokens.iter().any(|token| token == "provider-terminal-live") { + 1 + } else { + 0 + }; + write_json( + output, + &frame( + sequence, + "runtime.turn_terminal", + json!({"status":"completed","stopReason":"end_turn","backgroundTasks":{"live":live}}), + ), + )?; + Ok(sequence + 1) +} + fn write_turn_event( output: &mut impl Write, sequence: u64, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs index 1407ca9c356..d27871c36b1 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs @@ -404,6 +404,20 @@ pub fn project_acpx_state_event( }), ) } + AcpxProviderStateEvent::ProviderTurnStarted { turn_id } => { + require_projected_turn(context, turn_id)?; + one( + "turn.started", + EventPriority::P0, + json!({ + "provider": "acpx", + "providerTurnId": turn_id, + "status": "inProgress", + "turn": {"id": turn_id, "status": "inProgress"}, + "origin": crate::generated_acpx_sidecar_contract::GENERATED_ACPX_PROVIDER_TURN_ORIGIN, + }), + ) + } AcpxProviderStateEvent::Process(details) => one( "harness.diagnostic", EventPriority::P1, diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_quiet_turn_continuation.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_quiet_turn_continuation.rs index 0d69cc92bea..4663301d938 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_quiet_turn_continuation.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_quiet_turn_continuation.rs @@ -227,6 +227,196 @@ fn a_full_settled_turn_ledger_still_rotates_through_a_restart() { session.shutdown("test complete").unwrap(); } +fn provider_session(scratch: &Scratch, background: &str, agent: &str) -> AcpxProviderSession { + provider_session_in(scratch, background, agent, "turns-tool-result-terminal") +} + +fn provider_session_in( + scratch: &Scratch, + background: &str, + agent: &str, + mode: &str, +) -> AcpxProviderSession { + scratch.background(background); + AcpxProviderSession::start(&config(&scratch.0, mode, agent)).unwrap() +} + +fn arm(session: &mut AcpxProviderSession) -> bool { + session + .take_provider_turn_offer() + .is_some_and(|turn_id| session.arm_provider_turn(&turn_id)) +} + +fn poll_until_terminal( + session: &mut AcpxProviderSession, + turn_id: &str, +) -> Vec { + let mut seen = Vec::new(); + loop { + let events = session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); + let done = events.iter().any(|event| { + matches!(event, AcpxProviderStateEvent::TurnTerminal { turn_id: settled, .. } if settled == turn_id) + }); + seen.extend(events); + if done { + return seen; + } + } +} + +#[test] +fn live_shell_tasks_at_a_completed_claude_terminal_await_and_admit_the_provider_started_turn() { + let scratch = Scratch::new(); + let mut session = provider_session(&scratch, "live provider-turn", "claude"); + let pid = session.process_id(); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(arm(&mut session)); + assert!(!session.continue_after_quiet_turn()); + + let started = session + .poll_awaited_event(Duration::from_secs(1)) + .unwrap() + .unwrap(); + assert!(matches!( + started.as_slice(), + [AcpxProviderStateEvent::ProviderTurnStarted { turn_id }] if turn_id == "provider-1" + )); + assert_eq!(session.state().active_turn_id(), Some("provider-1")); + let provider = poll_until_terminal(&mut session, "provider-1"); + assert!(provider.iter().any(|event| matches!( + event, + AcpxProviderStateEvent::Activity(activity) if activity.event_type == "usage.reported" + ))); + assert_eq!(session.state().active_turn_id(), None); + assert!(session.take_provider_turn_offer().is_none()); + assert_eq!(session.process_id(), pid); + session.shutdown("test complete").unwrap(); +} + +#[test] +fn a_provider_started_turn_that_leaves_shells_running_awaits_again() { + let scratch = Scratch::new(); + let mut session = provider_session( + &scratch, + "live provider-turn provider-terminal-live", + "claude", + ); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(arm(&mut session)); + session + .poll_awaited_event(Duration::from_secs(1)) + .unwrap() + .unwrap(); + poll_until_terminal(&mut session, "provider-1"); + assert_eq!( + session.take_provider_turn_offer().as_deref(), + Some("provider-1") + ); + session.shutdown("test complete").unwrap(); +} + +#[test] +fn an_unexpected_provider_started_turn_fails_closed_like_any_idle_event() { + for (background, agent, expected) in [ + ( + "quiet provider-turn provider-turn-unarmed", + "claude", + "outside an awaited continuation", + ), + ( + "provider-turn provider-turn-unarmed", + "claude", + "outside an awaited continuation", + ), + ( + "live provider-turn provider-turn-unarmed", + "codex", + "outside an awaited continuation", + ), + ( + "live provider-turn provider-turn-id=turn-1", + "claude", + "reused a settled turn identity", + ), + ( + "live provider-turn provider-run=run-2", + "claude", + "bounded contract", + ), + ] { + let scratch = Scratch::new(); + let mut session = provider_session(&scratch, background, agent); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + let awaited = arm(&mut session); + let error = if awaited { + session.poll_awaited_event(Duration::from_secs(1)) + } else { + session.poll_event(Duration::from_secs(1)) + } + .unwrap_err(); + assert!( + error.to_string().contains(expected), + "{background} {agent}: {error}" + ); + let _ = session.shutdown("test complete"); + } +} + +#[test] +fn a_provider_started_turn_cannot_replay_a_settled_tool_call() { + let scratch = Scratch::new(); + let mut session = provider_session( + &scratch, + "live provider-turn provider-reuse-call=call-1", + "claude", + ); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(arm(&mut session)); + session + .poll_awaited_event(Duration::from_secs(1)) + .unwrap() + .unwrap(); + let error = session.poll_event(Duration::from_secs(1)).unwrap_err(); + assert!( + error + .to_string() + .contains("reused a completed tool call id"), + "{error}" + ); + let _ = session.shutdown("test complete"); +} + +#[test] +fn a_controller_turn_while_awaiting_restarts_the_provider_first() { + let scratch = Scratch::new(); + let mut session = provider_session(&scratch, "live provider-turn", "claude"); + let pid = session.process_id(); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(arm(&mut session)); + session + .start_turn("turn-2", "Paperclip's next turn", &std::env::temp_dir()) + .unwrap(); + assert_ne!(session.process_id(), pid); + assert!(session.turn_start_replaced_provider()); + assert!(session.take_provider_turn_offer().is_none()); + session.shutdown("test complete").unwrap(); +} + #[test] fn the_generated_reader_matches_the_shared_background_task_vectors() { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -248,3 +438,105 @@ fn the_generated_reader_matches_the_shared_background_task_vectors() { ); } } + +#[test] +fn a_turn_that_reported_its_result_never_awaits_even_with_live_shells() { + let scratch = Scratch::new(); + let mut session = provider_session_in( + &scratch, + "live provider-turn", + "claude", + "turns-reserved-result-terminal", + ); + session + .start_turn( + "turn-1", + "Finish with a shell still running", + &std::env::temp_dir(), + ) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(session.run_result_reported()); + assert!(session.take_provider_turn_offer().is_none()); + assert!(session + .poll_event(Duration::from_millis(200)) + .unwrap() + .is_none()); + session.shutdown("test complete").unwrap(); +} + +#[test] +fn a_provider_started_turn_that_reports_a_result_settles_and_does_not_await_again() { + let scratch = Scratch::new(); + let mut session = provider_session( + &scratch, + "live provider-turn provider-result provider-terminal-live", + "claude", + ); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(arm(&mut session)); + session + .poll_awaited_event(Duration::from_secs(1)) + .unwrap() + .unwrap(); + let events = poll_until_terminal(&mut session, "provider-1"); + assert!(events.iter().any(|event| matches!( + event, + AcpxProviderStateEvent::TurnTerminal { status, .. } + if *status == paperclip_runner_core::acpx_event_payload::AcpxTurnStatus::Completed + ))); + assert!(session.run_result_reported()); + assert!(session.take_provider_turn_offer().is_none()); + session.shutdown("test complete").unwrap(); +} + +#[test] +fn a_sidecar_that_cannot_arm_leaves_runnerd_not_awaiting() { + let scratch = Scratch::new(); + let mut session = + provider_session(&scratch, "live provider-turn provider-arm-reject", "claude"); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + let offer = session + .take_provider_turn_offer() + .expect("eligible terminal"); + assert!(!session.arm_provider_turn(&offer)); + session.shutdown("test complete").unwrap(); +} + +#[test] +fn cancelling_a_provider_started_turn_settles_it_cancelled_and_retires_the_provider() { + let scratch = Scratch::new(); + let mut session = provider_session(&scratch, "live provider-turn provider-turn-open", "claude"); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(arm(&mut session)); + session + .poll_awaited_event(Duration::from_secs(1)) + .unwrap() + .unwrap(); + assert_eq!(session.state().active_turn_id(), Some("provider-1")); + let response = session + .interrupt_turn("provider-1", "Operator stop") + .unwrap(); + assert_eq!(response["sessionClosed"], json!(true)); + assert!(session.runtime_retired()); + let events = poll_until_terminal(&mut session, "provider-1"); + assert!(events.iter().any(|event| matches!( + event, + AcpxProviderStateEvent::TurnTerminal { status, .. } + if *status == paperclip_runner_core::acpx_event_payload::AcpxTurnStatus::Cancelled + ))); + assert!(session.take_provider_turn_offer().is_none()); + assert!(session + .start_turn("turn-2", "Paperclip's next turn", &std::env::temp_dir()) + .is_err()); + let _ = session.shutdown("test complete"); +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/support/acpx_quiet_turn_restart.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/support/acpx_quiet_turn_restart.rs index 2f61aa3aa83..641f4150c01 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/support/acpx_quiet_turn_restart.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/support/acpx_quiet_turn_restart.rs @@ -75,8 +75,8 @@ fn a_quiet_completed_claude_turn_keeps_its_provider_for_the_next_turns() { } #[test] -fn live_or_unknown_background_work_restarts_at_the_turn_end_like_today() { - for mode in ["live", "", "malformed"] { +fn unknown_background_work_restarts_at_the_turn_end_like_today() { + for mode in ["", "malformed"] { let mut harness = Harness::new(); harness.fixture.background(mode); let pid = harness.opened["processId"].clone(); @@ -227,3 +227,643 @@ fn a_runner_restart_after_a_kept_quiet_provider_resumes_in_a_fresh_process() { ); harness.executor.shutdown().unwrap(); } + +fn complete_turn_accepting_provider_turns(harness: &mut Harness, turn: &str) -> Vec { + let started = harness.execute( + "turn.start", + json!({"turnId": turn, "text": "Complete the work.", "providerStartedTurns": true}), + ); + let seen = started + .events + .iter() + .map(|(kind, _, payload)| json!({"type":kind,"payload":payload})) + .collect::>(); + drain_until(harness, seen, "turn.completed") +} + +fn drain_until(harness: &mut Harness, mut seen: Vec, kind: &str) -> Vec { + let deadline = Instant::now() + Duration::from_secs(5); + while !seen.iter().any(|event| event["type"] == kind) { + assert!(Instant::now() < deadline, "expected {kind}: {seen:?}"); + seen.extend(harness.drain()); + } + seen +} + +#[test] +fn live_shells_at_a_completed_claude_turn_keep_the_provider_and_refuse_park() { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold"); + let pid = harness.opened["processId"].clone(); + let events = complete_turn_accepting_provider_turns(&mut harness, "first"); + let completed = events + .iter() + .find(|event| event["type"] == "turn.completed") + .unwrap(); + assert_eq!(completed["payload"]["awaitingProviderTurn"], json!(true)); + assert!(!events.iter().any(|event| event["type"] == "run.terminal")); + let before = harness.fixture.journal().len(); + harness.executor.prepare_next_turn().unwrap(); + assert_eq!( + commands_after(&harness.fixture.journal(), before), + Vec::::new() + ); + assert_eq!(live_providers(&harness), [pid.clone()]); + assert!(!harness.executor.host_park_ready().unwrap()); + + let (next, _) = run_turn(&mut harness, "second"); + assert!( + replaced(&next), + "a controller turn restarts the awaited provider first" + ); + assert!(!alive(&pid)); + harness.executor.shutdown().unwrap(); + assert_eq!(live_providers(&harness), Vec::::new()); +} + +#[test] +fn the_provider_started_turn_reaches_the_controller_as_ordinary_turn_events() { + let mut harness = Harness::new(); + harness.fixture.background("live provider-turn"); + let pid = harness.opened["processId"].clone(); + let first = complete_turn_accepting_provider_turns(&mut harness, "first"); + let events = drain_until(&mut harness, first, "run.terminal"); + let kinds = events + .iter() + .map(|event| event["type"].as_str().unwrap().to_owned()) + .collect::>(); + let first_completed = kinds + .iter() + .position(|kind| kind == "turn.completed") + .unwrap(); + let provider_started = events + .iter() + .position(|event| { + event["type"] == "turn.started" && event["payload"]["origin"] == "provider" + }) + .expect("provider-started turn"); + assert!(first_completed < provider_started); + assert_eq!( + events[provider_started]["payload"]["providerTurnId"], + json!("provider-1") + ); + let provider_completed = events + .iter() + .rposition(|event| event["type"] == "turn.completed") + .unwrap(); + assert!(provider_started < provider_completed); + assert_eq!( + events[provider_completed]["payload"]["providerTurnId"], + json!("provider-1") + ); + assert!(events[provider_completed]["payload"] + .get("awaitingProviderTurn") + .is_none()); + assert!(events[provider_started..provider_completed] + .iter() + .any(|event| event["type"] == "usage.reported")); + let run_terminal = kinds + .iter() + .position(|kind| kind == "run.terminal") + .unwrap(); + assert!(run_terminal > provider_completed); + assert_eq!( + kinds.iter().filter(|kind| *kind == "run.terminal").count(), + 1 + ); + harness.executor.prepare_next_turn().unwrap(); + assert!(harness.executor.host_park_ready().unwrap()); + assert_eq!(live_providers(&harness).len(), 1); + assert!(!alive(&pid) || live_providers(&harness) == [pid.clone()]); + harness.executor.shutdown().unwrap(); + assert_eq!(live_providers(&harness), Vec::::new()); +} + +#[test] +fn a_runner_restart_while_awaiting_resumes_like_today() { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold"); + let pid = harness.opened["processId"].clone(); + complete_turn_accepting_provider_turns(&mut harness, "first"); + harness.executor.prepare_next_turn().unwrap(); + assert_eq!(live_providers(&harness), [pid.clone()]); + harness.executor.shutdown().unwrap(); + assert!(!alive(&pid)); + harness.executor = + NativeProviderCommandExecutor::with_runner_config(&harness.fixture.0, &harness.config); + harness.sequence = 90; + harness.fixture.background("quiet"); + let resumed = harness.drain(); + assert_eq!( + resumed + .iter() + .filter(|event| event["type"] == "run.terminal") + .count(), + 1, + "{resumed:?}" + ); + let (events, _) = run_turn(&mut harness, "second"); + assert!(events.iter().any(|event| event["type"] == "turn.completed")); + assert_eq!(harness.fixture.process_starts().len(), 2); + harness.executor.shutdown().unwrap(); + assert_eq!(live_providers(&harness), Vec::::new()); +} + +fn controller_view(events: &[Value]) -> Vec { + events + .iter() + .map(|event| { + let mut event = event.clone(); + for volatile in ["processId", "previousProcessId", "providerTurnId"] { + if let Some(payload) = event["payload"].as_object_mut() { + payload.remove(volatile); + } + } + event + }) + .collect() +} + +#[test] +fn without_the_controller_capability_live_shells_behave_exactly_as_before() { + let outcome = |background: &str| { + let mut harness = Harness::new(); + harness.fixture.background(background); + let pid = harness.opened["processId"].clone(); + let events = harness.complete_turn("first"); + let park_ready_before_restart = harness.executor.host_park_ready().unwrap(); + let before = harness.fixture.journal().len(); + harness.executor.prepare_next_turn().unwrap(); + let commands = commands_after(&harness.fixture.journal(), before); + let restarted = !alive(&pid); + let park_ready = harness.executor.host_park_ready().unwrap(); + harness.executor.shutdown().unwrap(); + ( + controller_view(&events), + commands, + restarted, + park_ready, + park_ready_before_restart, + ) + }; + let live = outcome("live provider-turn"); + let unknown = outcome(""); + assert_eq!(live.0, unknown.0); + assert_eq!(live.1, unknown.1); + assert!(live.2 && unknown.2); + assert_eq!(live.3, unknown.3); + assert_eq!(live.4, unknown.4); + assert!(live.4); + assert!(live.0.iter().any(|event| event["type"] == "run.terminal")); + assert!(!live + .0 + .iter() + .any(|event| event["payload"].get("awaitingProviderTurn").is_some())); + assert!(!live + .1 + .iter() + .any(|command| command == "session.provider_turn")); +} + +#[test] +fn a_provider_lost_while_awaiting_ends_the_wait_with_the_held_run_terminal() { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold provider-lost-after-arm"); + let pid = harness.opened["processId"].clone(); + let first = complete_turn_accepting_provider_turns(&mut harness, "first"); + let events = drain_until(&mut harness, first, "run.terminal"); + let completed = events + .iter() + .position(|event| event["type"] == "turn.completed") + .unwrap(); + assert_eq!( + events[completed]["payload"]["awaitingProviderTurn"], + json!(true) + ); + let terminals = events + .iter() + .filter(|event| event["type"] == "run.terminal") + .collect::>(); + assert_eq!(terminals.len(), 1); + assert_eq!( + terminals[0]["payload"]["turnTerminalState"], + json!("completed") + ); + assert!(events + .iter() + .any(|event| event["payload"]["code"] == "ACPX_PROVIDER_PROCESS_LOST")); + harness.executor.prepare_next_turn().unwrap(); + assert!( + !alive(&pid), + "the lost provider is replaced at the turn end" + ); + harness.executor.shutdown().unwrap(); +} + +#[test] +fn a_sidecar_that_dies_while_awaiting_ends_the_wait_with_the_held_run_terminal() { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold provider-sidecar-exit-after-arm"); + let first = complete_turn_accepting_provider_turns(&mut harness, "first"); + let events = drain_until(&mut harness, first, "run.terminal"); + let terminals = events + .iter() + .filter(|event| event["type"] == "run.terminal") + .collect::>(); + assert_eq!(terminals.len(), 1); + assert_eq!( + terminals[0]["payload"]["turnTerminalState"], + json!("completed") + ); + assert!(!events.iter().any(|event| event["type"] == "turn.failed")); + let _ = harness.executor.shutdown(); +} + +#[test] +fn cancelling_or_stopping_the_run_while_awaiting_disarms_and_publishes_the_held_run_terminal() { + for command in ["run.cancel", "turn.interrupt", "turn.stop"] { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold provider-turn-after-disarm"); + let first = complete_turn_accepting_provider_turns(&mut harness, "first"); + assert!( + !first.iter().any(|event| event["type"] == "run.terminal"), + "{command}" + ); + let before = harness.fixture.journal().len(); + let cancelled = harness.execute( + command, + json!({"turnId": "first", "reason": "Operator stop"}), + ); + assert_eq!( + cancelled.result["status"], + json!("already_settled"), + "{command}" + ); + let mut later = cancelled + .events + .iter() + .map(|(kind, _, payload)| json!({"type":kind,"payload":payload})) + .collect::>(); + let journal = harness.fixture.journal(); + assert!( + journal[before..].iter().any(|line| { + line["command"] == "session.provider_turn" + && line["params"]["armed"] == json!(false) + }), + "{command}" + ); + let deadline = Instant::now() + Duration::from_millis(500); + while Instant::now() < deadline { + match harness.executor.poll_events() { + Ok(events) => { + later.extend( + events + .iter() + .map(|event| json!({"type":event.event_type,"payload":event.payload})), + ); + let _ = harness.executor.acknowledge_events(events.len()); + } + Err(_) => break, + } + } + assert!( + !later + .iter() + .any(|event| event["payload"]["origin"] == "provider"), + "{command}: {later:?}" + ); + let terminals = later + .iter() + .filter(|event| event["type"] == "run.terminal") + .collect::>(); + assert_eq!(terminals.len(), 1, "{command}: {later:?}"); + assert_eq!( + terminals[0]["payload"]["turnTerminalState"], + json!("completed"), + "{command}" + ); + let _ = harness.executor.shutdown(); + } +} + +#[test] +fn an_active_goal_at_a_live_shell_turn_end_neither_marks_nor_arms_a_wait() { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold"); + harness.fixture.fault("goal"); + let before = harness.fixture.journal().len(); + let events = complete_turn_accepting_provider_turns(&mut harness, "first"); + let events = [events, harness.drain()].concat(); + let completed = events + .iter() + .find(|event| event["type"] == "turn.completed") + .unwrap(); + assert!(completed["payload"].get("awaitingProviderTurn").is_none()); + assert!(!events.iter().any(|event| event["type"] == "run.terminal")); + assert!(!commands_after(&harness.fixture.journal(), before) + .iter() + .any(|command| command == "session.provider_turn")); + harness.executor.shutdown().unwrap(); +} + +#[test] +fn a_crash_after_cancelling_or_stopping_a_wait_still_delivers_the_held_run_terminal_once() { + for command in ["run.cancel", "turn.interrupt", "turn.stop"] { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold"); + complete_turn_accepting_provider_turns(&mut harness, "first"); + let lost_with_the_crash = harness.execute( + command, + json!({"turnId": "first", "reason": "Operator stop"}), + ); + assert_eq!( + lost_with_the_crash.result["status"], + json!("already_settled"), + "{command}" + ); + harness.executor = + NativeProviderCommandExecutor::with_runner_config(&harness.fixture.0, &harness.config); + harness.fixture.background("quiet"); + let recovered = harness.drain(); + assert_eq!( + recovered + .iter() + .filter(|event| event["type"] == "run.terminal") + .count(), + 1, + "{command}: {recovered:?}" + ); + let _ = harness.executor.shutdown(); + } +} + +fn awaiting_and_quiesced(harness: &mut Harness) { + harness + .fixture + .background("live provider-turn provider-turn-hold"); + let first = complete_turn_accepting_provider_turns(harness, "first"); + let events = [first, harness.drain()].concat(); + assert!(!events.iter().any(|event| event["type"] == "run.terminal")); + assert!(harness.fixture.state().get("providerTurnWait").is_some()); + harness.quiesce(); +} + +fn drain_briefly(harness: &mut Harness) -> Vec { + let mut seen = Vec::new(); + let deadline = Instant::now() + Duration::from_millis(500); + while Instant::now() < deadline { + seen.extend(harness.drain()); + } + seen +} + +#[test] +fn a_different_run_attaching_while_awaiting_ends_the_wait_without_the_old_run_terminal() { + let mut harness = Harness::new(); + awaiting_and_quiesced(&mut harness); + let pid = harness.opened["processId"].clone(); + let before = harness.fixture.journal().len(); + let (attached, _) = harness.attach("run-2", |_| {}); + let state = harness.fixture.state(); + assert_eq!(state["descriptor"]["runId"], "run-2"); + assert!(state.get("providerTurnWait").is_none(), "{state}"); + assert!( + !state["pendingEvents"] + .as_array() + .unwrap() + .iter() + .any(|event| event.to_string().contains("run.terminal")), + "{state}" + ); + let mut received = attached + .events + .iter() + .map(|(kind, _, payload)| json!({"type":kind,"payload":payload})) + .collect::>(); + received.extend(drain_briefly(&mut harness)); + assert!( + !received.iter().any(|event| event["type"] == "run.terminal"), + "{received:?}" + ); + let commands = commands_after(&harness.fixture.journal(), before); + assert!( + commands.iter().any(|command| command == "session.close"), + "{commands:?}" + ); + assert!( + !commands + .iter() + .any(|command| command == "session.provider_turn"), + "{commands:?}" + ); + assert!(!alive(&pid)); + let _ = harness.executor.shutdown(); +} + +#[test] +fn a_failed_different_run_attach_while_awaiting_keeps_the_wait_and_its_run_terminal() { + for ending in [ + None, + Some("run.cancel"), + Some("turn.interrupt"), + Some("turn.stop"), + ] { + let mut harness = Harness::new(); + awaiting_and_quiesced(&mut harness); + let pid = harness.opened["processId"].clone(); + std::process::Command::new("kill") + .args(["-9", &pid.to_string()]) + .status() + .unwrap(); + std::thread::sleep(Duration::from_millis(200)); + let payload = harness.attach_payload("run-2", |_| {}); + harness.sequence += 1; + let error = harness + .executor + .execute(&command(harness.sequence, "run.attach", payload)) + .unwrap_err(); + assert!( + format!("{error:?}").contains("failed to checkpoint ACPX before attaching a new run"), + "{ending:?}: {error:?}" + ); + let state = harness.fixture.state(); + assert_eq!(state["descriptor"]["runId"], "run-1", "{ending:?}"); + assert!( + state.get("providerTurnWait").is_some(), + "{ending:?}: {state}" + ); + let mut recovered = Vec::new(); + if let Some(ending) = ending { + let ended = harness.execute( + ending, + json!({"turnId": "first", "reason": "Operator stop"}), + ); + assert_eq!(ended.result["status"], json!("already_settled"), "{ending}"); + recovered.extend( + ended + .events + .iter() + .map(|(kind, _, payload)| json!({"type":kind,"payload":payload})), + ); + } + recovered.extend(drain_briefly(&mut harness)); + let terminals = recovered + .iter() + .filter(|event| event["type"] == "run.terminal") + .collect::>(); + assert_eq!(terminals.len(), 1, "{ending:?}: {recovered:?}"); + assert_eq!( + terminals[0]["payload"]["turnTerminalState"], + json!("completed"), + "{ending:?}" + ); + let state = harness.fixture.state(); + assert_eq!(state["descriptor"]["runId"], "run-1", "{ending:?}"); + assert!(state.get("providerTurnWait").is_none(), "{ending:?}"); + let _ = harness.executor.shutdown(); + } +} + +#[test] +fn a_same_run_attach_while_awaiting_publishes_the_held_run_terminal_once() { + let mut harness = Harness::new(); + awaiting_and_quiesced(&mut harness); + let attached = harness.execute( + "run.attach", + json!({"provider": harness.descriptor.clone()}), + ); + let mut received = attached + .events + .iter() + .map(|(kind, _, payload)| json!({"type":kind,"payload":payload})) + .collect::>(); + received.extend(drain_briefly(&mut harness)); + let terminals = received + .iter() + .filter(|event| event["type"] == "run.terminal") + .collect::>(); + assert_eq!(terminals.len(), 1, "{received:?}"); + assert_eq!( + terminals[0]["payload"]["turnTerminalState"], + json!("completed") + ); + let _ = harness.executor.shutdown(); +} + +fn execution_events(execution: &CommandExecution) -> Vec { + execution + .events + .iter() + .map(|(kind, _, payload)| json!({"type":kind,"payload":payload})) + .collect() +} + +fn run_terminals(events: &[Value]) -> usize { + events + .iter() + .filter(|event| event["type"] == "run.terminal") + .count() +} + +#[test] +fn a_lost_disarm_reply_is_bounded_and_still_publishes_the_held_run_terminal_once() { + for command in ["run.cancel", "turn.interrupt"] { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold provider-disarm-reply-lost"); + let pid = harness.opened["processId"].clone(); + let first = complete_turn_accepting_provider_turns(&mut harness, "first"); + let first = [first, harness.drain()].concat(); + assert_eq!(run_terminals(&first), 0, "{command}"); + let started = Instant::now(); + let ended = harness.execute( + command, + json!({"turnId": "first", "reason": "Operator stop"}), + ); + let elapsed = started.elapsed(); + assert_eq!( + ended.result["status"], + json!("already_settled"), + "{command}" + ); + assert!( + elapsed >= Duration::from_secs(4) && elapsed < Duration::from_secs(10), + "{command}: {elapsed:?}" + ); + let mut later = execution_events(&ended); + later.extend(drain_briefly(&mut harness)); + assert_eq!(run_terminals(&later), 1, "{command}: {later:?}"); + assert_eq!( + later + .iter() + .find(|event| event["type"] == "run.terminal") + .unwrap()["payload"]["turnTerminalState"], + json!("completed"), + "{command}" + ); + assert!( + harness.fixture.state().get("providerTurnWait").is_none(), + "{command}" + ); + let started = Instant::now(); + let closed = harness.execute("session.close", json!({})); + assert!( + started.elapsed() < Duration::from_secs(2), + "{command}: {:?}", + started.elapsed() + ); + assert_eq!(closed.result["status"], json!("closed"), "{command}"); + later.extend(execution_events(&closed)); + later.extend(drain_briefly(&mut harness)); + assert_eq!(run_terminals(&later), 1, "{command}: {later:?}"); + assert!(!alive(&pid), "{command}"); + let _ = harness.executor.shutdown(); + } +} + +#[test] +fn a_lost_arm_reply_is_bounded_and_the_turn_ends_with_its_run_terminal() { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-arm-reply-lost"); + let started = Instant::now(); + let mut seen = execution_events(&harness.execute( + "turn.start", + json!({"turnId": "first", "text": "Complete the work.", "providerStartedTurns": true}), + )); + let deadline = started + Duration::from_secs(15); + while run_terminals(&seen) == 0 { + assert!(Instant::now() < deadline, "{seen:?}"); + seen.extend(harness.drain()); + } + assert!( + started.elapsed() < Duration::from_secs(10), + "{:?}", + started.elapsed() + ); + seen.extend(drain_briefly(&mut harness)); + let completed = seen + .iter() + .find(|event| event["type"] == "turn.completed") + .unwrap(); + assert!(completed["payload"].get("awaitingProviderTurn").is_none()); + assert_eq!(run_terminals(&seen), 1, "{seen:?}"); + assert!(harness.fixture.state().get("providerTurnWait").is_none()); + let _ = harness.executor.shutdown(); +} diff --git a/packages/paperclip-runner/src/live/runnerd-acpx-provider-turn-wait.test.ts b/packages/paperclip-runner/src/live/runnerd-acpx-provider-turn-wait.test.ts new file mode 100644 index 00000000000..967258fba11 --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-acpx-provider-turn-wait.test.ts @@ -0,0 +1,174 @@ +import { createHash } from "node:crypto"; +import { execFileSync, spawn } from "node:child_process"; +import { closeSync, openSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +import { expect, it, vi } from "vitest"; + +import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import { QUALIFIED_ACPX_PROFILES } from "../drivers/acpx/qualified-profiles.js"; +import { createRunnerProviderLaunch, runnerLaunchRequirementsDigest, type RunnerProviderLaunchRequirements } from "./runner-provider-launch.js"; +import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding, type CapabilityRunnerdCodexTransportOptions } from "./runnerd-codex-transport.js"; + +const sha = (value: string | Buffer) => `sha256:${createHash("sha256").update(value).digest("hex")}`; +const quote = (value: string) => `'${value.replaceAll("'", "'\"'\"'")}'`; +const kill = (pid: number) => { try { process.kill(-pid, "SIGKILL"); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } }; + +type CoreEvent = { eventType: string; envelope: { payload?: Record } }; + +async function awaitedTurn(tokens: string, providerStartedTurns: boolean) { + const root = await realpath(await mkdtemp(join(tmpdir(), "runner-acpx-provider-wait-"))); + const journal = join(root, "calls"), fault = join(root, "fault"), wrapper = join(root, "sidecar"), script = join(root, "sidecar-entry"); + const background = join(root, "background"); + const binary = resolve(import.meta.dirname, "../../runner/target/debug/paperclip-runnerd"), sidecar = resolve(import.meta.dirname, "../../runner/target/debug/fake-acpx-sidecar"); + await mkdir(join(root, "acpx"), { mode: 0o700 }); await mkdir(join(root, "diagnostics"), { mode: 0o700 }); + await writeFile(script, "qualified fixture entry"); + await writeFile(background, tokens); + await writeFile(wrapper, `#!/bin/sh\nexec ${quote(sidecar)} --mode source-reconcile --profile-digest ${quote(QUALIFIED_ACPX_PROFILES.claude.commandDigest)} --journal ${quote(journal)} --source-fault ${quote(fault)} --background-tasks ${quote(background)} --spawn-owned-child\n`, { mode: 0o700 }); + let authority: DurablePrpControlPlane | null = null; + const server = createServer(); + server.on("upgrade", (request, socket, head) => authority ? authority.handleUpgrade(request, socket, "/runner", head) : socket.destroy()); + await new Promise(done => server.listen(0, "127.0.0.1", done)); + const address = server.address(); if (!address || typeof address === "string") throw new Error("missing fixture listener"); + const identity = { runnerInstanceId: "runner-wait", environmentLeaseId: "lease-wait", runId: "run-wait", normalizedSessionId: "session-wait", turnId: "turn-wait", itemId: "item-wait" }; + const runnerPids = new Set(); + let requirements: RunnerProviderLaunchRequirements | undefined; + const options: CapabilityRunnerdCodexTransportOptions = { + provider: "acpx", acpxAgent: "claude", runnerBinary: binary, stateDirectory: root, prpIdentity: identity, + runnerArtifact: readRunnerdArtifactBinding(binary), codexSourceRecoveryVersion: 1, acpxSourceRecoveryVersion: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, runnerProcessReconnectGraceMs: null, runnerReconnectGraceMs: 3_000, + controlPlaneRegistration: async next => { authority = next; return { connectUrl: `ws://127.0.0.1:${address.port}/runner`, release: () => { authority = null; } }; }, + prepareLaunch: async launch => { + requirements ??= structuredClone(launch); + return { configurationId: `initial-${launch.identity.runId}`, requirementsDigest: runnerLaunchRequirementsDigest(launch), + provider: createRunnerProviderLaunch(launch, { options: {}, root, cwd: root, codexHome: root, codexCommand: "", providerNodeCommand: wrapper, opencodeProxyPath: "", acpxSidecarPath: script }) }; + }, + runnerProcessLauncher: spec => { + const stderr = openSync(join(root, "native.stderr"), "a", 0o600); + const child = spawn(binary, [...spec.args, "--acpx-launch-authority-digest", sha("fixture authority"), + "--acpx-sidecar-command", wrapper, "--acpx-sidecar-command-sha256", sha(execFileSync("cat", [wrapper])), + "--acpx-sidecar-script", script, "--acpx-sidecar-script-sha256", sha("qualified fixture entry")], + { env: { PATH: process.env.PATH, HOME: root, ...spec.environment }, stdio: ["ignore", "ignore", stderr], detached: true }); + closeSync(stderr); + if (child.pid) runnerPids.add(child.pid); + return { child, completion: new Promise((done, reject) => { child.once("error", reject); child.once("exit", (code, signal) => done({ code, signal, stdout: "", stderr: "" })); }) }; + }, + }; + const bundles = [createCapabilityRunnerdCodexTransport(options)]; + const bundle = bundles[0]!; + const calls = async () => (await readFile(journal, "utf8").catch(() => "")).trim().split("\n").filter(Boolean).map(line => JSON.parse(line) as Record); + const coreEvents = async () => { + const core = JSON.parse(await readFile(join(root, "control-plane", "control-plane-state.json"), "utf8")) as { committedEvents?: CoreEvent[] }; + return (core.committedEvents ?? []).map(event => ({ eventType: event.eventType, + payload: ((event.envelope.payload ?? {}) as { payload?: Record }).payload ?? {} })); + }; + try { + const opened = await bundle.transport.request("thread/start", { cwd: root, model: "explicit-source-model", dynamicTools: [] }); + const thread = opened.thread as { id: string; sessionId: string }; + authority!.queueCommand("turn.start", { text: "Install and test.", turnId: "turn-own-1", ...(providerStartedTurns ? { providerStartedTurns: true } : {}) }, undefined, true); + await vi.waitFor(async () => expect((await coreEvents()).some(event => event.eventType === "turn.completed")).toBe(true), { timeout: 10_000, interval: 50 }); + const restartRunner = async () => { + const providerIdentity = JSON.parse(await readFile(join(root, "runner", "acpx-provider-state.json"), "utf8")).identity; + const pid = bundle.evidence().runnerPid!; + await bundle.detachControllerForRestart(); kill(pid); + for (const entry of await calls()) if (typeof entry.processStart === "number") kill(entry.processStart); + const recovered = createCapabilityRunnerdCodexTransport({ ...options, recoveryLaunch: { requirements: requirements! }, + resumeProviderSession: { driverSessionId: thread.id, providerSessionId: thread.sessionId, providerIdentity }, + onRecoveryAccountingEvent: async () => {}, authorityRunId: identity.runId }); + bundles.push(recovered); + await recovered.transport.request("thread/read", {}); + return recovered; + }; + return { coreEvents, calls, root, authority: authority!, restartRunner }; + } catch (error) { + throw new Error(`${String(error)}\n${await readFile(join(root, "native.stderr"), "utf8").catch(() => "")}`, { cause: error }); + } finally { + cleanups.push(async () => { + for (const opened of bundles.reverse()) await opened.transport.close().catch(() => undefined); + for (const pid of runnerPids) kill(pid); + for (const entry of await calls()) if (typeof entry.processStart === "number") kill(entry.processStart); + server.closeAllConnections(); await new Promise(done => server.close(() => done())); + await rm(root, { recursive: true, force: true }); + }); + } +} + +const cleanups: Array<() => Promise> = []; + +const alive = (pid: number) => { + try { process.kill(pid, 0); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; throw error; } + try { return !execFileSync("ps", ["-p", String(pid), "-o", "stat="], { encoding: "utf8" }).trim().startsWith("Z"); } + catch (error) { if ((error as { status?: number }).status === 1) return false; throw error; } +}; + +const runTerminals = (events: Array<{ eventType: string; payload: Record }>) => events.filter(event => event.eventType === "run.terminal"); +const commands = async (calls: () => Promise>>) => (await calls()).map(entry => entry.command).filter(Boolean); + +it.skipIf(process.platform === "win32")("ends a wait through the real runnerd when the sidecar exits while a descendant keeps its output open", async () => { + try { + const run = await awaitedTurn("live provider-turn provider-turn-hold provider-sidecar-exit-after-arm", true); + const awaited = (await run.coreEvents()).find(event => event.eventType === "turn.completed"); + expect(awaited?.payload.awaitingProviderTurn).toBe(true); + await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 10_000, interval: 100 }); + const events = await run.coreEvents(); + expect(runTerminals(events)[0]!.payload).toMatchObject({ turnTerminalState: "completed", runTerminalState: "succeeded" }); + expect(events.findIndex(event => event.eventType === "run.terminal")).toBeGreaterThan(events.findIndex(event => event.eventType === "turn.completed")); + expect(await commands(run.calls)).toContain("session.provider_turn"); + } finally { for (const cleanup of cleanups.splice(0)) await cleanup(); } +}, 60_000); + +it.skipIf(process.platform === "win32")("ends a wait through the real runnerd when the sidecar reports the provider lost", async () => { + try { + const run = await awaitedTurn("live provider-turn provider-turn-hold provider-lost-after-arm", true); + await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 10_000, interval: 100 }); + const events = await run.coreEvents(); + expect(events.find(event => event.eventType === "turn.completed")?.payload.awaitingProviderTurn).toBe(true); + expect(events.some(event => event.payload.code === "ACPX_PROVIDER_PROCESS_LOST")).toBe(true); + } finally { for (const cleanup of cleanups.splice(0)) await cleanup(); } +}, 60_000); + +it.skipIf(process.platform === "win32")("emits run.terminal with the completed turn and never arms without the controller opt-in", async () => { + try { + const run = await awaitedTurn("live provider-turn provider-turn-hold provider-sidecar-exit-after-arm", false); + await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 5_000, interval: 50 }); + const events = await run.coreEvents(); + expect(events.find(event => event.eventType === "turn.completed")?.payload.awaitingProviderTurn).toBeUndefined(); + expect(await commands(run.calls)).not.toContain("session.provider_turn"); + } finally { for (const cleanup of cleanups.splice(0)) await cleanup(); } +}, 60_000); + +it.skipIf(process.platform === "win32")("cancelling the run during a wait disarms the provider, publishes the held terminal and admits no later provider turn", async () => { + try { + const run = await awaitedTurn("live provider-turn provider-turn-hold provider-turn-after-disarm", true); + expect(runTerminals(await run.coreEvents())).toHaveLength(0); + run.authority.queueCommand("run.cancel", { reason: "Operator stop" }, undefined, true); + await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 10_000, interval: 100 }); + expect(runTerminals(await run.coreEvents())[0]!.payload).toMatchObject({ turnTerminalState: "completed" }); + expect((await run.calls()).some(entry => entry.command === "session.provider_turn" && (entry.params as { armed?: unknown }).armed === false)).toBe(true); + await new Promise(done => setTimeout(done, 1_000)); + const events = await run.coreEvents(); + expect(events.some(event => event.payload.origin === "provider")).toBe(false); + expect(runTerminals(events)).toHaveLength(1); + run.authority.queueCommand("session.close", { reason: "Run cancelled" }, undefined, true); + const processes = (await run.calls()).flatMap(entry => [entry.processStart, entry.childPid]).filter((pid): pid is number => typeof pid === "number"); + expect(processes.length).toBeGreaterThan(0); + await vi.waitFor(() => { for (const pid of processes) expect(alive(pid)).toBe(false); }, { timeout: 10_000, interval: 100 }); + } finally { for (const cleanup of cleanups.splice(0)) await cleanup(); } +}, 60_000); + +it.skipIf(process.platform === "win32")("publishes the held run.terminal exactly once when runnerd restarts during the wait", async () => { + try { + const run = await awaitedTurn("live provider-turn provider-turn-hold", true); + expect((await run.coreEvents()).find(event => event.eventType === "turn.completed")?.payload.awaitingProviderTurn).toBe(true); + expect(runTerminals(await run.coreEvents())).toHaveLength(0); + await run.restartRunner(); + await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 10_000, interval: 100 }); + expect(runTerminals(await run.coreEvents())[0]!.payload).toMatchObject({ turnTerminalState: "completed", runTerminalState: "succeeded" }); + await new Promise(done => setTimeout(done, 1_000)); + expect(runTerminals(await run.coreEvents())).toHaveLength(1); + } finally { for (const cleanup of cleanups.splice(0)) await cleanup(); } +}, 60_000); + diff --git a/packages/paperclip-runner/src/native-session-runtime.provider-turn.test.ts b/packages/paperclip-runner/src/native-session-runtime.provider-turn.test.ts new file mode 100644 index 00000000000..04fe166ad75 --- /dev/null +++ b/packages/paperclip-runner/src/native-session-runtime.provider-turn.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { NativeExecutionInputV1 } from "./contracts/native-execution.js"; +import type { NativeSession } from "./contracts/native-session-backend.js"; +import { executeNativeSession } from "./native-session-runtime.js"; +import type { PrpEvent } from "./protocol/replay-contract.js"; + +const identity = { + runId: "run-provider-turn", + sessionId: "session-provider-turn", + companyId: "company-provider-turn", + issueId: "issue-provider-turn", + agentId: "agent-provider-turn", +}; + +const input: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { companyId: identity.companyId, runId: identity.runId, issueId: identity.issueId, agentId: identity.agentId, + executionWorkspaceId: "workspace-provider-turn" }, + task: { identifier: "PAP-PROVIDER", title: "Wait for a background install", description: null, + prompt: "# PAP-PROVIDER: Wait for a background install", workMode: "standard" }, + workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: identity.sessionId, driverKind: "codex_app_server", protocolVersion: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + provider: { kind: "codex", model: null }, + completionContract: { id: "contract-provider-turn", sha256: "contract-provider-turn-sha", schemaVersion: "paperclip.completion-contract.v1", + contract: { revision: "1", objective: "Install and test", criteria: [{ id: "objective", requirement: "Install and test" }] } }, + interactionResponses: [], + credentialBindings: [], +}; + +const event = (sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record): PrpEvent => ({ + schema: "paperclip.prp.event.v1", + sourceEventId: `runner-provider-turn:${identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "runner-provider-turn", + sourceKind: "runner", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-1", + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: "2026-11-12T00:00:00.000Z", + payload, +}); + +const usage = { provider: "acpx", model: null, providerSessionId: null, providerRequestId: null, + cumulative: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, activeSeconds: 0, requests: 1, providerCostUsd: 0 }, + runDeltaAvailable: true, + runDelta: { inputTokens: 6, outputTokens: 4, cacheReadTokens: 0, cacheWriteTokens: 0, activeSeconds: 0, requests: 1, providerCostUsd: 0 } }; + +const turnOne = (awaiting: boolean): PrpEvent[] => [ + event(1, "turn.started", { provider: "acpx", providerTurnId: "turn-1", status: "inProgress", turn: { id: "turn-1", status: "inProgress" } }), + event(2, "usage.reported", usage), + event(3, "turn.completed", { provider: "acpx", providerTurnId: "turn-1", status: "completed", error: null, + ...(awaiting ? { awaitingProviderTurn: true } : {}) }), + ...(awaiting ? [] : [event(4, "run.terminal", { schema: "paperclip.prp.terminal.v1", status: "succeeded", turnTerminalState: "completed", + runTerminalState: "succeeded", reportedWorkDisposition: "done", provider: "acpx" })]), +]; + +const providerTurn: PrpEvent[] = [ + event(4, "turn.started", { provider: "acpx", providerTurnId: "provider-1", status: "inProgress", + turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }), + event(5, "turn.completed", { provider: "acpx", providerTurnId: "provider-1", status: "completed", error: null }), +]; + +async function finalize(stream: PrpEvent[]) { + const appended: PrpEvent[] = []; + const completeRun = vi.fn(async () => undefined); + const capabilities = { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + const session: NativeSession = { + identity: () => identity, + async capabilities() { return capabilities; }, + async *events() { for (const value of stream) yield structuredClone(value); }, + async startTurn() { return { turnId: "turn-1" }; }, + async result() { return null; }, + async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, + providerSessionId: "provider-session", cursor: String(stream.length), activeTurnId: "turn-1", pendingRuntimeRequests: [], lineage: [] }; }, + async close() {}, + }; + const outcome = await executeNativeSession({ + input, + backend: { async descriptor() { return { kind: "mock", name: "provider-turn", version: "1", capabilities }; }, async openSession() { return session; } }, + controlPlane: { + async openRun() {}, async checkpointSession() {}, + async appendEvent(value: PrpEvent) { + appended.push(structuredClone(value)); + return { cursor: value.sourceSeq, highestContiguousSourceSeq: value.sourceSeq, disposition: "committed" }; + }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + completeRun, + }, + runnerInstanceId: "runner-provider-turn", controlPlaneInstanceId: "control-provider-turn", timeoutMs: 1000, + resolveMissingResult: async () => null, + }).then(() => "finalized", (error: unknown) => String(error instanceof Error ? error.message : error)); + return { outcome, appended, completeRun }; +} + +describe("today's controller consumer with provider-started turn sequences", () => { + it("finalizes a no-result turn with live shells exactly as before when the controller did not opt in", async () => { + const withoutOptIn = await finalize(turnOne(false)); + expect(withoutOptIn.outcome).toContain("native_finalization_missing"); + expect(withoutOptIn.completeRun).not.toHaveBeenCalled(); + expect(withoutOptIn.appended.map(value => value.eventType)).toEqual(["turn.started", "usage.reported", "turn.completed"]); + }); + + it("would end the run at the awaited terminal and never consume the provider-started turn, so the opt-in must stay off", async () => { + const optedIn = await finalize([...turnOne(true), ...providerTurn]); + expect(optedIn.outcome).toContain("native_finalization_missing"); + expect(optedIn.appended.some(value => value.payload.origin === "provider")).toBe(false); + }); +});