diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 24feeca3dbd..ad2b42ec320 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1322,6 +1322,13 @@ future execution until Resume, and missing stop proof still blocks continuation. A permanently unusable native runner session may be replaced only with evidence that its predecessor is stopped and fenced, completed results and workspace state are preserved, required task history is available, and pending effects have been reconciled. A provider-native shell command or external write without a reliable outcome receipt is unknown. Unknown effects, integrity failures, and unverified process ownership never authorize speculative replay. Once automatic recovery is ruled out, Paperclip selects a conservative default: preserve recorded work, stop the affected task, and retain a durable no-replay hold. Unknown action outcomes remain unknown. No reconciliation form or user diagnosis is required. +An ACPX sidecar transport failure carries its redacted cause and code in the +terminal event's structured `error` field. The session facade must retain that +cause instead of reporting a failed turn with `error: null`. A transport failure +does not prove that the provider stopped or that a retry is safe; it remains +non-recoverable until the existing ownership and recovery checks establish the +next permitted action. + Local Codex crash replacement can use a complete interrupted-turn inventory, authenticated process-stop evidence, and unchanged retained-state fingerprints. Only text and an exactly receipted task-completion call qualify for this path; diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index 995ab7c4711..ef5c2dc136d 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -218,6 +218,12 @@ deadline. The cases also retain failed-welcome ticket fencing, identity checks, and reconnect replay without repeated provider effects. These transport cases do not establish hosted provider acceptance. +The [native provider regression](../../packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs) +sends malformed activity from a real ACPX sidecar process. It checks a structured +transport error, one failed terminal, a closed session, and no provider reopening. +It proves the error contract and retained failure boundary, not the cause of a +historical provider outage or successful hosted task recovery. + Active remote process supervision distinguishes a failed health read from a confirmed exit. The [executor tests](../../server/src/services/native-runtime/native-session-executor.test.ts) exercise timeout recovery with a real listening process, unchanged ownership, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index 212852476cd..69540277da3 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -1998,9 +1998,11 @@ impl AcpxCommandExecutor { state.lifecycle = "closed".to_owned(); state.provider_exit_unconfirmed = true; if let Some(turn_id) = failed_turn { + let message = crate::durable::redact_text(&error.to_string()); state.push(NormalizedProviderEvent { event_type: "turn.failed".to_owned(), priority: EventPriority::P0, payload: json!({"provider":"acpx","providerTurnId":turn_id,"status":"failed","providerTerminalObserved":false, - "code":"acpx_provider_transport_failed","message":crate::durable::redact_text(&error.to_string())}) })?; + "code":"acpx_provider_transport_failed","message":message, + "error":{"code":"acpx_provider_transport_failed","message":message,"recoverable":false}}) })?; state.push(NormalizedProviderEvent { event_type: "run.terminal".to_owned(), priority: EventPriority::P0, payload: json!({"schema":"paperclip.prp.terminal.v1","status":"failed","turnTerminalState":"failed","runTerminalState":"failed","reportedWorkDisposition":"unknown","provider":"acpx"}) })?; } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs index 29468749552..1e9eadd2971 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs @@ -327,6 +327,7 @@ fn run() -> Result<(), Box> { | "turns-unauthorized-tool" | "turns-permission" | "turns-provider-death" + | "turns-malformed-activity" | "permissions-forged-origin" | "turns-retired" | "turns-retired-terminal-first" @@ -377,6 +378,10 @@ fn run() -> Result<(), Box> { } } write_json(&mut stdout, &response)?; + if command == "turn.start" && mode == "turns-malformed-activity" { + stdout.write_all(b"{invalid sidecar activity\n")?; + stdout.flush()?; + } let params = request.get("params").unwrap_or(&Value::Null); let turn_id = params .get("turnId") diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs index 615a2e74839..2e99a0bb35c 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs @@ -1103,6 +1103,67 @@ fn routes_reserved_completion_feedback_with_the_full_controller_catalog() { fs::remove_dir_all(directory).unwrap(); } +#[test] +fn acpx_transport_failure_retains_its_error_without_reopening_the_provider() { + let directory = temporary_directory("acpx-transport-failure"); + let mode = "turns-malformed-activity"; + let config = acpx_config(&directory, mode); + let mut executor = NativeProviderCommandExecutor::with_runner_config(&directory, &config); + executor + .execute(&command( + 1, + "run.prepare", + prepare_payload_with_mode(&directory, "codex", mode), + )) + .unwrap(); + executor + .execute(&command(2, "session.open", json!({}))) + .unwrap(); + executor + .execute(&command( + 3, + "turn.start", + json!({"text":"Read the saved question", "turnId":"provider-turn-transport"}), + )) + .unwrap(); + let mut events = Vec::new(); + for _ in 0..20 { + let batch = executor.poll_events().unwrap(); + executor.acknowledge_events(batch.len()).unwrap(); + events.extend(batch); + if events + .iter() + .any(|event| event.event_type == "run.terminal") + { + break; + } + std::thread::sleep(Duration::from_millis(5)); + } + let failures: Vec<_> = events + .iter() + .filter(|event| event.event_type == "turn.failed") + .collect(); + assert_eq!(failures.len(), 1); + let failure = &failures[0].payload; + assert_eq!(failure["providerTurnId"], "provider-turn-transport"); + assert_eq!(failure["providerTerminalObserved"], false); + assert_eq!(failure["error"]["code"], "acpx_provider_transport_failed"); + assert!(failure["error"]["message"] + .as_str() + .unwrap() + .contains("ACPX sidecar emitted invalid JSON")); + assert_eq!(failure["error"]["recoverable"], false); + let snapshot = executor + .execute(&command(4, "session.snapshot", json!({}))) + .unwrap() + .result; + assert_eq!(snapshot["status"], "closed"); + assert_eq!(snapshot["activeProviderTurnId"], Value::Null); + assert!(executor.poll_events().unwrap().is_empty()); + executor.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); +} + #[test] fn provider_death_expires_input_and_commits_failure_without_querying_the_dead_provider() { let directory = temporary_directory("acpx-provider-death");