diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs index f4a692695f9..088370b8cbf 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs @@ -48,6 +48,7 @@ impl AcpxCommandExecutor { // proven gone, then it is saved suspended, so the next run resumes the // same conversation instead of starting a new one. self.failed_turn_end_restart = Some(error.to_string()); + self.unreported_turn_end_restart_failure = Some(error.to_string()); self.recover_failed_generation_restart() } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index dfbeb6f1c83..d3ac1ce7436 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -969,6 +969,8 @@ pub struct AcpxCommandExecutor { failed_turn_end_restart: Option, /// The failed turn-end restart already failed a turn of this run closed. turn_end_restart_failed_turn: bool, + /// The failed turn-end restart's cause, until the Runner records it. + unreported_turn_end_restart_failure: Option, } // The durable command's turn and the live provider callback have separate IDs. @@ -1010,6 +1012,7 @@ impl AcpxCommandExecutor { recovery_pending_effects: None, failed_turn_end_restart: None, turn_end_restart_failed_turn: false, + unreported_turn_end_restart_failure: None, restore_error: None, persistence_error: None, #[cfg(test)] @@ -2939,6 +2942,10 @@ impl CommandExecutor for AcpxCommandExecutor { self.restart_session_after_turn() } + fn take_turn_end_restart_failure(&mut self) -> Option { + self.unreported_turn_end_restart_failure.take() + } + fn execute(&mut self, command: &Command) -> Result { self.restore()?; if self diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs index d786da4a324..2386f4218d7 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs @@ -327,6 +327,12 @@ pub trait CommandExecutor { Ok(()) } + /// The cause of a provider restart after a turn that failed and was + /// recovered, once; recorded as a runner diagnostic. + fn take_turn_end_restart_failure(&mut self) -> Option { + None + } + fn execute(&mut self, command: &Command) -> Result; fn observe_recovery_state(&mut self, _state: &DurableState) -> Result<(), DurableRunnerError> { @@ -2175,6 +2181,10 @@ fn poll_executor_events_after_controller_ack( poll_executor_events(state, store, config, executor)?; if state.acked_source_seq >= state.highest_source_seq() { executor.prepare_next_turn()?; + if let Some(failure) = executor.take_turn_end_restart_failure() { + state.record_diagnostic(format!("turn-end provider restart failed: {failure}")); + store.save(state)?; + } } Ok(()) } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs index 13551eb72b9..d906e5ffa03 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs @@ -351,6 +351,13 @@ impl CommandExecutor for NativeProviderCommandExecutor { } } + fn take_turn_end_restart_failure(&mut self) -> Option { + match self.selected.as_mut() { + Some(SelectedExecutor::Acpx(executor)) => executor.take_turn_end_restart_failure(), + _ => None, + } + } + fn observe_recovery_state( &mut self, state: &crate::durable::DurableState, diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/support/acpx_turn_end_restart.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/support/acpx_turn_end_restart.rs index 887358b289b..9548c625bde 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/support/acpx_turn_end_restart.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/support/acpx_turn_end_restart.rs @@ -356,6 +356,15 @@ fn the_next_run_resumes_the_conversation_after_a_failed_turn_end_restart() { harness.fixture.fault("reject-open"); harness.executor.prepare_next_turn().unwrap(); assert!(!alive(&old_pid)); + // The Runner records the cause once as a diagnostic. + let reported = harness.executor.take_turn_end_restart_failure(); + assert!( + reported + .as_deref() + .is_some_and(|cause| cause.contains("session.open")), + "{reported:?}" + ); + assert_eq!(harness.executor.take_turn_end_restart_failure(), None); let saved = harness.fixture.state(); assert_eq!(saved["lifecycle"], "suspended", "{saved}"); assert_eq!(saved["providerExitUnconfirmed"], false);