fix(runner): record why a turn-end provider restart failed

A failed turn-end restart was recovered silently: its cause surfaced only if
another turn started in the same run, which a single-turn run never does.
The executor now reports the cause once and the Runner records it as a
redacted runner diagnostic, so a kept instance shows why the restart failed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-09 09:01:30 -06:00
1 parent c33ba2c42c
commit 9c5cf6ece8
5 files changed
+34

No files matched your search

@@ -48,6 +48,7 @@ impl AcpxCommandExecutor {
// proven gone, then it is saved suspended, so the next run resumes the
// same conversation instead of starting a new one.
self.failed_turn_end_restart = Some(error.to_string());
self.unreported_turn_end_restart_failure = Some(error.to_string());
self.recover_failed_generation_restart()
}
@@ -969,6 +969,8 @@ pub struct AcpxCommandExecutor {
failed_turn_end_restart: Option<String>,
/// The failed turn-end restart already failed a turn of this run closed.
turn_end_restart_failed_turn: bool,
/// The failed turn-end restart's cause, until the Runner records it.
unreported_turn_end_restart_failure: Option<String>,
}
// The durable command's turn and the live provider callback have separate IDs.
@@ -1010,6 +1012,7 @@ impl AcpxCommandExecutor {
recovery_pending_effects: None,
failed_turn_end_restart: None,
turn_end_restart_failed_turn: false,
unreported_turn_end_restart_failure: None,
restore_error: None,
persistence_error: None,
#[cfg(test)]
@@ -2939,6 +2942,10 @@ impl CommandExecutor for AcpxCommandExecutor {
self.restart_session_after_turn()
}
fn take_turn_end_restart_failure(&mut self) -> Option<String> {
self.unreported_turn_end_restart_failure.take()
}
fn execute(&mut self, command: &Command) -> Result<CommandExecution, DurableRunnerError> {
self.restore()?;
if self
@@ -327,6 +327,12 @@ pub trait CommandExecutor {
Ok(())
}
/// The cause of a provider restart after a turn that failed and was
/// recovered, once; recorded as a runner diagnostic.
fn take_turn_end_restart_failure(&mut self) -> Option<String> {
None
}
fn execute(&mut self, command: &Command) -> Result<CommandExecution, DurableRunnerError>;
fn observe_recovery_state(&mut self, _state: &DurableState) -> Result<(), DurableRunnerError> {
@@ -2175,6 +2181,10 @@ fn poll_executor_events_after_controller_ack<E: CommandExecutor>(
poll_executor_events(state, store, config, executor)?;
if state.acked_source_seq >= state.highest_source_seq() {
executor.prepare_next_turn()?;
if let Some(failure) = executor.take_turn_end_restart_failure() {
state.record_diagnostic(format!("turn-end provider restart failed: {failure}"));
store.save(state)?;
}
}
Ok(())
}
@@ -351,6 +351,13 @@ impl CommandExecutor for NativeProviderCommandExecutor {
}
}
fn take_turn_end_restart_failure(&mut self) -> Option<String> {
match self.selected.as_mut() {
Some(SelectedExecutor::Acpx(executor)) => executor.take_turn_end_restart_failure(),
_ => None,
}
}
fn observe_recovery_state(
&mut self,
state: &crate::durable::DurableState,
@@ -356,6 +356,15 @@ fn the_next_run_resumes_the_conversation_after_a_failed_turn_end_restart() {
harness.fixture.fault("reject-open");
harness.executor.prepare_next_turn().unwrap();
assert!(!alive(&old_pid));
// The Runner records the cause once as a diagnostic.
let reported = harness.executor.take_turn_end_restart_failure();
assert!(
reported
.as_deref()
.is_some_and(|cause| cause.contains("session.open")),
"{reported:?}"
);
assert_eq!(harness.executor.take_turn_end_restart_failure(), None);
let saved = harness.fixture.state();
assert_eq!(saved["lifecycle"], "suspended", "{saved}");
assert_eq!(saved["providerExitUnconfirmed"], false);