diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 0affca3c77c..3b7616bab9a 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2285,6 +2285,25 @@ unknown operations or partial transcripts do not. Replacement uses a fresh session and retires only the exact predecessor's obsolete recovery hold while recording the proof and successor lineage. Retained provider files are not edited. +A same-run resume does not open a fresh replacement session after a +deterministic refusal that left the saved state unchanged. The runner reports +these refusals by closed classification, never by message text: +`provider_attach_refused` (attaching the new run would change the durable +provider profile or a protected launch argument) and +`provider_side_effect_pending` (the saved session still owns a tool call or +runtime input whose result was not delivered to the runner). The first ends the run with +`native_provider_state_incompatible`; the second ends it with +`native_provider_side_effect_pending`. Both stop automatic retries and give the +task to an operator with the saved local and remote continuity state intact. +`native_provider_side_effect_pending` is an execution-reconciliation cause, so a +generic Retry is not offered: the operator records whether the action ran, and +the reconciled continuation starts a fresh provider session in the same +workspace. A run with a pending side-effecting call never reaches replacement. Transport +drops and other unclassified resume failures keep the existing retry and +replacement behavior. When a replacement does archive remote continuity state +and the remote move fails, the local state is not moved, and the run error keeps +the bounded, redacted exit status and output of that move. + Bootstrap retries, exact-checkpoint resumes, and fresh replacement sessions share three total provider attempts, including the original attempt. Linked run IDs, controller restarts, and duplicate wakes do not reset this budget. A resume after a graceful shutdown drain suspended the run for reattachment does not consume an attempt, because the run did not cause that restart. A resume after a crash, which leaves no drain record, still consumes one, unless the previous restart resume made progress before the crash. Each restart claim records the run's event-log position. Progress means that, after that position, the provider accepted a new turn and then completed work in that turn: a tool execution or a completed item other than a usage report. Restore activity, failed turns, and accepted turns without completed work are not progress, so a crash loop without progress and a resume that fails each time still use up the budget. This credit stops after 12 controller generations, so a run that makes progress and then crashes the controller each time is still bounded. Exact-checkpoint resumes retain the 30-second delay. Replacement scheduling and predecessor lineage commit together, with one successor per predecessor and admission through the normal task locks, authorization, pause, approval, and budget gates. A due failed remote retry can retain its provider PID after heartbeat completion diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index 4fb4f33444b..b2748091865 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -112,6 +112,13 @@ after a failed resume, it opens a fresh provider session as before. Otherwise th run fails with `native_provider_state_incompatible` and is not retried. A board Retry then continues the task in a fresh provider session in the same workspace. +A Codex `run.attach` that would change the durable provider profile or a +protected launch argument fails with the closed classification +`provider_attach_refused`. An attach blocked by a pending tool call fails with +`provider_side_effect_pending`. Neither refusal changes the saved state. The +controller never opens a replacement session after either class, whatever the +checkpoint's recovery policy, and stops the run for an operator. + Saved OpenCode and Cursor Cloud provider state binds the digest of its own qualified launch profile (command, proxy script and, for OpenCode, the executable, each by path and SHA-256). Cursor Cloud does not re-bind. A changed diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs index 20e21bd2a4a..14b4077918b 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs @@ -42,6 +42,15 @@ pub fn redact_diagnostic_text(input: &str) -> String { /// build: retrying the same restore cannot succeed. pub const PROVIDER_STATE_INCOMPATIBLE: &str = "provider_state_incompatible"; +/// The runner refused to attach a new run to a saved provider session because +/// the durable provider profile or a protected launch argument would change. +/// The refusal leaves the saved state unchanged and repeats on every retry. +pub const PROVIDER_ATTACH_REFUSED: &str = "provider_attach_refused"; + +/// The saved provider session still owns a side-effecting tool call without a +/// recorded result. Replacing the session could repeat or lose that action. +pub const PROVIDER_SIDE_EFFECT_PENDING: &str = "provider_side_effect_pending"; + #[derive(Clone, Debug, PartialEq, Eq)] pub struct DurableRunnerError(String, Option, Option<&'static str>); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index 8e71e9662f4..c414061d6fb 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -23,7 +23,7 @@ use crate::durable::{ validate_semantic_tool_input, verify_private_directory, Command, CommandExecution, CommandExecutor, CursorCloudLaunchProfile, DurableRunnerConfig, DurableRunnerError, EventPriority, OpenCodeLaunchProfile, PolledEvent, TerminalDeliveryReconciliation, - PROVIDER_STATE_INCOMPATIBLE, + PROVIDER_ATTACH_REFUSED, PROVIDER_SIDE_EFFECT_PENDING, PROVIDER_STATE_INCOMPATIBLE, }; use crate::provider_bridge::{ authorized_tool_catalog_digest, semantic_value_digest, AuthorizedToolSet, DurableReplayFilter, @@ -3208,10 +3208,17 @@ impl CodexCommandExecutor { blockers.push("durable_active_goal"); } if !blockers.is_empty() { - return Err(DurableRunnerError::invalid(format!( + let error = DurableRunnerError::invalid(format!( "run.attach requires a settled Codex provider session: {}", blockers.join(", ") - ))); + )); + // A pending tool call is a side effect without a recorded result. + // The controller must not replace this session to get past it. + return Err(if blockers.contains(&"durable_pending_tool_calls") { + error.with_classification(PROVIDER_SIDE_EFFECT_PENDING) + } else { + error + }); } let runtime_launch_args: Option> = payload .get("runtimeLaunchArgs") @@ -3260,7 +3267,8 @@ impl CodexCommandExecutor { if config != next_state.config { return Err(DurableRunnerError::invalid( "run.attach cannot change the durable Codex provider profile", - )); + ) + .with_classification(PROVIDER_ATTACH_REFUSED)); } } let runtime_launch_changed = if let Some(args) = runtime_launch_args { @@ -3270,7 +3278,8 @@ impl CodexCommandExecutor { { return Err(DurableRunnerError::invalid( "run.attach cannot change protected launch arguments", - )); + ) + .with_classification(PROVIDER_ATTACH_REFUSED)); } let changed = args != next_state.config.args; next_state.config.args = args; diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index 7de236dfd31..91a9e256d0b 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -4578,6 +4578,62 @@ fn drain_completed_turn(executor: &mut CodexCommandExecutor, label: &str) { panic!("{label}: the turn did not complete before the deadline"); } +#[test] +fn attach_with_a_pending_tool_call_is_classified_as_a_pending_side_effect() { + // A provider tool call without a recorded result is an external action in + // flight. The refusal carries a closed class so the controller never opens + // a replacement session over it. + let directory = temporary_directory("pending-side-effect-attach"); + let config = provider_config(&directory, &["--durable-turn-ids", "--emit-tool-call"]); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config, "authorizedTools": task_context_tool_set()}), + )) + .expect("prepare the durable tool catalog"); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open the provider session"); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Call the tool."}), + )) + .expect("start the provider turn"); + let deadline = event_deadline(); + let mut input_seen = false; + while !input_seen && std::time::Instant::now() < deadline { + input_seen = poll_and_ack(&mut executor) + .expect("poll the semantic input") + .iter() + .any(|event| event.event_type == "semantic_tool.input"); + } + assert!(input_seen, "the provider requests its tool"); + + let error = executor + .execute(&command( + "attach", + 4, + "run.attach", + json!({"authorizedTools": task_context_tool_set()}), + )) + .expect_err("attachment must wait for the pending tool call"); + assert!( + error.to_string().contains("durable_pending_tool_calls"), + "unexpected attach error: {error}" + ); + assert_eq!(error.classification(), Some("provider_side_effect_pending")); + + executor.shutdown().expect("stop the provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn saved_codex_thread_attaches_after_a_release_changes_its_feature_toggles() { // A thread saved by an earlier release keeps that release's feature @@ -4633,6 +4689,9 @@ fn saved_codex_thread_attaches_after_a_release_changes_its_feature_toggles() { error.to_string().contains("protected launch arguments"), "unexpected attach error: {error}" ); + // The refusal is a closed class, so the controller stops before opening a + // replacement session and keeps this saved state for an operator. + assert_eq!(error.classification(), Some("provider_attach_refused")); let attached = executor .execute(&command( diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts index bc1193167e7..f9af4a42915 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -1,7 +1,13 @@ import { describe, expect, it, vi } from "vitest"; import type { HarnessDriver, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js"; -import { PROVIDER_STATE_INCOMPATIBLE, runnerRecoveryFailureClass } from "../contracts/harness-driver.js"; +import { + NativeRunnerSourceReconciliationRequiredError, + PROVIDER_ATTACH_REFUSED, + PROVIDER_SIDE_EFFECT_PENDING, + PROVIDER_STATE_INCOMPATIBLE, + runnerRecoveryFailureClass, +} from "../contracts/harness-driver.js"; import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; import { NativeSessionProtocolIntegrityError, type NativeSession } from "../contracts/native-session-backend.js"; import { HarnessDriverBackend } from "./harness-driver-backend.js"; @@ -344,6 +350,19 @@ describe("HarnessDriverBackend", () => { enumerable: false, }); expect(runnerRecoveryFailureClass(unclassified)).toBeUndefined(); + for (const classification of [PROVIDER_ATTACH_REFUSED, PROVIDER_SIDE_EFFECT_PENDING]) { + const refused = Object.defineProperty(new Error("PRP command run.attach failed"), "runnerCommandResult", { + value: { status: "failed", result: { code: "command_execution_failed", classification } }, + enumerable: false, + }); + expect(runnerRecoveryFailureClass(new Error("wrapped", { cause: refused }))).toBe(classification); + } + const pendingSource = new NativeRunnerSourceReconciliationRequiredError("semantic"); + expect(pendingSource.message).toBe("native_runner_source_semantic_reconciliation_required"); + expect(runnerRecoveryFailureClass(pendingSource)).toBe(PROVIDER_SIDE_EFFECT_PENDING); + expect(runnerRecoveryFailureClass(new Error("wrapped", { cause: new NativeRunnerSourceReconciliationRequiredError("runtime") }))) + .toBe(PROVIDER_SIDE_EFFECT_PENDING); + expect(runnerRecoveryFailureClass(new Error("native_runner_source_semantic_reconciliation_required"))).toBeUndefined(); const backend = new HarnessDriverBackend({ ...driver, diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index e4d4387a54e..6e8e6ee4cc5 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -500,25 +500,58 @@ export interface PersistedHarnessSession { /** Runner proof that saved provider state cannot be restored by this runner * build. Restoring the same state again fails the same way. */ export const PROVIDER_STATE_INCOMPATIBLE = "provider_state_incompatible"; +/** Runner refused to attach a new run to the saved session (for example, a + * protected launch argument or the durable provider profile would change). + * The refusal leaves the saved state unchanged and repeats on every retry. */ +export const PROVIDER_ATTACH_REFUSED = "provider_attach_refused"; +/** The saved session still owns a side-effecting call without a recorded + * result. Replacing the session could repeat or lose that external action. */ +export const PROVIDER_SIDE_EFFECT_PENDING = "provider_side_effect_pending"; + +export type RunnerRecoveryFailureClass = + | typeof PROVIDER_STATE_INCOMPATIBLE + | typeof PROVIDER_ATTACH_REFUSED + | typeof PROVIDER_SIDE_EFFECT_PENDING; + +const RUNNER_RECOVERY_FAILURE_CLASSES: ReadonlySet = new Set([ + PROVIDER_STATE_INCOMPATIBLE, + PROVIDER_ATTACH_REFUSED, + PROVIDER_SIDE_EFFECT_PENDING, +]); + +/** A predecessor's semantic tool call or bridged runtime input has no settled + * result, so the source runner cannot hand its session to a new run. */ +export class NativeRunnerSourceReconciliationRequiredError extends Error { + readonly code: `native_runner_source_${"semantic" | "runtime"}_reconciliation_required`; + constructor(readonly pending: "semantic" | "runtime") { + super(`native_runner_source_${pending}_reconciliation_required`); + this.name = "NativeRunnerSourceReconciliationRequiredError"; + this.code = `native_runner_source_${pending}_reconciliation_required`; + } +} export interface HarnessSessionRecoveryResult { recovered: boolean; session?: HarnessSession; reason?: string; /** Closed runner class of a deterministic recovery failure. */ - failureClass?: typeof PROVIDER_STATE_INCOMPATIBLE; + failureClass?: RunnerRecoveryFailureClass; } -/** Reads the closed class of an authenticated runner command failure. */ +/** Reads the closed class of an authenticated runner command failure, or of a + * typed source-reconciliation refusal. Never derived from message text. */ export function runnerRecoveryFailureClass( error: unknown, depth = 0, -): typeof PROVIDER_STATE_INCOMPATIBLE | undefined { +): RunnerRecoveryFailureClass | undefined { if (!error || typeof error !== "object" || depth > 5) return undefined; + if (error instanceof NativeRunnerSourceReconciliationRequiredError) return PROVIDER_SIDE_EFFECT_PENDING; const value = error as Record; const result = value.runnerCommandResult as Record | undefined; const detail = result?.result as Record | undefined; - if (detail?.classification === PROVIDER_STATE_INCOMPATIBLE) return PROVIDER_STATE_INCOMPATIBLE; + if (typeof detail?.classification === "string" && RUNNER_RECOVERY_FAILURE_CLASSES.has(detail.classification)) { + return detail.classification as RunnerRecoveryFailureClass; + } return runnerRecoveryFailureClass(value.cause, depth + 1); } diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index 77b8431eb89..0fee816ce94 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -307,6 +307,17 @@ export class NativeProviderStateIncompatibleError extends Error { } } +/** The saved provider session still owns a side-effecting call without a + * recorded result. A replacement session could repeat or lose that action, so + * the run stops for an operator and the saved state stays intact. */ +export class NativeProviderSideEffectPendingError extends Error { + readonly code = "native_provider_side_effect_pending"; + constructor(reason: string) { + super(`native_session_recovery_failed: ${reason}`); + this.name = "NativeProviderSideEffectPendingError"; + } +} + /** A provider failed terminal is not a missing completion proposal. */ export class NativeProviderTerminalFailure extends Error { readonly code = "native_provider_terminal_failed"; diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index d1c83fc9221..05351de090f 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -69,6 +69,7 @@ import type { HarnessRuntimeRequestResolution, PersistedHarnessProviderIdentity, } from "../contracts/harness-driver.js"; +import { NativeRunnerSourceReconciliationRequiredError } from "../contracts/harness-driver.js"; import { DurablePrpControlPlane, inspectStagedRunAttachment, @@ -4637,12 +4638,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { throw new Error("native_runner_prp_run_rotation_unavailable"); } if (retainingSource && this.#bridgedRuntimeInputs.size > 0) { - throw new Error("native_runner_source_runtime_reconciliation_required"); + throw new NativeRunnerSourceReconciliationRequiredError("runtime"); } if (this.#releaseEpisode) await this.#resumeReleasedRunner(input.runId); await this.#awaitWarmRunAttachmentReady(); if (retainingSource && !core.semanticToolResultsSettled()) { - throw new Error("native_runner_source_semantic_reconciliation_required"); + throw new NativeRunnerSourceReconciliationRequiredError("semantic"); } this.#assertStartupOpen(); const currentRunGrant = input.currentRunGrant @@ -6844,7 +6845,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } if (sourceAttachmentTarget) { await this.#awaitWarmRunAttachmentReady(); - if (!core.semanticToolResultsSettled()) throw new Error("native_runner_source_semantic_reconciliation_required"); + if (!core.semanticToolResultsSettled()) throw new NativeRunnerSourceReconciliationRequiredError("semantic"); if (runtimeContext !== null) writeFileSync(localRuntimeContextPath, `${JSON.stringify(runtimeContext)}\n`, { mode: 0o600 }); await this.#attachRunWork(sourceAttachmentTarget); this.#sourceRecoveryIdentity = null; @@ -6861,7 +6862,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { async #recoverSemanticToolInput(call: Parameters>[0]) { if (this.#retainsSourceInputs() || (this.#authorityRunId() !== undefined && call.correlation.runId !== this.#authorityRunId())) return null; - if (this.#sourceRecoveryIdentity) throw new Error("native_runner_source_semantic_reconciliation_required"); + if (this.#sourceRecoveryIdentity) throw new NativeRunnerSourceReconciliationRequiredError("semantic"); if (call.operationId === "paperclip_finish" || call.operationId === "paperclip_block") { return this.#handleSemanticToolInput(call); } @@ -6874,7 +6875,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { NonNullable >[0], ) { - if (this.#sourceRecoveryIdentity) throw new Error("native_runner_source_semantic_reconciliation_required"); + if (this.#sourceRecoveryIdentity) throw new NativeRunnerSourceReconciliationRequiredError("semantic"); try { this.#throwIfFailed(); } catch { diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index a54f6a98050..0f2cc6840d2 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -17,13 +17,14 @@ import type { PersistedNativeSession, } from "./contracts/native-session-backend.js"; import { + NativeProviderSideEffectPendingError, NativeProviderStateIncompatibleError, NativeSessionCloseUnrecoverableError, NativeSessionCleanupQuarantinedError, NativeSessionProtocolIntegrityError, nativeRestartInterruptedTurnId, } from "./contracts/native-session-backend.js"; -import { PROVIDER_STATE_INCOMPATIBLE } from "./contracts/harness-driver.js"; +import { PROVIDER_ATTACH_REFUSED, PROVIDER_SIDE_EFFECT_PENDING, PROVIDER_STATE_INCOMPATIBLE } from "./contracts/harness-driver.js"; import type { PrpEvent, PrpStructuredRunResult, @@ -6282,6 +6283,71 @@ describe("executeNativeSession recovery", () => { } }); + it("never replaces a session after a deterministic, non-destructive recovery refusal", async () => { + const outcomes = [ + { failureClass: PROVIDER_ATTACH_REFUSED, reason: "run.attach cannot change protected launch arguments", + expected: NativeProviderStateIncompatibleError }, + { failureClass: PROVIDER_SIDE_EFFECT_PENDING, reason: "native_runner_source_semantic_reconciliation_required", + expected: NativeProviderSideEffectPendingError }, + ] as const; + for (const outcome of outcomes) { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", sessionId: "driver-refused", identity, + providerSessionId: "provider-refused", providerRecoveryPolicy: "allow_replacement_after_resume_failure", + cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], + }; + const openRun = vi.fn(async () => undefined); + const openSession = vi.fn(); + const openReplacementSession = vi.fn(); + const backend: NativeSessionBackend = { + descriptor: async () => ({ kind: "mock", name: "refused", version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true } }), + openSession, openReplacementSession, + recoverSession: async () => ({ recovered: false, reason: outcome.reason, failureClass: outcome.failureClass }), + }; + const port: ControlPlanePort = { + openRun, appendEvent: vi.fn(), completeRun: vi.fn(), + loadSessionCheckpoint: async () => structuredClone(checkpoint), + replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), + }; + const failure = await executeNativeSession({ + input, backend, controlPlane: port, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", + }).catch((error: unknown) => error); + expect(failure, outcome.failureClass).toBeInstanceOf(outcome.expected); + expect((failure as Error).message).toBe(`native_session_recovery_failed: ${outcome.reason}`); + expect(openReplacementSession, outcome.failureClass).not.toHaveBeenCalled(); + expect(openSession, outcome.failureClass).not.toHaveBeenCalled(); + expect(openRun, outcome.failureClass).not.toHaveBeenCalled(); + } + }); + + it("still replaces a session after an unclassified resume failure when the checkpoint allows it", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", sessionId: "driver-transient", identity, + providerSessionId: "provider-transient", providerRecoveryPolicy: "allow_replacement_after_resume_failure", + cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], + }; + const replacementAttempted = new Error("replacement attempted"); + const openReplacementSession = vi.fn(async () => { throw replacementAttempted; }); + const backend: NativeSessionBackend = { + descriptor: async () => ({ kind: "mock", name: "transient", version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true } }), + openSession: vi.fn(), openReplacementSession, + recoverSession: async () => ({ recovered: false, reason: "runnerd transport closed while waiting for run.attach" }), + }; + const port: ControlPlanePort = { + openRun: vi.fn(async () => undefined), appendEvent: vi.fn(), completeRun: vi.fn(), + loadSessionCheckpoint: async () => structuredClone(checkpoint), + replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), + }; + await expect(executeNativeSession({ + input, backend, controlPlane: port, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", + })).rejects.toBe(replacementAttempted); + expect(openReplacementSession).toHaveBeenCalledOnce(); + }); + it("bounds paginated recovery replay with one signal and observes a late rejection", async () => { vi.useFakeTimers(); let rejectStalledReplay = (_error: Error) => {}; diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 913b4aca042..471236db587 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -19,8 +19,14 @@ import type { NativeSessionBackend, } from "./contracts/native-session-backend.js"; import type { PersistedNativeSession } from "./contracts/native-session-backend.js"; -import { PROVIDER_STATE_INCOMPATIBLE, type HarnessThreadGoal } from "./contracts/harness-driver.js"; import { + PROVIDER_ATTACH_REFUSED, + PROVIDER_SIDE_EFFECT_PENDING, + PROVIDER_STATE_INCOMPATIBLE, + type HarnessThreadGoal, +} from "./contracts/harness-driver.js"; +import { + NativeProviderSideEffectPendingError, NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, NativeSessionCloseUnrecoverableError, @@ -2422,6 +2428,16 @@ export async function executeNativeSession( reason: "driver does not support recovery", }; if (!recovery.recovered || !recovery.session) { + // A deterministic refusal left the saved state unchanged and repeats on + // every retry. A replacement would archive that state and, with a pending + // side-effecting call, could repeat or lose the external action. Stop here + // for an operator regardless of the checkpoint's replacement policy. + if (recovery.failureClass === PROVIDER_SIDE_EFFECT_PENDING) { + throw new NativeProviderSideEffectPendingError(recovery.reason ?? "unknown"); + } + if (recovery.failureClass === PROVIDER_ATTACH_REFUSED) { + throw new NativeProviderStateIncompatibleError(recovery.reason ?? "unknown"); + } if (!replacementAllowed) { if (recovery.failureClass === PROVIDER_STATE_INCOMPATIBLE) { throw new NativeProviderStateIncompatibleError(recovery.reason ?? "unknown"); diff --git a/packages/shared/src/types/execution-projection.ts b/packages/shared/src/types/execution-projection.ts index 7818cc27f97..8ec31e60c0e 100644 --- a/packages/shared/src/types/execution-projection.ts +++ b/packages/shared/src/types/execution-projection.ts @@ -73,6 +73,7 @@ export const EXECUTION_RECONCILIATION_CAUSES = [ "native_session_retry_exhausted", "native_restart_recovery_blocked", "native_continuation_requires_reconciliation", + "native_provider_side_effect_pending", "legacy_execution_requires_reconciliation", ] as const; export function requiresExecutionReconciliation( diff --git a/server/src/services/heartbeat/restart-interrupted-turn.ts b/server/src/services/heartbeat/restart-interrupted-turn.ts index ab0205a3ec5..ba4f1ca07b1 100644 --- a/server/src/services/heartbeat/restart-interrupted-turn.ts +++ b/server/src/services/heartbeat/restart-interrupted-turn.ts @@ -12,6 +12,12 @@ import { logger } from "../../middleware/logger.js"; const REATTACHMENT_LOST_ERRORS = ["native_remote_recovery_lease_mismatch", "native_plugin_recovery_lease_mismatch", "runner_remote_process_identity_unavailable", "runner_continuity_break_archive_failed"]; +/** A failure message is one of the closed codes, optionally followed by + * ": " and bounded diagnostic detail (for example, the archive's exit status). */ +function isReattachmentLoss(error: string | null): boolean { + return error !== null && REATTACHMENT_LOST_ERRORS.some(code => error === code || error.startsWith(`${code}: `)); +} + const TURN_STARTED_EVENTS = ["turn.submitted", "turn.started", "turn.accepted"]; const TURN_SETTLED_EVENTS = ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]; @@ -50,7 +56,7 @@ export async function requestRestartInterruptedTurnRetry(db: Db, claim: NativeRe export async function requestInterruptedNativeTurnRetry(db: Db, runId: string): Promise { const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); - if (!run || run.runtimeMode !== "native" || run.status !== "failed" || !REATTACHMENT_LOST_ERRORS.includes(run.error ?? "")) + if (!run || run.runtimeMode !== "native" || run.status !== "failed" || !isReattachmentLoss(run.error)) return { requested: false, reason: "not_reattachment_loss" }; const context = run.contextSnapshot ?? {}; const issueId = run.nativeIssueId ?? run.issueId ?? (typeof context.issueId === "string" ? context.issueId : null); diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index 716fd9b68c5..599f874aea8 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -186,7 +186,9 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( it("replays saved user intent through normal Retry after a first-attempt retained runner failure", async () => { for (const mode of ["unattached", "archive_failed", "provider_started", "system_wake", "stopped", "stopped_failed", "promoted"] as const) { const f = await fixture({ runtimeMode: mode === "promoted" ? "legacy" : "native" }, `CMP${randomUUID().slice(0, 6)}`); - const failureMessage = mode === "archive_failed" ? "runner_continuity_break_archive_failed" : "runner_remote_process_identity_unavailable"; + const failureMessage = mode === "archive_failed" + ? "runner_continuity_break_archive_failed: exit=1 timedOut=false mv: cannot remove 'runner': Directory not empty" + : "runner_remote_process_identity_unavailable"; const retryExpected = mode === "unattached" || mode === "archive_failed" || mode === "promoted"; const [issue] = await db.insert(issues).values({ companyId: f.run.companyId, title: "Continue saved work", assigneeAgentId: f.agent.id, status: "in_progress" }).returning(); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index e251185499b..0a0df1b620c 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -87,9 +87,11 @@ import { AGENT_FILES_CONTRACT } from "../agent-file-store.js"; import { captureDirectorySnapshot, directorySnapshotSha256, serializeDirectorySnapshot } from "@paperclipai/adapter-utils/workspace-restore-merge"; import { agentInstructionWorkingCopies } from "@paperclipai/db"; import * as noLaunchProofModule from "./native-maintenance-no-launch.js"; +import { requiresExecutionReconciliation } from "@paperclipai/shared"; import { NativeSessionCleanupQuarantinedError, NativeSessionCloseUnrecoverableError, + NativeProviderSideEffectPendingError, NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, NativeSessionProtocolIntegrityError, @@ -13147,6 +13149,23 @@ describe("native session bounded recovery", () => { expect(nativeSessionFailureSourceCode(new Error(failure.message))).toBe("native_session_interrupted"); }); + it("stops same-run retries for an operator while a saved side-effecting call has no result", () => { + const failure = new NativeProviderSideEffectPendingError( + "Error: native_runner_source_semantic_reconciliation_required", + ); + const code = nativeSessionFailureSourceCode(failure); + expect(code).toBe("native_provider_side_effect_pending"); + const disposition = nativeSessionFailureDisposition(1, new Date(), code); + expect(disposition).toEqual({ phase: "terminal_failure", failureCode: code, nextAttemptAt: null }); + expect(nativeSessionRecoveryProjection({ ...disposition, agentId: "agent" })).toMatchObject({ + issueStatus: "blocked", recoveryOwner: { kind: "board" }, recoveryActionCause: code, + }); + // An operator reconciles the pending action; a generic Retry is not offered. + expect(requiresExecutionReconciliation(code)).toBe(true); + // The same text without the typed class stays an ordinary interruption. + expect(nativeSessionFailureSourceCode(new Error(failure.message))).toBe("native_session_interrupted"); + }); + it("makes only typed operator-required cleanup quarantine terminal on the first attempt", () => { const code = nativeSessionFailureSourceCode( new NativeSessionCleanupQuarantinedError(), @@ -15468,8 +15487,12 @@ describe("runnerd provider runtime wiring", () => { state.createBackend.mockClear(); state.createTransport.mockClear(); const openSession = vi.fn(async () => ({ close: async () => undefined })); state.createBackend.mockReturnValueOnce({ kind: "test", openSession } as never); - const execute = vi.fn(async (request: { args: string[] }) => ({ exitCode: request.args[1]?.includes(".continuity-break-") && mode === "failed" ? 1 : 0, - stdout: "", stderr: "", timedOut: Boolean(request.args[1]?.includes(".continuity-break-") && mode === "timeout") })); + const archiving = (request: { args: string[] }) => Boolean(request.args[1]?.includes(".continuity-break-")); + const execute = vi.fn(async (request: { args: string[] }) => ({ exitCode: archiving(request) && mode === "failed" ? 1 : 0, + stdout: "", + stderr: archiving(request) && mode === "failed" + ? "mv: cannot remove '/runtime/sessions/digest/runner': Directory not empty\nGH_TOKEN=ghp_abcdefghijklmnopqrstuvwxyz0123456789" : "", + timedOut: archiving(request) && mode === "timeout" })); const backend = await createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId: "archive-controller", runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", environmentId: "environment", leaseId: "lease", providerKey: "daytona", sandboxLeaseAcquisition: { outcome: "created", providerLeaseId: "sandbox" }, @@ -15490,7 +15513,15 @@ describe("runnerd provider runtime wiring", () => { .toBe("Retained provider checkpoint"); expect(existsSync(join(root, "runner"))).toBe(false); } else { - await expect(replacement).rejects.toThrow("runner_continuity_break_archive_failed"); + const failure = await replacement.then(() => new Error("replacement unexpectedly succeeded"), (error: unknown) => error as Error); + expect(failure.message).toMatch(/^runner_continuity_break_archive_failed: /); + if (mode === "failed") { + expect(failure.message).toContain("exit=1 timedOut=false"); + expect(failure.message).toContain("cannot remove '/runtime/sessions/digest/runner': Directory not empty"); + expect(failure.message).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz0123456789"); + } else { + expect(failure.message).toContain("timedOut=true"); + } expect(await readFile(join(root, "runner", "retained.txt"), "utf8")).toBe("Retained provider checkpoint"); expect(await readFile(join(root, "control-plane", "retained.txt"), "utf8")).toBe("Retained controller journal"); expect(openSession).not.toHaveBeenCalled(); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index a50289f7f8c..7808a39dbde 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -162,8 +162,10 @@ import type { PrpStructuredRunResult, } from "../../vendor/paperclip-runner/index.js"; import { + NativeProviderSideEffectPendingError, NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, + NativeRunnerSourceReconciliationRequiredError, NativeSessionCleanupQuarantinedError, NativeSessionCloseUnrecoverableError, NativeSessionProtocolIntegrityError, @@ -246,7 +248,7 @@ import { renderNativeRunnerStagedAttachmentPrompt, stageNativeRunnerWakeAttachments, } from "./native-runner-file-handoff.js"; -import { NATIVE_PROVIDER_STATE_INCOMPATIBLE, nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; +import { NATIVE_PROVIDER_SIDE_EFFECT_PENDING, NATIVE_PROVIDER_STATE_INCOMPATIBLE, nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; import { verifyRetainedMaintenanceNoLaunch } from "./native-maintenance-no-launch.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { reportRunFailure } from "../run-failure-report.js"; @@ -7623,6 +7625,7 @@ export function nativeSessionFailureDisposition( sourceFailureCode === "runner_remote_provider_verification_unavailable" || sourceFailureCode === "native_provider_terminal_failed" || sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE || + sourceFailureCode === NATIVE_PROVIDER_SIDE_EFFECT_PENDING || sourceFailureCode === NATIVE_CURRENT_WAKE_COMMENTS_UNREAD || sourceFailureCode === NATIVE_CURRENT_WAKE_COMMENTS_CHANGED_AFTER_READ || sourceFailureCode === "native_session_cleanup_quarantined" || @@ -7674,6 +7677,7 @@ export function nativeSessionFailureSourceCode( | "native_permission_declined" | "native_provider_terminal_failed" | typeof NATIVE_PROVIDER_STATE_INCOMPATIBLE + | typeof NATIVE_PROVIDER_SIDE_EFFECT_PENDING | "native_provider_approval_required" | "native_provider_usage_limit" | "native_session_cleanup_quarantined" @@ -7705,6 +7709,7 @@ export function nativeSessionFailureSourceCode( | "native_session_interrupted" { if (error instanceof NativePermissionDeclinedError) return "native_permission_declined"; if (error instanceof NativeProviderStateIncompatibleError) return NATIVE_PROVIDER_STATE_INCOMPATIBLE; + if (error instanceof NativeProviderSideEffectPendingError) return NATIVE_PROVIDER_SIDE_EFFECT_PENDING; if (error instanceof NativeProviderTerminalFailure) { if (error.providerCode === "approval_required") return "native_provider_approval_required"; // Failed terminals retain their security meaning across the provider facade. @@ -7823,6 +7828,9 @@ export function nativeSessionFailureSourceCode( return "native_session_interrupted"; } +const NATIVE_PROVIDER_SIDE_EFFECT_PENDING_ACTION = + "The saved provider session has a tool call whose result never reached the runner, so Paperclip did not replace the session or retry it. Reconcile the action: check whether it ran and record the outcome. The task then continues in a fresh provider session in the same workspace. The saved session and workspace are unchanged."; + const NATIVE_PROVIDER_STATE_INCOMPATIBLE_ACTION = "The current runner cannot restore this task's saved provider session, so retrying this run cannot succeed. Retry the task to continue in a fresh provider session with the same workspace. The earlier provider conversation is not carried over."; @@ -11382,6 +11390,8 @@ async function executePaperclipNativeSessionWithinScope( ? NATIVE_STARTUP_MODEL_REJECTED_ACTION : sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE ? NATIVE_PROVIDER_STATE_INCOMPATIBLE_ACTION + : sourceFailureCode === NATIVE_PROVIDER_SIDE_EFFECT_PENDING + ? NATIVE_PROVIDER_SIDE_EFFECT_PENDING_ACTION : recoveryEvidence.recoveryMode === "ambiguous_state" ? "Inspect the original provider failure and durable events; state is ambiguous and a replacement provider session is forbidden." : integrityFailure @@ -11636,6 +11646,8 @@ async function executePaperclipNativeSessionWithinScope( ? NATIVE_STARTUP_MODEL_REJECTED_ACTION : sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE ? NATIVE_PROVIDER_STATE_INCOMPATIBLE_ACTION + : sourceFailureCode === NATIVE_PROVIDER_SIDE_EFFECT_PENDING + ? NATIVE_PROVIDER_SIDE_EFFECT_PENDING_ACTION : recoveryEvidence.recoveryMode === "ambiguous_state" ? "Inspect the original provider failure and explicitly resolve the ambiguous session state; do not open a replacement provider session." : integrityFailure @@ -15644,7 +15656,13 @@ async function createRunnerdBackendWithinSessionClaim( timeoutMs: 30_000, }); if (result.exitCode !== 0 || result.timedOut) { - throw new Error("runner_continuity_break_archive_failed"); + // The local continuity state has not moved yet. Keep the bounded, + // redacted remote output: it is the only evidence of why the move + // failed and how much of the remote session folder it changed. + const output = redactSensitiveText(`${result.stderr ?? ""}\n${result.stdout ?? ""}`).trim().slice(-512); + throw new Error( + `runner_continuity_break_archive_failed: exit=${result.exitCode} timedOut=${result.timedOut}${output ? ` ${output}` : ""}`, + ); } } noteNativeRunnerEvidence(archivedAt); @@ -15702,7 +15720,7 @@ async function createRunnerdBackendWithinSessionClaim( if (!current) throw new Error("native_session_tool_authority_unavailable"); const attachmentRunId = runnerControlPlane?.store?.state.identity.runId; if (attachmentRunId && attachmentRunId !== current.runId) { - throw new Error("native_runner_source_semantic_reconciliation_required"); + throw new NativeRunnerSourceReconciliationRequiredError("semantic"); } return current.execute(call); }; diff --git a/server/src/services/native-runtime/native-session-resume.test.ts b/server/src/services/native-runtime/native-session-resume.test.ts index 9765563ee98..df3c17a2a2d 100644 --- a/server/src/services/native-runtime/native-session-resume.test.ts +++ b/server/src/services/native-runtime/native-session-resume.test.ts @@ -62,6 +62,7 @@ import { buildNativeExecutionInput } from "./native-execution-input.js"; import { buildNativeExecutionWithCheckpoint, findNativeSessionResumeRun, + NATIVE_PROVIDER_SIDE_EFFECT_PENDING, NATIVE_PROVIDER_STATE_INCOMPATIBLE, NATIVE_TOOL_CONTRACT_FINGERPRINT, isUnusedLegacyNativeRetryReplacement, @@ -279,6 +280,14 @@ describe("capability-gated connection tool refresh", () => { }); }); +describe("a source stopped with a pending side-effecting call", () => { + it("does not rebind the saved session that still refuses attachment", () => { + const current = execution(currentRunId); + const source = { ...previousRun(), errorCode: NATIVE_PROVIDER_SIDE_EFFECT_PENDING }; + expect(rebindNativeSessionCheckpoint({ previousRun: source, currentExecution: current })).toBeNull(); + }); +}); + describe("a source whose saved provider session the runner proved incompatible", () => { it("continues the task in a fresh provider session with the same workspace", () => { const current = execution(currentRunId); diff --git a/server/src/services/native-runtime/native-session-resume.ts b/server/src/services/native-runtime/native-session-resume.ts index fcf2a3e0856..08d2e9287e2 100644 --- a/server/src/services/native-runtime/native-session-resume.ts +++ b/server/src/services/native-runtime/native-session-resume.ts @@ -97,6 +97,9 @@ export const NATIVE_TOOL_CONTRACT_FINGERPRINT = /** The runner proved that this run's saved provider session cannot be * restored by the current runner build. Its checkpoint is not resumable. */ export const NATIVE_PROVIDER_STATE_INCOMPATIBLE = "native_provider_state_incompatible"; +/** The saved session owned a side-effecting call without a delivered result. + * Recovery requires an explicit reconciliation of that action. */ +export const NATIVE_PROVIDER_SIDE_EFFECT_PENDING = "native_provider_side_effect_pending"; function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) @@ -613,6 +616,10 @@ export function rebindNativeSessionCheckpoint(input: { // The source is still the progress barrier. Its provider session cannot be // restored, so the next run starts a fresh session in the same workspace. if (input.previousRun.errorCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE) return null; + // The saved session still holds the unsettled call and refuses attachment. + // Generic Retry is blocked until an operator reconciles that action; the + // reconciled continuation then starts a fresh session in the same workspace. + if (input.previousRun.errorCode === NATIVE_PROVIDER_SIDE_EFFECT_PENDING) return null; const previousProfile = record(input.previousRun.runnerProfileJson); if ( previousProfile.nativeToolContractFingerprint !== diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index e78bb85d50b..e6aa843d972 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -187,6 +187,8 @@ export const validatePrpStructuredRunResult = export const NativeProviderTerminalFailure = runner.NativeProviderTerminalFailure; export const NativeProviderStateIncompatibleError = runner.NativeProviderStateIncompatibleError; +export const NativeProviderSideEffectPendingError = runner.NativeProviderSideEffectPendingError; +export const NativeRunnerSourceReconciliationRequiredError = runner.NativeRunnerSourceReconciliationRequiredError; export const nativeRestartInterruptedTurnId = runner.nativeRestartInterruptedTurnId; export const completeTerminatedRemoteNativeSessionCleanup = runner.completeTerminatedRemoteNativeSessionCleanup;