diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 50df9a08fcd..83f248ee413 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1799,7 +1799,13 @@ only while the old Runner is alive: the launch fence stops that the launch closed when it cannot prove them gone. A Runner that already exited mid-turn can leave provider or tool processes that the fence cannot find, so it keeps the fail-closed path below, and the task continues in a fresh session from -the saved work. The fresh Runner +the saved work. + +The first unanswered read can come on the last of the three provider attempts, +for example when a server restart used the second attempt. The run then gets one +more attempt for the replacement only: the failure stays retryable, the claim +admits attempt 4 with `replaceUnansweredRunner`, and a live Runner is replaced as +above. That attempt never gets another one; any failure on it ends the run. The fresh Runner restores the provider thread; its in-flight turn is reconciled from history and never submitted again. The provider attempt budget still bounds these attempts. diff --git a/server/src/services/native-runtime/native-remote-retry-admission.test.ts b/server/src/services/native-runtime/native-remote-retry-admission.test.ts index f34349e5ab1..0a67db1d02f 100644 --- a/server/src/services/native-runtime/native-remote-retry-admission.test.ts +++ b/server/src/services/native-runtime/native-remote-retry-admission.test.ts @@ -461,6 +461,46 @@ describeEmbeddedPostgres("remote native retry execution admission", () => { processPid: 1386, runnerProfileJson: { sessionCheckpoint: f.checkpoint } }); }); + it("executes the one replacement attempt after the last attempt's first unanswered recovery read, and no further one", async () => { + const f = await failedRemoteRun("remote"); + await f.db.update(issues).set({ executionRunId: f.runId }).where(eq(issues.id, f.issueId)); + const [issue] = await f.db.select().from(issues).where(eq(issues.id, f.issueId)); + const [contract] = await f.db.select().from(completionContracts).where(eq(completionContracts.issueId, f.issueId)); + const execution: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { companyId: f.companyId, runId: f.runId, issueId: f.issueId, agentId: f.agentId, executionWorkspaceId: f.runId }, + provider: { kind: "codex", model: "gpt-5.6-luna" }, + task: { identifier: issue.identifier!, title: issue.title, description: null, prompt: "Continue retained work.", workMode: "standard" }, + workspace: { cwd: environment.current.runtimeRoot, repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: f.native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", + contract: buildNativeCompletionContract(issue, { revision: contract.revision }) }, + interactionResponses: [], credentialBindings: [], + }; + await f.db.update(heartbeatRuns).set({ runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) + || ${JSON.stringify({ nativeExecutionInput: execution })}::jsonb` }).where(eq(heartbeatRuns.id, f.runId)); + await f.db.update(nativeRunFinalizations).set({ attempt: 3, failureDetail: { + message: "native_session_recovery_failed: HarnessCapabilityUnavailableError: read is unavailable: Error: provider_transport_failed: PRP command session.snapshot timed out (runner connected; pending ahead 0; last runner event seq 412 84s ago)", + recoveryMode: "exact_checkpoint_resume", providerSessionEstablished: true, checkpointExists: true, + providerEventsExist: true, recoverable: true, adoptedRunnerUnanswered: true, + } }).where(eq(nativeRunFinalizations.runId, f.runId)); + provider.execute.mockReset().mockRejectedValue(new Error("replacement attempt reached the provider session")); + const controller = await currentNativeControllerIdentity(); + const [claim] = await claimNativeRestartRecoveries({ db: f.db, controller, restartKind: "hard", runIds: [f.runId] }); + expect(claim).toMatchObject({ kind: "reattach_remote_runner", providerAttempt: 4, replaceUnansweredRunner: true }); + if (claim.kind !== "reattach_remote_runner") throw new Error("Expected the replacement claim"); + await expect(executePaperclipNativeSession({ db: f.db, execution, runnerInstanceId: f.native.runnerInstanceId, + leaseOwner: claim.leaseOwner, restartRecovery: claim, + backend: { descriptor: async () => { throw new Error("Unexpected descriptor call"); }, + openSession: async () => { throw new Error("Unexpected session launch"); } }, + })).rejects.toThrow("replacement attempt reached the provider session"); + expect(provider.execute).toHaveBeenCalledTimes(1); + expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]) + .toMatchObject({ attempt: 4, phase: "terminal_failure", failureCode: "native_session_retry_exhausted", nextAttemptAt: null }); + expect(await claimNativeRestartRecoveries({ db: f.db, controller, restartKind: "hard", runIds: [f.runId] })).toEqual([]); + }); + it.each((["remote", "plugin"] as const).flatMap(storage => (["released", "retained"] as const).map(lock => [storage, lock] as const))) ("does not admit another failed %s retry when the persisted provider budget is already exhausted with a %s task lock", async (storage, lock) => { const f = await failedRemoteRun(storage); diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 76ea996d941..cd63032c87a 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -1252,7 +1252,9 @@ export async function claimNativeRestartRecoveries(input: { const lockLostBootstrapRetry = row.run.status === "failed" && row.coordinator.phase === "retryable_failure" && row.coordinator.failureDetail?.recoveryMode === "bootstrap_retry" && row.issueExecutionRunId !== row.run.id; const releasedRemoteRetry = failedRemoteRetry && row.issueExecutionRunId === null; - const exhaustedRemoteRetry = failedRemoteRetry && row.coordinator.attempt >= 3; + const unansweredRunnerReplacement = row.coordinator.failureDetail?.adoptedRunnerUnanswered === true + && row.coordinator.attempt === NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT; + const exhaustedRemoteRetry = failedRemoteRetry && row.coordinator.attempt >= 3 && !unansweredRunnerReplacement; const needsResumptionAdmission = releasedRemoteRetry || exhaustedRemoteRetry || (failedRemoteRetry && ["done", "cancelled"].includes(issue.status)); const admission = { exhausted: null as { reason: string; policy: SystemRecoveryState } | null }; diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 625ea711774..cbbe0422a00 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -1575,6 +1575,42 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); expect(run?.processPid).toBe(remotePid); }); + it.each([true, false])("gives a Runner whose first unanswered read came on the last attempt one replacement attempt: %s", async (unanswered) => { + const fixture = await seedRun(`remote-final-unanswered-${unanswered}`); + const now = new Date(); + await fixture.db.update(heartbeatRuns).set({ status: "failed", finishedAt: now, + nativePhase: "retryable_failure", runnerProfileJson: { ...fixture.preparedProfile, + nativeWorkspaceSync: remotePreparedWorkspace(), + sessionCheckpoint: { identity: { companyId, agentId, issueId: fixture.issueId, runId: fixture.runId, + sessionId: fixture.native.normalizedSessionId }, providerSessionId: "final-unanswered-session" }, + } }).where(eq(heartbeatRuns.id, fixture.runId)); + await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: successor.bootId, + leaseOwner: "predecessor-final-unanswered", leaseExpiresAt: new Date(now.getTime() + 60_000) }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + const authority = new DurablePrpControlPlane({ stateDirectory: resolve(runtimeRoot, fixture.runId), + identity: transportOptions(fixture, runtimeRoot).prpIdentity, + expectedRunnerVersion: "final-unanswered-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + try { + await registerNativeRunnerCheckpoint({ db: fixture.db, + binding: { companyId, issueId: fixture.issueId, runId: fixture.runId }, bootId: successor.bootId, authority }); + } finally { await authority.stop(); } + await fixture.db.update(nativeRunFinalizations).set({ phase: "retryable_failure", attempt: 3, + leaseOwner: null, leaseExpiresAt: null, nextAttemptAt: new Date(0), failureCode: "native_session_interrupted", + failureDetail: { message: "native_session_recovery_failed: HarnessCapabilityUnavailableError: read is unavailable: Error: provider_transport_failed: PRP command session.snapshot timed out (runner connected; pending ahead 0; last runner event seq 412 84s ago)", + recoveryMode: "exact_checkpoint_resume", checkpointExists: true, recoverable: true, + providerSessionEstablished: true, providerEventsExist: true, + ...(unanswered ? { adoptedRunnerUnanswered: true } : {}) } }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + const [claim] = await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId], now }); + if (unanswered) { + expect(claim).toMatchObject({ kind: "reattach_remote_runner", runId: fixture.runId, providerAttempt: 4, + replaceUnansweredRunner: true }); + } else { + expect(claim?.kind).not.toBe("reattach_remote_runner"); + } + }); + it.each([true, false])("asks the same-run reattachment to replace a Runner that left its recovery read unanswered: %s", async (unanswered) => { const fixture = await seedRun(`remote-unanswered-${unanswered}`); await fixture.db.update(heartbeatRuns).set({ diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 1f05f6623b9..52518b89559 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -477,6 +477,8 @@ import { stopStaleRemoteRunnerBeforeBootstrap, verifyRemoteRunnerOrSettledExit, nativeRecoveryCommandUnanswered, + nativeUnansweredRunnerReplacementDisposition, + nativeAdmittedProviderAttemptLimit, readRemoteProviderPackManifest, verifyRemoteProviderPackManifest, prepareRemoteProviderPackOnTarget, @@ -1172,6 +1174,35 @@ describe("remote controller restart adoption", () => { await expect(verifyRemoteRunnerOrSettledExit({ claim: unanswered, target: target as never, identity, runId: "run", normalizedSessionId: "session", providerAttempt: 2 })).rejects.toThrow("runner_remote_recovery_identity_mismatch"); }); + it.each([ + ["the last attempt's first unanswered read", { adoptedRunnerUnanswered: true, replacing: false }, 3, "retryable_failure"], + ["a replacement that also went unanswered", { adoptedRunnerUnanswered: true, replacing: true }, 3, "terminal_failure"], + ["another failure on the last attempt", { adoptedRunnerUnanswered: false, replacing: false }, 3, "terminal_failure"], + ["the extra replacement attempt", { adoptedRunnerUnanswered: true, replacing: false }, 4, "terminal_failure"], + ] as const)("bounds the replacement after %s", (_case, input, attempt, phase) => { + const now = new Date("2026-11-13T06:37:26.000Z"); + const disposition = nativeUnansweredRunnerReplacementDisposition( + nativeSessionFailureDisposition(attempt, now, "provider_transport_failed"), { ...input, attempt, now }); + expect(disposition.phase).toBe(phase); + if (phase === "retryable_failure") { + expect(disposition).toEqual({ phase, failureCode: "native_session_interrupted", nextAttemptAt: new Date(now.getTime() + 30_000) }); + } + }); + it("admits provider attempt 4 at execution only for the replacement claim", () => { + const claim = { kind: "reattach_remote_runner" as const, runId: "run", leaseOwner: "owner", controllerGeneration: 6, + providerAttempt: 4, restartKind: "hard" as const, recoveryRequestId: null, + remote: { providerLeaseId: "sandbox", remoteCwd: "/workspace" } }; + expect(nativeAdmittedProviderAttemptLimit({ ...claim, replaceUnansweredRunner: true })).toBe(4); + expect(nativeAdmittedProviderAttemptLimit(claim)).toBe(3); + expect(nativeAdmittedProviderAttemptLimit({ kind: "resume_dead_runner", runId: "run", leaseOwner: "owner", + controllerGeneration: 6, providerAttempt: 4, restartKind: "hard", recoveryRequestId: null })).toBe(3); + expect(nativeAdmittedProviderAttemptLimit(null)).toBe(3); + }); + it("replaces the live Runner on the extra replacement attempt", async () => { + const { target } = fixture(midTurn); + await expect(verifyRemoteRunnerOrSettledExit({ claim: unanswered, target: target as never, identity, + runId: "run", normalizedSessionId: "session", providerAttempt: 4 })).resolves.toBeUndefined(); + }); it.each([ "native_session_recovery_failed: HarnessReconciliationError: codex_history_read_failed: thread/turns/list requires supported paginated history; Error: provider_transport_failed: PRP command session.snapshot timed out (runner connected; pending ahead 0; last runner event seq 338 76s ago)", "native_session_recovery_failed: HarnessCapabilityUnavailableError: read is unavailable: Error: runnerd exited while waiting for session.snapshot", diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 00ac5f87600..f74002866cd 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -8797,6 +8797,21 @@ export function nativeSessionFailureDisposition( }; } +export function nativeAdmittedProviderAttemptLimit(claim: NativeRestartRecoveryClaim | null | undefined): number { + return claim?.kind === "reattach_remote_runner" && claim.replaceUnansweredRunner === true + ? NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT + 1 : NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT; +} + +export function nativeUnansweredRunnerReplacementDisposition( + disposition: ReturnType, + input: { attempt: number; now: Date; adoptedRunnerUnanswered: boolean; replacing: boolean }, +): ReturnType { + if (!input.adoptedRunnerUnanswered || input.replacing || input.attempt !== NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT + || disposition.failureCode !== "native_session_retry_exhausted") return disposition; + return { phase: "retryable_failure", failureCode: "native_session_interrupted", + nextAttemptAt: new Date(input.now.getTime() + 30_000) }; +} + export function nativeSessionRecoveryProjection(input: { phase: "retryable_failure" | "terminal_failure"; failureCode: string; @@ -11134,7 +11149,7 @@ async function executePaperclipNativeSessionAttempt( recovering?.kind, !recovering || !("chargesProviderAttempt" in recovering) || recovering.chargesProviderAttempt !== false, ); - if (nextAttempt > 3) + if (nextAttempt > nativeAdmittedProviderAttemptLimit(recovering)) throw new Error("native_session_retry_exhausted"); const scheduledResumeDecision = await scheduledResumeContinuationAllowed(tx as unknown as Db, { companyId: input.execution.binding.companyId, @@ -12937,11 +12952,17 @@ async function executePaperclipNativeSessionAttempt( sourceFailureCode, bootstrap, }); - const disposition = nativeSessionFailureDisposition( + const adoptedRunnerUnanswered = input.runnerExecutionTarget?.kind === "remote" + && input.restartRecovery?.kind === "reattach_remote_runner" + && input.restartRecovery.remote.kind !== "plugin_task" && nativeRecoveryCommandUnanswered(error); + const disposition = nativeUnansweredRunnerReplacementDisposition(nativeSessionFailureDisposition( attempt, now, sourceFailureCode, - ); + ), { + attempt, now, adoptedRunnerUnanswered, + replacing: input.restartRecovery?.kind === "reattach_remote_runner" && input.restartRecovery.replaceUnansweredRunner === true, + }); const phase = recoveryEvidence.recoveryMode === "ambiguous_state" ? ("terminal_failure" as const) @@ -13075,9 +13096,7 @@ async function executePaperclipNativeSessionAttempt( ? { runnerExitConfirmed: true } : {}), ...(input.runnerExecutionTarget?.kind === "remote" && nativeRunnerExitedBeforeSessionOpen(error) ? { runnerExitedBeforeSessionOpen: true } : {}), - ...(input.runnerExecutionTarget?.kind === "remote" && input.restartRecovery?.kind === "reattach_remote_runner" - && input.restartRecovery.remote.kind !== "plugin_task" && nativeRecoveryCommandUnanswered(error) - ? { adoptedRunnerUnanswered: true } : {}), + ...(adoptedRunnerUnanswered ? { adoptedRunnerUnanswered: true } : {}), ...(providerCooldownAt ? { providerCooldownAt: providerCooldownAt.toISOString() } : {}), ...(providerLimit?.plan.resetAt ? { providerLimitResetAt: providerLimit.plan.resetAt.toISOString() } : {}), recoveryOwner: recoveryProjection.recoveryOwner, @@ -15304,7 +15323,7 @@ export async function verifyRemoteRunnerOrSettledExit( const stall = input.stall ?? remoteRunnerPluginStall(input.recoveryDeadline); const target = input.target; const replaceUnanswered = input.claim.replaceUnansweredRunner === true - && input.providerAttempt <= NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT + && input.providerAttempt <= NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT + 1 && target?.kind === "remote" && target.transport === "sandbox" && Boolean(target.runner); let verified: Awaited>; try {