diff --git a/server/src/services/execution-recovery-resolution.test.ts b/server/src/services/execution-recovery-resolution.test.ts index a7c0fc57143..854b92fc58f 100644 --- a/server/src/services/execution-recovery-resolution.test.ts +++ b/server/src/services/execution-recovery-resolution.test.ts @@ -9,6 +9,13 @@ describe("executionRecoverySettlementNote", () => { expect(note).toContain("after a server restart"); }); + it("does not claim a server restart for a hold a same-run retry created", () => { + const note = executionRecoverySettlementNote({ current: true, cause: "native_restart_recovery_blocked", + workspaceRestoreFailed: false, trigger: "same_run_retry" }); + expect(note).not.toContain("server restart"); + expect(note).toContain("Automatic retry of this run stopped"); + }); + it("keeps the workspace repair note for other holds with a restore failure", () => { expect(executionRecoverySettlementNote({ current: true, cause: "execution_finalization_deadline_exceeded", workspaceRestoreFailed: true })).toContain("Workspace repair required"); diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index 2e917f39b69..e59f05c18f5 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -46,8 +46,13 @@ import { alias } from "drizzle-orm/pg-core"; */ export function executionRecoverySettlementNote(input: { current: boolean; cause: string; workspaceRestoreFailed: boolean; + /** The hold's recorded trigger; a same-run retry is not a server restart. */ + trigger?: unknown; }): string { if (!input.current) return "Recovery closed because the task's owner, execution, or status changed. No work was replayed."; + if (input.cause === "native_restart_recovery_blocked" && input.trigger === "same_run_retry") { + return "Automatic retry of this run stopped. Saved work is preserved; actions with unverified outcomes will not be repeated."; + } if (input.cause === "native_restart_recovery_blocked") { return "Automatic recovery stopped after a server restart because the previous execution could not be verified. Saved work is preserved; actions with unverified outcomes will not be repeated."; } @@ -696,7 +701,7 @@ export async function settleUnrecoverableExecutions( (!task.executionRunId || task.executionRunId === run.id) && (!task.checkoutRunId || task.checkoutRunId === run.id); const note = executionRecoverySettlementNote({ current, cause: action.cause, - workspaceRestoreFailed: hasWorkspaceRestoreFailure(run.resultJson) }); + workspaceRestoreFailed: hasWorkspaceRestoreFailure(run.resultJson), trigger: action.evidence.trigger }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: task.companyId, issueId: task.id }); let nativeFailureBlock = action.evidence.nativeFailureBlock; let nativeBootstrapFailureBlock = action.evidence.nativeBootstrapFailureBlock; diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index bc7a277b294..244d7120c77 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -1807,6 +1807,10 @@ export async function claimNativeRestartRecoveries(input: { evidence: { runId: row.run.id, reason, + // A saved retryable failure is a same-run retry, not a restart + // hold. Read from the coordinator so a deferred or post-restart + // check of that retry records the same trigger. + ...(row.coordinator.phase === "retryable_failure" ? { trigger: "same_run_retry" } : {}), providerAttempt: row.coordinator.attempt, ...(hold ? { automaticAdmissionExhausted: true, executorStopEvidence: "unverified", providerPid: row.run.processPid, checkpointPreserved: hasCheckpoint } : {}), diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 348fcb7c29c..63678a59f2a 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -40,6 +40,7 @@ import { } from "@paperclipai/db"; import { beforeAll, describe, expect, it, vi } from "vitest"; import { heartbeatService } from "../heartbeat.js"; +import { executionRecoverySettlementNote } from "../execution-recovery-resolution.js"; import { canonicalNativeRuntimeContextDigest, @@ -3156,6 +3157,25 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); kind: "blocked", runId: fixture.runId, reason })]); }); + it("records that a held same-run retry was not a server restart", async () => { + // Staging run 61d2e06e: a remote session.open rejection saved by an older + // release, held when its due retry re-entered restart recovery. No server + // restarted, so the hold must not say one did. + const fixture = await seedExitedRemoteRunner("SAME-RUN-RETRY-HOLD", { + originalFailureCode: "provider_session_open_rejected" }, 1); + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, restartKind: "hard", + runIds: [fixture.runId] })).toEqual([expect.objectContaining({ + kind: "blocked", runId: fixture.runId, reason: "ambiguous_remote_recovery_evidence" })]); + const [action] = await fixture.db.select().from(issueRecoveryActions) + .where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId)); + expect(action).toMatchObject({ cause: "native_restart_recovery_blocked", + evidence: expect.objectContaining({ trigger: "same_run_retry" }) }); + const note = executionRecoverySettlementNote({ current: true, cause: action.cause, + workspaceRestoreFailed: false, trigger: action.evidence.trigger }); + expect(note).not.toContain("server restart"); + expect(note).toContain("Automatic retry of this run stopped"); + }); + it.each<[string, Parameters[1], { kind: string; reason: string }]>([ ["a runner process start was requested", { bootstrapRetry: true, event: { eventType: "native.process_start_requested" } }, { kind: "blocked", reason: "ambiguous_remote_recovery_evidence" }],