diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 19fed8d72d4..50df9a08fcd 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1842,7 +1842,12 @@ first, and the Retry starts a fresh provider session over the saved work, so the turn is never replayed. Activity records the request as `restart-recovery` on behalf of that user. Admission keeps every gate: saved-work recovery, stop proof, approvals, pauses, budget and ownership. A Retry that the -server requested gets no second automatic attempt. A system wake (for example a +server requested gets no second automatic attempt, unless a server restart or +deploy took that continuation's controller while it ran (a shutdown +`native.recovery.transition` with `restartKind: "graceful"`, or a restart +scan's claim that is not a same-run retry). Each such deploy earns one more +fresh-session continuation and spends no budget, so a task that only deploys +interrupt never runs out. A system wake (for example a monitor) continues this way only with a positive record that no turn was ever issued: the refused reattachment proved the Runner process gone, and the run's controller journal, which records each turn command diff --git a/server/src/services/heartbeat/restart-interrupted-turn.ts b/server/src/services/heartbeat/restart-interrupted-turn.ts index 533d7b3cdc7..a80de052b18 100644 --- a/server/src/services/heartbeat/restart-interrupted-turn.ts +++ b/server/src/services/heartbeat/restart-interrupted-turn.ts @@ -83,7 +83,7 @@ async function recordUnknownTurnOutcome(db: Db, run: { companyId: string; id: st ...(typeof runnerExit.runnerLifecycle === "string" ? { runnerLifecycle: runnerExit.runnerLifecycle } : {}) } } }); } -/** Automatic continuation is requested at most once per chain of runs. */ +/** Automatic continuation is requested at most once per chain of runs, plus once for each deploy that interrupts a continuation. */ export const RESTART_RETRY_KEY_PREFIX = "restart-interrupted-turn:"; const MAX_RETRY_CHAIN = 32; @@ -108,6 +108,19 @@ async function chainUsedAutomaticRetry(db: Db, run: { companyId: string; id: str return visited.size >= MAX_RETRY_CHAIN; } +const RESTART_CLAIM_DISPOSITIONS = ["reattach_remote_runner", "reattach_existing_runner", "resume_dead_runner", + "bootstrap_incomplete"]; + +async function lostControllerToRestart(db: Db, run: { companyId: string; id: string }) { + const [evidence] = await db.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and( + eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), + eq(heartbeatRunEvents.eventType, "native.recovery.transition"), + sql`(${heartbeatRunEvents.payload}->>'restartKind' = 'graceful' or (${heartbeatRunEvents.payload}->>'trigger' is null + and ${heartbeatRunEvents.payload}->>'runnerDisposition' in (${sql.join(RESTART_CLAIM_DISPOSITIONS.map(kind => sql`${kind}`), sql`, `)})))`, + )).limit(1); + return evidence !== undefined; +} + type PrepareChatRetry = NonNullable[0]["prepareChatRetry"]>; /** A connected conversation's Retry goes through the chat service, which the @@ -188,15 +201,17 @@ export async function requestInterruptedNativeTurnRetry(db: Db, runId: string): if (!wake || reason === "task_retry" || !run.responsibleUserId || !(owedRedelivery || await hasProvenNoProviderTurn(db, run, runnerExit))) return { requested: false, reason: "not_user_request" }; - if (owedRedelivery && await chainUsedAutomaticRetry(db, run)) return { requested: false, reason: "automatic_retry_used" }; + if (owedRedelivery && await chainUsedAutomaticRetry(db, run) && !await lostControllerToRestart(db, run)) + return { requested: false, reason: "automatic_retry_used" }; actorId = run.responsibleUserId; } else if (reason === "task_retry") { - // A Retry this hook already requested gets no second automatic attempt. + // A Retry this hook already requested gets no second automatic attempt unless a deploy interrupted it. const retryId = typeof context.taskRetryRequestId === "string" ? context.taskRetryRequestId : null; const [receipt] = retryId ? await db.select({ key: agentWakeupRequests.idempotencyKey }).from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, run.companyId), sql`${agentWakeupRequests.id}::text = ${retryId}`)) : []; if (!receipt) return { requested: false, reason: "retry_source_missing" }; - if (receipt.key?.startsWith(`task-retry:${RESTART_RETRY_KEY_PREFIX}`)) return { requested: false, reason: "automatic_retry_used" }; + if (receipt.key?.startsWith(`task-retry:${RESTART_RETRY_KEY_PREFIX}`) && !await lostControllerToRestart(db, run)) + return { requested: false, reason: "automatic_retry_used" }; } else if (isUserCommentContinuationReason(reason)) { const commentId = typeof context.wakeCommentId === "string" ? context.wakeCommentId : typeof wake!.payload?.commentId === "string" ? wake!.payload.commentId : null; diff --git a/server/src/services/runner-lost-mid-turn-continuation.test.ts b/server/src/services/runner-lost-mid-turn-continuation.test.ts index 83d4eaf4110..5605d7070be 100644 --- a/server/src/services/runner-lost-mid-turn-continuation.test.ts +++ b/server/src/services/runner-lost-mid-turn-continuation.test.ts @@ -170,6 +170,112 @@ describe("a remote Runner that died in the middle of a provider turn", () => { eq(agentWakeupRequests.reason, "task_retry")))).filter(retry => retry.id !== hookIntent.id)).toHaveLength(0); }, 30_000); + describe("an automatic continuation that the next deploy interrupted", () => { + async function deployDuringRun(f: { companyId: string; runId: string; agentId: string }) { + for (const [message, payload] of [ + ["Server shutdown suspended native controller ownership without cancelling provider work", { signal: "SIGTERM", + processPid: 1048, restartKind: "graceful", processGroupId: null, retryRunCreated: false, + processStartedAt: "2026-11-13T05:28:12.307Z", runnerDisposition: "awaiting_runner_reattach" }], + ["Recovering the existing native runner process after server restart", { restartKind: "hard", providerAttempt: 2, + recoveryRequestId: null, runnerDisposition: "reattach_remote_runner", controllerGeneration: 3 }], + ["Reattaching to the existing remote runner to resume this run", { trigger: "same_run_retry", restartKind: "hard", + providerAttempt: 3, recoveryRequestId: null, runnerDisposition: "reattach_remote_runner", controllerGeneration: 4 }], + ] as const) await appendHeartbeatRunEvent(db, { companyId: f.companyId, runId: f.runId, agentId: f.agentId, + eventType: "native.recovery.transition", stream: "system", level: "info", message, payload }); + } + + async function automaticContinuation(input: { deploy: boolean; earlierContinuations?: number; systemRetries?: boolean }) { + const f = await runnerLostMidTurn({ wake: "user_retry", events: ["turn.submitted"], refusalPayload: { + remoteRunnerRecovery: { stage: "process_verification", reason: "process_not_running", runnerSettled: false, + runnerLifecycle: "ready", runnerDiagnostics: ["WebSocket peer closed the connection"], + runnerExitEvidence: "== runnerd.process-stderr.log" } } }); + let previousRunId: string | null = null; + for (let index = 0; index <= (input.earlierContinuations ?? 0); index += 1) { + const [lost]: Array = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, + status: "failed", runtimeMode: "native", error: "runner_remote_process_identity_unavailable", + createdAt: new Date(Date.now() - (120 - index) * 60_000), finishedAt: new Date(Date.now() - (110 - index) * 60_000), + errorCode: "native_continuation_requires_reconciliation", contextSnapshot: { issueId: f.issueId, + wakeReason: index === 0 ? "issue_assigned" : "task_retry", ...(previousRunId ? { previousRunId } : {}) } }).returning(); + if (previousRunId) { + const [hookIntent]: Array = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, + source: "on_demand", triggerDetail: "manual", reason: "task_retry", status: "completed", + idempotencyKey: `task-retry:restart-interrupted-turn:${previousRunId}`, requestedByActorType: "user", + requestedByActorId: "local-board", payload: { issueId: f.issueId } }).returning(); + await db.update(heartbeatRuns).set({ contextSnapshot: { ...lost.contextSnapshot, taskRetryRequestId: hookIntent.id } }) + .where(eq(heartbeatRuns.id, lost.id)); + if (input.systemRetries) { + const [systemRetry]: Array = await db.insert(heartbeatRuns).values({ + companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, status: "failed", runtimeMode: "native", + retryOfRunId: lost.id, error: "runner_remote_process_identity_unavailable", + createdAt: new Date(Date.now() - (120 - index) * 60_000 + 30_000), + finishedAt: new Date(Date.now() - (110 - index) * 60_000 + 30_000), + errorCode: "native_continuation_requires_reconciliation", contextSnapshot: { issueId: f.issueId, + wakeReason: "transient_failure_retry", taskRetryRequestId: hookIntent.id, retryOfRunId: lost.id } }).returning(); + previousRunId = systemRetry.id; + continue; + } + } + previousRunId = lost.id; + } + await db.update(agentWakeupRequests).set({ idempotencyKey: `task-retry:restart-interrupted-turn:${previousRunId}` }) + .where(eq(agentWakeupRequests.id, f.retryIntent!.id)); + const current = await run(f.runId); + await db.update(heartbeatRuns).set({ contextSnapshot: { ...current.contextSnapshot, previousRunId } }) + .where(eq(heartbeatRuns.id, f.runId)); + if (input.deploy) await deployDuringRun(f); + return f; + } + + it("continues again in a fresh session when a deploy took the continuation's Runner mid-turn", async () => { + const f = await automaticContinuation({ deploy: true }); + expect(await requestRestartInterruptedTurnRetry(db, f.claim)).toEqual({ requested: true, requestId: expect.any(String) }); + expect(await requestInterruptedNativeTurnRetry(db, f.runId)).toMatchObject({ requested: true }); + expect(await unknownOutcomeEvents(f.runId)).toEqual([expect.objectContaining({ payload: expect.objectContaining({ + lastTurnEvent: "turn.submitted", replayed: false }) })]); + await vi.waitFor(async () => { + await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, current => f.service.resumeRemoteStopComments(current)) ?? []); + expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete"); + }, { timeout: 10_000, interval: 10 }); + await vi.waitFor(async () => expect(await queued(f.companyId)).toHaveLength(1), { timeout: 10_000, interval: 50 }); + const [successor] = await queued(f.companyId); + expect(successor.contextSnapshot).toMatchObject({ previousRunId: f.runId, wakeReason: "task_retry", forceFreshSession: true }); + const retries = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, f.companyId), + eq(agentWakeupRequests.idempotencyKey, `task-retry:restart-interrupted-turn:${f.runId}`))); + expect(retries).toEqual([expect.objectContaining({ requestedByActorType: "user", requestedByActorId: "local-board" })]); + }, 30_000); + + it("continues a system retry of owed work that inherited the continuation when a deploy took its Runner", async () => { + const f = await runnerLostMidTurn({ wake: "system_retry" }); + const current = await run(f.runId); + const [hookIntent] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, + source: "on_demand", triggerDetail: "manual", reason: "task_retry", status: "completed", + idempotencyKey: `task-retry:restart-interrupted-turn:${randomUUID()}`, requestedByActorType: "user", + requestedByActorId: "local-board", payload: { issueId: f.issueId } }).returning(); + await db.update(heartbeatRuns).set({ contextSnapshot: { ...current.contextSnapshot, taskRetryRequestId: hookIntent.id } }) + .where(eq(heartbeatRuns.id, f.runId)); + await deployDuringRun(f); + expect(await requestRestartInterruptedTurnRetry(db, f.claim)).toEqual({ requested: true, requestId: expect.any(String) }); + expect(await unknownOutcomeEvents(f.runId)).toHaveLength(1); + }, 30_000); + + it("never runs out when only deploys interrupted a long chain whose system retries inherit each continuation", async () => { + const f = await automaticContinuation({ deploy: true, earlierContinuations: 12, systemRetries: true }); + expect(await requestRestartInterruptedTurnRetry(db, f.claim)).toEqual({ requested: true, requestId: expect.any(String) }); + expect(await unknownOutcomeEvents(f.runId)).toHaveLength(1); + }, 30_000); + + it.each([ + { title: "no deploy interrupted it", earlierContinuations: 0 }, + { title: "no deploy interrupted it after earlier deploy continuations", earlierContinuations: 4 }, + ])("keeps the visible Retry when $title", async ({ earlierContinuations }) => { + const f = await automaticContinuation({ deploy: false, earlierContinuations, systemRetries: true }); + expect(await requestRestartInterruptedTurnRetry(db, f.claim)).toEqual({ requested: false, reason: "automatic_retry_used" }); + expect(await unknownOutcomeEvents(f.runId)).toHaveLength(0); + expect(await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, f.companyId), + eq(agentWakeupRequests.idempotencyKey, `task-retry:restart-interrupted-turn:${f.runId}`)))).toHaveLength(0); + }, 30_000); + }); + // The staging chat task runs through a connected Slack conversation, whose // endpoint restricts external execution: its Retry goes through the chat service. describe("an external chat message in a connected conversation", () => {