From 7f7803ab0c9ac9ac5fa6fe22d6857f6d86b38c16 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 10 Nov 2026 14:45:13 -0600 Subject: [PATCH] fix(native): keep crediting proven unstarted restarts past the generation limit Past controller generation 12 a restart charged an attempt even when a durable record proved the provider never started. In a busy restart wave a long task then ran out of budget and needed a manual Retry. Each restart claim now records how many consecutive restarts made no provider progress. Past the generation limit a proven unstarted attempt keeps its credit until that count reaches 12. Provider work and unprovable cases still charge. Proven progress resets the count. Claims saved by earlier releases read the count as zero. Co-Authored-By: Claude --- ...07-durable-ownership-and-reconciliation.md | 8 + .../native-runtime/native-restart-recovery.ts | 69 +++++-- ...unner-restart-recovery.integration.test.ts | 179 ++++++++++++++++-- 3 files changed, 222 insertions(+), 34 deletions(-) diff --git a/doc/plans/2026-11-07-durable-ownership-and-reconciliation.md b/doc/plans/2026-11-07-durable-ownership-and-reconciliation.md index 46b73a402a2..5a8011b4999 100644 --- a/doc/plans/2026-11-07-durable-ownership-and-reconciliation.md +++ b/doc/plans/2026-11-07-durable-ownership-and-reconciliation.md @@ -218,6 +218,14 @@ Charge an attempt only on a recorded outcome that proves provider work started. The rules from #19720 and #19747 then become one rule over these outcome records, instead of watermark heuristics. The on-demand crash-point suite asserts this at the pre-spawn and pre-accept crash points. +**Crash-loop bound past the generation limit (2026-11-10, approved by Dotta).** Uncharged restarts used to stop at controller generation 12: from then on every restart charged, before the work rule was checked. In a concurrency-3 restart chaos wave one long task saw 12–15 restarts, most caused by other tasks. A restart that hit before the provider accepted any turn was charged, the budget ran out, and a person had to click Retry (wave `w-20261110T183431-s9106`, t03). The rule is now: + +- Each restart claim records `noProgressRestarts` in its `recovery_history` entry: 0 when the interrupted attempt is proven to have made provider progress, otherwise one more than the previous claim's count. A clean-drain claim carries the previous count unchanged. +- Before generation 12 nothing changes. +- From generation 12 on, a restart is not charged only when a durable record proves the interrupted attempt never started provider work (the same `provider_not_started` proof as above, or a resume that never reached execution) and the count is still below `NATIVE_NO_PROGRESS_RESTART_LIMIT` (12). Provider work (progress, or an accepted turn without progress) and unprovable cases still charge, as before. +- When the count reaches the limit, every further restart charges, so the attempt budget runs out and recovery stops with `execution_recovery_budget_exhausted`, which offers Retry. The limit equals the generation limit, so a run that never makes progress is bounded exactly as before; only proven progress resets the count. +- Claims saved by earlier releases have no count and are read as zero. A mid-run deploy therefore gives at most 12 more proven-unstarted restarts before the bound applies, and never charges a restart that the old rule would have credited. + ### G. Generation stamping The runner stamps `generation` and `turnId` on every event. Driver and session code treat an older generation as history. #19718, #19722, #19772 and #19783 each special-cased one variant of this. diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 28a6505dea8..2f1382c1f45 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -180,14 +180,29 @@ export function suspendedByCleanDrain(coordinator: { * before the crash. Each claim records the run's event-log position, so the * next claim judges only what happened after it. A resume the controller died * before starting, a runner that never admitted its provider, and a turn.start - * the provider did not accept are not charged. Credit stops after - * NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT controller generations, so a crash - * loop that never starts the provider, or that makes progress and then crashes - * the controller every time, is still bounded. + * the provider did not accept are not charged. After + * NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT controller generations only a proven + * unstarted attempt keeps credit, and only until NATIVE_NO_PROGRESS_RESTART_LIMIT + * consecutive restarts made no provider progress, so a crash loop that never + * starts the provider, or that makes progress and then crashes the controller + * every time, is still bounded. */ export const NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT = PROVIDER_ATTEMPT_CREDIT_GENERATION_LIMIT; +export const NATIVE_NO_PROGRESS_RESTART_LIMIT = PROVIDER_ATTEMPT_CREDIT_GENERATION_LIMIT; -type RecoveryClaimProgressEntry = { disposition?: unknown; progressWatermarkSeq?: unknown; progressAttempt?: unknown }; +type RecoveryClaimProgressEntry = { + disposition?: unknown; progressWatermarkSeq?: unknown; progressAttempt?: unknown; noProgressRestarts?: unknown; +}; + +function previousNoProgressRestarts(recoveryHistory: ReadonlyArray): number { + for (let index = recoveryHistory.length - 1; index >= 0; index -= 1) { + const entry = recoveryHistory[index]; + if (!RECOVERY_CLAIM_DISPOSITIONS.has(entry?.disposition)) continue; + const count = entry?.noProgressRestarts; + return typeof count === "number" && Number.isSafeInteger(count) && count >= 0 ? count : 0; + } + return 0; +} /** * The most recent recovery claim's event-log position and the provider attempt @@ -261,7 +276,9 @@ async function nativeRunEventWatermark(tx: Pick, runId: string): P * charge decision was never applied, so this claim carries it: the attempt * ends at the attempt that resume was to run as, never above it. * - A claim history written before claims recorded their position charges. - * At or after NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT every restart charges. + * At or after NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT a restart charges unless + * the interrupted attempt is proven unstarted and fewer than + * NATIVE_NO_PROGRESS_RESTART_LIMIT consecutive restarts made no progress. */ export async function restartResumeKeepsAttempt(tx: Pick, input: { runId: string; @@ -270,14 +287,29 @@ export async function restartResumeKeepsAttempt(tx: Pick, input: { controllerGeneration: number; recoveryHistory: ReadonlyArray; }; -}): Promise { - if (input.coordinator.attempt <= 0) return false; - if (input.coordinator.controllerGeneration >= NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT) return false; +}): Promise<{ keepsAttempt: boolean; noProgressRestarts: number }> { + const outcome = await interruptedAttemptOutcome(tx, input); + const noProgressRestarts = outcome === "provider_progressed" ? 0 + : previousNoProgressRestarts(input.coordinator.recoveryHistory) + 1; + const keepsAttempt = input.coordinator.controllerGeneration < NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT + ? outcome !== "charged" + : outcome === "provider_not_started" && noProgressRestarts < NATIVE_NO_PROGRESS_RESTART_LIMIT; + return { keepsAttempt, noProgressRestarts }; +} + +async function interruptedAttemptOutcome(tx: Pick, input: { + runId: string; + coordinator: { + attempt: number; + recoveryHistory: ReadonlyArray; + }; +}): Promise<"provider_not_started" | "provider_progressed" | "charged"> { + if (input.coordinator.attempt <= 0) return "charged"; const previous = previousRecoveryClaim(input.coordinator.recoveryHistory); - if (previous === "unrecorded") return false; + if (previous === "unrecorded") return "charged"; if (previous !== "none" && previous.launchesRunner && !await nativeExecutionStartedSince(tx, input.runId, previous.watermark)) - return input.coordinator.attempt >= previous.attempt; + return input.coordinator.attempt >= previous.attempt ? "provider_not_started" : "charged"; const claimedAt = previous === "none" ? 0 : previous.watermark; // A reattachment adopted a provider that had already started, and it keeps // the attempt; a normal retry after it starts a new attempt. @@ -287,13 +319,14 @@ export async function restartResumeKeepsAttempt(tx: Pick, input: { : await providerAttemptWindowStart(tx, { runId: input.runId, sinceSeq: claimedAt, providerAttempt: input.coordinator.attempt, }); - if (window === "unstarted") return true; + if (window === "unstarted") return "provider_not_started"; const outcome = await readProviderAttemptOutcome(tx, { runId: input.runId, sinceSeq: window.seq, adoptedProviderWork, }); - return !providerAttemptChargesBudget(outcome); + if (providerAttemptChargesBudget(outcome)) return "charged"; + return outcome === "provider_progressed" ? outcome : "provider_not_started"; } @@ -1546,9 +1579,10 @@ export async function claimNativeRestartRecoveries(input: { // Read before this claim appends its own history entry. const cleanDrainSuspension = suspendedByCleanDrain(row.coordinator); - const resumeKeepsAttempt = cleanDrainSuspension || await restartResumeKeepsAttempt(tx, { - runId: row.run.id, coordinator: row.coordinator, - }); + const restartCredit = cleanDrainSuspension + ? { keepsAttempt: true, noProgressRestarts: previousNoProgressRestarts(row.coordinator.recoveryHistory) } + : await restartResumeKeepsAttempt(tx, { runId: row.run.id, coordinator: row.coordinator }); + const resumeKeepsAttempt = restartCredit.keepsAttempt; const classification = classifyNativeRunnerRecoveryEvidence({ remoteSandbox: remoteSandbox || Boolean(pluginTask && durableAuthority), unclaimedBootstrap, @@ -1781,7 +1815,8 @@ export async function claimNativeRestartRecoveries(input: { : "bootstrap_incomplete", recoveryRequestId: input.recoveryRequestId ?? null, recoveryHistory: appendBoundedRecoveryHistory({ - ...event, progressWatermarkSeq, progressAttempt, ...(attemptKeptForProgress ? { attemptKeptForProgress } : {}), + ...event, progressWatermarkSeq, progressAttempt, noProgressRestarts: restartCredit.noProgressRestarts, + ...(attemptKeptForProgress ? { attemptKeptForProgress } : {}), }), failureCode: null, nextAttemptAt: null, diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index ba518e9e8b5..82b7d4d6b80 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -93,7 +93,9 @@ vi.mock("../../sentry.js", async () => { captureRunFailure: mockCaptureRunFailure, }; }); -import { claimNativeRestartRecoveries, currentNativeControllerIdentity, NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT, NATIVE_RESTART_RECOVERY_FAILED, nextNativeProviderAttempt, type NativeControllerIdentity } from "./native-restart-recovery.js"; +import { claimNativeRestartRecoveries, currentNativeControllerIdentity, NATIVE_NO_PROGRESS_RESTART_LIMIT, NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT, NATIVE_RESTART_RECOVERY_FAILED, nextNativeProviderAttempt, type NativeControllerIdentity } from "./native-restart-recovery.js"; +import { taskInterruptionPresentation } from "../task-interruption-presentation.js"; +import { requestTaskRetry } from "../task-retry.js"; // Exact v5 wire prompt from before the task-monitor upgrade (#15446). const historicalPrompt = { @@ -3026,15 +3028,19 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); kind: "bootstrap_incomplete", runId: fixture.runId, providerAttempt: 3, chargesProviderAttempt: false })]); }); - it("blocks an exited remote runner at its last attempt once the controller generation cap is reached", async () => { - // Repeated exits before provider work stay bounded by the generation cap. - const fixture = await seedExitedRemoteRunner("EXITED-GEN-CAP", {}, 3); - await fixture.db.update(nativeRunFinalizations).set({ controllerGeneration: NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT }) - .where(eq(nativeRunFinalizations.runId, fixture.runId)); - expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, - restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ - kind: "blocked", runId: fixture.runId, reason: "execution_recovery_budget_exhausted" })]); - }); + it.each([ + { recorded: null, expected: { kind: "bootstrap_incomplete", chargesProviderAttempt: false } }, + { recorded: NATIVE_NO_PROGRESS_RESTART_LIMIT - 2, expected: { kind: "bootstrap_incomplete", chargesProviderAttempt: false } }, + { recorded: NATIVE_NO_PROGRESS_RESTART_LIMIT - 1, expected: { kind: "blocked", reason: "execution_recovery_budget_exhausted" } }, + ])("bounds an exited remote runner at its last attempt past the generation cap by consecutive no-progress restarts ($recorded)", + async ({ recorded, expected }) => { + const fixture = await seedExitedRemoteRunner(`EXITED-GEN-CAP-${recorded}`, {}, 3); + await fixture.db.update(nativeRunFinalizations).set({ controllerGeneration: NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + if (recorded !== null) await recordUnstartedClaim(fixture, recorded); + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ runId: fixture.runId, ...expected })]); + }); it.each(["a provider event committed after the failure", "a session checkpoint saved after the failure"])( "keeps an exited remote runner's retry held when %s contradicts the saved failure", async (label) => { @@ -3093,13 +3099,20 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); it.each([ ["claimed remote bootstrap", (label: string) => seedClaimedRemoteBootstrap(label, { attempt: 3 })], ["remote bootstrap retry", (label: string) => seedUnlaunchedRemoteClaim(label, { bootstrapRetry: true, attempt: 3 })], - ] as const)("blocks a %s at its last attempt once the controller generation cap is reached", async (label, seed) => { - const fixture = await seed(`GEN-CAP-${label.replace(/\W+/g, "-")}`); - await fixture.db.update(nativeRunFinalizations).set({ controllerGeneration: NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT }) - .where(eq(nativeRunFinalizations.runId, fixture.runId)); - expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, - restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ - kind: "blocked", runId: fixture.runId, reason: "execution_recovery_budget_exhausted" })]); + ] as const)("bounds a %s at its last attempt past the generation cap by consecutive no-progress restarts", async (label, seed) => { + const uncapped = await seed(`GEN-CAP-OPEN-${label.replace(/\W+/g, "-")}`); + await uncapped.db.update(nativeRunFinalizations).set({ controllerGeneration: NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT }) + .where(eq(nativeRunFinalizations.runId, uncapped.runId)); + expect(await claimNativeRestartRecoveries({ db: uncapped.db, controller: successor, + restartKind: "hard", runIds: [uncapped.runId] })).toEqual([expect.objectContaining({ + kind: "bootstrap_incomplete", runId: uncapped.runId, providerAttempt: 3, chargesProviderAttempt: false })]); + const capped = await seed(`GEN-CAP-${label.replace(/\W+/g, "-")}`); + await capped.db.update(nativeRunFinalizations).set({ controllerGeneration: NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT }) + .where(eq(nativeRunFinalizations.runId, capped.runId)); + await recordUnstartedClaim(capped, NATIVE_NO_PROGRESS_RESTART_LIMIT - 1); + expect(await claimNativeRestartRecoveries({ db: capped.db, controller: successor, + restartKind: "hard", runIds: [capped.runId] })).toEqual([expect.objectContaining({ + kind: "blocked", runId: capped.runId, reason: "execution_recovery_budget_exhausted" })]); }); it("keeps a paused agent's remote bootstrap retry waiting instead of starting it", async () => { @@ -3557,6 +3570,138 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); .toEqual([2, ...Array(NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT - 2).fill(2), 3]); }); + async function recordUnstartedClaim(fixture: Awaited>, noProgressRestarts: number) { + const [last] = await fixture.db.select({ seq: sql`coalesce(max(${heartbeatRunEvents.seq}), 0)::int` }) + .from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, fixture.runId)); + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + await fixture.db.update(nativeRunFinalizations).set({ recoveryHistory: [...coordinator.recoveryHistory, { + disposition: "bootstrap_incomplete", progressWatermarkSeq: last?.seq ?? 0, progressAttempt: coordinator.attempt, + noProgressRestarts, + }] }).where(eq(nativeRunFinalizations.runId, fixture.runId)); + } + + async function seedPastGenerationLimit(label: string) { + const fixture = await seedInterruptedCheckpoint(label); + await fixture.db.update(nativeRunFinalizations).set({ controllerGeneration: NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + await appendRunEvents(fixture, [{ eventType: "tool.execution.completed", turnId: "interrupted-turn" }]); + return fixture; + } + + async function lastClaimEntry(fixture: Awaited>) { + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + return (coordinator.recoveryHistory as Array>).at(-1)!; + } + + const providerNeverStarted = [{ eventType: "native.process_start_requested" }, + { eventType: "native.process_identity_recorded" }, { eventType: "session.resumed" }]; + const providerProgressed = (round: number) => [{ eventType: "native.process_start_requested" }, + { eventType: "turn.accepted", turnId: `past-limit-${round}` }, + { eventType: "tool.execution.completed", turnId: `past-limit-${round}` }]; + + it("does not charge a restart past the generation limit whose resume never started the provider", async () => { + const fixture = await seedPastGenerationLimit("PAST-LIMIT-UNSTARTED"); + expect(await resumeThenCrash(fixture)).toMatchObject({ + disposition: { kind: "resume_dead_runner", chargesProviderAttempt: true }, attempt: 2 }); + await appendRunEvents(fixture, providerProgressed(1)); + expect(await resumeThenCrash(fixture)).toMatchObject({ + disposition: { kind: "resume_dead_runner", chargesProviderAttempt: true }, attempt: 3 }); + await appendRunEvents(fixture, providerNeverStarted); + expect(await resumeThenCrash(fixture)).toMatchObject({ + disposition: { kind: "resume_dead_runner", chargesProviderAttempt: false }, attempt: 3 }); + expect(await lastClaimEntry(fixture)).toMatchObject({ noProgressRestarts: 1 }); + }); + + it("stops after consecutive restarts with no provider progress and leaves a working Retry", async () => { + const fixture = await seedPastGenerationLimit("PAST-LIMIT-NO-PROGRESS-LOOP"); + expect(await resumeThenCrash(fixture)).toMatchObject({ attempt: 2 }); + const outcomes: Array<{ kind?: string; attempt?: number }> = []; + for (let round = 1; round <= NATIVE_NO_PROGRESS_RESTART_LIMIT + 2; round += 1) { + await appendRunEvents(fixture, providerNeverStarted); + const { disposition, attempt } = await resumeThenCrash(fixture); + outcomes.push({ kind: disposition?.kind, attempt }); + if (disposition?.kind !== "resume_dead_runner") break; + } + expect(outcomes).toEqual([ + ...Array(NATIVE_NO_PROGRESS_RESTART_LIMIT - 1).fill({ kind: "resume_dead_runner", attempt: 2 }), + { kind: "resume_dead_runner", attempt: 3 }, + { kind: "blocked", attempt: undefined }, + ]); + const [blocked] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(blocked).toMatchObject({ phase: "terminal_failure", failureCode: "native_restart_recovery_blocked", + leaseOwner: null, nextAttemptAt: null }); + const [issue] = await fixture.db.select().from(issues).where(eq(issues.id, fixture.issueId)); + expect(await taskInterruptionPresentation(fixture.db, issue)).toMatchObject({ + state: "failed", canRetry: true, observedRunId: fixture.runId }); + const personId = randomUUID(); + await fixture.db.insert(authUsers).values({ id: personId, name: "Retry person", email: `${personId}@fixture.invalid`, + emailVerified: true, createdAt: new Date(), updatedAt: new Date() }); + await fixture.db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: personId, + status: "active", membershipRole: "member" }); + expect(await requestTaskRetry({ db: fixture.db, companyId, issueId: fixture.issueId, actorId: personId, + actorSource: "session", idempotencyKey: `no-progress-loop-${fixture.runId}`, observedRunId: fixture.runId })) + .toMatchObject({ accepted: true }); + }); + + it("resets the consecutive no-progress count when a restart proves provider progress", async () => { + const fixture = await seedPastGenerationLimit("PAST-LIMIT-PROGRESS-RESETS"); + const counts: unknown[] = []; + const attempts: Array = []; + const round = async (events: Parameters[1]) => { + await appendRunEvents(fixture, events); + attempts.push((await resumeThenCrash(fixture)).attempt); + counts.push((await lastClaimEntry(fixture)).noProgressRestarts); + }; + await round([]); + for (let index = 0; index < 3; index += 1) await round(providerNeverStarted); + await round(providerProgressed(1)); + await round(providerNeverStarted); + expect(counts).toEqual([0, 1, 2, 3, 0, 1]); + expect(attempts).toEqual([2, 2, 2, 2, 3, 3]); + }); + + it("still charges a restart past the generation limit whose provider accepted a turn without progress", async () => { + const fixture = await seedPastGenerationLimit("PAST-LIMIT-ACCEPTED-NO-PROGRESS"); + expect(await resumeThenCrash(fixture)).toMatchObject({ attempt: 2 }); + await appendRunEvents(fixture, [{ eventType: "native.process_start_requested" }, + { eventType: "turn.accepted", turnId: "accepted" }, { eventType: "item.completed", turnId: "accepted", kind: "usage" }]); + expect(await resumeThenCrash(fixture)).toMatchObject({ + disposition: { kind: "resume_dead_runner", chargesProviderAttempt: true }, attempt: 3 }); + expect(await lastClaimEntry(fixture)).toMatchObject({ noProgressRestarts: 1 }); + }); + + it("still charges a restart past the generation limit when the previous claim did not record its position", async () => { + const fixture = await seedPastGenerationLimit("PAST-LIMIT-UNRECORDED"); + expect(await resumeThenCrash(fixture)).toMatchObject({ attempt: 2 }); + await fixture.db.update(nativeRunFinalizations).set({ + recoveryHistory: sql`jsonb_set(${nativeRunFinalizations.recoveryHistory}, + array[(jsonb_array_length(${nativeRunFinalizations.recoveryHistory}) - 1)::text], + (${nativeRunFinalizations.recoveryHistory} -> -1) - 'progressWatermarkSeq' - 'noProgressRestarts')`, + }).where(eq(nativeRunFinalizations.runId, fixture.runId)); + await appendRunEvents(fixture, providerNeverStarted); + expect(await resumeThenCrash(fixture)).toMatchObject({ + disposition: { kind: "resume_dead_runner", chargesProviderAttempt: true }, attempt: 3 }); + }); + + it("counts from zero for a claim saved before restarts recorded the count", async () => { + const fixture = await seedPastGenerationLimit("PAST-LIMIT-OLD-ROW"); + expect(await resumeThenCrash(fixture)).toMatchObject({ attempt: 2 }); + await appendRunEvents(fixture, providerNeverStarted); + await resumeThenCrash(fixture); + await fixture.db.update(nativeRunFinalizations).set({ + recoveryHistory: sql`jsonb_set(${nativeRunFinalizations.recoveryHistory}, + array[(jsonb_array_length(${nativeRunFinalizations.recoveryHistory}) - 1)::text], + (${nativeRunFinalizations.recoveryHistory} -> -1) - 'noProgressRestarts')`, + }).where(eq(nativeRunFinalizations.runId, fixture.runId)); + await appendRunEvents(fixture, providerNeverStarted); + expect(await resumeThenCrash(fixture)).toMatchObject({ + disposition: { kind: "resume_dead_runner", chargesProviderAttempt: false }, attempt: 2 }); + expect(await lastClaimEntry(fixture)).toMatchObject({ noProgressRestarts: 1 }); + }); + it("terminalizes the recorded failed-checkpoint incident atomically instead of resuming it on upgrade", async () => { const fixture = await seedRun("FAILED-CHECKPOINT"); await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { sessionCheckpoint: {