diff --git a/doc/plans/2026-11-07-durable-ownership-and-reconciliation.md b/doc/plans/2026-11-07-durable-ownership-and-reconciliation.md index 46b73a402a2..5a8011b4999 100644 --- a/doc/plans/2026-11-07-durable-ownership-and-reconciliation.md +++ b/doc/plans/2026-11-07-durable-ownership-and-reconciliation.md @@ -218,6 +218,14 @@ Charge an attempt only on a recorded outcome that proves provider work started. The rules from #19720 and #19747 then become one rule over these outcome records, instead of watermark heuristics. The on-demand crash-point suite asserts this at the pre-spawn and pre-accept crash points. +**Crash-loop bound past the generation limit (2026-11-10, approved by Dotta).** Uncharged restarts used to stop at controller generation 12: from then on every restart charged, before the work rule was checked. In a concurrency-3 restart chaos wave one long task saw 12–15 restarts, most caused by other tasks. A restart that hit before the provider accepted any turn was charged, the budget ran out, and a person had to click Retry (wave `w-20261110T183431-s9106`, t03). The rule is now: + +- Each restart claim records `noProgressRestarts` in its `recovery_history` entry: 0 when the interrupted attempt is proven to have made provider progress, otherwise one more than the previous claim's count. A clean-drain claim carries the previous count unchanged. +- Before generation 12 nothing changes. +- From generation 12 on, a restart is not charged only when a durable record proves the interrupted attempt never started provider work (the same `provider_not_started` proof as above, or a resume that never reached execution) and the count is still below `NATIVE_NO_PROGRESS_RESTART_LIMIT` (12). Provider work (progress, or an accepted turn without progress) and unprovable cases still charge, as before. +- When the count reaches the limit, every further restart charges, so the attempt budget runs out and recovery stops with `execution_recovery_budget_exhausted`, which offers Retry. The limit equals the generation limit, so a run that never makes progress is bounded exactly as before; only proven progress resets the count. +- Claims saved by earlier releases have no count and are read as zero. A mid-run deploy therefore gives at most 12 more proven-unstarted restarts before the bound applies, and never charges a restart that the old rule would have credited. + ### G. Generation stamping The runner stamps `generation` and `turnId` on every event. Driver and session code treat an older generation as history. #19718, #19722, #19772 and #19783 each special-cased one variant of this. diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index a24545c1ad6..7c825b3d318 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -180,14 +180,29 @@ export function suspendedByCleanDrain(coordinator: { * before the crash. Each claim records the run's event-log position, so the * next claim judges only what happened after it. A resume the controller died * before starting, a runner that never admitted its provider, and a turn.start - * the provider did not accept are not charged. Credit stops after - * NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT controller generations, so a crash - * loop that never starts the provider, or that makes progress and then crashes - * the controller every time, is still bounded. + * the provider did not accept are not charged. After + * NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT controller generations only a proven + * unstarted attempt keeps credit, and only until NATIVE_NO_PROGRESS_RESTART_LIMIT + * consecutive restarts made no provider progress, so a crash loop that never + * starts the provider, or that makes progress and then crashes the controller + * every time, is still bounded. */ export const NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT = PROVIDER_ATTEMPT_CREDIT_GENERATION_LIMIT; +export const NATIVE_NO_PROGRESS_RESTART_LIMIT = PROVIDER_ATTEMPT_CREDIT_GENERATION_LIMIT; -type RecoveryClaimProgressEntry = { disposition?: unknown; progressWatermarkSeq?: unknown; progressAttempt?: unknown }; +type RecoveryClaimProgressEntry = { + disposition?: unknown; progressWatermarkSeq?: unknown; progressAttempt?: unknown; noProgressRestarts?: unknown; +}; + +function previousNoProgressRestarts(recoveryHistory: ReadonlyArray): number { + for (let index = recoveryHistory.length - 1; index >= 0; index -= 1) { + const entry = recoveryHistory[index]; + if (!RECOVERY_CLAIM_DISPOSITIONS.has(entry?.disposition)) continue; + const count = entry?.noProgressRestarts; + return typeof count === "number" && Number.isSafeInteger(count) && count >= 0 ? count : 0; + } + return 0; +} /** * The most recent recovery claim's event-log position and the provider attempt @@ -261,7 +276,9 @@ async function nativeRunEventWatermark(tx: Pick, runId: string): P * charge decision was never applied, so this claim carries it: the attempt * ends at the attempt that resume was to run as, never above it. * - A claim history written before claims recorded their position charges. - * At or after NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT every restart charges. + * At or after NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT a restart charges unless + * the interrupted attempt is proven unstarted and fewer than + * NATIVE_NO_PROGRESS_RESTART_LIMIT consecutive restarts made no progress. */ export async function restartResumeKeepsAttempt(tx: Pick, input: { runId: string; @@ -270,14 +287,29 @@ export async function restartResumeKeepsAttempt(tx: Pick, input: { controllerGeneration: number; recoveryHistory: ReadonlyArray; }; -}): Promise { - if (input.coordinator.attempt <= 0) return false; - if (input.coordinator.controllerGeneration >= NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT) return false; +}): Promise<{ keepsAttempt: boolean; noProgressRestarts: number }> { + const outcome = await interruptedAttemptOutcome(tx, input); + const noProgressRestarts = outcome === "provider_progressed" ? 0 + : previousNoProgressRestarts(input.coordinator.recoveryHistory) + 1; + const keepsAttempt = input.coordinator.controllerGeneration < NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT + ? outcome !== "charged" + : outcome === "provider_not_started" && noProgressRestarts < NATIVE_NO_PROGRESS_RESTART_LIMIT; + return { keepsAttempt, noProgressRestarts }; +} + +async function interruptedAttemptOutcome(tx: Pick, input: { + runId: string; + coordinator: { + attempt: number; + recoveryHistory: ReadonlyArray; + }; +}): Promise<"provider_not_started" | "provider_progressed" | "charged"> { + if (input.coordinator.attempt <= 0) return "charged"; const previous = previousRecoveryClaim(input.coordinator.recoveryHistory); - if (previous === "unrecorded") return false; + if (previous === "unrecorded") return "charged"; if (previous !== "none" && previous.launchesRunner && !await nativeExecutionStartedSince(tx, input.runId, previous.watermark)) - return input.coordinator.attempt >= previous.attempt; + return input.coordinator.attempt >= previous.attempt ? "provider_not_started" : "charged"; const claimedAt = previous === "none" ? 0 : previous.watermark; // A reattachment adopted a provider that had already started, and it keeps // the attempt; a normal retry after it starts a new attempt. @@ -287,13 +319,14 @@ export async function restartResumeKeepsAttempt(tx: Pick, input: { : await providerAttemptWindowStart(tx, { runId: input.runId, sinceSeq: claimedAt, providerAttempt: input.coordinator.attempt, }); - if (window === "unstarted") return true; + if (window === "unstarted") return "provider_not_started"; const outcome = await readProviderAttemptOutcome(tx, { runId: input.runId, sinceSeq: window.seq, adoptedProviderWork, }); - return !providerAttemptChargesBudget(outcome); + if (providerAttemptChargesBudget(outcome)) return "charged"; + return outcome === "provider_progressed" ? outcome : "provider_not_started"; } @@ -1577,9 +1610,10 @@ export async function claimNativeRestartRecoveries(input: { // Read before this claim appends its own history entry. const cleanDrainSuspension = suspendedByCleanDrain(row.coordinator); - const resumeKeepsAttempt = cleanDrainSuspension || await restartResumeKeepsAttempt(tx, { - runId: row.run.id, coordinator: row.coordinator, - }); + const restartCredit = cleanDrainSuspension + ? { keepsAttempt: true, noProgressRestarts: previousNoProgressRestarts(row.coordinator.recoveryHistory) } + : await restartResumeKeepsAttempt(tx, { runId: row.run.id, coordinator: row.coordinator }); + const resumeKeepsAttempt = restartCredit.keepsAttempt; const classification = classifyNativeRunnerRecoveryEvidence({ remoteSandbox: remoteSandbox || Boolean(pluginTask && durableAuthority), unclaimedBootstrap, @@ -1812,7 +1846,8 @@ export async function claimNativeRestartRecoveries(input: { : "bootstrap_incomplete", recoveryRequestId: input.recoveryRequestId ?? null, recoveryHistory: appendBoundedRecoveryHistory({ - ...event, progressWatermarkSeq, progressAttempt, ...(attemptKeptForProgress ? { attemptKeptForProgress } : {}), + ...event, progressWatermarkSeq, progressAttempt, noProgressRestarts: restartCredit.noProgressRestarts, + ...(attemptKeptForProgress ? { attemptKeptForProgress } : {}), }), failureCode: null, nextAttemptAt: null, diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 71e32b5a3cf..7dcb86f8345 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -93,7 +93,9 @@ vi.mock("../../sentry.js", async () => { captureRunFailure: mockCaptureRunFailure, }; }); -import { claimNativeRestartRecoveries, currentNativeControllerIdentity, NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT, NATIVE_RESTART_RECOVERY_FAILED, nextNativeProviderAttempt, type NativeControllerIdentity } from "./native-restart-recovery.js"; +import { claimNativeRestartRecoveries, currentNativeControllerIdentity, NATIVE_NO_PROGRESS_RESTART_LIMIT, NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT, NATIVE_RESTART_RECOVERY_FAILED, nextNativeProviderAttempt, type NativeControllerIdentity } from "./native-restart-recovery.js"; +import { taskInterruptionPresentation } from "../task-interruption-presentation.js"; +import { requestTaskRetry } from "../task-retry.js"; // Exact v5 wire prompt from before the task-monitor upgrade (#15446). const historicalPrompt = { @@ -3107,15 +3109,19 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); kind: "bootstrap_incomplete", runId: fixture.runId, providerAttempt: 3, chargesProviderAttempt: false })]); }); - it("blocks an exited remote runner at its last attempt once the controller generation cap is reached", async () => { - // Repeated exits before provider work stay bounded by the generation cap. - const fixture = await seedExitedRemoteRunner("EXITED-GEN-CAP", {}, 3); - await fixture.db.update(nativeRunFinalizations).set({ controllerGeneration: NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT }) - .where(eq(nativeRunFinalizations.runId, fixture.runId)); - expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, - restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ - kind: "blocked", runId: fixture.runId, reason: "execution_recovery_budget_exhausted" })]); - }); + it.each([ + { recorded: null, expected: { kind: "bootstrap_incomplete", chargesProviderAttempt: false } }, + { recorded: NATIVE_NO_PROGRESS_RESTART_LIMIT - 2, expected: { kind: "bootstrap_incomplete", chargesProviderAttempt: false } }, + { recorded: NATIVE_NO_PROGRESS_RESTART_LIMIT - 1, expected: { kind: "blocked", reason: "execution_recovery_budget_exhausted" } }, + ])("bounds an exited remote runner at its last attempt past the generation cap by consecutive no-progress restarts ($recorded)", + async ({ recorded, expected }) => { + const fixture = await seedExitedRemoteRunner(`EXITED-GEN-CAP-${recorded}`, {}, 3); + await fixture.db.update(nativeRunFinalizations).set({ controllerGeneration: NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + if (recorded !== null) await recordUnstartedClaim(fixture, recorded); + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ runId: fixture.runId, ...expected })]); + }); it.each(["a provider event committed after the failure", "a session checkpoint saved after the failure"])( "keeps an exited remote runner's retry held when %s contradicts the saved failure", async (label) => { @@ -3174,13 +3180,20 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); it.each([ ["claimed remote bootstrap", (label: string) => seedClaimedRemoteBootstrap(label, { attempt: 3 })], ["remote bootstrap retry", (label: string) => seedUnlaunchedRemoteClaim(label, { bootstrapRetry: true, attempt: 3 })], - ] as const)("blocks a %s at its last attempt once the controller generation cap is reached", async (label, seed) => { - const fixture = await seed(`GEN-CAP-${label.replace(/\W+/g, "-")}`); - await fixture.db.update(nativeRunFinalizations).set({ controllerGeneration: NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT }) - .where(eq(nativeRunFinalizations.runId, fixture.runId)); - expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, - restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ - kind: "blocked", runId: fixture.runId, reason: "execution_recovery_budget_exhausted" })]); + ] as const)("bounds a %s at its last attempt past the generation cap by consecutive no-progress restarts", async (label, seed) => { + const uncapped = await seed(`GEN-CAP-OPEN-${label.replace(/\W+/g, "-")}`); + await uncapped.db.update(nativeRunFinalizations).set({ controllerGeneration: NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT }) + .where(eq(nativeRunFinalizations.runId, uncapped.runId)); + expect(await claimNativeRestartRecoveries({ db: uncapped.db, controller: successor, + restartKind: "hard", runIds: [uncapped.runId] })).toEqual([expect.objectContaining({ + kind: "bootstrap_incomplete", runId: uncapped.runId, providerAttempt: 3, chargesProviderAttempt: false })]); + const capped = await seed(`GEN-CAP-${label.replace(/\W+/g, "-")}`); + await capped.db.update(nativeRunFinalizations).set({ controllerGeneration: NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT }) + .where(eq(nativeRunFinalizations.runId, capped.runId)); + await recordUnstartedClaim(capped, NATIVE_NO_PROGRESS_RESTART_LIMIT - 1); + expect(await claimNativeRestartRecoveries({ db: capped.db, controller: successor, + restartKind: "hard", runIds: [capped.runId] })).toEqual([expect.objectContaining({ + kind: "blocked", runId: capped.runId, reason: "execution_recovery_budget_exhausted" })]); }); it("keeps a paused agent's remote bootstrap retry waiting instead of starting it", async () => { @@ -3638,6 +3651,138 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); .toEqual([2, ...Array(NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT - 2).fill(2), 3]); }); + async function recordUnstartedClaim(fixture: Awaited>, noProgressRestarts: number) { + const [last] = await fixture.db.select({ seq: sql`coalesce(max(${heartbeatRunEvents.seq}), 0)::int` }) + .from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, fixture.runId)); + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + await fixture.db.update(nativeRunFinalizations).set({ recoveryHistory: [...coordinator.recoveryHistory, { + disposition: "bootstrap_incomplete", progressWatermarkSeq: last?.seq ?? 0, progressAttempt: coordinator.attempt, + noProgressRestarts, + }] }).where(eq(nativeRunFinalizations.runId, fixture.runId)); + } + + async function seedPastGenerationLimit(label: string) { + const fixture = await seedInterruptedCheckpoint(label); + await fixture.db.update(nativeRunFinalizations).set({ controllerGeneration: NATIVE_PROGRESS_CREDIT_GENERATION_LIMIT }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + await appendRunEvents(fixture, [{ eventType: "tool.execution.completed", turnId: "interrupted-turn" }]); + return fixture; + } + + async function lastClaimEntry(fixture: Awaited>) { + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + return (coordinator.recoveryHistory as Array>).at(-1)!; + } + + const providerNeverStarted = [{ eventType: "native.process_start_requested" }, + { eventType: "native.process_identity_recorded" }, { eventType: "session.resumed" }]; + const providerProgressed = (round: number) => [{ eventType: "native.process_start_requested" }, + { eventType: "turn.accepted", turnId: `past-limit-${round}` }, + { eventType: "tool.execution.completed", turnId: `past-limit-${round}` }]; + + it("does not charge a restart past the generation limit whose resume never started the provider", async () => { + const fixture = await seedPastGenerationLimit("PAST-LIMIT-UNSTARTED"); + expect(await resumeThenCrash(fixture)).toMatchObject({ + disposition: { kind: "resume_dead_runner", chargesProviderAttempt: true }, attempt: 2 }); + await appendRunEvents(fixture, providerProgressed(1)); + expect(await resumeThenCrash(fixture)).toMatchObject({ + disposition: { kind: "resume_dead_runner", chargesProviderAttempt: true }, attempt: 3 }); + await appendRunEvents(fixture, providerNeverStarted); + expect(await resumeThenCrash(fixture)).toMatchObject({ + disposition: { kind: "resume_dead_runner", chargesProviderAttempt: false }, attempt: 3 }); + expect(await lastClaimEntry(fixture)).toMatchObject({ noProgressRestarts: 1 }); + }); + + it("stops after consecutive restarts with no provider progress and leaves a working Retry", async () => { + const fixture = await seedPastGenerationLimit("PAST-LIMIT-NO-PROGRESS-LOOP"); + expect(await resumeThenCrash(fixture)).toMatchObject({ attempt: 2 }); + const outcomes: Array<{ kind?: string; attempt?: number }> = []; + for (let round = 1; round <= NATIVE_NO_PROGRESS_RESTART_LIMIT + 2; round += 1) { + await appendRunEvents(fixture, providerNeverStarted); + const { disposition, attempt } = await resumeThenCrash(fixture); + outcomes.push({ kind: disposition?.kind, attempt }); + if (disposition?.kind !== "resume_dead_runner") break; + } + expect(outcomes).toEqual([ + ...Array(NATIVE_NO_PROGRESS_RESTART_LIMIT - 1).fill({ kind: "resume_dead_runner", attempt: 2 }), + { kind: "resume_dead_runner", attempt: 3 }, + { kind: "blocked", attempt: undefined }, + ]); + const [blocked] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(blocked).toMatchObject({ phase: "terminal_failure", failureCode: "native_restart_recovery_blocked", + leaseOwner: null, nextAttemptAt: null }); + const [issue] = await fixture.db.select().from(issues).where(eq(issues.id, fixture.issueId)); + expect(await taskInterruptionPresentation(fixture.db, issue)).toMatchObject({ + state: "failed", canRetry: true, observedRunId: fixture.runId }); + const personId = randomUUID(); + await fixture.db.insert(authUsers).values({ id: personId, name: "Retry person", email: `${personId}@fixture.invalid`, + emailVerified: true, createdAt: new Date(), updatedAt: new Date() }); + await fixture.db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: personId, + status: "active", membershipRole: "member" }); + expect(await requestTaskRetry({ db: fixture.db, companyId, issueId: fixture.issueId, actorId: personId, + actorSource: "session", idempotencyKey: `no-progress-loop-${fixture.runId}`, observedRunId: fixture.runId })) + .toMatchObject({ accepted: true }); + }); + + it("resets the consecutive no-progress count when a restart proves provider progress", async () => { + const fixture = await seedPastGenerationLimit("PAST-LIMIT-PROGRESS-RESETS"); + const counts: unknown[] = []; + const attempts: Array = []; + const round = async (events: Parameters[1]) => { + await appendRunEvents(fixture, events); + attempts.push((await resumeThenCrash(fixture)).attempt); + counts.push((await lastClaimEntry(fixture)).noProgressRestarts); + }; + await round([]); + for (let index = 0; index < 3; index += 1) await round(providerNeverStarted); + await round(providerProgressed(1)); + await round(providerNeverStarted); + expect(counts).toEqual([0, 1, 2, 3, 0, 1]); + expect(attempts).toEqual([2, 2, 2, 2, 3, 3]); + }); + + it("still charges a restart past the generation limit whose provider accepted a turn without progress", async () => { + const fixture = await seedPastGenerationLimit("PAST-LIMIT-ACCEPTED-NO-PROGRESS"); + expect(await resumeThenCrash(fixture)).toMatchObject({ attempt: 2 }); + await appendRunEvents(fixture, [{ eventType: "native.process_start_requested" }, + { eventType: "turn.accepted", turnId: "accepted" }, { eventType: "item.completed", turnId: "accepted", kind: "usage" }]); + expect(await resumeThenCrash(fixture)).toMatchObject({ + disposition: { kind: "resume_dead_runner", chargesProviderAttempt: true }, attempt: 3 }); + expect(await lastClaimEntry(fixture)).toMatchObject({ noProgressRestarts: 1 }); + }); + + it("still charges a restart past the generation limit when the previous claim did not record its position", async () => { + const fixture = await seedPastGenerationLimit("PAST-LIMIT-UNRECORDED"); + expect(await resumeThenCrash(fixture)).toMatchObject({ attempt: 2 }); + await fixture.db.update(nativeRunFinalizations).set({ + recoveryHistory: sql`jsonb_set(${nativeRunFinalizations.recoveryHistory}, + array[(jsonb_array_length(${nativeRunFinalizations.recoveryHistory}) - 1)::text], + (${nativeRunFinalizations.recoveryHistory} -> -1) - 'progressWatermarkSeq' - 'noProgressRestarts')`, + }).where(eq(nativeRunFinalizations.runId, fixture.runId)); + await appendRunEvents(fixture, providerNeverStarted); + expect(await resumeThenCrash(fixture)).toMatchObject({ + disposition: { kind: "resume_dead_runner", chargesProviderAttempt: true }, attempt: 3 }); + }); + + it("counts from zero for a claim saved before restarts recorded the count", async () => { + const fixture = await seedPastGenerationLimit("PAST-LIMIT-OLD-ROW"); + expect(await resumeThenCrash(fixture)).toMatchObject({ attempt: 2 }); + await appendRunEvents(fixture, providerNeverStarted); + await resumeThenCrash(fixture); + await fixture.db.update(nativeRunFinalizations).set({ + recoveryHistory: sql`jsonb_set(${nativeRunFinalizations.recoveryHistory}, + array[(jsonb_array_length(${nativeRunFinalizations.recoveryHistory}) - 1)::text], + (${nativeRunFinalizations.recoveryHistory} -> -1) - 'noProgressRestarts')`, + }).where(eq(nativeRunFinalizations.runId, fixture.runId)); + await appendRunEvents(fixture, providerNeverStarted); + expect(await resumeThenCrash(fixture)).toMatchObject({ + disposition: { kind: "resume_dead_runner", chargesProviderAttempt: false }, attempt: 2 }); + expect(await lastClaimEntry(fixture)).toMatchObject({ noProgressRestarts: 1 }); + }); + it("terminalizes the recorded failed-checkpoint incident atomically instead of resuming it on upgrade", async () => { const fixture = await seedRun("FAILED-CHECKPOINT"); await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { sessionCheckpoint: {