diff --git a/packages/db/src/schema/heartbeat_runs.ts b/packages/db/src/schema/heartbeat_runs.ts index 1d2e25466f0..247624e48c0 100644 --- a/packages/db/src/schema/heartbeat_runs.ts +++ b/packages/db/src/schema/heartbeat_runs.ts @@ -104,6 +104,9 @@ export const heartbeatRuns = pgTable( retryOfRunId: uuid("retry_of_run_id").references((): AnyPgColumn => heartbeatRuns.id, { onDelete: "set null", }), + // Legacy: no writer since process loss moved onto the durable failure + // retry budget (contextSnapshot.executionRetryAccounting). Kept so rows and + // readers from earlier releases stay valid; never gate retries on it. processLossRetryCount: integer("process_loss_retry_count").notNull().default(0), scheduledRetryAt: timestamp("scheduled_retry_at", { withTimezone: true }), scheduledRetryAttempt: integer("scheduled_retry_attempt").notNull().default(0), diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 86db90d91ad..518efef8187 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -1,4 +1,5 @@ import { persistHeartbeatRunProcessMetadata } from "../services/heartbeat/run-lifecycle.js"; +import { BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS } from "../services/heartbeat/retries.js"; import { enableRunFailureReporting } from "./helpers/run-failure-reporting.js"; import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { createSystemRecovery } from "../services/system-recovery-state.js"; @@ -499,6 +500,14 @@ async function spawnOrphanedProcessGroup() { }; } + +/** Failure retry accounting with every automatic failure retry already used. */ +const spentFailureRetryAccounting = { + version: 1, + failureRetries: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, + maxTurnContinuations: 0, +}; + describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { let db!: ReturnType; let tempDb: Awaited< @@ -851,6 +860,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { processPid?: number | null; processGroupId?: number | null; processLossRetryCount?: number; + failureRetryBudgetSpent?: boolean; runtimeMode?: "legacy" | "native"; includeIssue?: boolean; runErrorCode?: string | null; @@ -914,10 +924,12 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { triggerDetail: "system", status: input?.runStatus ?? "running", wakeupRequestId, - contextSnapshot: - input?.includeIssue === false + contextSnapshot: { + ...(input?.includeIssue === false ? (input?.contextSnapshot ?? {}) - : { ...(input?.contextSnapshot ?? {}), issueId }, + : { ...(input?.contextSnapshot ?? {}), issueId }), + ...(input?.failureRetryBudgetSpent ? { executionRetryAccounting: spentFailureRetryAccounting } : {}), + }, processPid: input?.processPid ?? null, processGroupId: input?.processGroupId ?? null, processLossRetryCount: input?.processLossRetryCount ?? 0, @@ -3542,6 +3554,39 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(mockAdapterExecute).not.toHaveBeenCalled(); }); + it("retries within the failure budget even when a legacy process-loss counter is set", async () => { + const { companyId, runId } = await seedRestartOrphanedLegacySetup("preparing"); + // Earlier releases wrote this counter; nothing writes it now. + await db.update(heartbeatRuns).set({ processLossRetryCount: 1 }).where(eq(heartbeatRuns.id, runId)); + + await heartbeatService(db).reapOrphanedRuns(); + + const retries = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.retryOfRunId, runId), + )); + expect(retries).toEqual([expect.objectContaining({ status: "scheduled_retry" })]); + }); + + it("does not promise a process-loss retry once the failure budget is spent", async () => { + const { companyId, runId } = await seedRestartOrphanedLegacySetup("preparing"); + const [seeded] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + await db.update(heartbeatRuns).set({ + contextSnapshot: { ...(seeded!.contextSnapshot ?? {}), executionRetryAccounting: spentFailureRetryAccounting }, + }).where(eq(heartbeatRuns.id, runId)); + + await heartbeatService(db).reapOrphanedRuns(); + + const [reaped] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(reaped).toMatchObject({ status: "failed", errorCode: "process_lost", + error: "Process lost -- server may have restarted" }); + expect(reaped?.resultJson).toMatchObject({ processLossDiagnostic: { retryEligible: false } }); + expect(await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.retryOfRunId, runId), + ))).toEqual([]); + }); + it("keeps an orphaned legacy run that reached adapter dispatch held for reconciliation", async () => { const { companyId, runId, issueId } = await seedRestartOrphanedLegacySetup("dispatching"); @@ -5110,7 +5155,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { adapterType: "process", agentStatus: "idle", processPid: 999_999_999, - processLossRetryCount: 1, + failureRetryBudgetSpent: true, }); const heartbeat = heartbeatService(db); await heartbeat.reapOrphanedRuns(); @@ -5135,7 +5180,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { adapterType: "process", agentStatus: "idle", processPid: 999_999_999, - processLossRetryCount: 1, + failureRetryBudgetSpent: true, runErrorCode: "process_lost", runError: "Authorization: Bearer sk-test-recovery-secret", }); @@ -5212,7 +5257,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const { companyId, agentId, runId, issueId } = await seedRunFixture({ agentStatus: "idle", processPid: 999_999_999, - processLossRetryCount: 1, + failureRetryBudgetSpent: true, }); await db.insert(issueTreeHolds).values({ companyId, diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index f52151df613..a12a74bfbfb 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -108,7 +108,7 @@ import type { environmentService } from "../environments.js"; import type { environmentRuntimeService, ProviderResourceDisposition } from "../environment-runtime.js"; import type { agentInstructionWorkingCopyService } from "../agent-instruction-working-copies.js"; import type { createHeartbeatRunState } from "./run-state.js"; -import type { HeartbeatRetryDependencies, createHeartbeatRetries } from "./retries.js"; +import { failedRunRetryBudgetSpent, type HeartbeatRetryDependencies, type createHeartbeatRetries } from "./retries.js"; import type { CancelRunOptions } from "./run-control.js"; import { requestRestartInterruptedTurnRetry } from "./restart-interrupted-turn.js"; @@ -2672,7 +2672,12 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD !revokedState.processGroupId; const shouldRetry = !await pendingCaseHandoffForRun(db, run.id, run.companyId) && - (run.processLossRetryCount ?? 0) < 1 && + // Process loss spends the durable failure retry budget that + // enqueueProcessLossRetry schedules against (execution-semantics + // "Shutdown, process loss, and provider failure"). Checking it here + // keeps "retrying once" honest and lets an exhausted chain fall + // through to the interaction continuation path below. + !failedRunRetryBudgetSpent(run) && ((tracksLegacyLocalChild && (!!run.processPid || !!run.processGroupId)) || restartInterruptedBeforeDispatch ||