diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 6ab0c7c27c4..078c3598b23 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3670,7 +3670,7 @@ another provider attempt, so restarts the run already survived cannot block it. A disposition request that also ends without a result consumes the marker and fails closed as before. -Provider execution and control-plane finalization have different clocks. A healthy provider can think or execute a long tool without output. Once execution settles, recovery and finalization control steps have a 60-second deadline, checked on startup and every 15 seconds. With a healthy database and scheduler, an abandoned transition must be repaired or surfaced within 90 seconds. Terminal persistence must not wait on provider cleanup or publication; a late finalizer cannot change a reassigned or closed task or release another run's locks. In particular, a done report from a run whose task now belongs to another agent, a user or nobody is kept as an accepted result but records `completion_owner_superseded` and preserves the task status for the current owner. Historical ambiguous runs are never automatically replayed after an upgrade. +Provider execution and control-plane finalization have different clocks. A healthy provider can think or execute a long tool without output. Once execution settles, recovery and finalization control steps have a 60-second deadline, checked on startup and every 15 seconds. With a healthy database and scheduler, an abandoned transition must be repaired or surfaced within 90 seconds. Terminal persistence must not wait on provider cleanup or publication; a late finalizer cannot change a reassigned or closed task or release another run's locks. In particular, a done report from a run whose task now belongs to another agent, a user or nobody is kept as an accepted result but records `completion_owner_superseded` and preserves the task status for the current owner. A finalizer retired because a newer run already committed the task's decision (`native_finalization_superseded`) also ends its run, so the run never holds the task or the runs queued behind it; the result is kept for reference and never applied, and the task's decision does not change. The native run projection worker ends a run that an older release left running in that state, at startup and whenever it is signalled. Historical ambiguous runs are never automatically replayed after an upgrade. A control deadline bounds the settlement of one attempt, never a later attempt of the same run. While a native run's finalization owner holds a live lease (for example, a controller that reclaimed the run after a server restart) or has a same-run resume scheduled, that owner is responsible for the run. The reconciler clears a run deadline left by an earlier attempt instead of failing the resumed work. When the live owner's own settlement deadline is due, the reconciler waits until that lease lapses before it surfaces the deadline. A run whose deadline genuinely expires fails with `execution_finalization_deadline_exceeded` and shows plain wording with Retry: its saved work is kept, and actions with unconfirmed outcomes are not repeated. Runs saved by earlier releases with the previous operator-facing wording show the same plain wording when the task is read; no migration is needed. diff --git a/server/src/__tests__/native-finalization-recovery.test.ts b/server/src/__tests__/native-finalization-recovery.test.ts index 2b8f36166b2..fb8bb5798ac 100644 --- a/server/src/__tests__/native-finalization-recovery.test.ts +++ b/server/src/__tests__/native-finalization-recovery.test.ts @@ -28,6 +28,7 @@ import { reconcileRetainedNativeSessionCleanups, } from "../services/native-runtime/native-finalization-reconciler.js"; import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js"; +import { endSupersededNativeFinalizationRuns } from "../services/native-runtime/native-run-finalizer.js"; import { assertRetainedNativeSourceArchiveSettled, reconcileRetainedNativeSessionCleanup } from "../services/native-runtime/native-session-executor.js"; const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_FINALIZATION_TEST_DATABASE_URL"); @@ -331,6 +332,46 @@ describe("P6-16/P6-25/P6-28 native finalization recovery", () => { .where(eq(issueRecoveryActions.sourceIssueId, staleIssueId))).resolves.toEqual([ expect.objectContaining({ status: "resolved", outcome: "false_positive" }), ]); + // The retired run ends, so it no longer holds the task and its queue. + const [ended] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, staleRunId)); + // Provider work succeeded; the result is kept for reference only. + expect(ended).toMatchObject({ status: "succeeded", nativePhase: "terminal_failure", finishedAt: expect.any(Date) }); + }); + + it("ends a run an earlier release left running after retiring its finalizer as superseded", async () => { + // The row an earlier release wrote after a failed workspace export: a + // terminal superseded coordinator, and the run still running. + await db.update(heartbeatRuns).set({ status: "running", finishedAt: null, error: null, errorCode: null }) + .where(eq(heartbeatRuns.id, staleRunId)); + // Written here, so the case does not depend on the previous test's outcome. + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null, leaseExpiresAt: null, + failureCode: "native_finalization_superseded", nextAttemptAt: null, failureDetail: { workspaceFinalizeAttempt: 1, + originalFailureCode: "native_workspace_sync_out_failed", recoveryOwner: { kind: "none", reason: "newer_native_decision" } } }) + .where(eq(nativeRunFinalizations.runId, staleRunId)); + const [issueBefore] = await db.select().from(issues).where(eq(issues.id, staleIssueId)); + const [coordinatorBefore] = await db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, staleRunId)); + expect(coordinatorBefore).toMatchObject({ phase: "terminal_failure", failureCode: "native_finalization_superseded" }); + + await expect(endSupersededNativeFinalizationRuns(db, [staleRunId])).resolves.toEqual([staleRunId]); + + const [ended] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, staleRunId)); + expect(ended).toMatchObject({ status: "failed", errorCode: "native_finalization_superseded", finishedAt: expect.any(Date) }); + expect(ended.error).toBe("A newer run already decided this task, so this run's result was kept for reference and not applied."); + // The durable verdict stands: no issue change, no re-applied result. + await expect(db.select().from(issues).where(eq(issues.id, staleIssueId))).resolves.toEqual([ + expect.objectContaining({ status: issueBefore.status, statusVersion: issueBefore.statusVersion, + lastStatusDecisionId: issueBefore.lastStatusDecisionId }), + ]); + await expect(db.select().from(statusDecisions).where(eq(statusDecisions.runId, staleRunId))).resolves.toHaveLength(0); + await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, staleRunId))) + .resolves.toEqual([expect.objectContaining({ phase: "terminal_failure", failureCode: "native_finalization_superseded", + resultId: coordinatorBefore.resultId })]); + // A second pass finds nothing left to end. + await expect(endSupersededNativeFinalizationRuns(db, [staleRunId])).resolves.toEqual([]); + await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, staleRunId))).resolves.toEqual([ + expect.objectContaining({ status: "failed" }), + ]); }); }); diff --git a/server/src/__tests__/native-run-projection-work.test.ts b/server/src/__tests__/native-run-projection-work.test.ts index 52bf0fb1924..84f80b02e34 100644 --- a/server/src/__tests__/native-run-projection-work.test.ts +++ b/server/src/__tests__/native-run-projection-work.test.ts @@ -365,6 +365,37 @@ describe("committed native run projection lifecycle", () => { expect(await work.load({ companyId, runId: binding.runId })).toEqual([]); }); + // Staging, 2026-11-12: an earlier release retired a finalizer as superseded + // after its workspace export failed, but left the run running. It held the + // task and five queued runs for over an hour, through an upgrade. + it("ends, at startup, a run an earlier release left running after retiring its finalizer as superseded", async () => { + const binding = await seed(); + await complete(binding, "control_plane"); + const before = await state(binding.runId); + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null, leaseExpiresAt: null, + failureCode: "native_finalization_superseded", failureDetail: { workspaceFinalizeAttempt: 1, + originalFailureCode: "native_workspace_sync_out_failed", recoveryOwner: { kind: "none", reason: "newer_native_decision" } } }) + .where(eq(nativeRunFinalizations.runId, binding.runId)); + await db.update(heartbeatRuns).set({ nativePhase: "terminal_failure" }).where(eq(heartbeatRuns.id, binding.runId)); + const wake = await claimedWake(binding); + const [issueBefore] = await db.select().from(issues).where(eq(issues.id, binding.issueId)); + + await start(); + await flush(); + + const ended = await read(binding.runId); + expect(ended).toMatchObject({ status: "failed", errorCode: "native_finalization_superseded", finishedAt: expect.any(Date), + error: "A newer run already decided this task, so this run's result was kept for reference and not applied." }); + expect(await readWake(wake.id)).toMatchObject({ status: "failed", finishedAt: ended.finishedAt }); + // The durable verdict stands: the task and the retired finalizer are unchanged. + expect(await state(binding.runId)).toMatchObject({ phase: "terminal_failure", + failureCode: "native_finalization_superseded", resultId: before.resultId }); + const [issueAfter] = await db.select().from(issues).where(eq(issues.id, binding.issueId)); + expect({ status: issueAfter.status, statusVersion: issueAfter.statusVersion, lastStatusDecisionId: issueAfter.lastStatusDecisionId }) + .toEqual({ status: issueBefore.status, statusVersion: issueBefore.statusVersion, lastStatusDecisionId: issueBefore.lastStatusDecisionId }); + expect(await nativeRunProjectionWork(db).load({ companyId, runId: binding.runId })).toEqual([]); + }); + it.each(["unbound", "newer-run", "queued", "cancelled", "foreign-company", "foreign-agent", "live-owner"] as const)( "does not settle a %s wake claim", async fault => { const binding = await accepted(), wake = await claimedWake(binding); diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index 6ec13047d12..338786cbec2 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -114,6 +114,7 @@ import { } from "../durable-chat-wakeup.js"; import { isNativeRunnerOwnershipHeld, + nativeRunnerOwnershipNotHeldCondition, } from "./native-runner-ownership.js"; import { crashPoint } from "@paperclipai/shared/crash-points"; @@ -332,11 +333,99 @@ function terminalFinalizationFailureMessage(failureCode: string, failureScope: " if (failureCode === "native_finalization_retry_exhausted") { return "Native run finalization failed after 3 attempts."; } + if (failureCode === NATIVE_FINALIZATION_SUPERSEDED) { + return "A newer run already decided this task, so this run's result was kept for reference and not applied."; + } return failureScope === "workspace" ? `Native workspace finalization failed (${failureCode}).` : `Native run finalization failed (${failureCode}).`; } +export const NATIVE_FINALIZATION_SUPERSEDED = "native_finalization_superseded"; + +/** The run columns that end a run whose finalization reached a terminal verdict. */ +function terminalFinalizationRunPatch(input: { + status: "succeeded" | "failed" | "cancelled"; failureCode: string; error: string; now: Date; +}) { + return { + // Stop can win after the failure's admission snapshot. The + // UPDATE locks and evaluates the current row, not that snapshot. + status: sql`case when ${heartbeatRuns.status} = 'cancelled' then ${heartbeatRuns.status} + else ${input.status} end`, + finishedAt: sql`case when ${heartbeatRuns.status} = 'cancelled' then ${heartbeatRuns.finishedAt} + else coalesce(${heartbeatRuns.finishedAt}, ${input.now.toISOString()}::timestamptz) end`, + // A failed run names its cause. Keep an earlier root cause, and + // leave a run that Stop already cancelled without a failure. + ...(input.status === "failed" ? explainFailedRun({ errorCode: input.failureCode, error: input.error }) : {}), + }; +} + +/** + * Ends runs that an earlier release left `running` after retiring their + * finalizer as superseded (`native_finalization_superseded`): the coordinator + * is terminal, but the run kept holding its task and the runs queued behind + * it. The durable verdict stands: the issue keeps the newer run's decision and + * this result is never applied. The run only gets the terminal status the + * retirement now writes itself. + */ +export async function endSupersededNativeFinalizationRuns(db: Db, runIds?: readonly string[]) { + if (runIds && runIds.length === 0) return []; + const candidates = await db.select({ runId: nativeRunFinalizations.runId }) + .from(nativeRunFinalizations) + .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, nativeRunFinalizations.runId), + eq(heartbeatRuns.companyId, nativeRunFinalizations.companyId))) + .where(and( + eq(heartbeatRuns.runtimeMode, "native"), + eq(heartbeatRuns.status, "running"), + eq(nativeRunFinalizations.phase, "terminal_failure"), + eq(nativeRunFinalizations.failureCode, NATIVE_FINALIZATION_SUPERSEDED), + nativeRunnerOwnershipNotHeldCondition(), + runIds ? inArray(nativeRunFinalizations.runId, [...runIds]) : undefined, + )) + .limit(25); + const ended: string[] = []; + for (const { runId } of candidates) { + const now = new Date(); + const updatedRun = await db.transaction(async (tx) => { + // Match the finalizer's coordinator -> issue -> run lock order. + const [coordinator] = await tx.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, runId)).for("update").limit(1); + if (!coordinator || coordinator.phase !== "terminal_failure" + || coordinator.failureCode !== NATIVE_FINALIZATION_SUPERSEDED) return null; + await tx.select({ id: issues.id }).from(issues).where(and(eq(issues.id, coordinator.issueId), + eq(issues.companyId, coordinator.companyId))).for("update").limit(1); + const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), + eq(heartbeatRuns.companyId, coordinator.companyId))).for("update").limit(1); + if (!run || run.runtimeMode !== "native" || run.status !== "running" || isNativeRunnerOwnershipHeld(run)) return null; + const detail = record(coordinator.failureDetail); + const workspace = typeof detail.workspaceFinalizeAttempt === "number"; + const accepted = record(run.resultJson).prpRunTerminalState; + const status = workspace ? "failed" as const + : accepted === "succeeded" ? "succeeded" as const : accepted === "cancelled" ? "cancelled" as const : "failed" as const; + await notifyNativeFinalizationWork(tx, { companyId: run.companyId, runId: run.id }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, run.companyId); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId: coordinator.issueId }); + await notifyInstructionCollection(tx, { companyId: run.companyId, runId: run.id }); + await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); + await notifyUnsafeWorkspaceExportRunChange(tx, run); + const [updated] = await tx.update(heartbeatRuns).set({ + executionStatusDeliveryId: randomUUID(), + ...terminalFinalizationRunPatch({ status, failureCode: NATIVE_FINALIZATION_SUPERSEDED, now, + error: terminalFinalizationFailureMessage(NATIVE_FINALIZATION_SUPERSEDED, workspace ? "workspace" : "provider") }), + updatedAt: now, + }).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"))).returning(); + if (updated) await settleNativeTerminalWake(tx as unknown as Db, updated); + return updated ?? null; + }); + if (!updatedRun) continue; + ended.push(runId); + await emitAgentTaskRun(db, updatedRun); + if (updatedRun.status === "failed") void reportRunFailure(db, updatedRun); + } + return ended; +} + /** * A full sandbox disk stops every copy-back the same way: the archive is built * in that sandbox, and nothing frees space between finalizer attempts. It is @@ -523,7 +612,7 @@ async function recordRetryableFailure(input: { ? ("terminal_failure" as const) : ("retryable_failure" as const); const failureCode = supersededByNewerRun - ? "native_finalization_superseded" + ? NATIVE_FINALIZATION_SUPERSEDED : input.permanent ? input.failureCode : exhausted @@ -571,9 +660,12 @@ async function recordRetryableFailure(input: { updatedAt: now, }) .where(eq(nativeRunFinalizations.runId, input.run.id)); + // A superseded finalizer is retired, so its run ends too: left running it + // would hold the task and every run queued behind it forever. The issue + // keeps the newer run's decision; this result is never applied. const projectsTerminalStatus = - exhausted && !supersededByNewerRun && input.projectRunStatus; - if (projectsTerminalStatus && input.failureScope === "workspace") { + (exhausted || supersededByNewerRun) && input.projectRunStatus; + if (projectsTerminalStatus && !supersededByNewerRun && input.failureScope === "workspace") { await preserveNativeWorkspaceExportLease(tx as unknown as Db, input.run, input.coordinator.resultId); } if (projectsTerminalStatus) await notifyInstructionCollection(tx, { companyId: input.run.companyId, runId: input.run.id }); @@ -585,22 +677,8 @@ async function recordRetryableFailure(input: { .set({ executionStatusDeliveryId: randomUUID(), ...(projectsTerminalStatus - ? { - // Stop can win after the failure's admission snapshot. The - // UPDATE locks and evaluates the current row, not that snapshot. - status: sql`case when ${heartbeatRuns.status} = 'cancelled' then ${heartbeatRuns.status} - else ${terminalRunStatus} end`, - finishedAt: sql`case when ${heartbeatRuns.status} = 'cancelled' then ${heartbeatRuns.finishedAt} - else coalesce(${heartbeatRuns.finishedAt}, ${now.toISOString()}::timestamptz) end`, - // A failed run names its cause. Keep an earlier root cause, and - // leave a run that Stop already cancelled without a failure. - ...(terminalRunStatus === "failed" - ? explainFailedRun({ - errorCode: failureCode, - error: terminalFinalizationFailureMessage(failureCode, input.failureScope, failureCause), - }) - : {}), - } + ? terminalFinalizationRunPatch({ status: terminalRunStatus, failureCode, now, + error: terminalFinalizationFailureMessage(failureCode, input.failureScope, failureCause) }) : {}), nativePhase: phase, nativePhaseUpdatedAt: now, diff --git a/server/src/services/native-runtime/native-run-projection-work.ts b/server/src/services/native-runtime/native-run-projection-work.ts index 54656c61c64..6ac8e078368 100644 --- a/server/src/services/native-runtime/native-run-projection-work.ts +++ b/server/src/services/native-runtime/native-run-projection-work.ts @@ -1,4 +1,4 @@ -import { and, asc, eq, gt, not, sql } from "drizzle-orm"; +import { and, asc, eq, gt, not, or, sql } from "drizzle-orm"; import { heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; @@ -6,21 +6,35 @@ import type { NativeRunProjectionScope } from "../native-run-projection-notifica import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; import { pendingNativeStatusEffectsCondition } from "./native-pending-effect-work.js"; import { pendingNativeRunProjectionCondition, projectCommittedRun } from "./native-run-projection.js"; +import { endSupersededNativeFinalizationRuns, NATIVE_FINALIZATION_SUPERSEDED } from "./native-run-finalizer.js"; +import { nativeRunnerOwnershipNotHeldCondition } from "./native-runner-ownership.js"; type Entry = { id: string; companyId: string; issueId: string; dueAt: number }; -export function nativeRunProjectionWork(db: Db, project = projectCommittedRun) { +/** + * A run an earlier release left running after retiring its finalizer as + * superseded. It gets the terminal status the retirement now writes itself. + */ +function supersededRunStillRunningCondition() { + return and(eq(nativeRunFinalizations.phase, "terminal_failure"), + eq(nativeRunFinalizations.failureCode, NATIVE_FINALIZATION_SUPERSEDED), + eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.status, "running"), + nativeRunnerOwnershipNotHeldCondition()); +} + +export function nativeRunProjectionWork(db: Db, project = projectCommittedRun, + endSuperseded: (db: Db, runIds: readonly string[]) => Promise = endSupersededNativeFinalizationRuns) { function candidates(scope?: NativeRunProjectionScope, after?: string) { return db.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, issueId: nativeRunFinalizations.issueId }) .from(nativeRunFinalizations) .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, nativeRunFinalizations.runId), eq(heartbeatRuns.companyId, nativeRunFinalizations.companyId), eq(heartbeatRuns.nativeIssueId, nativeRunFinalizations.issueId))) .innerJoin(issues, and(eq(issues.id, nativeRunFinalizations.issueId), eq(issues.companyId, nativeRunFinalizations.companyId))) - .where(and(eq(nativeRunFinalizations.phase, "committed"), + .where(and(or(and(eq(nativeRunFinalizations.phase, "committed"), pendingNativeRunProjectionCondition(db), - not(pendingNativeStatusEffectsCondition(db)), + not(pendingNativeStatusEffectsCondition(db))), supersededRunStillRunningCondition()), scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined, scope?.issueId ? eq(nativeRunFinalizations.issueId, scope.issueId) : undefined, scope?.runId ? eq(heartbeatRuns.id, scope.runId) : undefined, @@ -43,6 +57,15 @@ export function nativeRunProjectionWork(db: Db, project = projectCommittedRun) { const scope = { companyId: entry.companyId, issueId: entry.issueId, runId: entry.id }; const [pending] = await candidates(scope); if (!pending) return null; + const [superseded] = await db.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations) + .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, nativeRunFinalizations.runId), + eq(heartbeatRuns.companyId, nativeRunFinalizations.companyId))) + .where(and(eq(nativeRunFinalizations.runId, entry.id), eq(nativeRunFinalizations.companyId, entry.companyId), + supersededRunStillRunningCondition())); + if (superseded) { + await endSuperseded(db, [entry.id]); + return (await candidates(scope)).length ? { ...entry, dueAt: Date.now() + 30_000 } : null; + } const [run] = await db.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, status: heartbeatRuns.status, finishedAt: heartbeatRuns.finishedAt, nativePhase: heartbeatRuns.nativePhase, runtimeMode: heartbeatRuns.runtimeMode, nativeIssueId: heartbeatRuns.nativeIssueId,