From be43c23e2df9fc88392390569f5e3b1356cfe275 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sun, 27 Sep 2026 00:22:49 -0700 Subject: [PATCH] fix(recovery): preserve pending native result finalization (#14219) Preserve native coordinator ownership while accepted results await workspace copy-back, assessment, or arbitration. Check that ownership in the terminal update so a coordinator recorded after the liveness read is also protected. Keep terminal task authority and exhausted-retry cleanup unchanged. Validation: 28 focused recovery tests, local typecheck/build, 52 passing CI checks, and Greptile 5/5 with no open findings. Co-Authored-By: Paperclip --- doc/execution-semantics.md | 8 ++ .../recovery-stale-issue-lock-sweep.test.ts | 80 ++++++++++++++++++- server/src/services/recovery/service.ts | 57 +++++++------ 3 files changed, 118 insertions(+), 27 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 7b2c9ce59da..4674753f237 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -706,6 +706,14 @@ On startup and on the periodic recovery loop, Paperclip performs the following r The stranded-work pass closes the gap where issue state survives a crash but the wake/run path does not. The silent-run scan covers the separate case where a live process exists but has stopped producing observable output. +The stale-lock sweep leaves native runs with their finalization coordinator while +same-session recovery or accepted-result finalization is pending. A provider can +exit before workspace copy-back, assessment, or arbitration completes. An expired +coordinator lease or a delayed retry does not make that run orphaned. The sweep +checks coordinator ownership in its terminal update, so a result recorded after +the process check is also protected. Terminal task status remains authoritative; +exhausted finalization retries do not keep an otherwise orphaned run alive. + Automatic productivity reviews are retired. Run counts, missing comments, and elapsed task time do not create review tasks or impose continuation holds. Bounded continuation, provider recovery, budget limits, explicit blockers, and normal review/approval stages remain in force. Existing productivity-review tasks, comments, assignments, and dependencies remain unchanged and readable; their historical origins still identify them as recovery work for recursion suppression. ### Issue-thread interaction resolution diff --git a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts index aef5c24ae68..1fd20c88cd8 100644 --- a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts +++ b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts @@ -5,6 +5,7 @@ import { activityLog, agents, companies, + completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, @@ -12,6 +13,7 @@ import { issueRelations, issues, nativeRunFinalizations, + nativeRunResults, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, @@ -47,12 +49,14 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => { afterEach(async () => { mockTelemetryClient.track.mockClear(); await db.delete(nativeRunFinalizations); + await db.delete(nativeRunResults); await db.delete(issueComments); await db.delete(issueRelations); await db.delete(activityLog); - await db.delete(issues); await db.delete(heartbeatRunEvents); await db.delete(heartbeatRuns); + await db.delete(completionContracts); + await db.delete(issues); await db.delete(agents); await db.delete(companies); }); @@ -427,6 +431,80 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => { .resolves.toEqual([{ checkoutRunId: runningRunId, executionRunId: runningRunId }]); }); + async function seedPendingNativeResult(phase: string, options: { terminalIssue?: boolean; deferCoordinator?: boolean } = {}) { + const fixture = await seed(); + const { companyId, agentId, runningRunId } = fixture; + const issueId = randomUUID(); + const contractId = randomUUID(); + const resultId = randomUUID(); + await db.insert(issues).values({ + id: issueId, companyId, title: "Workspace finalization fixture", + status: options.terminalIssue ? "done" : "in_progress", assigneeAgentId: agentId, + executionRunId: runningRunId, checkoutRunId: runningRunId, + }); + await db.insert(completionContracts).values({ + id: contractId, companyId, issueId, revision: 1, + schemaVersion: "paperclip.completion-contract.v1", policyVersion: "fixture", + risk: "standard", completionAuthority: "server_arbiter", + incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: {}, + canonicalSha256: "fixture-contract", createdByActorType: "system", createdByActorId: "test", + }); + await db.update(heartbeatRuns).set({ + runtimeMode: "native", nativeIssueId: issueId, nativePhase: phase, + processPid: 2_000_000_000, completionContractId: contractId, + }).where(eq(heartbeatRuns.id, runningRunId)); + await db.insert(nativeRunResults).values({ + id: resultId, companyId, issueId, runId: runningRunId, completionContractId: contractId, + serverFingerprint: "fixture-result", schemaStatus: "accepted", resultJson: {}, canonicalSha256: "fixture-result", + }); + const recordCoordinator = () => db.insert(nativeRunFinalizations).values({ + runId: runningRunId, companyId, issueId, phase, resultId, attempt: 1, + nextAttemptAt: new Date(Date.now() + 30_000), + leaseOwner: "previous-controller", leaseExpiresAt: new Date(Date.now() - 30_000), + }); + if (!options.deferCoordinator) await recordCoordinator(); + return { ...fixture, issueId, recordCoordinator }; + } + + it.each(["observed", "workspace_finalizing", "ready_for_assessment", "arbitrating", "retryable_failure"])( + "preserves a completed native result in %s after its provider exits", async phase => { + const { runningRunId, issueId } = await seedPendingNativeResult(phase); + const result = await recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks(); + expect(result).toEqual({ cleared: 0, issueIds: [], terminalizedRunIds: [] }); + expect(await db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode }).from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "running", errorCode: null }]); + expect(await db.select({ executionRunId: issues.executionRunId }).from(issues) + .where(eq(issues.id, issueId))).toEqual([{ executionRunId: runningRunId }]); + expect(mockTelemetryClient.track).not.toHaveBeenCalled(); + }, + ); + + it("rechecks native finalization ownership when the coordinator appears after the liveness read", async () => { + const { runningRunId, recordCoordinator } = await seedPendingNativeResult("retryable_failure", { deferCoordinator: true }); + const beforeOrphanedRunTerminalWrite = vi.fn(async () => { await recordCoordinator(); }); + const result = await recoveryService(db, { enqueueWakeup: vi.fn(), beforeOrphanedRunTerminalWrite }).sweepStaleIssueLocks(); + expect(beforeOrphanedRunTerminalWrite).toHaveBeenCalledOnce(); + expect(result.terminalizedRunIds).toEqual([]); + expect(await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "running" }]); + }); + + it("still respects terminal issue authority while workspace finalization is pending", async () => { + const { runningRunId } = await seedPendingNativeResult("retryable_failure", { terminalIssue: true }); + const result = await recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks(); + expect(result.terminalizedRunIds).toContain(runningRunId); + expect(await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "succeeded" }]); + }); + + it("does not keep an orphan live after native finalization exhausts its retries", async () => { + const { runningRunId } = await seedPendingNativeResult("terminal_failure"); + const result = await recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks(); + expect(result.terminalizedRunIds).toContain(runningRunId); + expect(await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "interrupted" }]); + }); + it("preserves a process-less run while its in-process execution is still finalizing", async () => { const { companyId, agentId, runningRunId } = await seed(); const issueId = randomUUID(); diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 78110616c55..705297ac574 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -16,8 +16,10 @@ import { gt, gte, inArray, + isNotNull, isNull, not, + notExists, notInArray, or, sql, @@ -5910,32 +5912,6 @@ export function recoveryService( } } - // A result-less native run may intentionally have no live provider process - // while the native finalization coordinator waits to resume the same - // provider session. That coordinator, rather than this generic - // process-death backstop, owns retryable/resumed attempts. Preserve issue - // terminality as the stronger authority, but never interrupt coordinator- - // owned recovery merely because the provider process has exited. - if (!issueTerminalStatus && processGone && run.runtimeMode === "native") { - const coordinator = await db - .select({ - phase: nativeRunFinalizations.phase, - resultId: nativeRunFinalizations.resultId, - attempt: nativeRunFinalizations.attempt, - }) - .from(nativeRunFinalizations) - .where(eq(nativeRunFinalizations.runId, run.id)) - .limit(1) - .then((rows) => rows[0] ?? null); - const nativeResumeOwnsRun = - coordinator?.resultId === null && - (coordinator.phase === "retryable_failure" || - (coordinator.phase === "observed" && coordinator.attempt > 0)); - if (nativeResumeOwnsRun) { - return { terminalized: false, status: run.status }; - } - } - // Neither authority applies. The run is still live, so leave it alone. if (!issueTerminalStatus && !processGone) { return { terminalized: false, status: run.status }; @@ -5970,6 +5946,35 @@ export function recoveryService( eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.runtimeMode, run.runtimeMode), nativeRunnerOwnershipNotHeldCondition(), + // Provider exit is expected while the native coordinator resumes a + // session or copies its completed workspace back. The coordinator + // owns those retries, including expired leases and future attempts. + // Check at the write so a newly recorded result cannot be orphaned + // using the earlier liveness snapshot. Terminal issue status remains + // the stronger authority. + !issueTerminalStatus && run.runtimeMode === "native" + ? notExists(db.select({ runId: nativeRunFinalizations.runId }) + .from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.runId, heartbeatRuns.id), + eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), + or( + and( + isNotNull(nativeRunFinalizations.resultId), + inArray(nativeRunFinalizations.phase, [ + "observed", "workspace_finalizing", "ready_for_assessment", + "arbitrating", "retryable_failure", + ]), + ), + and( + isNull(nativeRunFinalizations.resultId), + or( + eq(nativeRunFinalizations.phase, "retryable_failure"), + and(eq(nativeRunFinalizations.phase, "observed"), gt(nativeRunFinalizations.attempt, 0)), + ), + ), + ), + ))) + : undefined, // Recheck ownership in the write: a controller can renew or claim // the run after the liveness read. An old snapshot cannot end a new // controller's run, even if that controller's lease later expires.