fix(recovery): preserve pending native result finalization (#14219)

Preserve native coordinator ownership while accepted results await workspace
copy-back, assessment, or arbitration. Check that ownership in the terminal
update so a coordinator recorded after the liveness read is also protected.
Keep terminal task authority and exhausted-retry cleanup unchanged.

Validation: 28 focused recovery tests, local typecheck/build, 52 passing CI
checks, and Greptile 5/5 with no open findings.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip authored and GitHub committed 2026-09-27 00:22:49 -07:00
1 parent 5ee9e751fb
commit be43c23e2d
3 files changed
+118 -27

No files matched your search

+8
View File
@@ -706,6 +706,14 @@ On startup and on the periodic recovery loop, Paperclip performs the following r
The stranded-work pass closes the gap where issue state survives a crash but the wake/run path does not. The silent-run scan covers the separate case where a live process exists but has stopped producing observable output.
The stale-lock sweep leaves native runs with their finalization coordinator while
same-session recovery or accepted-result finalization is pending. A provider can
exit before workspace copy-back, assessment, or arbitration completes. An expired
coordinator lease or a delayed retry does not make that run orphaned. The sweep
checks coordinator ownership in its terminal update, so a result recorded after
the process check is also protected. Terminal task status remains authoritative;
exhausted finalization retries do not keep an otherwise orphaned run alive.
Automatic productivity reviews are retired. Run counts, missing comments, and elapsed task time do not create review tasks or impose continuation holds. Bounded continuation, provider recovery, budget limits, explicit blockers, and normal review/approval stages remain in force. Existing productivity-review tasks, comments, assignments, and dependencies remain unchanged and readable; their historical origins still identify them as recovery work for recursion suppression.
### Issue-thread interaction resolution
@@ -5,6 +5,7 @@ import {
activityLog,
agents,
companies,
completionContracts,
createDb,
heartbeatRunEvents,
heartbeatRuns,
@@ -12,6 +13,7 @@ import {
issueRelations,
issues,
nativeRunFinalizations,
nativeRunResults,
} from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
@@ -47,12 +49,14 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => {
afterEach(async () => {
mockTelemetryClient.track.mockClear();
await db.delete(nativeRunFinalizations);
await db.delete(nativeRunResults);
await db.delete(issueComments);
await db.delete(issueRelations);
await db.delete(activityLog);
await db.delete(issues);
await db.delete(heartbeatRunEvents);
await db.delete(heartbeatRuns);
await db.delete(completionContracts);
await db.delete(issues);
await db.delete(agents);
await db.delete(companies);
});
@@ -427,6 +431,80 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => {
.resolves.toEqual([{ checkoutRunId: runningRunId, executionRunId: runningRunId }]);
});
async function seedPendingNativeResult(phase: string, options: { terminalIssue?: boolean; deferCoordinator?: boolean } = {}) {
const fixture = await seed();
const { companyId, agentId, runningRunId } = fixture;
const issueId = randomUUID();
const contractId = randomUUID();
const resultId = randomUUID();
await db.insert(issues).values({
id: issueId, companyId, title: "Workspace finalization fixture",
status: options.terminalIssue ? "done" : "in_progress", assigneeAgentId: agentId,
executionRunId: runningRunId, checkoutRunId: runningRunId,
});
await db.insert(completionContracts).values({
id: contractId, companyId, issueId, revision: 1,
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "fixture",
risk: "standard", completionAuthority: "server_arbiter",
incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: {},
canonicalSha256: "fixture-contract", createdByActorType: "system", createdByActorId: "test",
});
await db.update(heartbeatRuns).set({
runtimeMode: "native", nativeIssueId: issueId, nativePhase: phase,
processPid: 2_000_000_000, completionContractId: contractId,
}).where(eq(heartbeatRuns.id, runningRunId));
await db.insert(nativeRunResults).values({
id: resultId, companyId, issueId, runId: runningRunId, completionContractId: contractId,
serverFingerprint: "fixture-result", schemaStatus: "accepted", resultJson: {}, canonicalSha256: "fixture-result",
});
const recordCoordinator = () => db.insert(nativeRunFinalizations).values({
runId: runningRunId, companyId, issueId, phase, resultId, attempt: 1,
nextAttemptAt: new Date(Date.now() + 30_000),
leaseOwner: "previous-controller", leaseExpiresAt: new Date(Date.now() - 30_000),
});
if (!options.deferCoordinator) await recordCoordinator();
return { ...fixture, issueId, recordCoordinator };
}
it.each(["observed", "workspace_finalizing", "ready_for_assessment", "arbitrating", "retryable_failure"])(
"preserves a completed native result in %s after its provider exits", async phase => {
const { runningRunId, issueId } = await seedPendingNativeResult(phase);
const result = await recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks();
expect(result).toEqual({ cleared: 0, issueIds: [], terminalizedRunIds: [] });
expect(await db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode }).from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "running", errorCode: null }]);
expect(await db.select({ executionRunId: issues.executionRunId }).from(issues)
.where(eq(issues.id, issueId))).toEqual([{ executionRunId: runningRunId }]);
expect(mockTelemetryClient.track).not.toHaveBeenCalled();
},
);
it("rechecks native finalization ownership when the coordinator appears after the liveness read", async () => {
const { runningRunId, recordCoordinator } = await seedPendingNativeResult("retryable_failure", { deferCoordinator: true });
const beforeOrphanedRunTerminalWrite = vi.fn(async () => { await recordCoordinator(); });
const result = await recoveryService(db, { enqueueWakeup: vi.fn(), beforeOrphanedRunTerminalWrite }).sweepStaleIssueLocks();
expect(beforeOrphanedRunTerminalWrite).toHaveBeenCalledOnce();
expect(result.terminalizedRunIds).toEqual([]);
expect(await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "running" }]);
});
it("still respects terminal issue authority while workspace finalization is pending", async () => {
const { runningRunId } = await seedPendingNativeResult("retryable_failure", { terminalIssue: true });
const result = await recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks();
expect(result.terminalizedRunIds).toContain(runningRunId);
expect(await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "succeeded" }]);
});
it("does not keep an orphan live after native finalization exhausts its retries", async () => {
const { runningRunId } = await seedPendingNativeResult("terminal_failure");
const result = await recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks();
expect(result.terminalizedRunIds).toContain(runningRunId);
expect(await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "interrupted" }]);
});
it("preserves a process-less run while its in-process execution is still finalizing", async () => {
const { companyId, agentId, runningRunId } = await seed();
const issueId = randomUUID();
+31 -26
View File
@@ -16,8 +16,10 @@ import {
gt,
gte,
inArray,
isNotNull,
isNull,
not,
notExists,
notInArray,
or,
sql,
@@ -5910,32 +5912,6 @@ export function recoveryService(
}
}
// A result-less native run may intentionally have no live provider process
// while the native finalization coordinator waits to resume the same
// provider session. That coordinator, rather than this generic
// process-death backstop, owns retryable/resumed attempts. Preserve issue
// terminality as the stronger authority, but never interrupt coordinator-
// owned recovery merely because the provider process has exited.
if (!issueTerminalStatus && processGone && run.runtimeMode === "native") {
const coordinator = await db
.select({
phase: nativeRunFinalizations.phase,
resultId: nativeRunFinalizations.resultId,
attempt: nativeRunFinalizations.attempt,
})
.from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, run.id))
.limit(1)
.then((rows) => rows[0] ?? null);
const nativeResumeOwnsRun =
coordinator?.resultId === null &&
(coordinator.phase === "retryable_failure" ||
(coordinator.phase === "observed" && coordinator.attempt > 0));
if (nativeResumeOwnsRun) {
return { terminalized: false, status: run.status };
}
}
// Neither authority applies. The run is still live, so leave it alone.
if (!issueTerminalStatus && !processGone) {
return { terminalized: false, status: run.status };
@@ -5970,6 +5946,35 @@ export function recoveryService(
eq(heartbeatRuns.status, "running"),
eq(heartbeatRuns.runtimeMode, run.runtimeMode),
nativeRunnerOwnershipNotHeldCondition(),
// Provider exit is expected while the native coordinator resumes a
// session or copies its completed workspace back. The coordinator
// owns those retries, including expired leases and future attempts.
// Check at the write so a newly recorded result cannot be orphaned
// using the earlier liveness snapshot. Terminal issue status remains
// the stronger authority.
!issueTerminalStatus && run.runtimeMode === "native"
? notExists(db.select({ runId: nativeRunFinalizations.runId })
.from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.runId, heartbeatRuns.id),
eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId),
or(
and(
isNotNull(nativeRunFinalizations.resultId),
inArray(nativeRunFinalizations.phase, [
"observed", "workspace_finalizing", "ready_for_assessment",
"arbitrating", "retryable_failure",
]),
),
and(
isNull(nativeRunFinalizations.resultId),
or(
eq(nativeRunFinalizations.phase, "retryable_failure"),
and(eq(nativeRunFinalizations.phase, "observed"), gt(nativeRunFinalizations.attempt, 0)),
),
),
),
)))
: undefined,
// Recheck ownership in the write: a controller can renew or claim
// the run after the liveness read. An old snapshot cannot end a new
// controller's run, even if that controller's lease later expires.