From 6fda716f4340cc666c21f4aef07d65a3a0d26bb0 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 20 Nov 2026 17:45:48 -0800 Subject: [PATCH] fix: retry verified legacy adapter preparation failures Older preparers used adapter_failed before provider dispatch. Retry admitted these runs with full safety proof only when they were retry ancestors. An equivalent latest source stayed held. Use the existing bounded host-log, stopped-lease, settlement and workspace proof for both positions. Keep the empty-allocation shortcut restricted to setup failures. Add database regressions for one successor and refusals when the required evidence is missing. Tested: saved-work Retry, legacy preparation Retry, verified preparation logs, saved bootstrap continuation and workspace preparation gates. Co-Authored-By: Paperclip --- .../src/services/cancelled-native-startup.ts | 10 ++-- .../saved-work-retry-eligibility.test.ts | 49 ++++++++++++++++++- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/server/src/services/cancelled-native-startup.ts b/server/src/services/cancelled-native-startup.ts index 9358bc973c9..f88c48a006a 100644 --- a/server/src/services/cancelled-native-startup.ts +++ b/server/src/services/cancelled-native-startup.ts @@ -48,7 +48,11 @@ export async function isFailedNativePreparationRetryAncestor(db: Db, run: Run, c async function failedNativePreparation(db: Db, run: Run, coordinator: Coordinator | undefined, historicalRetryAncestor: boolean) { if (!historicalRetryAncestor && !coordinator && await isInterruptedNativePreparation(db, run)) return true; const recovery = run.resultJson?.executionRecovery as Record | undefined; - const preparationFailure = run.errorCode === "setup_failed" || historicalRetryAncestor && run.errorCode === "adapter_failed"; + // Older preparers used adapter_failed before provider dispatch. Apply the + // same full stop, host-log and saved-file proof whether that run is the + // latest Retry source or an ancestor behind a never-started recovery turn. + const historicalEvidence = historicalRetryAncestor || run.errorCode === "adapter_failed"; + const preparationFailure = run.errorCode === "setup_failed" || run.errorCode === "adapter_failed"; if (run.status !== "failed" || !run.finishedAt || run.runtimeMode !== "legacy" || run.runtimeModeResolvedAt || !preparationFailure || run.executionStage !== "preparing" || claimedAdapterType(run) !== "paperclip_runner" || coordinator @@ -64,11 +68,11 @@ async function failedNativePreparation(db: Db, run: Run, coordinator: Coordinato eq(nativeRunResults.companyId, run.companyId), eq(nativeRunResults.runId, run.id), )).limit(1); if (result || await nativeStartupHasExecutionEvidence(db, run)) return false; - if (!leases.length && run.runnerProfileJson?.remoteWorkspace == null) return !historicalRetryAncestor; + if (!leases.length && run.runnerProfileJson?.remoteWorkspace == null) return !historicalEvidence; const issueId = run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId; return typeof issueId === "string" && leases.length > 0 && leases.every(lease => lease.issueId === issueId && lease.metadata?.pendingCleanupInFlight !== true && hasRemoteTerminationReceipt(lease)) - && await hasSettledFailedPreparationEvidence(db, run, historicalRetryAncestor) + && await hasSettledFailedPreparationEvidence(db, run, historicalEvidence) && (!holdsPreparedFiles(run) || await hasVerifiedWorkspaceRepair(db, run)); } diff --git a/server/src/services/saved-work-retry-eligibility.test.ts b/server/src/services/saved-work-retry-eligibility.test.ts index fc2602a5868..efbbb324ea5 100644 --- a/server/src/services/saved-work-retry-eligibility.test.ts +++ b/server/src/services/saved-work-retry-eligibility.test.ts @@ -46,7 +46,7 @@ describe("Retry of a task whose saved work recovery settled", () => { return { companyId, issueId, agentId, workspaceId }; } - async function sourceUnavailable() { + async function sourceUnavailable({ heldRun = true } = {}) { const f = await base("SU"); const { companyId, issueId, agentId, workspaceId } = f; const firstId = randomUUID(), preparedId = randomUUID(), heldId = randomUUID(), actionId = randomUUID(); @@ -126,7 +126,7 @@ describe("Retry of a task whose saved work recovery settled", () => { workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [leaseId] }, historicalWorkspaceSource: { schema: "paperclip.historical-workspace-source.v1", runId: preparedId, leaseIds: [leaseId] }, previousWorkspaceRecovery: null } }); - await db.insert(heartbeatRuns).values({ id: heldId, companyId, agentId, issueId, status: "cancelled", runtimeMode: "legacy", + if (heldRun) await db.insert(heartbeatRuns).values({ id: heldId, companyId, agentId, issueId, status: "cancelled", runtimeMode: "legacy", errorCode: "execution_reconciliation_required", error: "Recovering saved work…", invocationSource: "automation", triggerDetail: "system", retryOfRunId: preparedId, createdAt: new Date(now - 680_000), finishedAt: new Date(now - 600_000), contextSnapshot: { issueId, taskId: issueId, source: "issue.continuation_recovery", wakeReason: "issue_continuation_needed", @@ -275,6 +275,51 @@ describe("Retry of a task whose saved work recovery settled", () => { expect(successor.contextSnapshot).toMatchObject({ taskRetryRequestId: wake.id }); }); + it("admits Retry when the latest run is the verified legacy adapter preparation itself", async () => { + const f = await sourceUnavailable({ heldRun: false }); + // Occupy the worker so this proves admission without starting a provider. + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running" }); + const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId, + observedRunId: f.preparedId, idempotencyKey: randomUUID(), actorSource: "local_implicit" }); + expect(receipt).toMatchObject({ accepted: true, requestId: expect.any(String) }); + const heartbeat = heartbeatService(db); + await heartbeat.resumeTaskRetry(f.companyId, receipt.requestId!); + await heartbeat.resumeTaskRetry(f.companyId, receipt.requestId!); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!)); + expect(wake).toMatchObject({ status: "coalesced", runId: expect.any(String) }); + const successors = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`)); + expect(successors).toHaveLength(1); + expect(successors[0]).toMatchObject({ id: wake.runId, status: "queued", + contextSnapshot: { previousRunId: f.preparedId, taskRetryRequestId: receipt.requestId } }); + }); + + it.each(["provider_event", "unverified_log", "live_controller", "unsaved_files", "missing_lease", "no_allocation"] as const)( + "refuses Retry of the latest legacy adapter preparation without proof: %s", async gate => { + const f = await sourceUnavailable({ heldRun: false }); + if (gate === "provider_event") await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, agentId: f.agentId, + runId: f.preparedId, seq: 5, eventType: "tool.execution.started", stream: "system" }); + if (gate === "unverified_log") logs.content.clear(); + if (gate === "live_controller") await db.update(heartbeatRuns) + .set({ controllerLeaseExpiresAt: new Date(Date.now() + 3_600_000) }).where(eq(heartbeatRuns.id, f.preparedId)); + if (gate === "unsaved_files") await db.update(executionWorkspaces) + .set({ metadata: sql`${executionWorkspaces.metadata} - 'remoteWorkspaceAbandonedEpochs'` }) + .where(eq(executionWorkspaces.id, f.workspaceId)); + if (gate === "missing_lease" || gate === "no_allocation") + await db.delete(environmentLeases).where(eq(environmentLeases.heartbeatRunId, f.preparedId)); + if (gate === "no_allocation") await db.update(heartbeatRuns).set({ + runnerProfileJson: sql`${heartbeatRuns.runnerProfileJson} - 'remoteWorkspace'`, + resultJson: sql`${heartbeatRuns.resultJson} - 'workspaceRestoreFailure' - 'workspaceRestoreRecovery'`, + }).where(eq(heartbeatRuns.id, f.preparedId)); + const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId, + observedRunId: f.preparedId, idempotencyKey: randomUUID(), actorSource: "local_implicit" }); + expect(receipt.requestId).toEqual(expect.any(String)); + await heartbeatService(db).resumeTaskRetry(f.companyId, receipt.requestId!); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!)); + expect(wake).toMatchObject({ status: "deferred_issue_execution", runId: null }); + expect(wake.payload?.executionWait).toMatchObject({ reason: "previous_execution_unverified" }); + }); + it("verifies the repair of a restored snapshot that the failed preparation never changed", async () => { const f = await restoredSnapshot(); const blocker = await getExecutionBlocker(db, f.companyId, f.issueId);