fix: retry verified legacy adapter preparation failures

Older preparers used adapter_failed before provider dispatch. Retry admitted
these runs with full safety proof only when they were retry ancestors. An
equivalent latest source stayed held.

Use the existing bounded host-log, stopped-lease, settlement and workspace
proof for both positions. Keep the empty-allocation shortcut restricted to
setup failures. Add database regressions for one successor and refusals
when the required evidence is missing.

Tested: saved-work Retry, legacy preparation Retry, verified preparation
logs, saved bootstrap continuation and workspace preparation gates.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip committed 2026-11-20 23:12:08 -08:00
1 parent 6bf312270c
commit 6fda716f43
2 files changed
+54 -5

No files matched your search

@@ -48,7 +48,11 @@ export async function isFailedNativePreparationRetryAncestor(db: Db, run: Run, c
async function failedNativePreparation(db: Db, run: Run, coordinator: Coordinator | undefined, historicalRetryAncestor: boolean) {
if (!historicalRetryAncestor && !coordinator && await isInterruptedNativePreparation(db, run)) return true;
const recovery = run.resultJson?.executionRecovery as Record<string, unknown> | undefined;
const preparationFailure = run.errorCode === "setup_failed" || historicalRetryAncestor && run.errorCode === "adapter_failed";
// Older preparers used adapter_failed before provider dispatch. Apply the
// same full stop, host-log and saved-file proof whether that run is the
// latest Retry source or an ancestor behind a never-started recovery turn.
const historicalEvidence = historicalRetryAncestor || run.errorCode === "adapter_failed";
const preparationFailure = run.errorCode === "setup_failed" || run.errorCode === "adapter_failed";
if (run.status !== "failed" || !run.finishedAt || run.runtimeMode !== "legacy" || run.runtimeModeResolvedAt
|| !preparationFailure || run.executionStage !== "preparing"
|| claimedAdapterType(run) !== "paperclip_runner" || coordinator
@@ -64,11 +68,11 @@ async function failedNativePreparation(db: Db, run: Run, coordinator: Coordinato
eq(nativeRunResults.companyId, run.companyId), eq(nativeRunResults.runId, run.id),
)).limit(1);
if (result || await nativeStartupHasExecutionEvidence(db, run)) return false;
if (!leases.length && run.runnerProfileJson?.remoteWorkspace == null) return !historicalRetryAncestor;
if (!leases.length && run.runnerProfileJson?.remoteWorkspace == null) return !historicalEvidence;
const issueId = run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId;
return typeof issueId === "string" && leases.length > 0 && leases.every(lease => lease.issueId === issueId
&& lease.metadata?.pendingCleanupInFlight !== true && hasRemoteTerminationReceipt(lease))
&& await hasSettledFailedPreparationEvidence(db, run, historicalRetryAncestor)
&& await hasSettledFailedPreparationEvidence(db, run, historicalEvidence)
&& (!holdsPreparedFiles(run) || await hasVerifiedWorkspaceRepair(db, run));
}
@@ -46,7 +46,7 @@ describe("Retry of a task whose saved work recovery settled", () => {
return { companyId, issueId, agentId, workspaceId };
}
async function sourceUnavailable() {
async function sourceUnavailable({ heldRun = true } = {}) {
const f = await base("SU");
const { companyId, issueId, agentId, workspaceId } = f;
const firstId = randomUUID(), preparedId = randomUUID(), heldId = randomUUID(), actionId = randomUUID();
@@ -126,7 +126,7 @@ describe("Retry of a task whose saved work recovery settled", () => {
workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [leaseId] },
historicalWorkspaceSource: { schema: "paperclip.historical-workspace-source.v1", runId: preparedId, leaseIds: [leaseId] },
previousWorkspaceRecovery: null } });
await db.insert(heartbeatRuns).values({ id: heldId, companyId, agentId, issueId, status: "cancelled", runtimeMode: "legacy",
if (heldRun) await db.insert(heartbeatRuns).values({ id: heldId, companyId, agentId, issueId, status: "cancelled", runtimeMode: "legacy",
errorCode: "execution_reconciliation_required", error: "Recovering saved work…", invocationSource: "automation",
triggerDetail: "system", retryOfRunId: preparedId, createdAt: new Date(now - 680_000), finishedAt: new Date(now - 600_000),
contextSnapshot: { issueId, taskId: issueId, source: "issue.continuation_recovery", wakeReason: "issue_continuation_needed",
@@ -275,6 +275,51 @@ describe("Retry of a task whose saved work recovery settled", () => {
expect(successor.contextSnapshot).toMatchObject({ taskRetryRequestId: wake.id });
});
it("admits Retry when the latest run is the verified legacy adapter preparation itself", async () => {
const f = await sourceUnavailable({ heldRun: false });
// Occupy the worker so this proves admission without starting a provider.
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running" });
const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId,
observedRunId: f.preparedId, idempotencyKey: randomUUID(), actorSource: "local_implicit" });
expect(receipt).toMatchObject({ accepted: true, requestId: expect.any(String) });
const heartbeat = heartbeatService(db);
await heartbeat.resumeTaskRetry(f.companyId, receipt.requestId!);
await heartbeat.resumeTaskRetry(f.companyId, receipt.requestId!);
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!));
expect(wake).toMatchObject({ status: "coalesced", runId: expect.any(String) });
const successors = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`));
expect(successors).toHaveLength(1);
expect(successors[0]).toMatchObject({ id: wake.runId, status: "queued",
contextSnapshot: { previousRunId: f.preparedId, taskRetryRequestId: receipt.requestId } });
});
it.each(["provider_event", "unverified_log", "live_controller", "unsaved_files", "missing_lease", "no_allocation"] as const)(
"refuses Retry of the latest legacy adapter preparation without proof: %s", async gate => {
const f = await sourceUnavailable({ heldRun: false });
if (gate === "provider_event") await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, agentId: f.agentId,
runId: f.preparedId, seq: 5, eventType: "tool.execution.started", stream: "system" });
if (gate === "unverified_log") logs.content.clear();
if (gate === "live_controller") await db.update(heartbeatRuns)
.set({ controllerLeaseExpiresAt: new Date(Date.now() + 3_600_000) }).where(eq(heartbeatRuns.id, f.preparedId));
if (gate === "unsaved_files") await db.update(executionWorkspaces)
.set({ metadata: sql`${executionWorkspaces.metadata} - 'remoteWorkspaceAbandonedEpochs'` })
.where(eq(executionWorkspaces.id, f.workspaceId));
if (gate === "missing_lease" || gate === "no_allocation")
await db.delete(environmentLeases).where(eq(environmentLeases.heartbeatRunId, f.preparedId));
if (gate === "no_allocation") await db.update(heartbeatRuns).set({
runnerProfileJson: sql`${heartbeatRuns.runnerProfileJson} - 'remoteWorkspace'`,
resultJson: sql`${heartbeatRuns.resultJson} - 'workspaceRestoreFailure' - 'workspaceRestoreRecovery'`,
}).where(eq(heartbeatRuns.id, f.preparedId));
const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId,
observedRunId: f.preparedId, idempotencyKey: randomUUID(), actorSource: "local_implicit" });
expect(receipt.requestId).toEqual(expect.any(String));
await heartbeatService(db).resumeTaskRetry(f.companyId, receipt.requestId!);
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!));
expect(wake).toMatchObject({ status: "deferred_issue_execution", runId: null });
expect(wake.payload?.executionWait).toMatchObject({ reason: "previous_execution_unverified" });
});
it("verifies the repair of a restored snapshot that the failed preparation never changed", async () => {
const f = await restoredSnapshot();
const blocker = await getExecutionBlocker(db, f.companyId, f.issueId);