diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 91d1bbca4cb..dd5ae58a64f 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -6,7 +6,7 @@ import { canSelectFreshWorkspace, freshReadinessSchema, retainedIssueWorkspaceFo import { verifiedWorkspaceSnapshotReceipt } from "./workspace-recovery-snapshot.js"; import { readTaskRetryIntent, retireTaskRetriesContinuedBy } from "./task-retry.js"; import { isPreDispatchExecutionWaitVerified } from "./pre-dispatch-execution-wait.js"; -import { readSystemRecovery } from "./system-recovery-state.js"; +import { readSystemRecovery, savedFilesUnrecoverable, unrecoverableSavedFilesFailure, unrecoverableSavedFilesNotice } from "./system-recovery-state.js"; import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; import { hasVerifiedWorkspaceRepair } from "./workspace-recovery-receipt.js"; import { readMigratedLegacyWorkspaceSource } from "./legacy-workspace-recovery-migration.js"; @@ -476,6 +476,7 @@ export async function admitExplicitNativeContinuation(input: { and ${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`, recoveryState: sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}'`, retryClass: sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,failure,retryClass}'`, + evidence: issueRecoveryActions.evidence, }).from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId), inArray(issueRecoveryActions.status, ["active", "escalated"]), @@ -486,11 +487,12 @@ export async function admitExplicitNativeContinuation(input: { // A spent recovery budget finishes nothing by itself. Another Retry // starts one new recovery generation (task-retry), so name that action // instead of an endless finalizing wait (RC-50b, s14). - if (live.length && live.every(row => row.recoveryState === "exhausted")) + if (!live.length || !live.every(row => row.recoveryState === "exhausted")) + return blocked("finalizing_saved_result", "Paperclip is finishing the saved result."); + if (!freshIntent || !live.every(row => savedFilesUnrecoverable(readSystemRecovery(row.evidence)))) return live.some(row => row.retryClass === "service_required") - ? blocked("saved_work_recovery_refused", "Paperclip refused to recover the saved files because a safety check on them cannot pass. Retry runs the same check again.") + ? blocked("saved_work_recovery_refused", "Paperclip refused to recover the saved files because a safety check on them cannot pass. Your Retry is saved.") : blocked("saved_work_recovery_failed", "Saving the agent's files failed. Retry to try again."); - return blocked("finalizing_saved_result", "Paperclip is finishing the saved result."); } } // A repaired export no longer blocks execution. Its machine receipt still @@ -716,12 +718,15 @@ export async function admitExplicitNativeContinuation(input: { failureDetail: sql`coalesce(${nativeRunFinalizations.failureDetail}, '{}'::jsonb) || ${JSON.stringify({ replacementDenied: "explicit_user_continuation" })}::jsonb`, updatedAt: new Date(), }).where(and(eq(nativeRunFinalizations.companyId, companyId), inArray(nativeRunFinalizations.runId, nativeSources.map(run => run.id)))); + const unrecoverable = actions.map(action => freshSources.has(String(action.evidence.runId)) + ? unrecoverableSavedFilesFailure(readSystemRecovery(action.evidence)) : null).find(failure => failure !== null) ?? null; if (actions.length) await db.transaction(async tx => { await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId }); for (const action of actions) { await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "cancelled", resolvedAt: new Date(), updatedAt: new Date(), - nextAction: "The user started a fresh conversation turn. Prior action outcomes remain recorded.", + nextAction: unrecoverable ? unrecoverableSavedFilesNotice(unrecoverable) + : "The user started a fresh conversation turn. Prior action outcomes remain recorded.", resolutionNote: "The user continued after the prior execution stopped. No action outcomes were inferred.", wakePolicy: null, monitorPolicy: null, evidence: { ...action.evidence, explicitUserContinuation: authorization, @@ -735,6 +740,11 @@ export async function admitExplicitNativeContinuation(input: { }, }).where(eq(issueRecoveryActions.id, action.id)); } + if (unrecoverable) { + const { issueService } = await import("./issues.js"); + await issueService(db).addComment(issueId, unrecoverableSavedFilesNotice(unrecoverable), {}, + { authorType: "system" }, tx); + } }); await persistActivity(db, { companyId, actorType: "user", actorId, action: "issue.execution_recovery_settled", entityType: "issue", entityId: issueId, diff --git a/server/src/services/native-runtime/native-source-retention.ts b/server/src/services/native-runtime/native-source-retention.ts index d4b18b7b786..b1541ce83b3 100644 --- a/server/src/services/native-runtime/native-source-retention.ts +++ b/server/src/services/native-runtime/native-source-retention.ts @@ -3,7 +3,7 @@ import { agentWakeupRequests, environmentLeases, heartbeatRuns, issueRecoveryAct import { persistActivity } from "../activity-log.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js"; -import { createSystemRecovery, readSystemRecovery, SystemRecoveryFailure, type SystemRecoveryState } from "../system-recovery-state.js"; +import { canAdoptNativeSourceRetention, createSystemRecovery, readSystemRecovery, SystemRecoveryFailure, type SystemRecoveryState } from "../system-recovery-state.js"; import { updateSystemRecovery } from "../system-recovery.js"; import { readTaskRetryIntent, taskRetryMutationAllowed } from "../task-retry.js"; import { inspectWorkspaceRecoverySource, type WorkspaceRecoverySourceIdentity } from "../workspace-recovery-source.js"; @@ -12,12 +12,7 @@ import { verifiedWorkspaceSnapshotReceipt } from "../workspace-recovery-snapshot import type { NativeWorkspaceFinalizationOwnership } from "./native-workspace-finalization-ownership.js"; import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; -export function canAdoptNativeSourceRetention(state: SystemRecoveryState) { - return state.state === "exhausted" && state.phase === "verify_execution" && state.attempts === 1 - && state.retriesUsed === 0 && state.failure?.code === "source_binding_invalid" - && (!state.failure.sourceVerification || state.failure.sourceVerification.reason === "lease_policy_not_retained") - && state.preservationOperationIds.length === 0 && !state.freshAttemptRunId; -} +export { canAdoptNativeSourceRetention }; export async function retainNativeRecoverySource(db: Db, input: WorkspaceRecoverySourceIdentity, previous: SystemRecoveryState, ownership: NativeWorkspaceFinalizationOwnership): Promise { diff --git a/server/src/services/system-recovery-state.test.ts b/server/src/services/system-recovery-state.test.ts index 4dae47be2e6..030d6dc36eb 100644 --- a/server/src/services/system-recovery-state.test.ts +++ b/server/src/services/system-recovery-state.test.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import { describe, expect, it } from "vitest"; -import { createSystemRecovery, failSystemRecovery, isDeterministicRecoveryRefusal, readSystemRecovery, reserveSystemRecoveryAttempt, SystemRecoveryFailure } from "./system-recovery-state.js"; +import { createSystemRecovery, failSystemRecovery, isDeterministicRecoveryRefusal, readSystemRecovery, reserveSystemRecoveryAttempt, savedFilesUnrecoverable, SystemRecoveryFailure } from "./system-recovery-state.js"; import { exhaustedSystemRecoveryAction } from "./system-recovery.js"; import { workspaceRecoveryDisplay } from "./workspace-recovery-receipt.js"; @@ -59,31 +59,48 @@ describe("durable system recovery budget", () => { message: "Automatic recovery of saved work stopped after 1 attempts (last failure: allocation_ownership_ambiguous). " + "Retry recovery to export and merge from the retained sandbox again; no provider turn is submitted. " + "If Retry keeps failing, inspect the run and its environment." } }, runId); - expect(display?.message).toContain("Paperclip refused to recover the saved files"); + expect(display?.message).toContain("Paperclip cannot recover the unsaved files from the previous run"); expect(display?.message).not.toMatch(manual); }); - it("states a deterministic refusal plainly and never claims retries that did not happen", () => { + it("states unrecoverable saved files plainly and says Retry continues from the recorded work", () => { const refused = failSystemRecovery(reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!, new SystemRecoveryFailure("allocation_ownership_ambiguous", "service_required"), { now: 0 }); expect(refused).toMatchObject({ state: "exhausted", attempts: 1, retriesUsed: 0 }); expect(isDeterministicRecoveryRefusal(refused)).toBe(true); + expect(savedFilesUnrecoverable(refused)).toBe(true); const text = exhaustedSystemRecoveryAction(refused); - expect(text).toContain("Paperclip refused to recover the saved files"); - expect(text).toContain("Paperclip cannot prove which runs used the sandbox that holds the saved files."); + expect(text).toContain("Paperclip cannot recover the unsaved files from the previous run: Paperclip cannot prove which runs used the sandbox that holds the saved files."); expect(text).not.toContain("allocation_ownership_ambiguous"); - expect(text).toContain("Retry runs the check again. The agent's result is saved"); - // Recovery never waits on a person or a release, and never promises a - // fallback the hold may not get. - expect(text).not.toMatch(/succeeds only after|Paperclip is updated|last saved version/); + expect(text).toContain("Retry continues the task in a new workspace from its recorded work, without those files"); + expect(text).not.toMatch(/succeeds only after|Paperclip is updated|runs the check again|last saved version/); expect(text).not.toMatch(/after automatic retries|stopped after|Retry to try again/); - const once = failSystemRecovery(reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!, + const missing = failSystemRecovery(reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!, new SystemRecoveryFailure("historical_source_missing", "use_alternative_strategy"), { now: 0 }); - expect(isDeterministicRecoveryRefusal(once)).toBe(false); - expect(exhaustedSystemRecoveryAction(once)).toBe("Paperclip could not recover the agent's saved files. " + expect(isDeterministicRecoveryRefusal(missing)).toBe(false); + expect(savedFilesUnrecoverable(missing)).toBe(true); + expect(exhaustedSystemRecoveryAction(missing)).toContain("Retry continues the task in a new workspace"); + + const transient = failSystemRecovery(reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!, + new SystemRecoveryFailure("export_interrupted", "user_action_required"), { now: 0 }); + expect(savedFilesUnrecoverable(transient)).toBe(false); + expect(exhaustedSystemRecoveryAction(transient)).toBe("Paperclip could not recover the agent's saved files. " + "Press Retry to try again; the agent will not repeat its work."); - expect(exhaustedSystemRecoveryAction({ ...once, attempts: 6 })).toContain("after 6 tries"); + expect(exhaustedSystemRecoveryAction({ ...transient, attempts: 6 })).toContain("after 6 tries"); + + for (const code of ["archive_unsafe", "snapshot_verification_failed"] as const) { + const unsafe = failSystemRecovery(reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!, + new SystemRecoveryFailure(code, "service_required"), { now: 0 }); + expect(savedFilesUnrecoverable(unsafe)).toBe(false); + expect(exhaustedSystemRecoveryAction(unsafe)).toContain("Retry runs the check again"); + } + + const adoptable = failSystemRecovery(reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!, + new SystemRecoveryFailure("source_binding_invalid", "service_required"), { now: 0 }); + expect(savedFilesUnrecoverable(adoptable)).toBe(false); + expect(savedFilesUnrecoverable({ ...adoptable, attempts: 0 })).toBe(false); + expect(savedFilesUnrecoverable({ ...refused, freshAttemptRunId: randomUUID() })).toBe(false); }); it("promises an automatic retry for an unproven stop only while a re-arm remains", () => { diff --git a/server/src/services/system-recovery-state.ts b/server/src/services/system-recovery-state.ts index fd250dfe189..3fdbe6edeae 100644 --- a/server/src/services/system-recovery-state.ts +++ b/server/src/services/system-recovery-state.ts @@ -75,6 +75,7 @@ export const systemRecoverySchema = z.object({ capturePolicyVersion: z.number().int().positive().optional(), }); export type SystemRecoveryState = z.infer; +type RecoveryFailure = NonNullable; export function readSystemRecovery(evidence: Record | null | undefined): SystemRecoveryState | null { const parsed = systemRecoverySchema.safeParse(evidence?.systemRecovery); @@ -197,6 +198,10 @@ export function exhaustedSystemRecoveryAction(state: SystemRecoveryState, eviden return "Saving the agent's files stopped because the sandbox disk is full. Paperclip deletes only inactive tool " + "caches it can prove safe, and that did not free enough space. Automatic recovery does not repeat a save that fails the same way. " + "Retry saves the files again. The agent's result is kept, and the agent will not repeat its work."; + const lostFiles = unrecoverableSavedFilesFailure(state); + if (lostFiles) + return `Paperclip cannot recover the unsaved files from the previous run: ${unrecoverableSavedFilesReason(lostFiles)}. ` + + "Retry continues the task in a new workspace from its recorded work, without those files. The agent will not repeat its completed actions."; if (isDeterministicRecoveryRefusal(state)) { const code = state.failure!.code; // Never promise a release. A retained-file hold the fallback settles @@ -210,6 +215,38 @@ export function exhaustedSystemRecoveryAction(state: SystemRecoveryState, eviden + "Press Retry to try again; the agent will not repeat its work."; } +export const UNVERIFIABLE_SOURCE_CODES = ["allocation_ownership_ambiguous", "source_binding_invalid", "source_unavailable", "historical_source_missing"] as const; +export const PERMANENT_SOURCE_RETRY_CLASSES = ["service_required", "use_alternative_strategy"] as const; +const UNRECOVERABLE_SOURCE_CODES = ["allocation_ownership_ambiguous", "source_unavailable", "historical_source_missing"] as const; + +export function canAdoptNativeSourceRetention(state: SystemRecoveryState) { + return state.state === "exhausted" && state.phase === "verify_execution" && state.attempts === 1 + && state.retriesUsed === 0 && state.failure?.code === "source_binding_invalid" + && (!state.failure.sourceVerification || state.failure.sourceVerification.reason === "lease_policy_not_retained") + && state.preservationOperationIds.length === 0 && !state.freshAttemptRunId; +} + +export function unrecoverableSavedFilesFailure(state: SystemRecoveryState | null | undefined): RecoveryFailure | null { + const failure = state?.failure; + if (!state || !failure || state.state !== "exhausted" || state.freshAttemptRunId || canAdoptNativeSourceRetention(state)) return null; + return (UNRECOVERABLE_SOURCE_CODES as readonly string[]).includes(failure.code) + && (PERMANENT_SOURCE_RETRY_CLASSES as readonly string[]).includes(failure.retryClass) ? failure : null; +} + +export function savedFilesUnrecoverable(state: SystemRecoveryState | null | undefined) { + return unrecoverableSavedFilesFailure(state) !== null; +} + +function unrecoverableSavedFilesReason(failure: RecoveryFailure) { + return recoveryRefusalReasons[failure.code] ?? "a safety check on the saved files did not pass"; +} + +export function unrecoverableSavedFilesNotice(failure: RecoveryFailure) { + return `Paperclip could not recover the unsaved files from the previous run: ${unrecoverableSavedFilesReason(failure)}. ` + + "Retry continued this task in a new workspace from its recorded work: comments, documents, attachments and pushed commits. " + + "Files that run did not save or push are not included. Its completed actions stay recorded and are not repeated."; +} + export function isDeterministicRecoveryRefusal(state: Pick | null | undefined) { return state?.state === "exhausted" && state.failure?.retryClass === "service_required"; } @@ -220,4 +257,5 @@ const recoveryRefusalReasons: Partial> = { snapshot_verification_failed: "the saved snapshot of the files did not pass verification", archive_unsafe: "the saved file archive contains entries that are not safe to apply", source_unavailable: "the saved files are no longer available where the run recorded them", + historical_source_missing: "Paperclip has no record of where the run kept the saved files", }; diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 2004d4387dc..2f3d1dc02bd 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -157,7 +157,7 @@ const retryWaitMessages: Record = { // Retry starts a new recovery generation; the agent does not repeat its work. saved_work_recovery_failed: "Saving the agent's files failed. Retry to try again; the agent will not repeat its work.", saved_work_recovery_refused: "Paperclip refused to recover the saved files because a safety check on them cannot pass. " - + "Retry runs the same check again and succeeds only after Paperclip is updated. The agent's result is saved, and the agent will not repeat its work.", + + "Retry checks the saved files again. The agent's result is saved, and the agent will not repeat its work.", workspace_owner_stop_required: "Retry is waiting until Paperclip confirms that the previous server stopped saving files. The agent's result and your Retry are saved.", cleanup_quarantined: NATIVE_CLEANUP_SETTLING_MESSAGE, cancelled_by_operator: "This attempt was stopped. Send a new message to continue from saved work.", diff --git a/server/src/services/task-retry.test.ts b/server/src/services/task-retry.test.ts index 6bd15aece7a..cae346c9192 100644 --- a/server/src/services/task-retry.test.ts +++ b/server/src/services/task-retry.test.ts @@ -1022,7 +1022,7 @@ describe("durable task Retry admission", () => { const input = await seed(); const first = await requestTaskRetry(input); const state = failSystemRecovery(createSystemRecovery(input.observedRunId, first.requestId!), - new SystemRecoveryFailure("source_unavailable", "service_required")); + new SystemRecoveryFailure("export_interrupted", "user_action_required")); const [action] = await db.insert(issueRecoveryActions).values({ companyId: input.companyId, sourceIssueId: input.issueId, kind: "active_run_watchdog", ownerType: "system", cause: "legacy_execution_requires_reconciliation", fingerprint: randomUUID(), @@ -1039,11 +1039,7 @@ describe("durable task Retry admission", () => { expect(wakes.find(row => row.id === first.requestId)?.status).toBe("failed"); expect(wakes.filter(row => row.status === "deferred_issue_execution")).toHaveLength(1); }); - it("accepts Retry for an old-release retained-export hold whose source ownership could not be verified", async () => { - // State saved by an earlier release: a board-owned export hold whose only - // attempt ended with a permanent ownership refusal. The task UI now offers - // Retry for it; Retry must open one new bounded generation that re-verifies - // the source, keep the hold in place, and never resolve or bypass it. + it("accepts Retry for an old-release retained-export hold whose source ownership could not be verified, without repeating the check", async () => { const input = await seed(); const previous = { ...failSystemRecovery({ ...createSystemRecovery(input.observedRunId), revision: 4 }, new SystemRecoveryFailure("allocation_ownership_ambiguous", "service_required")), revision: 5 }; @@ -1063,15 +1059,12 @@ describe("durable task Retry admission", () => { const [updated] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)); expect(updated.status).toBe("active"); expect(updated.resolvedAt).toBeNull(); - expect(readSystemRecovery(updated.evidence)).toMatchObject({ generationId: receipt.requestId, sourceRunId: input.observedRunId, - attempts: 0, failure: null, revision: 6, freshAttemptRunId: null, preservationOperationIds: [] }); - // File recovery is queued again for the worker, which re-verifies the source. - expect(updated.evidence.workspaceRecovery).toMatchObject({ runId: input.observedRunId, status: "queued", - stage: "verifying_stop", attemptLimit: 6, retryable: true }); - // A replayed click is idempotent and does not open another generation. + expect(readSystemRecovery(updated.evidence)).toMatchObject({ generationId: previous.generationId, intentId: receipt.requestId, + state: "exhausted", failure: { code: "allocation_ownership_ambiguous" }, freshAttemptRunId: null }); + expect(updated.evidence.workspaceRecovery).toMatchObject({ status: "blocked" }); expect((await requestTaskRetry(input)).requestId).toBe(receipt.requestId); const [replayed] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)); - expect(readSystemRecovery(replayed.evidence)?.revision).toBe(6); + expect(readSystemRecovery(replayed.evidence)?.generationId).toBe(previous.generationId); }); it("binds a key to its company, task and actor across all statuses", async () => { const input = await seed(); diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index 8d76021d72d..758418e40b0 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -11,7 +11,7 @@ import { and, desc, eq, inArray, isNull, or, sql } from "drizzle-orm"; import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, issueComments, runIdentityContexts, issueRecoveryActions, type Db } from "@paperclipai/db"; import { conflict, forbidden, notFound, HttpError } from "../errors.js"; import { persistActivity } from "./activity-log.js"; -import { createSystemRecovery, readSystemRecovery } from "./system-recovery-state.js"; +import { createSystemRecovery, readSystemRecovery, savedFilesUnrecoverable } from "./system-recovery-state.js"; import { adoptSystemRecovery, updateSystemRecovery } from "./system-recovery.js"; import { instanceSettingsService } from "./instance-settings.js"; import { notifyChatActionWork } from "./chat-work-notifications.js"; @@ -484,7 +484,7 @@ function admitTaskRetry(input: TaskRetryRequest) { // A completed preservation still owns its snapshot evidence. Bind the // new intent so sibling holds can select that same continuation, without // repeating recovery or renewing its attempt budget. - if (previous.state === "complete" || previous.state === "running" || (previous.state === "exhausted" + if (previous.state === "complete" || previous.state === "running" || savedFilesUnrecoverable(previous) || (previous.state === "exhausted" && previous.failure?.retryClass === "use_alternative_strategy" && !previous.freshAttemptRunId)) { await updateSystemRecovery(db, { companyId: issue.companyId, issueId: issue.id, runId: previous.sourceRunId, actionId: action.id }, previous, { ...previous, intentId: active?.id ?? id }); diff --git a/server/src/services/unrecoverable-saved-work-retry.test.ts b/server/src/services/unrecoverable-saved-work-retry.test.ts new file mode 100644 index 00000000000..360a0fe3847 --- /dev/null +++ b/server/src/services/unrecoverable-saved-work-retry.test.ts @@ -0,0 +1,120 @@ +import { randomUUID } from "node:crypto"; +import request from "supertest"; +import { and, eq } from "drizzle-orm"; +import { describe, expect, it, vi } from "vitest"; +import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { activityLog, agentWakeupRequests, environmentLeases, heartbeatRuns, issueComments, issueRecoveryActions, issues, workspaceOperations } from "@paperclipai/db"; +import { getExecutionBlocker } from "./execution-blocker.js"; +import { heartbeatService } from "./heartbeat.js"; +import { remoteTerminationReceipt } from "./remote-execution-termination.js"; +import { createSystemRecovery, failSystemRecovery, readSystemRecovery, reserveSystemRecoveryAttempt, + SystemRecoveryFailure } from "./system-recovery-state.js"; +import { taskInterruptionPresentation } from "./task-interruption-presentation.js"; +import { action, boardRecoveryApp, db, failedWarmSource, run, useWorkspaceRecoveryFixture } from "./__tests__/workspace-recovery-fixture.js"; + +const targets = vi.hoisted(() => new Map()); +vi.mock("./environment-execution-target.js", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, resolveEnvironmentExecutionTarget: async (input: Parameters[0]) => + (input.leaseId ? targets.get(input.leaseId) : undefined) ?? actual.resolveEnvironmentExecutionTarget(input) }; +}); + +type Shape = { code: "source_unavailable" | "allocation_ownership_ambiguous" | "archive_unsafe"; retryClass: "use_alternative_strategy" | "service_required"; + sandbox: "destroyed" | "stopped" }; + +const shapes: Array<[string, Shape]> = [ + ["the sandbox that held the saved files was destroyed", { code: "source_unavailable", retryClass: "use_alternative_strategy", sandbox: "destroyed" }], + ["Paperclip cannot prove which runs used the shared sandbox", { code: "allocation_ownership_ambiguous", retryClass: "service_required", sandbox: "stopped" }], +]; + +describe("Retry of a task whose saved files cannot be recovered", () => { + useWorkspaceRecoveryFixture(targets); + + async function exhaustedExport(shape: Shape) { + const f = await failedWarmSource(true); + const spent = failSystemRecovery(reserveSystemRecoveryAttempt(createSystemRecovery(f.runId, randomUUID()))!, + new SystemRecoveryFailure(shape.code, shape.retryClass), { now: Date.now() - 60_000 }); + const held = await action(f.actionId); + await db.update(issueRecoveryActions).set({ evidence: { ...held.evidence, systemRecovery: spent, + workspaceRecovery: { ...(held.evidence.workspaceRecovery as Record), status: "blocked", retryable: true } } }) + .where(eq(issueRecoveryActions.id, f.actionId)); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success", + leasePolicy: shape.sandbox === "destroyed" ? "retain_on_failure" : "reuse_by_environment", + metadata: { ...lease.metadata, remoteExecutionTermination: remoteTerminationReceipt(lease, + { providerLeaseId: f.providerLeaseId, state: shape.sandbox }) } }).where(eq(environmentLeases.id, lease.id)); + await db.update(agentWakeupRequests).set({ status: "cancelled", finishedAt: new Date() }) + .where(eq(agentWakeupRequests.id, f.retry.requestId!)); + const [sourceLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + return { ...f, sourceLease }; + } + + async function task(issueId: string) { + return (await db.select().from(issues).where(eq(issues.id, issueId)))[0]!; + } + + async function pressRetry(f: { companyId: string; issueId: string; runId: string }) { + const receipt = (await request(boardRecoveryApp()).post(`/api/issues/${f.issueId}/retry`) + .send({ observedRunId: f.runId, idempotencyKey: randomUUID() }).expect(202)).body as { requestId: string }; + await heartbeatService(db).resumeTaskRetry(f.companyId, receipt.requestId); + return (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId)))[0]!; + } + + it.each(shapes)("offers a Retry that continues the task when %s", async (_name, shape) => { + const f = await exhaustedExport(shape); + const blocker = await getExecutionBlocker(db, f.companyId, f.issueId); + expect(blocker).toMatchObject({ cause: "native_workspace_sync_out_retry_exhausted", canRetry: true }); + expect(blocker?.nextAction).toContain("Retry continues the task in a new workspace from its recorded work"); + expect(blocker?.nextAction).not.toMatch(/runs the check again|Paperclip is updated/); + expect(await taskInterruptionPresentation(db, await task(f.issueId))).toMatchObject({ canRetry: true }); + }, 60_000); + + it.each(shapes)("starts a fresh run from the recorded work when %s", async (_name, shape) => { + const f = await exhaustedExport(shape); + const wake = await pressRetry(f); + expect(wake.payload?.executionWait).toBeUndefined(); + expect(wake).toMatchObject({ status: "coalesced", runId: expect.any(String), + payload: { taskRetry: { strategy: "fresh_isolated_workspace" } } }); + const successor = await run(wake.runId!); + expect(["queued", "running"]).toContain(successor.status); + expect(successor.contextSnapshot).toMatchObject({ taskRetryRequestId: wake.id }); + + const settled = await action(f.actionId); + expect(settled).toMatchObject({ status: "resolved" }); + expect(readSystemRecovery(settled.evidence)).toMatchObject({ state: "exhausted", freshAttemptRunId: successor.id, + failure: { code: shape.code } }); + expect(settled.nextAction).toContain("could not recover the unsaved files"); + + const notices = await db.select().from(issueComments).where(and(eq(issueComments.issueId, f.issueId), + eq(issueComments.authorType, "system"))); + expect(notices).toHaveLength(1); + expect(notices[0]!.body).toContain("Paperclip could not recover the unsaved files from the previous run"); + expect(notices[0]!.body).not.toMatch(/[0-9a-f]{8}-[0-9a-f]{4}-/); + + const [sourceLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + expect(sourceLease).toMatchObject({ status: f.sourceLease.status, leasePolicy: f.sourceLease.leasePolicy, + providerLeaseId: f.sourceLease.providerLeaseId }); + expect(await db.select({ id: activityLog.id }).from(activityLog).where(and(eq(activityLog.entityId, f.issueId), + eq(activityLog.action, "issue.recovery_generation_replaced")))).toEqual([]); + expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + eq(heartbeatRuns.retryOfRunId, f.runId)))).toEqual([]); + }, 60_000); + + it("keeps a Retry out of a fresh attempt that would import a succeeded snapshot whose archive is unsafe", async () => { + const f = await exhaustedExport({ code: "archive_unsafe", retryClass: "service_required", sandbox: "stopped" }); + const [snapshot] = await db.insert(workspaceOperations).values({ companyId: f.companyId, issueId: f.issueId, + heartbeatRunId: f.runId, phase: "workspace_recovery_snapshot", status: "succeeded", + metadata: { schema: "paperclip.workspace-recovery-snapshot.v1", leaseId: f.lease.id, + archive: { objectKey: `${f.companyId}/workspace-recovery/${f.runId}/../../escape.tar`, sha256: "0".repeat(64), byteSize: 1 } } }).returning(); + const wake = await pressRetry(f); + expect(wake.runId).toBeNull(); + expect(wake.payload?.taskRetry).not.toMatchObject({ strategy: "fresh_isolated_workspace" }); + expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + eq(heartbeatRuns.issueId, f.issueId), eq(heartbeatRuns.status, "queued")))).toEqual([]); + expect(await db.select({ id: issueComments.id }).from(issueComments).where(and(eq(issueComments.issueId, f.issueId), + eq(issueComments.authorType, "system")))).toEqual([]); + expect(readSystemRecovery((await action(f.actionId)).evidence)?.freshAttemptRunId).toBeNull(); + const [kept] = await db.select().from(workspaceOperations).where(eq(workspaceOperations.id, snapshot.id)); + expect(kept.status).toBe("succeeded"); + }, 60_000); +}); diff --git a/server/src/services/workspace-recovery-fallback-discovery.ts b/server/src/services/workspace-recovery-fallback-discovery.ts index 8fff7463d39..61e18fc2845 100644 --- a/server/src/services/workspace-recovery-fallback-discovery.ts +++ b/server/src/services/workspace-recovery-fallback-discovery.ts @@ -1,13 +1,12 @@ import { sql } from "drizzle-orm"; import { issueRecoveryActions } from "@paperclipai/db"; import { REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION } from "./remote-workspace-snapshot-archive.js"; -import { CAPTURE_POLICY_REARM_CODES, FINISHED_COORDINATOR_PHASES, isExhaustedStopProofWait, STOP_PROOF_MAX_REARMS, STOP_PROOF_RECOVERY_KEY, STOP_PROOF_RECOVERY_SCHEMA, STOP_PROOF_WAIT_CODE, +import { CAPTURE_POLICY_REARM_CODES, FINISHED_COORDINATOR_PHASES, PERMANENT_SOURCE_RETRY_CLASSES, UNVERIFIABLE_SOURCE_CODES, isExhaustedStopProofWait, STOP_PROOF_MAX_REARMS, STOP_PROOF_RECOVERY_KEY, STOP_PROOF_RECOVERY_SCHEMA, STOP_PROOF_WAIT_CODE, stopProofRearmsSpent, type SystemRecoveryState } from "./system-recovery-state.js"; import { canonicalExecutionUuid } from "./execution-identity.js"; /** Source refusals that repeat on every retry (see workspace-recovery-fallback). */ -export const UNVERIFIABLE_SOURCE_CODES = ["allocation_ownership_ambiguous", "source_binding_invalid", "source_unavailable", "historical_source_missing"] as const; -export const PERMANENT_SOURCE_RETRY_CLASSES = ["service_required", "use_alternative_strategy"] as const; +export { PERMANENT_SOURCE_RETRY_CLASSES, UNVERIFIABLE_SOURCE_CODES }; export const RETAINED_SOURCE_FALLBACK_CHECK_KEY = "retainedSourceFallbackCheck"; /** Holds of an accepted result whose copy-back could not finish. */ export const EXPORT_HOLD_CAUSES = ["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"] as const; diff --git a/server/src/services/workspace-recovery-preparation-gates.test.ts b/server/src/services/workspace-recovery-preparation-gates.test.ts index e23443fb8e1..75de3f8dc10 100644 --- a/server/src/services/workspace-recovery-preparation-gates.test.ts +++ b/server/src/services/workspace-recovery-preparation-gates.test.ts @@ -410,7 +410,7 @@ describe("automatic retained workspace recovery", () => { expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]).toMatchObject({ status: "done" }); }, 60_000); - it("names a deterministic saved-files refusal behind Retry when no fallback can continue, and another Retry shows it again", async () => { + it("continues a Retry in a fresh workspace when saved files are refused and no fallback can continue, without running the check again", async () => { const f = await acceptedRemoteExport(); const issue = (await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]; const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); @@ -420,24 +420,21 @@ describe("automatic retained workspace recovery", () => { cleanupStatus: "success", releasedAt: new Date(), metadata: { remoteCwd: path.dirname(f.remote) } }); await db.insert(workspaceOperations).values({ companyId: f.companyId, executionWorkspaceId: f.workspaceId, heartbeatRunId: f.runId, issueId: f.issueId, phase: "remote_workspace_snapshot", status: "running", metadata: { boundary: "final" } }); - await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.source.agentId, status: "running" }); const app = boardRecoveryApp(); - for (const click of [1, 2]) { - const retry = (await request(app).post(`/api/issues/${f.issueId}/retry`) - .send({ observedRunId: f.runId, idempotencyKey: randomUUID() }).expect(202)).body; - await recoverRetainedWorkspace(f.input); - expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ state: "exhausted", attempts: 1, - failure: { code: "allocation_ownership_ambiguous", retryClass: "service_required" } }); - await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!); - expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, retry.requestId!)))[0], `click ${click}`) - .toMatchObject({ runId: null, payload: { executionWait: { reason: "saved_work_recovery_refused" } } }); - const shown = await taskInterruptionPresentation(db, issue); - expect(shown).toMatchObject({ state: "failed", canRetry: true, retryWait: { reason: "saved_work_recovery_refused" } }); - expect(shown?.retryWait?.message).toContain("Paperclip refused to recover the saved files"); - expect(shown?.retryWait?.message).not.toMatch(/after automatic retries/); - expect((await getExecutionBlocker(db, f.companyId, f.issueId))?.nextAction) - .toContain("Paperclip cannot prove which runs used the sandbox that holds the saved files"); - } + const first = (await request(app).post(`/api/issues/${f.issueId}/retry`) + .send({ observedRunId: f.runId, idempotencyKey: randomUUID() }).expect(202)).body; + await recoverRetainedWorkspace(f.input); + const refused = readSystemRecovery((await action(f.actionId)).evidence); + expect(refused).toMatchObject({ state: "exhausted", attempts: 1, + failure: { code: "allocation_ownership_ambiguous", retryClass: "service_required" } }); + await heartbeatService(db).resumeTaskRetry(f.companyId, first.requestId!); + const [admitted] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, first.requestId!)); + expect(admitted.payload?.executionWait).toBeUndefined(); + expect(admitted).toMatchObject({ status: "coalesced", runId: expect.any(String), + payload: { taskRetry: { strategy: "fresh_isolated_workspace" } } }); + expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ generationId: refused!.generationId, + freshAttemptRunId: admitted.runId }); + expect(await taskInterruptionPresentation(db, issue)).not.toMatchObject({ retryWait: { reason: "saved_work_recovery_refused" } }); }, 60_000); it("preserves an explicitly requested superseded remote export without applying its result over newer work", async () => { diff --git a/server/src/services/workspace-recovery-retry-copyback.test.ts b/server/src/services/workspace-recovery-retry-copyback.test.ts index 3a05b657318..aa6cfed0f27 100644 --- a/server/src/services/workspace-recovery-retry-copyback.test.ts +++ b/server/src/services/workspace-recovery-retry-copyback.test.ts @@ -539,7 +539,7 @@ describe("automatic retained workspace recovery", () => { const stable = (row: typeof after) => ({ ...row, updatedAt: null, ownerType: null, nextAction: null, evidence: { ...row.evidence, workspaceRecovery: { ...(row.evidence.workspaceRecovery as object), message: null } } }); if (spent) { - expect(after).toMatchObject({ ownerType: "board", nextAction: expect.stringMatching(/Paperclip refused to recover the saved files: .*Retry runs the check again/) }); + expect(after).toMatchObject({ ownerType: "board", nextAction: expect.stringMatching(/Paperclip refused to recover the saved files: .*Retry runs the check again|Paperclip cannot recover the unsaved files .*Retry continues the task in a new workspace/) }); expect(stable(after)).toEqual(stable(before)); } else expect(after).toEqual(before); expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0].leasePolicy).toBe("reuse_by_environment");