diff --git a/packages/shared/src/types/execution-projection.ts b/packages/shared/src/types/execution-projection.ts index fee5ade4c8a..ce2a1e3a67a 100644 --- a/packages/shared/src/types/execution-projection.ts +++ b/packages/shared/src/types/execution-projection.ts @@ -114,4 +114,6 @@ export interface TaskInterruptionPresentation { projectAccess?: { agentId: string; projectId: string | null; ready: boolean }; sourceRunIds: string[]; observedRunId: string | null; + /** The saved-work recovery step that must finish before the task continues. */ + recoveryStep?: "checking_previous_run" | "saving_previous_files" | "finishing_recovery"; } diff --git a/server/src/services/system-recovery-state.test.ts b/server/src/services/system-recovery-state.test.ts index 96357950a6f..ad9b4e7a1b6 100644 --- a/server/src/services/system-recovery-state.test.ts +++ b/server/src/services/system-recovery-state.test.ts @@ -32,6 +32,16 @@ describe("durable system recovery budget", () => { expect(failSystemRecovery(state, new SystemRecoveryFailure("provider_unavailable", "retry_transient"), { now: 0, retryAfterAt: 31 * 60_000 }).state).toBe("exhausted"); }); + it("keeps a closed-label failure cause through serialization and drops anything else", () => { + const state = reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!; + const failed = failSystemRecovery(state, new SystemRecoveryFailure("provider_unavailable", "retry_transient"), + { now: 0, cause: { label: "sandbox_not_running", code: "ECONNRESET", status: 503 } }); + expect(readSystemRecovery(JSON.parse(JSON.stringify({ systemRecovery: failed })))?.failure) + .toMatchObject({ code: "provider_unavailable", cause: { label: "sandbox_not_running", code: "ECONNRESET", status: 503 } }); + const unsafe = failSystemRecovery(state, new SystemRecoveryFailure("provider_unavailable", "retry_transient"), + { now: 0, cause: { label: "/home/daytona/secret", code: null, status: null } }); + expect(unsafe.failure).not.toHaveProperty("cause"); + }); it("reconciles a recorded attempt after restart without spending another retry", () => { const running = reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!; const restarted = readSystemRecovery(JSON.parse(JSON.stringify({ systemRecovery: running })))!; diff --git a/server/src/services/system-recovery-state.ts b/server/src/services/system-recovery-state.ts index c1777a1a8f6..411f2cc43f2 100644 --- a/server/src/services/system-recovery-state.ts +++ b/server/src/services/system-recovery-state.ts @@ -36,6 +36,14 @@ export const workspaceSourceVerificationFailureSchema = z.object({ observedLeasePolicy: z.enum([...ENVIRONMENT_LEASE_POLICIES, "unrecognized"]).optional(), }); +/** Why an attempt failed, in closed labels only. Provider text never appears. */ +export const recoveryFailureCauseSchema = z.object({ + label: z.string().regex(/^[a-z][a-z0-9_]{0,47}$/), + code: z.string().regex(/^[A-Z][A-Z0-9_]{1,31}$/).nullable(), + status: z.number().int().min(100).max(599).nullable(), +}); +export type RecoveryFailureCause = z.infer; + export const systemRecoverySchema = z.object({ schema: z.literal("paperclip.system-recovery.v1"), policyVersion: z.literal(SYSTEM_RECOVERY_POLICY_VERSION), @@ -57,6 +65,7 @@ export const systemRecoverySchema = z.object({ retryClass: z.enum(recoveryRetryClasses), evidenceFingerprint: z.string(), sourceVerification: workspaceSourceVerificationFailureSchema.optional(), + cause: recoveryFailureCauseSchema.optional(), }).nullable(), preservationOperationIds: z.array(z.string().uuid()), freshAttemptRunId: z.string().uuid().nullable(), @@ -94,7 +103,7 @@ export function reserveSystemRecoveryAttempt(state: SystemRecoveryState, now = D /** The caller persists this timestamp. A restart must not recalculate jitter. */ export function failSystemRecovery( state: SystemRecoveryState, failure: SystemRecoveryFailure, - options: { now?: number; retryAfterAt?: number; evidenceFingerprint?: string } = {}, + options: { now?: number; retryAfterAt?: number; evidenceFingerprint?: string; cause?: RecoveryFailureCause } = {}, ): SystemRecoveryState { const now = options.now ?? Date.now(); const firstFailureAt = state.firstFailureAt ?? now; @@ -106,12 +115,14 @@ export function failSystemRecovery( const retryable = ["retry_transient", "recheck_evidence"].includes(failure.retryClass); const retry = retryable && retryNumber <= SYSTEM_RECOVERY_MAX_RETRIES && nextAttemptAt < deadlineAt; const sourceVerification = workspaceSourceVerificationFailureSchema.safeParse(failure.sourceVerification); + const cause = recoveryFailureCauseSchema.safeParse(options.cause); return { ...state, revision: state.revision + 1, firstFailureAt, deadlineAt, providerCooldownAt: options.retryAfterAt ?? null, state: retry ? failure.retryClass === "recheck_evidence" ? "awaiting_evidence" : "scheduled" : "exhausted", nextAttemptAt: retry ? nextAttemptAt : null, failure: { code: failure.code, retryClass: failure.retryClass, ...(sourceVerification.success ? { sourceVerification: sourceVerification.data } : {}), + ...(cause.success ? { cause: cause.data } : {}), evidenceFingerprint: options.evidenceFingerprint ?? createHash("sha256").update(`${state.sourceRunId}:${failure.code}`).digest("hex") }, }; } diff --git a/server/src/services/system-recovery.ts b/server/src/services/system-recovery.ts index bfc8e36f0fb..5b93c297570 100644 --- a/server/src/services/system-recovery.ts +++ b/server/src/services/system-recovery.ts @@ -48,7 +48,8 @@ export async function adoptSystemRecovery(db: Db, source: WorkspaceRecoverySourc } export function exhaustedSystemRecoveryAction(state: SystemRecoveryState): string { - const reason = state.failure?.code ? ` (last failure: ${state.failure.code})` : ""; + const cause = state.failure?.cause ? `, cause: ${state.failure.cause.label}` : ""; + const reason = state.failure?.code ? ` (last failure: ${state.failure.code}${cause})` : ""; return `Automatic recovery of saved work stopped after ${state.attempts} attempts${reason}. ` + "Retry recovery to export and merge from the retained sandbox again; no provider turn is submitted. " + "If Retry keeps failing, inspect the run and its environment."; diff --git a/server/src/services/task-interruption-presentation.test.ts b/server/src/services/task-interruption-presentation.test.ts index 0b491a1d008..1a01b402c14 100644 --- a/server/src/services/task-interruption-presentation.test.ts +++ b/server/src/services/task-interruption-presentation.test.ts @@ -174,3 +174,42 @@ describe("task project-access interruption", () => { .toMatchObject({ state: "resolved" }); }); }); + +describe("saved-work recovery step", () => { + const now = Date.parse("2026-01-01T12:00:00Z"); + const failed = { ...run, errorCode: "setup_failed", resultJson: null }; + const system = (phase: string, state: string, nextAttemptAt: number | null) => ({ + schema: "paperclip.system-recovery.v1", policyVersion: 1, sourceRunId: "33333333-3333-4333-8333-333333333333", revision: 3, + generationId: "44444444-4444-4444-8444-444444444444", intentId: null, phase, state, attemptId: null, attempts: 2, + retriesUsed: 1, firstFailureAt: now - 60_000, nextAttemptAt, deadlineAt: now + 600_000, providerCooldownAt: null, + failure: null, preservationOperationIds: [], freshAttemptRunId: null }); + const action = (evidence: Record) => ({ evidence }) as never; + + it("names the step a scheduled recovery will retry", () => { + expect(projectTaskInterruption(issue, { run: failed, agent, paused: false, approval: false, + action: action({ systemRecovery: system("verify_execution", "scheduled", now + 30_000) }) }, now)) + .toMatchObject({ state: "retry_scheduled", recoveryStep: "checking_previous_run" }); + }); + + it("names the step a running recovery is on", () => { + expect(projectTaskInterruption(issue, { run: failed, agent, paused: false, approval: false, + action: action({ systemRecovery: system("preserve_files", "running", null), systemRecoveryActiveAt: now - 1_000 }) }, now)) + .toMatchObject({ state: "working", recoveryStep: "saving_previous_files" }); + }); + + it("names recovery over a pending chat preparation and keeps failed preparation authoritative", () => { + const wait = { id: "retry", requestedAt: new Date(now), sourceRunId: failed.id, preparationStatus: "issued", preparationRetryAt: now + 250 }; + const running = action({ systemRecovery: system("preserve_files", "running", null), systemRecoveryActiveAt: now - 1_000 }); + expect(projectTaskInterruption(issue, { run: failed, agent, wait, paused: false, approval: false, action: running }, now)) + .toMatchObject({ state: "working", recoveryStep: "saving_previous_files" }); + expect(projectTaskInterruption(issue, { run: failed, agent, wait: { ...wait, preparationStatus: "failed" }, + paused: false, approval: false, action: running }, now)).toMatchObject({ state: "failed" }); + expect(projectTaskInterruption(issue, { run: failed, agent, wait, paused: false, approval: false }, now)) + .toMatchObject({ state: "retry_scheduled", nextRetryAt: new Date(now + 250).toISOString() }); + }); + + it("does not name a step for an ordinary scheduled retry", () => { + expect(projectTaskInterruption(issue, { run: { ...failed, scheduledRetryAt: new Date(now + 30_000) }, agent, + paused: false, approval: false }, now)).not.toHaveProperty("recoveryStep"); + }); +}); diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 91e074fd518..0878f2dad04 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -108,18 +108,27 @@ export function projectTaskInterruption(issue: ProjectionIssue, input: Projectio if (!ready) return { ...base, responsibility: "user", state: "waiting_for_user", userAction: "project_access", projectAccess }; } + const recoveryStep: TaskInterruptionPresentation["recoveryStep"] = !system ? undefined + : system.phase === "verify_execution" ? "checking_previous_run" + : system.phase === "preserve_files" ? "saving_previous_files" : "finishing_recovery"; + const activeObservation = action?.evidence.systemRecoveryActiveAt; + const recoveryRunning = system?.state === "running" && typeof activeObservation === "number" + && activeObservation > now - 45_000; + const recoveryScheduled = Boolean(system?.nextAttemptAt && system.nextAttemptAt > now); if (retryWait?.sourceRunId === run.id && retryWait.preparationStatus) { const deadline = retryWait.preparationRetryAt == null ? NaN : Number(retryWait.preparationRetryAt); - if (["issued", "processing"].includes(retryWait.preparationStatus) && Number.isFinite(deadline)) + const pending = ["issued", "processing"].includes(retryWait.preparationStatus) && Number.isFinite(deadline); + // Saved-work recovery gates a pending preparation, so it names the wait. + // Failed or cancelled preparation stays authoritative. + if (pending && !recoveryRunning && !recoveryScheduled) return { ...base, state: "retry_scheduled", nextRetryAt: new Date(deadline).toISOString() }; - return { ...base, state: "failed", canRetry: retryWait.preparationStatus !== "cancelled" }; + if (!pending) return { ...base, state: "failed", canRetry: retryWait.preparationStatus !== "cancelled" }; } const retryAt = system?.nextAttemptAt ?? run.scheduledRetryAt?.getTime(); if (retryAt && retryAt > now) return { ...base, state: "retry_scheduled", - nextRetryAt: new Date(retryAt).toISOString(), canRetry: true }; - const activeObservation = action?.evidence.systemRecoveryActiveAt; - if (system?.state === "running" && typeof activeObservation === "number" && - activeObservation > now - 45_000) return { ...base, state: "working" }; + nextRetryAt: new Date(retryAt).toISOString(), canRetry: true, + ...(system?.nextAttemptAt && recoveryStep ? { recoveryStep } : {}) }; + if (recoveryRunning) return { ...base, state: "working", ...(recoveryStep ? { recoveryStep } : {}) }; if (retryWait?.sourceRunId === run.id && retryWait.executionWaitReason) { const reason = Object.hasOwn(retryWaitMessages, retryWait.executionWaitReason) ? retryWait.executionWaitReason : "execution_recovery"; // Keep the latest input receipt so message submission can settle independently. diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index 635091541d0..659533fe842 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -4298,7 +4298,7 @@ describe("automatic retained workspace recovery", () => { expect(lease.metadata).not.toHaveProperty("sandboxStopAndRetain"); expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ state: "exhausted", attempts, failure: failure === "disk_full" ? { code: "source_unavailable", retryClass: "use_alternative_strategy" } - : { code: "provider_unavailable", retryClass: "retry_transient" } }); + : { code: "provider_unavailable", retryClass: "retry_transient", cause: { label: "provider_error", code: null, status: null } } }); expect(f.runtime.resumeRunLease).toHaveBeenCalledTimes(attempts); expect(await readFile(path.join(f.remote, "saved.txt"), "utf8")).toBe("finished work\n"); }, 60_000); diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index 955a20a222b..a2f98a733bd 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -10,7 +10,7 @@ import { beginIdleTrackedWork } from "./task-admission.js"; import { writeEnvironmentCleanup } from "./environment-cleanup-notifications.js"; import { resumePreservedResultFinalization } from "./preserved-result-finalization.js"; import { reclaimInactiveNativeProviderCache } from "./native-runtime/retained-provider-cache.js"; -import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL } from "./remote-workspace-transport.js"; +import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL, sandboxProviderFailureLabel } from "./remote-workspace-transport.js"; import { and, asc, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm"; import { agentWakeupRequests, environmentLeases, executionWorkspaces, heartbeatRuns, issues, issueRecoveryActions, nativeRunFinalizations, workspaceOperations, type Db } from "@paperclipai/db"; import { remoteWorkspaceRunReferenceSchema, type Environment, type EnvironmentLease } from "@paperclipai/shared"; @@ -420,12 +420,16 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ ? new SystemRecoveryFailure("source_unavailable", "use_alternative_strategy") : new SystemRecoveryFailure(current!.phase === "verify_execution" ? "provider_unavailable" : "export_interrupted", "retry_transient"); const retryAfterAt = (error as { retryAfterAt?: unknown })?.retryAfterAt; + // A generic transient code says only that the step can be retried. Keep + // why it failed in closed labels so a repeated deferral is diagnosable. + const cause = error instanceof SystemRecoveryFailure ? undefined : sandboxProviderFailureLabel(error); await updateSystemRecovery(db, input, current!, failSystemRecovery(current!, failure, { ...(typeof retryAfterAt === "number" && Number.isFinite(retryAfterAt) ? { retryAfterAt } : {}), + ...(cause ? { cause } : {}), })); if (failure.retryClass === "recheck_evidence" && ["agent_stop_unconfirmed", "copyback_stop_unconfirmed"].includes(failure.code)) await recheckStoppedWorkspaceRecovery(db, input, false); - logger.warn({ err: error, runId: input.runId, failureCode: failure.code, phase: current!.phase }, "Workspace preservation deferred; source retained"); + logger.warn({ err: error, runId: input.runId, failureCode: failure.code, failureCause: cause?.label, phase: current!.phase }, "Workspace preservation deferred; source retained"); } finally { clearInterval(observer); await observing; } }; const owned = await withNativeWorkspaceFinalizationOwnership({ db, companyId: input.companyId, runId: input.runId, diff --git a/ui/src/components/ExecutionBlockerNotice.test.tsx b/ui/src/components/ExecutionBlockerNotice.test.tsx index b4ac75f8575..6386dbe7e32 100644 --- a/ui/src/components/ExecutionBlockerNotice.test.tsx +++ b/ui/src/components/ExecutionBlockerNotice.test.tsx @@ -41,6 +41,25 @@ describe("ordinary task interruption", () => { it.each(["working", "resolved"] as const)("does not duplicate the thread indicator for %s", async state => { await render({ ...failed, state }); expect(container.textContent).toBe(""); }); + it("names the saved-work recovery step instead of a bare queue", async () => { + await render({ ...failed, state: "working", canRetry: false, recoveryStep: "saving_previous_files" }); + expect(container.textContent).toBe("Saving files from the previous attempt…"); + expect(container.querySelector("button")).toBeNull(); + }); + it("names the step a scheduled recovery will retry", async () => { + vi.useFakeTimers(); vi.setSystemTime(new Date("2026-10-20T00:00:00Z")); + await render({ ...failed, state: "retry_scheduled", nextRetryAt: "2026-10-20T00:00:30Z", recoveryStep: "checking_previous_run" }); + expect(container.textContent).toContain("Checking that the previous attempt stopped. Trying again in 30s…"); + expect(container.querySelector("button")?.textContent).toBe("Retry"); + }); + it("names the recovery step inside a provider failure notice", async () => { + vi.useFakeTimers(); vi.setSystemTime(new Date("2026-10-20T00:00:00Z")); + await render({ ...failed, state: "retry_scheduled", nextRetryAt: "2026-10-20T00:00:30Z", recoveryStep: "checking_previous_run", + providerFailure: { schema: "paperclip.provider-failure.v1", provider: "openai", phase: "startup", category: "overloaded", + retryability: "unknown", message: "Try later." } }); + expect(container.textContent).toContain("Checking that the previous attempt stopped. Trying again in 30s…"); + expect(container.textContent).not.toContain("Paperclip will retry automatically."); + }); it("leaves genuine user decisions to their specific action surface", async () => { await render({ ...failed, responsibility: "user", state: "waiting_for_user", userAction: "approval", canRetry: false }); expect(container.textContent).toBe(""); diff --git a/ui/src/components/ExecutionBlockerNotice.tsx b/ui/src/components/ExecutionBlockerNotice.tsx index 31209cc4e97..583fe53bca2 100644 --- a/ui/src/components/ExecutionBlockerNotice.tsx +++ b/ui/src/components/ExecutionBlockerNotice.tsx @@ -7,6 +7,12 @@ import { useSecondTick } from "../hooks/useSecondTick"; import { TaskProjectAccessNotice } from "./TaskProjectAccessNotice"; import { ProviderFailureNotice } from "./ProviderFailureNotice"; +const recoveryStepText: Record, string> = { + checking_previous_run: "Checking that the previous attempt stopped", + saving_previous_files: "Saving files from the previous attempt", + finishing_recovery: "Finishing recovery of saved work", +}; + /** One ordinary interruption surface. Technical records remain in diagnostics. */ export function ExecutionBlockerNotice({ presentation, canRetry, onRetry, optimistic, takingLonger, error, className, issueId, agentName, projectName }: { presentation: TaskInterruptionPresentation | null | undefined; @@ -26,15 +32,22 @@ export function ExecutionBlockerNotice({ presentation, canRetry, onRetry, optimi canRetry={canRetry && presentation.canRetry} onRetry={onRetry} error={error} className={className} />; if (error && !optimistic && !takingLonger) return
{describeError(error).body}{canRetry && }
; if (takingLonger) return
Taking longer than usual…{canRetry && }
; + const step = !optimistic && presentation?.responsibility === "system" && presentation.recoveryStep + && ["working", "retry_scheduled"].includes(presentation.state) ? recoveryStepText[presentation.recoveryStep] : null; + const seconds = Math.max(0, Math.ceil((Date.parse(presentation?.nextRetryAt ?? "") - Date.now()) / 1000)); + const retryIn = `Trying again in ${seconds > 60 ? `${Math.ceil(seconds / 60)}m` : `${seconds}s`}…`; if (!optimistic && !presentation?.retryWait && presentation?.providerFailure && ["failed", "retry_scheduled", "waiting_for_user"].includes(presentation.state)) { + // Saved-work recovery is what the task waits for, so name it over the + // generic automatic-retry line while keeping the provider explanation. return Retry : undefined}> - {presentation.state === "retry_scheduled" &&

Paperclip will retry automatically.

} + {presentation.state === "retry_scheduled" &&

{step ? `${step}. ${retryIn}` : "Paperclip will retry automatically."}

} ; } + if (step && presentation?.state === "working") return
{`${step}…`}
; if (optimistic || !presentation || presentation.responsibility !== "system" || !["failed", "retry_scheduled"].includes(presentation.state)) return null; - const seconds = Math.max(0, Math.ceil((Date.parse(presentation.nextRetryAt ?? "") - Date.now()) / 1000)); return (
{presentation.state === "failed" &&