diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index aaf5361224e..a454d073ee1 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -4565,7 +4565,15 @@ That transfer can stream without another sandbox archive. Provider exclusions ca also omit a nested entry with the same name; credentials remain excluded. Other copy failures and timeouts still fail. Workspace capture itself still needs free sandbox space. Its failed save operation names `remote_workspace_sandbox_disk_full` -when that capacity is exhausted, without changing retry or ownership rules. +when that capacity is exhausted, without changing ownership rules. Finalization +does not retry that copy-back: nothing frees the sandbox between its attempts. +The first full-disk failure ends finalization as +`native_workspace_sync_out_retry_exhausted` and keeps the accepted result. Saved-work +recovery then reclaims the retained sandbox's inactive provider cache and saves the +files again with no provider turn. The export Retry stays available. A read-only +fallback that only lost its connection (a closed transport, a provider 5xx or rate +limit, or a sandbox state change) reports its own cause instead, and finalization +keeps its bounded retry for it. Native assigned app tools save governed responses above 500,000 encoded bytes before returning a bounded reference through PRP. `read_tool_result` retrieves the saved JSON in diff --git a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts index 7ec2ea198b1..213fae33bf7 100644 --- a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts +++ b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts @@ -66,6 +66,7 @@ import { restoreNativeWorkspaceBestEffort } from "./native-workspace-best-effort import { reconcileNativeFinalizations } from "./native-finalization-reconciler.js"; import { recoverLegacyUnsafeWorkspaceExports } from "./native-workspace-export-recovery.js"; import { finalizeNativeRun, recordNativeFinalizationFailure } from "./native-run-finalizer.js"; +import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL } from "../remote-workspace-transport.js"; import { waitForPendingRunFailureReports } from "../run-failure-report.js"; import { deliverExecutionStatuses } from "../execution-status-delivery.js"; import { commitNativeStatusDecision } from "./status-decision-committer.js"; @@ -902,6 +903,52 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a expect(run?.error).toMatch(/recovery action/); }); + it("hands a full sandbox disk to saved-work recovery on the first copy-back failure", async () => { + // PAP-4253: three copy-back attempts each hit the same full disk (about 15 + // minutes apiece across deploys). Nothing frees the sandbox between them; + // saved-work recovery reclaims the inactive provider cache, then saves. + const fixture = await seedNativeRun(); + await driveToCompleteResult(fixture); + const [accepted] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, fixture.runId)); + expect(accepted).toBeDefined(); + const wakeupsBefore = await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests) + .where(eq(agentWakeupRequests.agentId, agentId)); + const failure = await recordNativeFinalizationFailure({ + db, runId: fixture.runId, error: new Error("native_workspace_sync_out_failed"), + projectRunStatus: true, failureScope: "workspace", failureCause: REMOTE_WORKSPACE_SANDBOX_DISK_FULL, + }); + expect(failure).toMatchObject({ phase: "terminal_failure", failureCode: "native_workspace_sync_out_retry_exhausted", + nextAttemptAt: null }); + const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(coordinator.failureDetail).toMatchObject({ recoveryOwner: { kind: "board" }, workspaceFinalizeAttempt: 1, + failureCause: REMOTE_WORKSPACE_SANDBOX_DISK_FULL }); + // The accepted result stays linked for the export recovery. + expect(coordinator.resultId).toBe(accepted.id); + expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, fixture.runId))).toEqual([accepted]); + const run = await readRunError(fixture.runId); + expect(run).toMatchObject({ status: "failed", errorCode: "native_workspace_sync_out_retry_exhausted" }); + expect(run?.error).toMatch(/sandbox disk is full/); + expect(run?.error).not.toMatch(/after 3 attempts/); + // The export hold owns the next step; no provider turn is requested. + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId))) + .toEqual([expect.objectContaining({ status: "active", ownerType: "board", cause: "native_workspace_sync_out_retry_exhausted" })]); + expect(await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests) + .where(eq(agentWakeupRequests.agentId, agentId))).toEqual(wakeupsBefore); + }); + + it("keeps other copy-back failures on their bounded retry", async () => { + const fixture = await seedNativeRun(); + await db.insert(nativeRunFinalizations).values({ + runId: fixture.runId, companyId, issueId: fixture.issueId, phase: "observed", attempt: 1, + }); + const failure = await recordNativeFinalizationFailure({ + db, runId: fixture.runId, error: new Error("native_workspace_sync_out_failed"), + projectRunStatus: true, failureScope: "workspace", + failureCause: "remote_workspace_capture_failed: stage=archive code=ECONNRESET", + }); + expect(failure).toMatchObject({ phase: "retryable_failure", failureCode: "native_workspace_sync_out_failed" }); + }); + it("does not attach a finalization error to a provider-succeeded run when provider retries are exhausted", async () => { const fixture = await seedNativeRun(); await db.update(heartbeatRuns).set({ resultJson: { prpRunTerminalState: "succeeded", finalizationFailureCount: 2 } }) diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index a29e848e59c..74e5b1faf37 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -1,5 +1,6 @@ import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js"; import { isContextCompactionRun } from "../context-compaction.js"; +import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL_CODE } from "../remote-workspace-transport.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyNativeRecoveryNotices } from "../native-recovery-notice-notifications.js"; import { resolveCommittedNativeRecoveryNotices } from "./native-recovery-notices.js"; @@ -309,6 +310,11 @@ export function readWorkspaceFailureCause(value: unknown): string | null { function terminalFinalizationFailureMessage(failureCode: string, failureScope: "provider" | "workspace" | undefined, failureCause?: string | null) { + if (failureCode === "native_workspace_sync_out_retry_exhausted" && isSandboxStorageExhausted(failureCause)) { + return "Saving the workspace stopped because the sandbox disk is full. The agent's result is kept, and Paperclip " + + "saves the files again from the retained sandbox without repeating the agent's work; follow the run's recovery " + + `action if that cannot free enough space. Last cause: ${failureCause}`; + } if (failureCode === "native_workspace_sync_out_retry_exhausted") { // This code also covers local-directory and plugin workspaces, so the // message stays general and defers to the recovery action's next step. @@ -326,6 +332,16 @@ function terminalFinalizationFailureMessage(failureCode: string, failureScope: " : `Native run finalization failed (${failureCode}).`; } +/** + * A full sandbox disk stops every copy-back the same way: the archive is built + * in that sandbox, and nothing frees space between finalizer attempts. It is + * recognised on the first failure and handed to saved-work recovery, which can + * reclaim the retained sandbox's inactive provider cache before saving again. + */ +function isSandboxStorageExhausted(failureCause: string | null | undefined) { + return typeof failureCause === "string" && failureCause.startsWith(`${REMOTE_WORKSPACE_SANDBOX_DISK_FULL_CODE}:`); +} + const NATIVE_FINALIZATION_RETRY_LIMIT = 3; /** * Contention while applying a saved, accepted, provider-succeeded result is @@ -476,6 +492,7 @@ async function recordRetryableFailure(input: { : readFinalizationFailureCount(input.run.resultJson) + 1; const budgetExhausted = input.permanent === true || + (input.failureScope === "workspace" && isSandboxStorageExhausted(failureCause)) || (workspaceFinalizeAttempt !== null ? workspaceFinalizeAttempt >= 3 : finalizationFailureCount! >= NATIVE_FINALIZATION_RETRY_LIMIT); diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index 945dbcbcc37..1c46c97259e 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -16,6 +16,7 @@ import { prepareRemoteRepository, prepareRemoteWorkspaceDirectory, remoteWorkspaceFailureCause, + REMOTE_WORKSPACE_SANDBOX_DISK_FULL, removeRemoteRecoveryAttachment, restoreRemoteWorkspaceDirectory, type RemoteWorkspaceTarget, @@ -1056,6 +1057,28 @@ fs.unlinkSync(${JSON.stringify(kill)});process.kill(process.pid,'SIGKILL');}retu } }); + it.each([ + ["socket hang up", "transport_closed", true], + ["503 service unavailable", "provider_unavailable", true], + ["synthetic transfer unavailable", "provider_error", false], + ] as const)("keeps a full disk only when the read-only fallback did not just lose its connection (%s)", async (thrown, label, transient) => { + await fs.writeFile(path.join(workspace, "saved.txt"), "retained work\n"); + const { remote, execute } = target(); + execute.mockResolvedValueOnce({ exitCode: 1, stdout: "", stderr: REMOTE_WORKSPACE_SANDBOX_DISK_FULL, + timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() }); + remote.runner!.syncOut = async () => { throw new Error(thrown); }; + const failure = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }) + .then(() => { throw new Error("expected the capture to fail"); }, (error: Error) => error); + const fallback = `remote_workspace_readonly_capture_failed: stage=transfer reason=operation_failed transport=${label}`; + // Finalization retries a dropped connection; a full disk goes straight to saved-work recovery. + if (transient) { + expect(failure.message).toBe(fallback); + // The save records where the fallback stopped. + expect(remoteWorkspaceFailureCause(failure)).toBe(fallback); + } else expect(failure).toMatchObject({ message: REMOTE_WORKSPACE_SANDBOX_DISK_FULL, cause: { message: fallback } }); + expect(await fs.readFile(path.join(workspace, "saved.txt"), "utf8")).toBe("retained work\n"); + }); + it.each(["source changed", "copy changed", "extra file"])("rejects a read-only fallback when %s", async fault => { await fs.writeFile(path.join(workspace, "saved.txt"), "retained work\n"); const { remote, execute } = target(); diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 75248a2cf38..42ecdfd82bd 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -37,8 +37,9 @@ export function requireRemoteWorkspaceTarget(target: AdapterExecutionTarget | nu } /** The workspace archive is built inside the sandbox, so a full sandbox disk stops the save. */ +export const REMOTE_WORKSPACE_SANDBOX_DISK_FULL_CODE = "remote_workspace_sandbox_disk_full"; export const REMOTE_WORKSPACE_SANDBOX_DISK_FULL = - "remote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace"; + `${REMOTE_WORKSPACE_SANDBOX_DISK_FULL_CODE}: the sandbox disk is full; free space in the sandbox before saving the workspace`; // Node reports ENOSPC or EDQUOT. Git and tar child processes report the text. const DISK_FULL_CHECK = String.raw`const diskFull=e=>['ENOSPC','EDQUOT'].includes(e?.code)||/No space left on device|Disk quota exceeded/i.test(String(e?.stderr??'')+String(e?.message??''));`; const CAPTURE_FAILURE_CHECK = `const captureFailure=${remoteWorkspaceCaptureFailureMessage.toString()};const failureDetail=${remoteWorkspaceFailureDetail.toString()};`; @@ -88,11 +89,16 @@ export class RemoteWorkspaceOperationError extends Error { * The server-built diagnostic for a remote workspace failure, or null when the * error text is not one of these redacted forms and must stay off the record. */ +const READONLY_CAPTURE_FAILURE_CAUSE = + /^remote_workspace_readonly_capture_failed: stage=(?:inspect|transfer|verify|archive) reason=[a-z_]{1,32} transport=[a-z_]{1,48}(?: code=[A-Z][A-Z0-9_]{1,31})?$/; + export function remoteWorkspaceFailureCause(error: unknown): string | null { if (error instanceof RemoteWorkspaceOperationError || error instanceof RemoteWorkspaceOperationTimeoutError || error instanceof RemoteWorkspaceRestoreError) return error.message; if (!(error instanceof Error)) return null; if (remoteWorkspaceCaptureFailureMessage({ message: error.message }) === error.message) return error.message; + // A read-only capture failure is server-built from closed fields. + if (READONLY_CAPTURE_FAILURE_CAUSE.test(error.message)) return error.message; const operation = providerRefusals.get(error); return operation ? remoteWorkspaceProviderRefusalCause(operation, error) : null; } @@ -487,10 +493,23 @@ export async function captureRemoteWorkspaceArchive(input: { // it, then normalize and seal it on the controller. Failed verification // retains the original disk-full failure and never permits source cleanup. try { return await captureReadOnlyWorkspaceArchive(input); } - catch (cause) { throw new Error(error.message, { cause }); } + catch (cause) { + // The read-only copy writes nothing to the sandbox. When only its + // connection dropped, a later copy-back can succeed, so finalization + // keeps its bounded retry. Any other fallback failure keeps the + // disk-full code, which finalization does not retry. + if (isTransientReadOnlyCaptureFailure(cause)) throw cause; + throw new Error(error.message, { cause }); + } } } +const TRANSIENT_READONLY_CAPTURE_FAILURE = + /^remote_workspace_readonly_capture_failed: stage=[a-z]+ reason=operation_failed transport=(?:transport_closed|provider_unavailable|provider_rate_limited|sandbox_state_transition)(?: |$)/; +function isTransientReadOnlyCaptureFailure(error: unknown) { + return error instanceof Error && TRANSIENT_READONLY_CAPTURE_FAILURE.test(error.message); +} + /** Captures a sandbox directory without writing to the sandbox. The copy is * verified against an inventory taken before and after the transfer. */ export async function captureReadOnlyWorkspaceArchive(input: {