diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 5f4951a9620..5df3d2d2b83 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -4648,7 +4648,16 @@ recovery then reclaims the retained sandbox's inactive provider cache and saves files again with no provider turn. The export Retry stays available. A read-only fallback that only lost its connection (a closed transport, a provider 5xx or rate limit, or a sandbox state change) reports its own cause instead, and finalization -keeps its bounded retry for it. +keeps its bounded retry for it. When saved-work recovery's own save still finds the +disk full after a confirmed reclaim, or with nothing to reclaim, the recovery +generation ends at once as +`export_interrupted` with cause `sandbox_disk_full` (retry class +`user_action_required`) and goes to the board. It is not retried on the automatic +schedule, and it never falls back to the last save, so the run's unsaved files stay +in the retained sandbox and the workspace keeps its previous saved version. The +board's Retry saves again; once the save succeeds, the accepted Retry starts one run. +A reclaim whose outcome is unknown keeps the scheduled retry, because its journal +resumes on the next attempt and can still free space. Native assigned app tools save governed responses above 500,000 encoded bytes before returning a bounded reference through PRP. `read_tool_result` retrieves the saved JSON in diff --git a/server/src/services/native-runtime/retained-provider-cache.ts b/server/src/services/native-runtime/retained-provider-cache.ts index eccd1efcba4..2386ebcbf1f 100644 --- a/server/src/services/native-runtime/retained-provider-cache.ts +++ b/server/src/services/native-runtime/retained-provider-cache.ts @@ -162,10 +162,13 @@ function readInventory(value: unknown, runId: string, expected: string): Invento return parsed; } +export type NativeProviderCacheReclaim = { changed: boolean; confirmed: boolean }; + export async function reclaimInactiveNativeProviderCache(input: { db: Db; companyId: string; issueId: string; runId: string; workspaceId: string; target: RemoteWorkspaceTarget; assertOwned: () => Promise; -}): Promise { +}): Promise { + const nothing = { changed: false, confirmed: true } as const; await input.assertOwned(); const [source] = await input.db.select({ run: heartbeatRuns, coordinator: nativeRunFinalizations }) .from(heartbeatRuns).innerJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), @@ -175,9 +178,9 @@ export async function reclaimInactiveNativeProviderCache(input: { if (!source || source.run.status !== "failed" || source.run.runtimeMode !== "native" || (source.run.nativeIssueId ?? source.run.issueId ?? source.run.contextSnapshot?.issueId) !== input.issueId || source.coordinator.phase !== "terminal_failure" || source.coordinator.leaseOwner !== null - || record(source.coordinator.failureDetail).recoverable === true) return false; + || record(source.coordinator.failureDetail).recoverable === true) return nothing; const expected = verifiedCacheDigest(source.run, input.workspaceId, input.target); - if (!expected) return false; + if (!expected) return nothing; const scope = and(eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.issueId, input.issueId), eq(workspaceOperations.heartbeatRunId, input.runId), eq(workspaceOperations.executionWorkspaceId, input.workspaceId)); const execute = async (mode: "inventory" | "stage" | "reclaim", @@ -200,7 +203,7 @@ export async function reclaimInactiveNativeProviderCache(input: { .orderBy(asc(workspaceOperations.startedAt), asc(workspaceOperations.id)).limit(1); if (!prepared) { const inventory = readInventory(await execute("inventory"), input.runId, expected); - if (!inventory.targets.length) return false; + if (!inventory.targets.length) return nothing; await input.assertOwned(); [prepared] = await withDedicatedDbConnection(input.db, journal => journal.transaction(async tx => { await tx.execute(sql`select set_config('statement_timeout', '10000', true), set_config('lock_timeout', '2000', true)`); @@ -222,7 +225,7 @@ export async function reclaimInactiveNativeProviderCache(input: { eq(workspaceOperations.phase, "native_provider_cache_reclaim"), eq(workspaceOperations.status, "succeeded"), sql`${workspaceOperations.metadata}->>'preparationOperationId' = ${preparationOperationId}`, sql`${workspaceOperations.metadata}->>'outcome' = 'evicted'`)).limit(1); - if (completed) return false; + if (completed) return nothing; let staged: Staged = []; if (inventory.targets.some(target => target.generation.startsWith("c:"))) { if (prepared.metadata.staged === undefined) { @@ -257,5 +260,5 @@ export async function reclaimInactiveNativeProviderCache(input: { await input.db.insert(workspaceOperations).values({ companyId: input.companyId, issueId: input.issueId, heartbeatRunId: input.runId, executionWorkspaceId: input.workspaceId, phase: "native_provider_cache_reclaim", status: confirmed ? "succeeded" : "failed", startedAt, finishedAt: new Date(), metadata }); - return changed; + return { changed, confirmed }; } diff --git a/server/src/services/system-recovery-state.ts b/server/src/services/system-recovery-state.ts index e5492404584..ffe8e22dfa7 100644 --- a/server/src/services/system-recovery-state.ts +++ b/server/src/services/system-recovery-state.ts @@ -194,6 +194,10 @@ export function exhaustedSystemRecoveryAction(state: SystemRecoveryState, eviden return "Recovery of saved work is waiting to confirm that the previous run stopped. " + "Paperclip tries again automatically, and the agent will not repeat its work."; const cause = state.failure?.cause ? `, cause: ${state.failure.cause.label}` : ""; + if (state.failure?.cause?.label === "sandbox_disk_full") + return "Saving the agent's files stopped because the sandbox disk is full. Paperclip deletes only inactive tool " + + "caches it can prove safe, and that did not free enough space. Automatic recovery does not repeat a save that fails the same way. " + + "Retry saves the files again. The agent's result is kept, and the agent will not repeat its work."; if (isDeterministicRecoveryRefusal(state)) { const code = state.failure!.code; // Never promise a release. A retained-file hold the fallback settles diff --git a/server/src/services/workspace-recovery-disk-full.test.ts b/server/src/services/workspace-recovery-disk-full.test.ts new file mode 100644 index 00000000000..e954a781c75 --- /dev/null +++ b/server/src/services/workspace-recovery-disk-full.test.ts @@ -0,0 +1,100 @@ +import { randomUUID } from "node:crypto"; +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import request from "supertest"; +import { and, eq } from "drizzle-orm"; +import { agentWakeupRequests, executionWorkspaces, heartbeatRuns, issues } from "@paperclipai/db"; +import { heartbeatService } from "./heartbeat.js"; +import { describe, expect, it, vi } from "vitest"; +import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { readSystemRecovery } from "./system-recovery-state.js"; +import { recoverRetainedWorkspace } from "./workspace-recovery.js"; +import { readWorkspaceRecovery } from "./workspace-recovery-receipt.js"; +import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL } from "./remote-workspace-transport.js"; +import { getExecutionBlocker } from "./execution-blocker.js"; +import { taskInterruptionPresentation } from "./task-interruption-presentation.js"; +import { createSavedWorkspaceLoader, readRemoteWorkspaceAuthority } from "./remote-workspaces.js"; +import { useWorkspaceRecoveryFixture, acceptedRemoteExport, action, db, receiptOnlyRecoveryApp } from "./__tests__/workspace-recovery-fixture.js"; +const targets = vi.hoisted(() => new Map()); +vi.mock("./environment-execution-target.js", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, resolveEnvironmentExecutionTarget: async (input: Parameters[0]) => + (input.leaseId ? targets.get(input.leaseId) : undefined) ?? actual.resolveEnvironmentExecutionTarget(input) }; +}); + +describe("saved-work recovery on a full sandbox disk", () => { + useWorkspaceRecoveryFixture(targets); + + it("ends recovery at the first full-disk save with a named cause and a Retry that saves once there is room", async () => { + const f = await acceptedRemoteExport(); + const target = targets.get(f.lease.id)!, syncOut = target.runner!.syncOut!; + let full = true, saves = 0; + target.runner!.syncOut = async operations => { + saves++; + if (full) throw new Error(REMOTE_WORKSPACE_SANDBOX_DISK_FULL); + return syncOut(operations); + }; + const workspaceBefore = (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.workspaceId)))[0]; + + await recoverRetainedWorkspace(f.input); + const failed = readSystemRecovery((await action(f.actionId)).evidence); + expect(failed).toMatchObject({ state: "exhausted", nextAttemptAt: null, attempts: 1, failure: { + code: "export_interrupted", retryClass: "user_action_required", + cause: { label: "sandbox_disk_full", code: "ENOSPC", status: null } } }); + const savesAtFailure = saves; + expect(savesAtFailure).toBeGreaterThan(0); + + await recoverRetainedWorkspace(f.input); + await recoverRetainedWorkspace(f.input); + expect(saves).toBe(savesAtFailure); + const handedOff = await action(f.actionId); + expect(handedOff).toMatchObject({ status: "active", ownerType: "board" }); + expect(handedOff.nextAction).toMatch(/sandbox disk is full/); + expect(handedOff.nextAction).toMatch(/Retry saves the files again/); + + const workspaceAfter = (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.workspaceId)))[0]; + expect(readRemoteWorkspaceAuthority(workspaceAfter.metadata)?.snapshotId) + .toBe(readRemoteWorkspaceAuthority(workspaceBefore.metadata)?.snapshotId ?? null); + expect(await readFile(path.join(f.remote, "saved.txt"), "utf8")).toBe("finished work\n"); + const staging = path.join(f.remote, ".paperclip-runtime", "workspace-snapshots"); + for (const attempt of await readdir(staging).catch(() => [] as string[])) + expect((await readdir(path.join(staging, attempt))).filter(name => name.startsWith("capture-"))).toEqual([]); + + const issue = (await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]; + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ canRetry: true, observedRunId: f.runId }); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ runId: f.runId, + workspaceRecovery: { canRetry: true } }); + const runsBefore = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)); + + full = false; + const app = receiptOnlyRecoveryApp(); + const receipt = (await request(app).post(`/api/issues/${f.issueId}/retry`) + .send({ observedRunId: f.runId, idempotencyKey: randomUUID() }).expect(202)).body; + expect(receipt).toMatchObject({ accepted: true, observedRunId: f.runId }); + await recoverRetainedWorkspace(f.input); + const recovered = await action(f.actionId); + expect(readWorkspaceRecovery(recovered.evidence)?.status).toBe("recovered"); + expect(readSystemRecovery(recovered.evidence)?.preservationOperationIds).toHaveLength(1); + const authority = readRemoteWorkspaceAuthority((await db.select().from(executionWorkspaces) + .where(eq(executionWorkspaces.id, f.workspaceId)))[0].metadata)!; + const saved = await createSavedWorkspaceLoader(db)({ companyId: f.companyId, workspaceId: f.workspaceId, + snapshotId: authority.snapshotId! }); + try { expect(await readFile(path.join(saved.root, "saved.txt"), "utf8")).toBe("finished work\n"); } + finally { await saved.cleanup(); } + const savesAfterRetry = saves; + await recoverRetainedWorkspace(f.input); + expect(saves).toBe(savesAfterRetry); + expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))) + .toEqual(runsBefore); + const heartbeat = heartbeatService(db); + await Promise.all([heartbeat.resumeTaskRetry(f.companyId, receipt.requestId), + heartbeat.resumeTaskRetry(f.companyId, receipt.requestId)]); + await heartbeat.resumeTaskRetry(f.companyId, receipt.requestId); + const successors = (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))) + .filter(row => row.id !== f.runId); + expect(successors).toHaveLength(1); + expect(successors[0].contextSnapshot).toMatchObject({ previousRunId: f.runId, taskRetryRequestId: receipt.requestId }); + expect(await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, f.companyId), + eq(agentWakeupRequests.idempotencyKey, `task-retry-dispatch:${receipt.requestId}`)))).toHaveLength(1); + }, 60_000); +}); diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index 2bc45c172ab..31177ac4c57 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -44,6 +44,8 @@ import { settleRecoveredWorkspaceRestoreHolds } from "./workspace-restore-hold-s const exportCauses = ["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"]; const terminal = ["failed", "interrupted", "timed_out", "cancelled", "succeeded"]; const sandboxDiskFull = /No space left on device|Disk quota exceeded/i; +export const SANDBOX_DISK_FULL_RECOVERY_CAUSE = { label: "sandbox_disk_full", code: "ENOSPC", status: null } as const; +const isSandboxDiskFullSave = (error: unknown) => error instanceof Error && error.message === REMOTE_WORKSPACE_SANDBOX_DISK_FULL; type Source = { companyId: string; issueId: string; runId: string; actionId: string }; type StopEvidenceSource = Omit & { actionId?: string }; @@ -258,6 +260,7 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,attemptId}' = ${reserved.attemptId}`)); }; let observing = Promise.resolve(); + let cacheReclaimUnconfirmed = false; const observer = setInterval(() => { observing = observing.then(observeActive).catch(() => undefined); }, 15_000); try { await observeActive(); @@ -368,13 +371,18 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ let saved: Awaited>; try { saved = await save(); } catch (error) { - if (!(error instanceof Error) || error.message !== REMOTE_WORKSPACE_SANDBOX_DISK_FULL) throw error; + if (!isSandboxDiskFullSave(error)) throw error; let reclaimed = false; try { - reclaimed = await reclaimInactiveNativeProviderCache({ db, companyId: input.companyId, + const reclaim = await reclaimInactiveNativeProviderCache({ db, companyId: input.companyId, issueId: input.issueId, runId: input.runId, workspaceId: workspace.id, target, assertOwned: ownership.assertHeld }); - } catch { logger.warn({ runId: input.runId }, "Retained provider cache cleanup could not be confirmed"); } + reclaimed = reclaim.changed; + cacheReclaimUnconfirmed = !reclaim.confirmed; + } catch { + cacheReclaimUnconfirmed = true; + logger.warn({ runId: input.runId }, "Retained provider cache cleanup could not be confirmed"); + } if (!reclaimed) throw error; await ownership.assertHeld(); saved = await save(); @@ -471,7 +479,9 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ return; } const classification = classifyNativeWorkspaceFailure(error); + const diskFull = !(error instanceof SystemRecoveryFailure) && !cacheReclaimUnconfirmed && isSandboxDiskFullSave(error); const failure = error instanceof SystemRecoveryFailure ? error + : diskFull ? new SystemRecoveryFailure("export_interrupted", "user_action_required") : classifyWorkspaceRestoreFailure(error) === "restore_invalid_metadata" ? new SystemRecoveryFailure("snapshot_verification_failed", "service_required") : classification.code === "workspace_sync_out_unsafe_archive" @@ -485,7 +495,7 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ // A classified failure already names its subsystem. const generic = !(error instanceof SystemRecoveryFailure) && failure.retryClass === "retry_transient" && (failure.code === "provider_unavailable" || failure.code === "export_interrupted"); - const cause = generic ? sandboxProviderFailureLabel(error) : undefined; + const cause = diskFull ? { ...SANDBOX_DISK_FULL_RECOVERY_CAUSE } : generic ? sandboxProviderFailureLabel(error) : undefined; const failed = failSystemRecovery(current!, failure, { ...(typeof retryAfterAt === "number" && Number.isFinite(retryAfterAt) ? { retryAfterAt } : {}), ...(cause ? { cause } : {}),