diff --git a/server/src/__tests__/heartbeat-retry-scheduling.test.ts b/server/src/__tests__/heartbeat-retry-scheduling.test.ts index 55e75d888ec..e29d44ef727 100644 --- a/server/src/__tests__/heartbeat-retry-scheduling.test.ts +++ b/server/src/__tests__/heartbeat-retry-scheduling.test.ts @@ -265,6 +265,22 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0); }); + it("does not repeat a run that filled its sandbox disk on a fresh sandbox of the same size", async () => { + const runId = randomUUID(), companyId = randomUUID(), agentId = randomUUID(); + const now = new Date("2026-04-20T12:00:00.000Z"); + await seedRetryFixture({ runId, companyId, agentId, now, errorCode: "sandbox_disk_full" }); + expect(await heartbeat.scheduleBoundedRetry(runId, { now, random: () => 0 })) + .toMatchObject({ outcome: "not_scheduled", reason: expect.stringContaining("disk") }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0); + }); + + it("still retries a runner that exited for another reason", async () => { + const runId = randomUUID(), companyId = randomUUID(), agentId = randomUUID(); + const now = new Date("2026-04-20T12:00:00.000Z"); + await seedRetryFixture({ runId, companyId, agentId, now, errorCode: "native_runner_process_exited" }); + expect(await heartbeat.scheduleBoundedRetry(runId, { now, random: () => 0 })).toMatchObject({ outcome: "scheduled" }); + }); + it.each(["restore_unsafe_archive", "restore_lock_timeout"])("keeps the existing retry budget for %s", async (classification) => { const runId = randomUUID(), companyId = randomUUID(), agentId = randomUUID(); const now = new Date("2026-04-20T12:00:00.000Z"); diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 6e5ee8e85dc..fd7fdeefb83 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -91,6 +91,7 @@ import { import type { AppendHeartbeatRunEventInput } from "../heartbeat-run-events.js"; import type { evaluateAgentInvokabilityFromDb } from "../agent-invokability.js"; import { currentRunFailureText } from "../native-runtime/native-exact-resume-failure.js"; +import { NATIVE_SANDBOX_DISK_FULL, NATIVE_SANDBOX_DISK_FULL_ACTION } from "../native-runtime/native-sandbox-disk-full.js"; import type { createHeartbeatRunState } from "./run-state.js"; import type { createHeartbeatRunPreparation } from "./run-preparation.js"; import type { createRunDispatch, PostCommitEffect } from "../../modules/run-dispatch/index.js"; @@ -537,6 +538,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen || ["openai_dot", "muse"].includes(String(parseObject(parseObject(parseObject(run.runnerProfileJson).nativeExecutionInput).provider).kind))) { return { outcome: "not_scheduled" as const, reason: "Dot external execution must be reconciled before a new assignment; Paperclip cannot confirm its external stop.", issueId: readNonEmptyString(run.contextSnapshot?.issueId) }; } + if (run.errorCode === NATIVE_SANDBOX_DISK_FULL) { + return { outcome: "not_scheduled" as const, reason: NATIVE_SANDBOX_DISK_FULL_ACTION, + issueId: readNonEmptyString(run.contextSnapshot?.issueId) }; + } if (run.errorCode === "provider_tool_definition_invalid") { return { outcome: "not_scheduled" as const, reason: "Repair the invalid tool definitions before starting a new attempt.", diff --git a/server/src/services/native-runtime/native-sandbox-disk-full.test.ts b/server/src/services/native-runtime/native-sandbox-disk-full.test.ts new file mode 100644 index 00000000000..14e5acdcbe1 --- /dev/null +++ b/server/src/services/native-runtime/native-sandbox-disk-full.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { NativeProviderTerminalFailure, NativeSessionProtocolIntegrityError, SANDBOX_DISK_FULL_FAILURE_CODE } from "../../vendor/paperclip-runner/index.js"; +import { nativeSessionFailureDisposition, nativeSessionFailureSourceCode } from "./native-session-executor.js"; +import { NATIVE_SANDBOX_DISK_FULL } from "./native-sandbox-disk-full.js"; +import { NativeSessionDiagnostics } from "./native-session-diagnostics.js"; + +const turnLostOnRestore = () => new NativeProviderTerminalFailure("provider_turn_lost_on_restore", true); + +describe("a native run that ends because its sandbox disk filled", () => { + it("names the full disk and stops instead of retrying the same work", () => { + const code = nativeSessionFailureSourceCode(turnLostOnRestore(), true); + expect(code).toBe("sandbox_disk_full"); + expect(nativeSessionFailureDisposition(1, new Date(), code)).toEqual({ + phase: "terminal_failure", failureCode: "sandbox_disk_full", nextAttemptAt: null }); + }); + + it("names the full disk when the runner could not be relaunched on it", () => { + const launch = Object.assign(new Error("sandbox_disk_full: the sandbox disk is full, so the runner cannot be started"), + { code: SANDBOX_DISK_FULL_FAILURE_CODE }); + const transport = new Error("Notification transport failed closed: sandbox_disk_full: the sandbox disk is full", { cause: launch }); + expect(nativeSessionFailureSourceCode(transport)).toBe("sandbox_disk_full"); + }); + + it("names the full disk when a failed relaunch reaches the run as a failed event", () => { + const failed = new NativeProviderTerminalFailure("notification_transport_failed", false, + "Notification transport failed closed: sandbox_disk_full: the sandbox disk is full, so the runner cannot be started"); + expect(nativeSessionFailureSourceCode(failed)).toBe("sandbox_disk_full"); + const owner = { companyId: "company", sessionScope: "session", runId: "run-a" }; + const channel = new NativeSessionDiagnostics(owner, owner, () => undefined, () => undefined); + const lease = channel.activate(owner, async () => undefined); + channel.record("[paperclip-runner] runnerd diagnostic: sandbox_disk_full: the sandbox disk is full, so the runner cannot be started\n"); + expect(lease.sandboxDiskFullObserved()).toBe(true); + }); + + it("keeps other failures as they are", () => { + expect(nativeSessionFailureSourceCode(turnLostOnRestore())).toBe("native_provider_terminal_failed"); + expect(nativeSessionFailureSourceCode(new NativeSessionProtocolIntegrityError("semantic_input_digest_mismatch"), true)) + .toBe("native_event_replay_conflict"); + expect(nativeSessionFailureSourceCode(new NativeProviderTerminalFailure("approval_required", false, "Approval required"), true)) + .toBe("native_provider_approval_required"); + expect(nativeSessionFailureDisposition(1, new Date(), nativeSessionFailureSourceCode(new Error("runnerd exited")))) + .toMatchObject({ phase: "retryable_failure" }); + }); + + it.each(["runner_ingress_unavailable: PRP command session.open timed out", "provider_initialize_timeout", "provider_request_protocol_error", + "provider_process_status_failed"])("names the full disk for a later %s on the same run", message => { + expect(nativeSessionFailureSourceCode(new Error(message), true)).toBe("sandbox_disk_full"); + expect(nativeSessionFailureDisposition(1, new Date(), nativeSessionFailureSourceCode(new Error(message), true))) + .toMatchObject({ phase: "terminal_failure", failureCode: "sandbox_disk_full" }); + }); + + it("keeps a full-disk report on the run that saw it", () => { + const owner = { companyId: "company", sessionScope: "session", runId: "run-a" }; + const channel = new NativeSessionDiagnostics(owner, owner, () => undefined, () => undefined); + const sink = async () => undefined; + const first = channel.activate(owner, sink); + channel.record("[paperclip-runner] runnerd diagnostic: runner process disconnected; recovery is allowed for 420000ms\n"); + expect(first.sandboxDiskFullObserved()).toBe(false); + channel.record("[paperclip-runner] runnerd diagnostic: failed to write state: No space left on device (os error 28)\n"); + expect(first.sandboxDiskFullObserved()).toBe(true); + first.retire(); + channel.record("[paperclip-runner] runnerd diagnostic: late exit: No space left on device (os error 28)\n"); + const second = channel.activate({ ...owner, runId: "run-b" }, sink); + channel.record("[paperclip-runner] runnerd diagnostic: runner process restored its durable PRP session\n"); + expect(second.sandboxDiskFullObserved()).toBe(false); + }); + + it("uses the runner's own code", () => { + expect(NATIVE_SANDBOX_DISK_FULL).toBe(SANDBOX_DISK_FULL_FAILURE_CODE); + }); +}); diff --git a/server/src/services/native-runtime/native-sandbox-disk-full.ts b/server/src/services/native-runtime/native-sandbox-disk-full.ts new file mode 100644 index 00000000000..2ca342e055e --- /dev/null +++ b/server/src/services/native-runtime/native-sandbox-disk-full.ts @@ -0,0 +1,56 @@ +import { isSandboxDiskFullFailure } from "../../vendor/paperclip-runner/index.js"; +import { isSandboxDiskFullText } from "./remote-runner-disk-reserve.js"; + +export const NATIVE_SANDBOX_DISK_FULL = "sandbox_disk_full" as const; + +export const NATIVE_SANDBOX_DISK_FULL_ACTION = + "The agent filled the disk of its computer, so its runner stopped. Paperclip does not repeat this automatically: " + + "a new computer has the same disk size, and the same work would fill it again. Ask the agent to use less disk space, " + + "for example by deleting build output, caches or dependencies it no longer needs, then press Retry."; + +const CODES_THAT_KEEP_THEIR_MEANING: ReadonlySet = new Set([ + "native_permission_declined", + "native_provider_approval_required", + "native_provider_usage_limit", + "native_provider_model_rejected", + "native_provider_runtime_missing", + "native_provider_state_incompatible", + "native_provider_side_effect_pending", + "native_provider_startup_rejected", + "native_event_replay_conflict", + "native_session_cleanup_quarantined", + "native_governed_wait_settlement_pending", + "native_predecessor_retirement_unproven", + "native_adopted_runner_authentication_timeout", + "native_controller_journal_behind_runner", + "native_current_wake_comments_unread", + "native_current_wake_comments_changed_after_read", + "runner_task_admission_rejected", + "runner_ingress_access_rejected", + "runner_remote_provider_artifact_incompatible", + "planning_mode_unsupported", +]); + +const NAMED_DISK_FULL_FAILURE = /(?:^|: )sandbox_disk_full: /; + +export function isNativeSandboxDiskFullReport(text: string): boolean { + return isSandboxDiskFullText(text) || NAMED_DISK_FULL_FAILURE.test(text); +} + +function diskFullCause(error: unknown, depth = 0): boolean { + if (!error || typeof error !== "object" || depth > 5) return false; + if (isSandboxDiskFullFailure(error)) return true; + const value = error as { cause?: unknown; errors?: unknown; message?: unknown }; + if (typeof value.message === "string" && NAMED_DISK_FULL_FAILURE.test(value.message)) return true; + return diskFullCause(value.cause, depth + 1) + || (Array.isArray(value.errors) && value.errors.slice(0, 3).some(item => diskFullCause(item, depth + 1))); +} + +export function nativeSandboxDiskFullSourceCode( + code: T, + error: unknown, + runnerSandboxDiskFull: boolean, +): T | typeof NATIVE_SANDBOX_DISK_FULL { + if (CODES_THAT_KEEP_THEIR_MEANING.has(code)) return code; + return runnerSandboxDiskFull || diskFullCause(error) ? NATIVE_SANDBOX_DISK_FULL : code; +} diff --git a/server/src/services/native-runtime/native-session-diagnostics.ts b/server/src/services/native-runtime/native-session-diagnostics.ts index d824304eb48..d86b19669cb 100644 --- a/server/src/services/native-runtime/native-session-diagnostics.ts +++ b/server/src/services/native-runtime/native-session-diagnostics.ts @@ -1,3 +1,5 @@ +import { isNativeSandboxDiskFullReport } from "./native-sandbox-disk-full.js"; + export interface NativeDiagnosticOwner { companyId: string; sessionScope: string; @@ -6,12 +8,13 @@ export interface NativeDiagnosticOwner { export interface NativeDiagnosticLease { retire(): void; + sandboxDiskFullObserved(): boolean; } export type NativeIdleDiagnosticKind = "transport_close_requested" | "session_diagnostic"; type Sink = (stream: "stdout" | "stderr", chunk: string) => Promise; -type Epoch = { owner: NativeDiagnosticOwner; sink?: Sink }; +type Epoch = { owner: NativeDiagnosticOwner; sink?: Sink; sandboxDiskFull?: boolean }; const sessions = new WeakMap(); export class NativeSessionDiagnostics { @@ -39,6 +42,7 @@ export class NativeSessionDiagnostics { this.lastOwner = epoch.owner; return { retire: () => { if (this.active === epoch) this.active = undefined; }, + sandboxDiskFullObserved: () => epoch.sandboxDiskFull === true, }; } @@ -55,6 +59,7 @@ export class NativeSessionDiagnostics { record(chunk: string): void { const epoch = this.active; + if (epoch && isNativeSandboxDiskFullReport(chunk)) epoch.sandboxDiskFull = true; if (!epoch?.sink) { const kind = chunk.startsWith("[paperclip-runner] runnerd diagnostic: runner transport close requested:") ? "transport_close_requested" : "session_diagnostic"; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 86eb62b791d..d676b2f2d8b 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -6,6 +6,7 @@ import { retryTransientLockContention } from "../transient-lock-contention.js"; import { nativeLeaseHolder, notifyNativeTokenLeaseClaimed } from "./native-token-lease-settlement.js"; import { captureRemoteNativeProviderArtifacts, shareNativeProviderArtifactCapture } from "./native-provider-artifact-capture.js"; import { NativeSessionDiagnostics, nativeSessionDiagnostics, type NativeDiagnosticLease } from "./native-session-diagnostics.js"; +import { NATIVE_SANDBOX_DISK_FULL, NATIVE_SANDBOX_DISK_FULL_ACTION, nativeSandboxDiskFullSourceCode } from "./native-sandbox-disk-full.js"; import { recordAgentProviderRateLimits } from "../agent-provider-rate-limits.js"; import { nativePostResultFailureCategory, type NativePostResultFailureCategory } from "./native-post-result-failure.js"; import { logger } from "../../middleware/logger.js"; @@ -9064,7 +9065,8 @@ export function nativeSessionFailureDisposition( sourceFailureCode === "native_provider_usage_limit" || sourceFailureCode === NATIVE_PROVIDER_SESSION_CLOSED || sourceFailureCode === NATIVE_PROVIDER_STARTUP_REJECTED || - sourceFailureCode === NATIVE_CONTROLLER_JOURNAL_BEHIND_RUNNER; + sourceFailureCode === NATIVE_CONTROLLER_JOURNAL_BEHIND_RUNNER || + sourceFailureCode === NATIVE_SANDBOX_DISK_FULL; const permissionDeclined = sourceFailureCode === "native_permission_declined"; const exhausted = permanentFailure || permissionDeclined || attempt >= 3; return { @@ -9242,7 +9244,11 @@ function runnerClosedTurnStartSession(error: unknown, depth = 0): boolean { return runnerClosedTurnStartSession(value.cause, depth + 1); } -export function nativeSessionFailureSourceCode( +export function nativeSessionFailureSourceCode(error: unknown, runnerSandboxDiskFull = false) { + return nativeSandboxDiskFullSourceCode(classifyNativeSessionFailure(error), error, runnerSandboxDiskFull); +} + +function classifyNativeSessionFailure( error: unknown, ): | "native_permission_declined" @@ -10997,6 +11003,9 @@ async function executePaperclipNativeSessionAttempt( cleanupOnly = false, ownDiagnosticLease?: (lease: NativeDiagnosticLease) => void, ): Promise { + const attemptDiagnosticLeases: NativeDiagnosticLease[] = []; + const ownRunDiagnosticLease = ownDiagnosticLease; + ownDiagnosticLease = lease => { attemptDiagnosticLeases.push(lease); ownRunDiagnosticLease?.(lease); }; if (input.execution.provider.kind === "acpx" && input.execution.provider.agent === "antigravity") { const candidate = resolveAntigravityCandidateAdmission({ companyId: input.execution.binding.companyId, model: input.execution.provider.model, @@ -13277,7 +13286,8 @@ async function executePaperclipNativeSessionAttempt( throw error; } const now = new Date(); - const startupClassifiedFailureCode = nativeSessionFailureSourceCode(error); + const startupClassifiedFailureCode = nativeSessionFailureSourceCode(error, + attemptDiagnosticLeases.some(lease => lease.sandboxDiskFullObserved())); // A timed-out session.open may still have opened a provider session // before its runner stopped. The same-run retry must not reuse it. const classifiedFailureCode = startupClassifiedFailureCode === "provider_session_open_timeout" && @@ -13497,6 +13507,8 @@ async function executePaperclipNativeSessionAttempt( ? RUNNER_TASK_ADMISSION_REJECTED_ACTION : sourceFailureCode === "runner_ingress_access_rejected" ? "The computer provider rejected access. Restore the environment's credentials or permissions before retrying. Automatic retries are stopped." + : sourceFailureCode === NATIVE_SANDBOX_DISK_FULL + ? NATIVE_SANDBOX_DISK_FULL_ACTION : sourceFailureCode === "native_provider_terminal_failed" ? "The provider session is permanently unusable. Verify stopped execution, completed actions, and task context before starting a linked continuation." : sourceFailureCode === "native_governed_wait_settlement_pending" @@ -13776,6 +13788,8 @@ async function executePaperclipNativeSessionAttempt( ? RUNNER_TASK_ADMISSION_REJECTED_ACTION : sourceFailureCode === "runner_ingress_access_rejected" ? "The computer provider rejected access. Restore the environment's credentials or permissions before retrying. Automatic retries are stopped." + : sourceFailureCode === NATIVE_SANDBOX_DISK_FULL + ? NATIVE_SANDBOX_DISK_FULL_ACTION : sourceFailureCode === "native_provider_terminal_failed" ? "Verify that the failed provider stopped and reconcile its action outcomes. A linked continuation can proceed only after these checks succeed." : sourceFailureCode === "native_provider_model_rejected" && !recoveryEvidence.providerSessionEstablished diff --git a/ui/src/components/TaskChatThread.test.tsx b/ui/src/components/TaskChatThread.test.tsx index fbb312553dd..ef337fa382e 100644 --- a/ui/src/components/TaskChatThread.test.tsx +++ b/ui/src/components/TaskChatThread.test.tsx @@ -2434,6 +2434,42 @@ describe("TaskChatThread runtime transcript selection", () => { expect(container.querySelector('[data-testid="task-chat-run-failed-try-again"]')).toBeNull(); }); + it("explains a full sandbox disk without exposing its error code", async () => { + renderThread({ + comments: [], onAdd: async () => {}, issueStatus: "blocked", onRetryFailedRun: vi.fn(), + linkedRuns: [{ + runId: "native-disk-full", runtimeMode: "native", status: "failed", errorCode: "sandbox_disk_full", + agentId: "agent-1", agentName: "Runner", adapterType: "paperclip_runner", + createdAt: "2026-08-25T18:00:00.000Z", startedAt: "2026-08-25T18:00:00.000Z", finishedAt: "2026-08-25T18:00:02.000Z", + }], + }); + const marker = container.querySelector('[data-testid="task-chat-collapsible-marker"]'); + expect(marker?.textContent).toContain("Disk full"); + flushSync(() => marker!.querySelector('button[aria-expanded="false"]')!.click()); + const details = container.querySelector('[data-testid="task-chat-collapsible-marker-details"]'); + expect(details?.textContent).toContain("Ask the agent to use less disk space"); + expect(details?.textContent).not.toContain("sandbox_disk_full"); + }); + + it("explains a full sandbox disk beside the task's interruption without a second Retry", async () => { + renderThread({ + comments: [], onAdd: async () => {}, issueStatus: "blocked", onRetryFailedRun: vi.fn(), + interruption: { intentId: null, responsibility: "system", state: "failed", nextRetryAt: null, canRetry: true, + userAction: null, sourceRunIds: ["native-disk-full"], observedRunId: "native-disk-full" }, + linkedRuns: [{ + runId: "native-disk-full", runtimeMode: "native", status: "failed", errorCode: "sandbox_disk_full", + agentId: "agent-1", agentName: "Runner", adapterType: "paperclip_runner", + createdAt: "2026-08-25T18:00:00.000Z", startedAt: "2026-08-25T18:00:00.000Z", finishedAt: "2026-08-25T18:00:02.000Z", + }], + }); + const marker = container.querySelector('[data-testid="task-chat-collapsible-marker"]'); + expect(marker?.textContent).toContain("Disk full"); + flushSync(() => marker!.querySelector('button[aria-expanded="false"]')!.click()); + expect(container.querySelector('[data-testid="task-chat-collapsible-marker-details"]')?.textContent) + .toContain("Ask the agent to use less disk space"); + expect(container.querySelector('[data-testid="task-chat-run-failed-try-again"]')).toBeNull(); + }); + it("explains a native provider usage limit without exposing its error code", async () => { const onRetryFailedRun = vi.fn(); renderThread({ diff --git a/ui/src/components/TaskChatThread.tsx b/ui/src/components/TaskChatThread.tsx index cf7ec215898..24736265263 100644 --- a/ui/src/components/TaskChatThread.tsx +++ b/ui/src/components/TaskChatThread.tsx @@ -1865,6 +1865,8 @@ export function TaskChatThread(props: TaskChatThreadProps) { ? "Approval required" : code === "native_provider_usage_limit" && source.status === "failed" ? "Usage limit reached" + : code === "sandbox_disk_full" && source.status === "failed" + ? "Disk full" : source.status === "cancelled" ? "Run cancelled" : source.status === "interrupted" @@ -1891,6 +1893,8 @@ export function TaskChatThread(props: TaskChatThreadProps) { : code === "native_provider_usage_limit" && source.status === "failed" ? "The model provider has reached its current usage limit. Try again after the limit resets." + : code === "sandbox_disk_full" && source.status === "failed" + ? "The agent filled the disk of its computer, so the run stopped. Ask the agent to use less disk space, then retry." : code === "provider_frame_too_large" ? "Provider output exceeded the safe limit." : source.status === "timed_out" @@ -1902,7 +1906,8 @@ export function TaskChatThread(props: TaskChatThreadProps) { : undefined; const finishedAt = meta?.finishedAt ?? meta?.startedAt ?? meta?.createdAt; - if (!historical && (!props.interruption || credentialAccess || + const diskFullBesideInterruption = Boolean(props.interruption) && code === "sandbox_disk_full" && source.status === "failed"; + if (!historical && (!props.interruption || credentialAccess || diskFullBesideInterruption || (props.interruption.responsibility === "user" && ["native_provider_model_rejected", "native_provider_approval_required"].includes(code)))) entriesWithFailures.push({ ms: toMs(finishedAt), @@ -1915,7 +1920,7 @@ export function TaskChatThread(props: TaskChatThreadProps) { tone: source.status === "cancelled" ? "neutral" : "error", label, detail, - retryable: code !== "native_session_cleanup_quarantined", + retryable: code !== "native_session_cleanup_quarantined" && !diskFullBesideInterruption, ...(credentialAccess ? { credentialAccess, retryable: false } : {}), collapsible: true, runId: source.id,