From b4cea5b5a4e9b307ef325299b11eda63f54e286a Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 20 Nov 2026 17:08:31 -0800 Subject: [PATCH] fix(runner): show disk exhaustion before recovery errors (#22448) Show the confirmed sandbox disk failure before a later lost-turn or restore error in saved run errors and both run detail views. Preserve the original diagnostics and leave scarce inodes available when preparing the existing runner relaunch reserve. Verified focused regressions, scoped typechecks, and full CI on the reviewed revision, including Runner checks and all eight end-to-end shards. Greptile is 5/5 with all review findings resolved. Co-Authored-By: Paperclip --- doc/runner-launch-preparation.md | 14 ++++++++++ packages/shared/src/run-failure-cause.test.ts | 9 ++++++- packages/shared/src/run-failure-cause.ts | 8 ++++++ .../native-runtime/native-session-executor.ts | 6 +++-- .../remote-runner-disk-reserve.test.ts | 12 +++++++++ .../remote-runner-disk-reserve.ts | 8 +++--- .../runner-e2e/source-sdk-public-inputs.json | 4 +-- .../AgentDetail.native-retry-stop.test.tsx | 27 +++++++++++++++++++ ui/src/pages/AgentDetail.production.tsx | 5 ++-- ui/src/pages/AgentDetail.tsx | 8 +++--- 10 files changed, 87 insertions(+), 14 deletions(-) diff --git a/doc/runner-launch-preparation.md b/doc/runner-launch-preparation.md index c96c355adb9..f2b68ac6a9c 100644 --- a/doc/runner-launch-preparation.md +++ b/doc/runner-launch-preparation.md @@ -10,6 +10,20 @@ The built-in TypeScript implementation constructs the provider descriptor in By default, the SDK prepares local files. It stages remote files through the existing execution driver. +## Managed sandbox storage + +Core holds a 256 MiB runner relaunch reserve only in managed sandboxes. It +creates the reserve when there is at least 2 GiB of additional free space and +1,024 free inodes. It leaves scarce capacity available to an existing agent. +A relaunch releases that reserve before it stages the replacement runner. +This reserve is recovery headroom; it does not limit the agent's downloads. + +When runner diagnostics confirm a full sandbox disk, the saved run error and +run details show that cause ahead of a later lost-turn or restore error. Raw +failure diagnostics remain in the run's diagnostic evidence. Older saved runs +with `sandbox_disk_full` get the same display without a data migration. +Download budgets and image policy belong to the deployment environment. + ## Environment-owned preparation An execution driver can supply `prepareLaunch`. diff --git a/packages/shared/src/run-failure-cause.test.ts b/packages/shared/src/run-failure-cause.test.ts index 68a726ac3e3..fdf56d4c6f3 100644 --- a/packages/shared/src/run-failure-cause.test.ts +++ b/packages/shared/src/run-failure-cause.test.ts @@ -1,7 +1,14 @@ import { describe, expect, it } from "vitest"; -import { normalizeRunFailureCause, readRunFailureCause, RUN_FAILURE_CAUSE_MAX_LENGTH, withoutSqlStatementText } from "./run-failure-cause.js"; +import { normalizeRunFailureCause, readRunFailureCause, RUN_FAILURE_CAUSE_MAX_LENGTH, withoutSqlStatementText, runFailureMessage } from "./run-failure-cause.js"; describe("run failure cause contract", () => { + it("leads with the confirmed full disk instead of the later restore failure", () => { + expect(runFailureMessage({ errorCode: "sandbox_disk_full", error: "The previous turn is no longer active." })) + .toMatch(/^The agent's computer ran out of disk space/); + expect(runFailureMessage({ errorCode: "sandbox_disk_full", error: null })).toContain("Retry"); + expect(runFailureMessage({ errorCode: "process_lost", error: "Process lost" })).toBe("Process lost"); + expect(runFailureMessage({ errorCode: null, error: null })).toBeNull(); + }); it("accepts bounded identifier codes", () => { for (const code of ["40001", "ECONNRESET", "provider_http_429", "TypeError", "signal_SIGKILL", "a".repeat(RUN_FAILURE_CAUSE_MAX_LENGTH)]) { expect(normalizeRunFailureCause(code)).toBe(code); diff --git a/packages/shared/src/run-failure-cause.ts b/packages/shared/src/run-failure-cause.ts index 6f7c646d715..33c31b748be 100644 --- a/packages/shared/src/run-failure-cause.ts +++ b/packages/shared/src/run-failure-cause.ts @@ -11,6 +11,14 @@ export const RUN_FAILURE_CAUSE_MAX_LENGTH = 80; export const RUN_FAILURE_CAUSE_PATTERN = /^[A-Za-z0-9_.:-]{1,80}$/; +export function runFailureMessage(run: { errorCode?: string | null; error?: string | null }): string | null { + if (run.errorCode === "sandbox_disk_full") { + return "The agent's computer ran out of disk space, so its runner stopped. " + + "Ask the agent to use less disk space, then select Retry. Saved task updates remain available."; + } + return run.error ?? null; +} + /** The stored cause when it has the closed shape, else `null` (absent, legacy or malformed). */ export function normalizeRunFailureCause(value: unknown): string | null { return typeof value === "string" && RUN_FAILURE_CAUSE_PATTERN.test(value) ? value : null; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 7c9a0275ace..1f4c0fb60cf 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -7,6 +7,7 @@ import { nativeLeaseHolder, notifyNativeTokenLeaseClaimed } from "./native-token import { captureRemoteNativeProviderArtifacts, shareNativeProviderArtifactCapture } from "./native-provider-artifact-capture.js"; import { NativeSessionDiagnostics, nativeSessionDiagnostics, type NativeDiagnosticLease } from "./native-session-diagnostics.js"; import { NATIVE_SANDBOX_DISK_FULL, NATIVE_SANDBOX_DISK_FULL_ACTION, nativeSandboxDiskFullSourceCode } from "./native-sandbox-disk-full.js"; +import { runFailureMessage } from "@paperclipai/shared"; import { recordAgentProviderRateLimits } from "../agent-provider-rate-limits.js"; import { nativePostResultFailureCategory, type NativePostResultFailureCategory } from "./native-post-result-failure.js"; import { logger } from "../../middleware/logger.js"; @@ -13494,10 +13495,11 @@ async function executePaperclipNativeSessionAttempt( const { exhausted } = recoveryProjection; const integrityFailure = sourceFailureCode === "native_event_replay_conflict"; - const message = redactOutputText( + const originalMessage = redactOutputText( error instanceof Error ? error.message : String(error), ).slice(0, 2_000); - const sanitizedStderrTail = redactOutputText(message).slice(-4_096); + const message = runFailureMessage({ errorCode: sourceFailureCode, error: originalMessage })!; + const sanitizedStderrTail = originalMessage.slice(-4_096); const remoteRunnerRecovery = readRemoteRunnerRecoveryError(error); // Set inside the transaction only when the write below genuinely // transitions the run into "failed". Read after the transaction diff --git a/server/src/services/native-runtime/remote-runner-disk-reserve.test.ts b/server/src/services/native-runtime/remote-runner-disk-reserve.test.ts index 080d744a6fd..dc5748e9424 100644 --- a/server/src/services/native-runtime/remote-runner-disk-reserve.test.ts +++ b/server/src/services/native-runtime/remote-runner-disk-reserve.test.ts @@ -284,4 +284,16 @@ describe("remote runner disk reserve scripts", () => { .toBe("/home/daytona/ws/.paperclip-runtime/paperclip-runner/disk-reserve"); expect(remoteRunnerDiskReservePath({ transport: "ssh", remoteCwd: "/home/me/ws" })).toBeUndefined(); }); + + it("leaves scarce inodes available to the running agent", async () => { + const root = await mkdtemp(join(tmpdir(), "runner-disk-reserve-")); + const reserve = join(root, "disk-reserve"); + try { + await holdRemoteRunnerDiskReserve({ execute: localExecute }, reserve, + { bytes: 1024 * 1024, minimumFreeBytes: 0, minimumFreeInodes: Number.MAX_SAFE_INTEGER }); + expect(existsSync(reserve)).toBe(false); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); }); diff --git a/server/src/services/native-runtime/remote-runner-disk-reserve.ts b/server/src/services/native-runtime/remote-runner-disk-reserve.ts index 631d42f7fb2..a17471ab0cf 100644 --- a/server/src/services/native-runtime/remote-runner-disk-reserve.ts +++ b/server/src/services/native-runtime/remote-runner-disk-reserve.ts @@ -15,10 +15,12 @@ const MINIMUM_FREE_BYTES = 2 * 1024 * 1024 * 1024; type Runner = Pick; const HOLD_SCRIPT = [ - 'set -u; reserve=$1; bytes=$2; minimum_free_kib=$3; [ -f "$reserve" ] && exit 0', + 'set -u; reserve=$1; bytes=$2; minimum_free_kib=$3; minimum_free_inodes=$4; [ -f "$reserve" ] && exit 0', 'directory=$(dirname -- "$reserve"); mkdir -p -- "$directory" || exit 1', `free_kib=$(df -Pk -- "$directory" | awk 'NR == 2 { print $4 }')`, '[ -n "$free_kib" ] && [ "$free_kib" -ge "$minimum_free_kib" ] || exit 0', + `free_inodes=$(df -Pi -- "$directory" | awk 'NR == 2 { print $(NF - 2) }')`, + '[ -n "$free_inodes" ] && [ "$free_inodes" -ge "$minimum_free_inodes" ] || exit 0', 'partial="$reserve.partial"; rm -f -- "$partial"', 'fallocate -l "$bytes" "$partial" 2>/dev/null || dd if=/dev/zero of="$partial" bs=65536 count=$((bytes / 65536)) 2>/dev/null || { rm -f -- "$partial"; exit 1; }', 'mv -f -- "$partial" "$reserve"', @@ -31,14 +33,14 @@ const RELEASE_SCRIPT = export async function holdRemoteRunnerDiskReserve( runner: Runner, reservePath: string, - options: { bytes?: number; minimumFreeBytes?: number } = {}, + options: { bytes?: number; minimumFreeBytes?: number; minimumFreeInodes?: number } = {}, ): Promise { const bytes = options.bytes ?? REMOTE_RUNNER_DISK_RESERVE_BYTES; const minimumFreeBytes = options.minimumFreeBytes ?? MINIMUM_FREE_BYTES; await runner.execute({ command: "sh", args: ["-c", HOLD_SCRIPT, "paperclip-runner-disk-reserve-hold", reservePath, - String(bytes), String(Math.ceil((minimumFreeBytes + bytes) / 1024))], + String(bytes), String(Math.ceil((minimumFreeBytes + bytes) / 1024)), String(options.minimumFreeInodes ?? 1024)], bypassSession: true, timeoutMs: 30_000, }); diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index bcdc78a96f7..1d36afd3004 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -349,8 +349,8 @@ "packages/shared/src/routine-suggestions.ts": "4895e6090faae8c36cd509667e69bc571e90f217", "packages/shared/src/routine-variables.test.ts": "3cd4ea5e1a279eacdc525a3eb8e7bf317170c1ec", "packages/shared/src/routine-variables.ts": "43d57551aaf8fcd32c10733af3198b6932b0d5d4", - "packages/shared/src/run-failure-cause.test.ts": "68a726ac3e3b9be394ffa32b65acc6119e06424c", - "packages/shared/src/run-failure-cause.ts": "6f7c646d715f384ee0a01805d5507b0c7a0669a2", + "packages/shared/src/run-failure-cause.test.ts": "fdf56d4c6f3e29eb0c13c2e4a8b2b8b83111ca85", + "packages/shared/src/run-failure-cause.ts": "33c31b748be37f6ce171cb11ae68c6bb0f288872", "packages/shared/src/runner-configuration-warnings.ts": "7d8db761ee51bec746c425320d5900ef3505de1b", "packages/shared/src/runner-goal.test.ts": "05814124d8f592d6bf4ca78541474e6f354411d1", "packages/shared/src/runner-goal.ts": "dc61b33fd36d2ed9f68fc3a5630a81d1e001333b", diff --git a/ui/src/pages/AgentDetail.native-retry-stop.test.tsx b/ui/src/pages/AgentDetail.native-retry-stop.test.tsx index b04d2eb9b25..9d429d27100 100644 --- a/ui/src/pages/AgentDetail.native-retry-stop.test.tsx +++ b/ui/src/pages/AgentDetail.native-retry-stop.test.tsx @@ -50,6 +50,33 @@ const states = [ { status: "running", eligibility: undefined, label: "Cancel", driverKind: "grok_bot_mcp" }, { status: "queued", eligibility: undefined, label: "Cancel", driverKind: "grok_bot_mcp" }, ]; + +it.each([RunsTab, ProductionRunsTab].flatMap(Tab => [null, "The previous turn is no longer active."].map(error => ({ Tab, error }))))( + "shows the confirmed full disk for historical run details (%#)", async ({ Tab, error }) => { + const run = { id: "disk-full-run", companyId: "company-1", agentId: "agent-1", status: "failed", + errorCode: "sandbox_disk_full", error, resultJson: null, contextSnapshot: null, usageJson: null, + createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z", invocationSource: "manual", + startedAt: null, finishedAt: null, logRef: null, exitCode: 1, + execution: { canStop: false, phase: "completed", label: "Continued in another run" }, + } as unknown as HeartbeatRun; + get.mockResolvedValue(run); + const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + const element = document.createElement("div"); document.body.append(element); + const root = createRoot(element); + try { + await act(async () => root.render( + + )); + await vi.waitFor(async () => { + await act(async () => {}); + expect(element.textContent).toContain("computer ran out of disk space"); + }); + expect(element.textContent).not.toContain("previous turn is no longer active"); + expect(cancel).not.toHaveBeenCalled(); + } finally { await act(async () => root.unmount()); client.clear(); element.remove(); } + }, +); + it.each([RunsTab, ProductionRunsTab].flatMap(Tab => states.map(state => ({ Tab, ...state }))))("renders and submits only eligible run cancellation (%#)", async ({ Tab, status, eligibility, label, ...state }) => { if (state.local) { actor.id = "local-board"; actor.source = "local_implicit"; } const run = { id: state.priorSession ? `persisted-run-${Tab === RunsTab ? "main" : "production"}` : "failed-native-run", companyId: "company-1", agentId: "agent-1", issueId: "issue-1", status, driverKind: state.driverKind, invocationSource: "manual", diff --git a/ui/src/pages/AgentDetail.production.tsx b/ui/src/pages/AgentDetail.production.tsx index d502334c323..b4144a0fe13 100644 --- a/ui/src/pages/AgentDetail.production.tsx +++ b/ui/src/pages/AgentDetail.production.tsx @@ -66,6 +66,7 @@ import { readSourceResolvedWatchdogFold } from "../lib/source-resolved-watchdog- import { formatCents, formatDate, relativeTime, formatTokens, visibleRunCostUsd, visibleRunTokenTotal } from "../lib/utils"; import { cn } from "../lib/utils"; import { ProviderLimitMessage } from "../components/ProviderLimitMessage"; +import { runFailureMessage } from "@paperclipai/shared"; import { describeRunRetryState } from "../lib/runRetryState"; import { Button } from "@/components/ui/button"; import { Skeleton } from "@/components/ui/skeleton"; @@ -3039,9 +3040,9 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig } )} )} - {run.error && ( + {runFailureMessage(run) && (
- {run.error} + {runFailureMessage(run)!} {run.errorCode && ({run.errorCode})}
)} diff --git a/ui/src/pages/AgentDetail.tsx b/ui/src/pages/AgentDetail.tsx index 086560673fa..d46a657a60c 100644 --- a/ui/src/pages/AgentDetail.tsx +++ b/ui/src/pages/AgentDetail.tsx @@ -6,7 +6,7 @@ import { GrokBotWorkSummary } from "@/components/GrokBotWorkSummary"; import { PrimaryAgentCrownButton } from "@/components/primary-agent/PrimaryAgentPresentation"; import { AiConnectionPoolRunDetails } from "@/components/ai-connections/AiConnectionPoolRunDetails"; import { ProviderFailureNotice } from "@/components/ProviderFailureNotice"; -import { readProviderFailure } from "@paperclipai/shared"; +import { readProviderFailure, runFailureMessage } from "@paperclipai/shared"; import { AgentConnectionInstructions } from "@/features/connections/ConnectionInstructions"; import type { AgentInstructionCandidate, AgentInstructionsBundle } from "@paperclipai/shared"; import { InstructionHistory } from "../components/InstructionHistory"; @@ -3362,11 +3362,11 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig } )} - {readProviderFailure(run.resultJson?.providerFailure) ? ( + {run.errorCode !== "sandbox_disk_full" && readProviderFailure(run.resultJson?.providerFailure) ? ( - ) : run.error && ( + ) : runFailureMessage(run) && (
- {run.error} + {runFailureMessage(run)!} {run.errorCode && ({run.errorCode})}
)}