diff --git a/doc/runner-launch-preparation.md b/doc/runner-launch-preparation.md index c96c355adb9..f2b68ac6a9c 100644 --- a/doc/runner-launch-preparation.md +++ b/doc/runner-launch-preparation.md @@ -10,6 +10,20 @@ The built-in TypeScript implementation constructs the provider descriptor in By default, the SDK prepares local files. It stages remote files through the existing execution driver. +## Managed sandbox storage + +Core holds a 256 MiB runner relaunch reserve only in managed sandboxes. It +creates the reserve when there is at least 2 GiB of additional free space and +1,024 free inodes. It leaves scarce capacity available to an existing agent. +A relaunch releases that reserve before it stages the replacement runner. +This reserve is recovery headroom; it does not limit the agent's downloads. + +When runner diagnostics confirm a full sandbox disk, the saved run error and +run details show that cause ahead of a later lost-turn or restore error. Raw +failure diagnostics remain in the run's diagnostic evidence. Older saved runs +with `sandbox_disk_full` get the same display without a data migration. +Download budgets and image policy belong to the deployment environment. + ## Environment-owned preparation An execution driver can supply `prepareLaunch`. diff --git a/packages/shared/src/run-failure-cause.test.ts b/packages/shared/src/run-failure-cause.test.ts index 68a726ac3e3..fdf56d4c6f3 100644 --- a/packages/shared/src/run-failure-cause.test.ts +++ b/packages/shared/src/run-failure-cause.test.ts @@ -1,7 +1,14 @@ import { describe, expect, it } from "vitest"; -import { normalizeRunFailureCause, readRunFailureCause, RUN_FAILURE_CAUSE_MAX_LENGTH, withoutSqlStatementText } from "./run-failure-cause.js"; +import { normalizeRunFailureCause, readRunFailureCause, RUN_FAILURE_CAUSE_MAX_LENGTH, withoutSqlStatementText, runFailureMessage } from "./run-failure-cause.js"; describe("run failure cause contract", () => { + it("leads with the confirmed full disk instead of the later restore failure", () => { + expect(runFailureMessage({ errorCode: "sandbox_disk_full", error: "The previous turn is no longer active." })) + .toMatch(/^The agent's computer ran out of disk space/); + expect(runFailureMessage({ errorCode: "sandbox_disk_full", error: null })).toContain("Retry"); + expect(runFailureMessage({ errorCode: "process_lost", error: "Process lost" })).toBe("Process lost"); + expect(runFailureMessage({ errorCode: null, error: null })).toBeNull(); + }); it("accepts bounded identifier codes", () => { for (const code of ["40001", "ECONNRESET", "provider_http_429", "TypeError", "signal_SIGKILL", "a".repeat(RUN_FAILURE_CAUSE_MAX_LENGTH)]) { expect(normalizeRunFailureCause(code)).toBe(code); diff --git a/packages/shared/src/run-failure-cause.ts b/packages/shared/src/run-failure-cause.ts index 6f7c646d715..33c31b748be 100644 --- a/packages/shared/src/run-failure-cause.ts +++ b/packages/shared/src/run-failure-cause.ts @@ -11,6 +11,14 @@ export const RUN_FAILURE_CAUSE_MAX_LENGTH = 80; export const RUN_FAILURE_CAUSE_PATTERN = /^[A-Za-z0-9_.:-]{1,80}$/; +export function runFailureMessage(run: { errorCode?: string | null; error?: string | null }): string | null { + if (run.errorCode === "sandbox_disk_full") { + return "The agent's computer ran out of disk space, so its runner stopped. " + + "Ask the agent to use less disk space, then select Retry. Saved task updates remain available."; + } + return run.error ?? null; +} + /** The stored cause when it has the closed shape, else `null` (absent, legacy or malformed). */ export function normalizeRunFailureCause(value: unknown): string | null { return typeof value === "string" && RUN_FAILURE_CAUSE_PATTERN.test(value) ? value : null; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 7c9a0275ace..1f4c0fb60cf 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -7,6 +7,7 @@ import { nativeLeaseHolder, notifyNativeTokenLeaseClaimed } from "./native-token import { captureRemoteNativeProviderArtifacts, shareNativeProviderArtifactCapture } from "./native-provider-artifact-capture.js"; import { NativeSessionDiagnostics, nativeSessionDiagnostics, type NativeDiagnosticLease } from "./native-session-diagnostics.js"; import { NATIVE_SANDBOX_DISK_FULL, NATIVE_SANDBOX_DISK_FULL_ACTION, nativeSandboxDiskFullSourceCode } from "./native-sandbox-disk-full.js"; +import { runFailureMessage } from "@paperclipai/shared"; import { recordAgentProviderRateLimits } from "../agent-provider-rate-limits.js"; import { nativePostResultFailureCategory, type NativePostResultFailureCategory } from "./native-post-result-failure.js"; import { logger } from "../../middleware/logger.js"; @@ -13494,10 +13495,11 @@ async function executePaperclipNativeSessionAttempt( const { exhausted } = recoveryProjection; const integrityFailure = sourceFailureCode === "native_event_replay_conflict"; - const message = redactOutputText( + const originalMessage = redactOutputText( error instanceof Error ? error.message : String(error), ).slice(0, 2_000); - const sanitizedStderrTail = redactOutputText(message).slice(-4_096); + const message = runFailureMessage({ errorCode: sourceFailureCode, error: originalMessage })!; + const sanitizedStderrTail = originalMessage.slice(-4_096); const remoteRunnerRecovery = readRemoteRunnerRecoveryError(error); // Set inside the transaction only when the write below genuinely // transitions the run into "failed". Read after the transaction diff --git a/server/src/services/native-runtime/remote-runner-disk-reserve.test.ts b/server/src/services/native-runtime/remote-runner-disk-reserve.test.ts index 080d744a6fd..dc5748e9424 100644 --- a/server/src/services/native-runtime/remote-runner-disk-reserve.test.ts +++ b/server/src/services/native-runtime/remote-runner-disk-reserve.test.ts @@ -284,4 +284,16 @@ describe("remote runner disk reserve scripts", () => { .toBe("/home/daytona/ws/.paperclip-runtime/paperclip-runner/disk-reserve"); expect(remoteRunnerDiskReservePath({ transport: "ssh", remoteCwd: "/home/me/ws" })).toBeUndefined(); }); + + it("leaves scarce inodes available to the running agent", async () => { + const root = await mkdtemp(join(tmpdir(), "runner-disk-reserve-")); + const reserve = join(root, "disk-reserve"); + try { + await holdRemoteRunnerDiskReserve({ execute: localExecute }, reserve, + { bytes: 1024 * 1024, minimumFreeBytes: 0, minimumFreeInodes: Number.MAX_SAFE_INTEGER }); + expect(existsSync(reserve)).toBe(false); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); }); diff --git a/server/src/services/native-runtime/remote-runner-disk-reserve.ts b/server/src/services/native-runtime/remote-runner-disk-reserve.ts index 631d42f7fb2..a17471ab0cf 100644 --- a/server/src/services/native-runtime/remote-runner-disk-reserve.ts +++ b/server/src/services/native-runtime/remote-runner-disk-reserve.ts @@ -15,10 +15,12 @@ const MINIMUM_FREE_BYTES = 2 * 1024 * 1024 * 1024; type Runner = Pick; const HOLD_SCRIPT = [ - 'set -u; reserve=$1; bytes=$2; minimum_free_kib=$3; [ -f "$reserve" ] && exit 0', + 'set -u; reserve=$1; bytes=$2; minimum_free_kib=$3; minimum_free_inodes=$4; [ -f "$reserve" ] && exit 0', 'directory=$(dirname -- "$reserve"); mkdir -p -- "$directory" || exit 1', `free_kib=$(df -Pk -- "$directory" | awk 'NR == 2 { print $4 }')`, '[ -n "$free_kib" ] && [ "$free_kib" -ge "$minimum_free_kib" ] || exit 0', + `free_inodes=$(df -Pi -- "$directory" | awk 'NR == 2 { print $(NF - 2) }')`, + '[ -n "$free_inodes" ] && [ "$free_inodes" -ge "$minimum_free_inodes" ] || exit 0', 'partial="$reserve.partial"; rm -f -- "$partial"', 'fallocate -l "$bytes" "$partial" 2>/dev/null || dd if=/dev/zero of="$partial" bs=65536 count=$((bytes / 65536)) 2>/dev/null || { rm -f -- "$partial"; exit 1; }', 'mv -f -- "$partial" "$reserve"', @@ -31,14 +33,14 @@ const RELEASE_SCRIPT = export async function holdRemoteRunnerDiskReserve( runner: Runner, reservePath: string, - options: { bytes?: number; minimumFreeBytes?: number } = {}, + options: { bytes?: number; minimumFreeBytes?: number; minimumFreeInodes?: number } = {}, ): Promise { const bytes = options.bytes ?? REMOTE_RUNNER_DISK_RESERVE_BYTES; const minimumFreeBytes = options.minimumFreeBytes ?? MINIMUM_FREE_BYTES; await runner.execute({ command: "sh", args: ["-c", HOLD_SCRIPT, "paperclip-runner-disk-reserve-hold", reservePath, - String(bytes), String(Math.ceil((minimumFreeBytes + bytes) / 1024))], + String(bytes), String(Math.ceil((minimumFreeBytes + bytes) / 1024)), String(options.minimumFreeInodes ?? 1024)], bypassSession: true, timeoutMs: 30_000, }); diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index bcdc78a96f7..1d36afd3004 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -349,8 +349,8 @@ "packages/shared/src/routine-suggestions.ts": "4895e6090faae8c36cd509667e69bc571e90f217", "packages/shared/src/routine-variables.test.ts": "3cd4ea5e1a279eacdc525a3eb8e7bf317170c1ec", "packages/shared/src/routine-variables.ts": "43d57551aaf8fcd32c10733af3198b6932b0d5d4", - "packages/shared/src/run-failure-cause.test.ts": "68a726ac3e3b9be394ffa32b65acc6119e06424c", - "packages/shared/src/run-failure-cause.ts": "6f7c646d715f384ee0a01805d5507b0c7a0669a2", + "packages/shared/src/run-failure-cause.test.ts": "fdf56d4c6f3e29eb0c13c2e4a8b2b8b83111ca85", + "packages/shared/src/run-failure-cause.ts": "33c31b748be37f6ce171cb11ae68c6bb0f288872", "packages/shared/src/runner-configuration-warnings.ts": "7d8db761ee51bec746c425320d5900ef3505de1b", "packages/shared/src/runner-goal.test.ts": "05814124d8f592d6bf4ca78541474e6f354411d1", "packages/shared/src/runner-goal.ts": "dc61b33fd36d2ed9f68fc3a5630a81d1e001333b", diff --git a/ui/src/pages/AgentDetail.native-retry-stop.test.tsx b/ui/src/pages/AgentDetail.native-retry-stop.test.tsx index b04d2eb9b25..9d429d27100 100644 --- a/ui/src/pages/AgentDetail.native-retry-stop.test.tsx +++ b/ui/src/pages/AgentDetail.native-retry-stop.test.tsx @@ -50,6 +50,33 @@ const states = [ { status: "running", eligibility: undefined, label: "Cancel", driverKind: "grok_bot_mcp" }, { status: "queued", eligibility: undefined, label: "Cancel", driverKind: "grok_bot_mcp" }, ]; + +it.each([RunsTab, ProductionRunsTab].flatMap(Tab => [null, "The previous turn is no longer active."].map(error => ({ Tab, error }))))( + "shows the confirmed full disk for historical run details (%#)", async ({ Tab, error }) => { + const run = { id: "disk-full-run", companyId: "company-1", agentId: "agent-1", status: "failed", + errorCode: "sandbox_disk_full", error, resultJson: null, contextSnapshot: null, usageJson: null, + createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z", invocationSource: "manual", + startedAt: null, finishedAt: null, logRef: null, exitCode: 1, + execution: { canStop: false, phase: "completed", label: "Continued in another run" }, + } as unknown as HeartbeatRun; + get.mockResolvedValue(run); + const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + const element = document.createElement("div"); document.body.append(element); + const root = createRoot(element); + try { + await act(async () => root.render( + + )); + await vi.waitFor(async () => { + await act(async () => {}); + expect(element.textContent).toContain("computer ran out of disk space"); + }); + expect(element.textContent).not.toContain("previous turn is no longer active"); + expect(cancel).not.toHaveBeenCalled(); + } finally { await act(async () => root.unmount()); client.clear(); element.remove(); } + }, +); + it.each([RunsTab, ProductionRunsTab].flatMap(Tab => states.map(state => ({ Tab, ...state }))))("renders and submits only eligible run cancellation (%#)", async ({ Tab, status, eligibility, label, ...state }) => { if (state.local) { actor.id = "local-board"; actor.source = "local_implicit"; } const run = { id: state.priorSession ? `persisted-run-${Tab === RunsTab ? "main" : "production"}` : "failed-native-run", companyId: "company-1", agentId: "agent-1", issueId: "issue-1", status, driverKind: state.driverKind, invocationSource: "manual", diff --git a/ui/src/pages/AgentDetail.production.tsx b/ui/src/pages/AgentDetail.production.tsx index d502334c323..b4144a0fe13 100644 --- a/ui/src/pages/AgentDetail.production.tsx +++ b/ui/src/pages/AgentDetail.production.tsx @@ -66,6 +66,7 @@ import { readSourceResolvedWatchdogFold } from "../lib/source-resolved-watchdog- import { formatCents, formatDate, relativeTime, formatTokens, visibleRunCostUsd, visibleRunTokenTotal } from "../lib/utils"; import { cn } from "../lib/utils"; import { ProviderLimitMessage } from "../components/ProviderLimitMessage"; +import { runFailureMessage } from "@paperclipai/shared"; import { describeRunRetryState } from "../lib/runRetryState"; import { Button } from "@/components/ui/button"; import { Skeleton } from "@/components/ui/skeleton"; @@ -3039,9 +3040,9 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig } )} )} - {run.error && ( + {runFailureMessage(run) && (
- {run.error} + {runFailureMessage(run)!} {run.errorCode && ({run.errorCode})}
)} diff --git a/ui/src/pages/AgentDetail.tsx b/ui/src/pages/AgentDetail.tsx index 086560673fa..d46a657a60c 100644 --- a/ui/src/pages/AgentDetail.tsx +++ b/ui/src/pages/AgentDetail.tsx @@ -6,7 +6,7 @@ import { GrokBotWorkSummary } from "@/components/GrokBotWorkSummary"; import { PrimaryAgentCrownButton } from "@/components/primary-agent/PrimaryAgentPresentation"; import { AiConnectionPoolRunDetails } from "@/components/ai-connections/AiConnectionPoolRunDetails"; import { ProviderFailureNotice } from "@/components/ProviderFailureNotice"; -import { readProviderFailure } from "@paperclipai/shared"; +import { readProviderFailure, runFailureMessage } from "@paperclipai/shared"; import { AgentConnectionInstructions } from "@/features/connections/ConnectionInstructions"; import type { AgentInstructionCandidate, AgentInstructionsBundle } from "@paperclipai/shared"; import { InstructionHistory } from "../components/InstructionHistory"; @@ -3362,11 +3362,11 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig } )} - {readProviderFailure(run.resultJson?.providerFailure) ? ( + {run.errorCode !== "sandbox_disk_full" && readProviderFailure(run.resultJson?.providerFailure) ? ( - ) : run.error && ( + ) : runFailureMessage(run) && (
- {run.error} + {runFailureMessage(run)!} {run.errorCode && ({run.errorCode})}
)}