fix(runner): show disk exhaustion before recovery errors (#22448)

Show the confirmed sandbox disk failure before a later lost-turn or restore error in saved run errors and both run detail views. Preserve the original diagnostics and leave scarce inodes available when preparing the existing runner relaunch reserve.

Verified focused regressions, scoped typechecks, and full CI on the reviewed revision, including Runner checks and all eight end-to-end shards. Greptile is 5/5 with all review findings resolved.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip authored and GitHub committed 2026-11-20 17:08:31 -08:00
1 parent 611dc68581
commit b4cea5b5a4
10 files changed
+87 -14

No files matched your search

@@ -7,6 +7,7 @@ import { nativeLeaseHolder, notifyNativeTokenLeaseClaimed } from "./native-token
import { captureRemoteNativeProviderArtifacts, shareNativeProviderArtifactCapture } from "./native-provider-artifact-capture.js";
import { NativeSessionDiagnostics, nativeSessionDiagnostics, type NativeDiagnosticLease } from "./native-session-diagnostics.js";
import { NATIVE_SANDBOX_DISK_FULL, NATIVE_SANDBOX_DISK_FULL_ACTION, nativeSandboxDiskFullSourceCode } from "./native-sandbox-disk-full.js";
import { runFailureMessage } from "@paperclipai/shared";
import { recordAgentProviderRateLimits } from "../agent-provider-rate-limits.js";
import { nativePostResultFailureCategory, type NativePostResultFailureCategory } from "./native-post-result-failure.js";
import { logger } from "../../middleware/logger.js";
@@ -13494,10 +13495,11 @@ async function executePaperclipNativeSessionAttempt(
const { exhausted } = recoveryProjection;
const integrityFailure =
sourceFailureCode === "native_event_replay_conflict";
const message = redactOutputText(
const originalMessage = redactOutputText(
error instanceof Error ? error.message : String(error),
).slice(0, 2_000);
const sanitizedStderrTail = redactOutputText(message).slice(-4_096);
const message = runFailureMessage({ errorCode: sourceFailureCode, error: originalMessage })!;
const sanitizedStderrTail = originalMessage.slice(-4_096);
const remoteRunnerRecovery = readRemoteRunnerRecoveryError(error);
// Set inside the transaction only when the write below genuinely
// transitions the run into "failed". Read after the transaction
@@ -284,4 +284,16 @@ describe("remote runner disk reserve scripts", () => {
.toBe("/home/daytona/ws/.paperclip-runtime/paperclip-runner/disk-reserve");
expect(remoteRunnerDiskReservePath({ transport: "ssh", remoteCwd: "/home/me/ws" })).toBeUndefined();
});
it("leaves scarce inodes available to the running agent", async () => {
const root = await mkdtemp(join(tmpdir(), "runner-disk-reserve-"));
const reserve = join(root, "disk-reserve");
try {
await holdRemoteRunnerDiskReserve({ execute: localExecute }, reserve,
{ bytes: 1024 * 1024, minimumFreeBytes: 0, minimumFreeInodes: Number.MAX_SAFE_INTEGER });
expect(existsSync(reserve)).toBe(false);
} finally {
await rm(root, { recursive: true, force: true });
}
});
});
@@ -15,10 +15,12 @@ const MINIMUM_FREE_BYTES = 2 * 1024 * 1024 * 1024;
type Runner = Pick<CommandManagedRuntimeRunner, "execute">;
const HOLD_SCRIPT = [
'set -u; reserve=$1; bytes=$2; minimum_free_kib=$3; [ -f "$reserve" ] && exit 0',
'set -u; reserve=$1; bytes=$2; minimum_free_kib=$3; minimum_free_inodes=$4; [ -f "$reserve" ] && exit 0',
'directory=$(dirname -- "$reserve"); mkdir -p -- "$directory" || exit 1',
`free_kib=$(df -Pk -- "$directory" | awk 'NR == 2 { print $4 }')`,
'[ -n "$free_kib" ] && [ "$free_kib" -ge "$minimum_free_kib" ] || exit 0',
`free_inodes=$(df -Pi -- "$directory" | awk 'NR == 2 { print $(NF - 2) }')`,
'[ -n "$free_inodes" ] && [ "$free_inodes" -ge "$minimum_free_inodes" ] || exit 0',
'partial="$reserve.partial"; rm -f -- "$partial"',
'fallocate -l "$bytes" "$partial" 2>/dev/null || dd if=/dev/zero of="$partial" bs=65536 count=$((bytes / 65536)) 2>/dev/null || { rm -f -- "$partial"; exit 1; }',
'mv -f -- "$partial" "$reserve"',
@@ -31,14 +33,14 @@ const RELEASE_SCRIPT =
export async function holdRemoteRunnerDiskReserve(
runner: Runner,
reservePath: string,
options: { bytes?: number; minimumFreeBytes?: number } = {},
options: { bytes?: number; minimumFreeBytes?: number; minimumFreeInodes?: number } = {},
): Promise<void> {
const bytes = options.bytes ?? REMOTE_RUNNER_DISK_RESERVE_BYTES;
const minimumFreeBytes = options.minimumFreeBytes ?? MINIMUM_FREE_BYTES;
await runner.execute({
command: "sh",
args: ["-c", HOLD_SCRIPT, "paperclip-runner-disk-reserve-hold", reservePath,
String(bytes), String(Math.ceil((minimumFreeBytes + bytes) / 1024))],
String(bytes), String(Math.ceil((minimumFreeBytes + bytes) / 1024)), String(options.minimumFreeInodes ?? 1024)],
bypassSession: true,
timeoutMs: 30_000,
});