mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-21 02:43:25 +01:00
fix(runner): show disk exhaustion before recovery errors (#22448)
Show the confirmed sandbox disk failure before a later lost-turn or restore error in saved run errors and both run detail views. Preserve the original diagnostics and leave scarce inodes available when preparing the existing runner relaunch reserve. Verified focused regressions, scoped typechecks, and full CI on the reviewed revision, including Runner checks and all eight end-to-end shards. Greptile is 5/5 with all review findings resolved. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
611dc68581
commit
b4cea5b5a4
10 files changed
+87
-14
No files matched your search
@@ -10,6 +10,20 @@ The built-in TypeScript implementation constructs the provider descriptor in
|
||||
By default, the SDK prepares local files. It stages remote files through the
|
||||
existing execution driver.
|
||||
|
||||
## Managed sandbox storage
|
||||
|
||||
Core holds a 256 MiB runner relaunch reserve only in managed sandboxes. It
|
||||
creates the reserve when there is at least 2 GiB of additional free space and
|
||||
1,024 free inodes. It leaves scarce capacity available to an existing agent.
|
||||
A relaunch releases that reserve before it stages the replacement runner.
|
||||
This reserve is recovery headroom; it does not limit the agent's downloads.
|
||||
|
||||
When runner diagnostics confirm a full sandbox disk, the saved run error and
|
||||
run details show that cause ahead of a later lost-turn or restore error. Raw
|
||||
failure diagnostics remain in the run's diagnostic evidence. Older saved runs
|
||||
with `sandbox_disk_full` get the same display without a data migration.
|
||||
Download budgets and image policy belong to the deployment environment.
|
||||
|
||||
## Environment-owned preparation
|
||||
|
||||
An execution driver can supply `prepareLaunch`.
|
||||
|
||||
@@ -1,7 +1,14 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeRunFailureCause, readRunFailureCause, RUN_FAILURE_CAUSE_MAX_LENGTH, withoutSqlStatementText } from "./run-failure-cause.js";
|
||||
import { normalizeRunFailureCause, readRunFailureCause, RUN_FAILURE_CAUSE_MAX_LENGTH, withoutSqlStatementText, runFailureMessage } from "./run-failure-cause.js";
|
||||
|
||||
describe("run failure cause contract", () => {
|
||||
it("leads with the confirmed full disk instead of the later restore failure", () => {
|
||||
expect(runFailureMessage({ errorCode: "sandbox_disk_full", error: "The previous turn is no longer active." }))
|
||||
.toMatch(/^The agent's computer ran out of disk space/);
|
||||
expect(runFailureMessage({ errorCode: "sandbox_disk_full", error: null })).toContain("Retry");
|
||||
expect(runFailureMessage({ errorCode: "process_lost", error: "Process lost" })).toBe("Process lost");
|
||||
expect(runFailureMessage({ errorCode: null, error: null })).toBeNull();
|
||||
});
|
||||
it("accepts bounded identifier codes", () => {
|
||||
for (const code of ["40001", "ECONNRESET", "provider_http_429", "TypeError", "signal_SIGKILL", "a".repeat(RUN_FAILURE_CAUSE_MAX_LENGTH)]) {
|
||||
expect(normalizeRunFailureCause(code)).toBe(code);
|
||||
|
||||
@@ -11,6 +11,14 @@
|
||||
export const RUN_FAILURE_CAUSE_MAX_LENGTH = 80;
|
||||
export const RUN_FAILURE_CAUSE_PATTERN = /^[A-Za-z0-9_.:-]{1,80}$/;
|
||||
|
||||
export function runFailureMessage(run: { errorCode?: string | null; error?: string | null }): string | null {
|
||||
if (run.errorCode === "sandbox_disk_full") {
|
||||
return "The agent's computer ran out of disk space, so its runner stopped. "
|
||||
+ "Ask the agent to use less disk space, then select Retry. Saved task updates remain available.";
|
||||
}
|
||||
return run.error ?? null;
|
||||
}
|
||||
|
||||
/** The stored cause when it has the closed shape, else `null` (absent, legacy or malformed). */
|
||||
export function normalizeRunFailureCause(value: unknown): string | null {
|
||||
return typeof value === "string" && RUN_FAILURE_CAUSE_PATTERN.test(value) ? value : null;
|
||||
|
||||
@@ -7,6 +7,7 @@ import { nativeLeaseHolder, notifyNativeTokenLeaseClaimed } from "./native-token
|
||||
import { captureRemoteNativeProviderArtifacts, shareNativeProviderArtifactCapture } from "./native-provider-artifact-capture.js";
|
||||
import { NativeSessionDiagnostics, nativeSessionDiagnostics, type NativeDiagnosticLease } from "./native-session-diagnostics.js";
|
||||
import { NATIVE_SANDBOX_DISK_FULL, NATIVE_SANDBOX_DISK_FULL_ACTION, nativeSandboxDiskFullSourceCode } from "./native-sandbox-disk-full.js";
|
||||
import { runFailureMessage } from "@paperclipai/shared";
|
||||
import { recordAgentProviderRateLimits } from "../agent-provider-rate-limits.js";
|
||||
import { nativePostResultFailureCategory, type NativePostResultFailureCategory } from "./native-post-result-failure.js";
|
||||
import { logger } from "../../middleware/logger.js";
|
||||
@@ -13494,10 +13495,11 @@ async function executePaperclipNativeSessionAttempt(
|
||||
const { exhausted } = recoveryProjection;
|
||||
const integrityFailure =
|
||||
sourceFailureCode === "native_event_replay_conflict";
|
||||
const message = redactOutputText(
|
||||
const originalMessage = redactOutputText(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
).slice(0, 2_000);
|
||||
const sanitizedStderrTail = redactOutputText(message).slice(-4_096);
|
||||
const message = runFailureMessage({ errorCode: sourceFailureCode, error: originalMessage })!;
|
||||
const sanitizedStderrTail = originalMessage.slice(-4_096);
|
||||
const remoteRunnerRecovery = readRemoteRunnerRecoveryError(error);
|
||||
// Set inside the transaction only when the write below genuinely
|
||||
// transitions the run into "failed". Read after the transaction
|
||||
|
||||
@@ -284,4 +284,16 @@ describe("remote runner disk reserve scripts", () => {
|
||||
.toBe("/home/daytona/ws/.paperclip-runtime/paperclip-runner/disk-reserve");
|
||||
expect(remoteRunnerDiskReservePath({ transport: "ssh", remoteCwd: "/home/me/ws" })).toBeUndefined();
|
||||
});
|
||||
|
||||
it("leaves scarce inodes available to the running agent", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "runner-disk-reserve-"));
|
||||
const reserve = join(root, "disk-reserve");
|
||||
try {
|
||||
await holdRemoteRunnerDiskReserve({ execute: localExecute }, reserve,
|
||||
{ bytes: 1024 * 1024, minimumFreeBytes: 0, minimumFreeInodes: Number.MAX_SAFE_INTEGER });
|
||||
expect(existsSync(reserve)).toBe(false);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -15,10 +15,12 @@ const MINIMUM_FREE_BYTES = 2 * 1024 * 1024 * 1024;
|
||||
type Runner = Pick<CommandManagedRuntimeRunner, "execute">;
|
||||
|
||||
const HOLD_SCRIPT = [
|
||||
'set -u; reserve=$1; bytes=$2; minimum_free_kib=$3; [ -f "$reserve" ] && exit 0',
|
||||
'set -u; reserve=$1; bytes=$2; minimum_free_kib=$3; minimum_free_inodes=$4; [ -f "$reserve" ] && exit 0',
|
||||
'directory=$(dirname -- "$reserve"); mkdir -p -- "$directory" || exit 1',
|
||||
`free_kib=$(df -Pk -- "$directory" | awk 'NR == 2 { print $4 }')`,
|
||||
'[ -n "$free_kib" ] && [ "$free_kib" -ge "$minimum_free_kib" ] || exit 0',
|
||||
`free_inodes=$(df -Pi -- "$directory" | awk 'NR == 2 { print $(NF - 2) }')`,
|
||||
'[ -n "$free_inodes" ] && [ "$free_inodes" -ge "$minimum_free_inodes" ] || exit 0',
|
||||
'partial="$reserve.partial"; rm -f -- "$partial"',
|
||||
'fallocate -l "$bytes" "$partial" 2>/dev/null || dd if=/dev/zero of="$partial" bs=65536 count=$((bytes / 65536)) 2>/dev/null || { rm -f -- "$partial"; exit 1; }',
|
||||
'mv -f -- "$partial" "$reserve"',
|
||||
@@ -31,14 +33,14 @@ const RELEASE_SCRIPT =
|
||||
export async function holdRemoteRunnerDiskReserve(
|
||||
runner: Runner,
|
||||
reservePath: string,
|
||||
options: { bytes?: number; minimumFreeBytes?: number } = {},
|
||||
options: { bytes?: number; minimumFreeBytes?: number; minimumFreeInodes?: number } = {},
|
||||
): Promise<void> {
|
||||
const bytes = options.bytes ?? REMOTE_RUNNER_DISK_RESERVE_BYTES;
|
||||
const minimumFreeBytes = options.minimumFreeBytes ?? MINIMUM_FREE_BYTES;
|
||||
await runner.execute({
|
||||
command: "sh",
|
||||
args: ["-c", HOLD_SCRIPT, "paperclip-runner-disk-reserve-hold", reservePath,
|
||||
String(bytes), String(Math.ceil((minimumFreeBytes + bytes) / 1024))],
|
||||
String(bytes), String(Math.ceil((minimumFreeBytes + bytes) / 1024)), String(options.minimumFreeInodes ?? 1024)],
|
||||
bypassSession: true,
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
|
||||
@@ -349,8 +349,8 @@
|
||||
"packages/shared/src/routine-suggestions.ts": "4895e6090faae8c36cd509667e69bc571e90f217",
|
||||
"packages/shared/src/routine-variables.test.ts": "3cd4ea5e1a279eacdc525a3eb8e7bf317170c1ec",
|
||||
"packages/shared/src/routine-variables.ts": "43d57551aaf8fcd32c10733af3198b6932b0d5d4",
|
||||
"packages/shared/src/run-failure-cause.test.ts": "68a726ac3e3b9be394ffa32b65acc6119e06424c",
|
||||
"packages/shared/src/run-failure-cause.ts": "6f7c646d715f384ee0a01805d5507b0c7a0669a2",
|
||||
"packages/shared/src/run-failure-cause.test.ts": "fdf56d4c6f3e29eb0c13c2e4a8b2b8b83111ca85",
|
||||
"packages/shared/src/run-failure-cause.ts": "33c31b748be37f6ce171cb11ae68c6bb0f288872",
|
||||
"packages/shared/src/runner-configuration-warnings.ts": "7d8db761ee51bec746c425320d5900ef3505de1b",
|
||||
"packages/shared/src/runner-goal.test.ts": "05814124d8f592d6bf4ca78541474e6f354411d1",
|
||||
"packages/shared/src/runner-goal.ts": "dc61b33fd36d2ed9f68fc3a5630a81d1e001333b",
|
||||
|
||||
@@ -50,6 +50,33 @@ const states = [
|
||||
{ status: "running", eligibility: undefined, label: "Cancel", driverKind: "grok_bot_mcp" },
|
||||
{ status: "queued", eligibility: undefined, label: "Cancel", driverKind: "grok_bot_mcp" },
|
||||
];
|
||||
|
||||
it.each([RunsTab, ProductionRunsTab].flatMap(Tab => [null, "The previous turn is no longer active."].map(error => ({ Tab, error }))))(
|
||||
"shows the confirmed full disk for historical run details (%#)", async ({ Tab, error }) => {
|
||||
const run = { id: "disk-full-run", companyId: "company-1", agentId: "agent-1", status: "failed",
|
||||
errorCode: "sandbox_disk_full", error, resultJson: null, contextSnapshot: null, usageJson: null,
|
||||
createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z", invocationSource: "manual",
|
||||
startedAt: null, finishedAt: null, logRef: null, exitCode: 1,
|
||||
execution: { canStop: false, phase: "completed", label: "Continued in another run" },
|
||||
} as unknown as HeartbeatRun;
|
||||
get.mockResolvedValue(run);
|
||||
const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } });
|
||||
const element = document.createElement("div"); document.body.append(element);
|
||||
const root = createRoot(element);
|
||||
try {
|
||||
await act(async () => root.render(<QueryClientProvider client={client}><MemoryRouter><PanelProvider><ToastProvider>
|
||||
<Tab runs={[]} companyId="company-1" agentId="agent-1" agentRouteId="agent-1" selectedRunId={run.id} adapterType="paperclip_runner" adapterConfig={{}} />
|
||||
</ToastProvider></PanelProvider></MemoryRouter></QueryClientProvider>));
|
||||
await vi.waitFor(async () => {
|
||||
await act(async () => {});
|
||||
expect(element.textContent).toContain("computer ran out of disk space");
|
||||
});
|
||||
expect(element.textContent).not.toContain("previous turn is no longer active");
|
||||
expect(cancel).not.toHaveBeenCalled();
|
||||
} finally { await act(async () => root.unmount()); client.clear(); element.remove(); }
|
||||
},
|
||||
);
|
||||
|
||||
it.each([RunsTab, ProductionRunsTab].flatMap(Tab => states.map(state => ({ Tab, ...state }))))("renders and submits only eligible run cancellation (%#)", async ({ Tab, status, eligibility, label, ...state }) => {
|
||||
if (state.local) { actor.id = "local-board"; actor.source = "local_implicit"; }
|
||||
const run = { id: state.priorSession ? `persisted-run-${Tab === RunsTab ? "main" : "production"}` : "failed-native-run", companyId: "company-1", agentId: "agent-1", issueId: "issue-1", status, driverKind: state.driverKind, invocationSource: "manual",
|
||||
|
||||
@@ -66,6 +66,7 @@ import { readSourceResolvedWatchdogFold } from "../lib/source-resolved-watchdog-
|
||||
import { formatCents, formatDate, relativeTime, formatTokens, visibleRunCostUsd, visibleRunTokenTotal } from "../lib/utils";
|
||||
import { cn } from "../lib/utils";
|
||||
import { ProviderLimitMessage } from "../components/ProviderLimitMessage";
|
||||
import { runFailureMessage } from "@paperclipai/shared";
|
||||
import { describeRunRetryState } from "../lib/runRetryState";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
@@ -3039,9 +3040,9 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig }
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{run.error && (
|
||||
{runFailureMessage(run) && (
|
||||
<div className="text-xs">
|
||||
<span className="text-red-600 dark:text-red-400"><ProviderLimitMessage>{run.error}</ProviderLimitMessage></span>
|
||||
<span className="text-destructive"><ProviderLimitMessage>{runFailureMessage(run)!}</ProviderLimitMessage></span>
|
||||
{run.errorCode && <span className="text-muted-foreground ml-1">({run.errorCode})</span>}
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -6,7 +6,7 @@ import { GrokBotWorkSummary } from "@/components/GrokBotWorkSummary";
|
||||
import { PrimaryAgentCrownButton } from "@/components/primary-agent/PrimaryAgentPresentation";
|
||||
import { AiConnectionPoolRunDetails } from "@/components/ai-connections/AiConnectionPoolRunDetails";
|
||||
import { ProviderFailureNotice } from "@/components/ProviderFailureNotice";
|
||||
import { readProviderFailure } from "@paperclipai/shared";
|
||||
import { readProviderFailure, runFailureMessage } from "@paperclipai/shared";
|
||||
import { AgentConnectionInstructions } from "@/features/connections/ConnectionInstructions";
|
||||
import type { AgentInstructionCandidate, AgentInstructionsBundle } from "@paperclipai/shared";
|
||||
import { InstructionHistory } from "../components/InstructionHistory";
|
||||
@@ -3362,11 +3362,11 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig }
|
||||
</div>
|
||||
)}
|
||||
<AgentFileRunNotice resultJson={run.resultJson} />
|
||||
{readProviderFailure(run.resultJson?.providerFailure) ? (
|
||||
{run.errorCode !== "sandbox_disk_full" && readProviderFailure(run.resultJson?.providerFailure) ? (
|
||||
<ProviderFailureNotice failure={readProviderFailure(run.resultJson?.providerFailure)!} />
|
||||
) : run.error && (
|
||||
) : runFailureMessage(run) && (
|
||||
<div className="text-xs">
|
||||
<span className="text-red-600 dark:text-red-400"><ProviderLimitMessage>{run.error}</ProviderLimitMessage></span>
|
||||
<span className="text-destructive"><ProviderLimitMessage>{runFailureMessage(run)!}</ProviderLimitMessage></span>
|
||||
{run.errorCode && <span className="text-muted-foreground ml-1">({run.errorCode})</span>}
|
||||
</div>
|
||||
)}
|
||||
|
||||
Reference in new issue
Block a user