test(chaos): add DF1, a sandbox disk-full fault on the final workspace save

A test-only server hook, armed only by PAPERCLIP_DISK_FULL_FAULT_CONTROL and
never under NODE_ENV=production, makes an armed task's final workspace saves
get what a full sandbox returns. The harness arms DF1 tasks at launch, clears
them at the first rest, and grades the sandbox_disk_full rest, the offered
Retry and whether that Retry finished the task.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
DottaandClaude committed 2026-11-11 07:12:51 -06:00
1 parent 556b249897
commit 57e8ebe355
14 files changed
+548 -13

No files matched your search

+4 -1
View File
@@ -9,6 +9,7 @@ import os from "node:os";
import path from "node:path";
import { and, eq, inArray, isNull, or, sql } from "drizzle-orm";
import { crashPoint } from "@paperclipai/shared/crash-points";
import { sandboxDiskFullFaultTarget } from "./sandbox-disk-full-fault.js";
import {
companies, environmentLeases, executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issues, workspaceOperations, type Db,
} from "@paperclipai/db";
@@ -912,7 +913,9 @@ export async function saveRemoteWorkspace(input: {
const existingUpload = snapshot.status === "uploaded" ? await getWorkspaceSnapshot(input.db, { companyId: input.workspace.companyId,
workspaceId: input.workspace.id, snapshotId: snapshot.id }) : null;
if (existingUpload) timing.reusedUpload = true;
captured = existingUpload ? null : await captureRemoteWorkspaceArchive({ target, remoteCwd: expected.location.remoteCwd,
captured = existingUpload ? null : await captureRemoteWorkspaceArchive({ remoteCwd: expected.location.remoteCwd,
// Test-only DF1 hook (sandbox-disk-full-fault.ts): the target itself unless a harness armed this save.
target: sandboxDiskFullFaultTarget(target, { boundary, issueId: operation.issueId, runId: input.runId }),
operationId: operation.id, timing });
await input.assertOwned();
const uploadStarted = performance.now();
@@ -0,0 +1,78 @@
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL, captureRemoteWorkspaceArchive, type RemoteWorkspaceTarget } from "./remote-workspace-transport.js";
import {
DISK_FULL_FAULT_CONTROL_ENV, DISK_FULL_FAULT_RECEIPT_ENV, DISK_FULL_FAULT_SYNC_OUT_MESSAGE,
readDiskFullFaultGate, sandboxDiskFullFaultTarget, setDiskFullFaultGateForTest,
} from "./sandbox-disk-full-fault.js";
const ISSUE = "11111111-1111-4111-8111-111111111111";
const RUN = "22222222-2222-4222-8222-222222222222";
const OPERATION = "33333333-3333-4333-8333-333333333333";
function sandboxTarget(calls: string[]): RemoteWorkspaceTarget {
return {
kind: "remote", transport: "sandbox", remoteCwd: "/workspace", providerKey: "test",
runner: {
execute: async () => { calls.push("execute"); return { exitCode: 0, signal: null, timedOut: false, stdout: "{}", stderr: "", pid: 1, startedAt: null }; },
syncIn: async () => { calls.push("syncIn"); return {} as never; },
syncOut: async () => { calls.push("syncOut"); return {} as never; },
},
} as unknown as RemoteWorkspaceTarget;
}
describe("sandbox disk-full fault gate", () => {
let restore: (() => void) | null = null;
const dirs: string[] = [];
afterEach(() => {
restore?.(); restore = null;
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
const scratch = () => { const dir = mkdtempSync(path.join(os.tmpdir(), "disk-full-fault-")); dirs.push(dir); return dir; };
it("is off unless the harness switch is set, and never on in production", () => {
expect(readDiskFullFaultGate({})).toBeNull();
expect(readDiskFullFaultGate({ [DISK_FULL_FAULT_CONTROL_ENV]: " " })).toBeNull();
expect(readDiskFullFaultGate({ NODE_ENV: "production", [DISK_FULL_FAULT_CONTROL_ENV]: "/tmp/control" })).toBeNull();
expect(readDiskFullFaultGate({ NODE_ENV: "test", [DISK_FULL_FAULT_CONTROL_ENV]: "/tmp/control", [DISK_FULL_FAULT_RECEIPT_ENV]: "/tmp/receipt" }))
.toEqual({ controlPath: "/tmp/control", receiptPath: "/tmp/receipt" });
});
it("returns the save's own target when the process was started without the switch", () => {
const target = sandboxTarget([]);
expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: ISSUE, runId: RUN })).toBe(target);
});
it("faults only a final save whose issue or run is listed, and stops when cleared", async () => {
const dir = scratch();
const control = path.join(dir, "control"), receipt = path.join(dir, "receipts.jsonl");
restore = setDiskFullFaultGateForTest({ controlPath: control, receiptPath: receipt });
const calls: string[] = [];
const target = sandboxTarget(calls);
// No control file yet: disarmed.
expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: ISSUE, runId: RUN })).toBe(target);
writeFileSync(control, `other-issue\n${ISSUE}\n`);
expect(sandboxDiskFullFaultTarget(target, { boundary: "initial", issueId: ISSUE, runId: RUN })).toBe(target);
expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: "other", runId: "other-run" })).toBe(target);
const faulted = sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: ISSUE, runId: RUN });
expect(faulted).not.toBe(target);
expect(await faulted.runner!.execute({ command: "node" })).toMatchObject({ exitCode: 1, stderr: REMOTE_WORKSPACE_SANDBOX_DISK_FULL });
await expect(faulted.runner!.syncOut!([])).rejects.toThrow(DISK_FULL_FAULT_SYNC_OUT_MESSAGE);
await faulted.runner!.syncIn!([]);
expect(calls).toEqual(["syncIn"]);
expect(JSON.parse(readFileSync(receipt, "utf8").trim())).toMatchObject({ issueId: ISSUE, runId: RUN, boundary: "final" });
// The save's own capture path turns the sandbox's answer into the named disk-full failure.
await expect(captureRemoteWorkspaceArchive({ target: faulted, remoteCwd: "/workspace", operationId: OPERATION }))
.rejects.toThrow(REMOTE_WORKSPACE_SANDBOX_DISK_FULL);
// A run id arms it too; clearing the file disarms the next save.
writeFileSync(control, `${RUN}\n`);
expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: null, runId: RUN })).not.toBe(target);
writeFileSync(control, "");
expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: ISSUE, runId: RUN })).toBe(target);
});
});
@@ -0,0 +1,82 @@
/**
* Test-only sandbox disk-full fault (restart-chaos DF1, tests/restart-chaos/disk-full-faults.ts).
*
* A real full sandbox disk cannot be made on demand, locally or on Daytona. This hook makes a
* final workspace save see what a full sandbox returns, and leaves every decision after that to
* the product: the save's own failure handling, finalization, saved-work recovery and Retry.
*
* Arming (test mode only), in the same style as `@paperclipai/shared/crash-points`:
*
* PAPERCLIP_DISK_FULL_FAULT_CONTROL=<file> each final save reads this file. A line that equals
* the save's issue id or run id arms that save. No file,
* or no matching line, means disarmed.
* PAPERCLIP_DISK_FULL_FAULT_RECEIPT=<file> optional; one JSON line is appended per faulted save.
*
* While a save is armed, every sandbox command its capture runs exits 1 with the exact message the
* in-sandbox capture script writes on ENOSPC, and the provider's file download fails with tar's
* "No space left on device". Restore and other operations are untouched.
*
* The variable is read once, when this module is first imported, and is ignored when
* `NODE_ENV=production`. Disarmed, the hook is one comparison against a module constant and
* returns the save's own target. Only the chaos harness sets the variable, on the server it starts.
*
* This is test instrumentation, not Telemetry or Observability: nothing is recorded or sent
* anywhere unless a test arms a save.
*/
import { appendFileSync, readFileSync } from "node:fs";
import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL, type RemoteWorkspaceTarget } from "./remote-workspace-transport.js";
export const DISK_FULL_FAULT_CONTROL_ENV = "PAPERCLIP_DISK_FULL_FAULT_CONTROL";
export const DISK_FULL_FAULT_RECEIPT_ENV = "PAPERCLIP_DISK_FULL_FAULT_RECEIPT";
/** The provider download's failure on a full disk: tar cannot write its archive. */
export const DISK_FULL_FAULT_SYNC_OUT_MESSAGE = "tar: workspace archive: Cannot write: No space left on device";
export type DiskFullFaultReceipt = { issueId: string | null; runId: string; boundary: string; pid: number; at: string };
type Gate = { controlPath: string; receiptPath: string | null };
export function readDiskFullFaultGate(env: NodeJS.ProcessEnv | undefined): Gate | null {
if (!env || env.NODE_ENV === "production") return null;
const controlPath = env[DISK_FULL_FAULT_CONTROL_ENV]?.trim();
if (!controlPath) return null;
return { controlPath, receiptPath: env[DISK_FULL_FAULT_RECEIPT_ENV]?.trim() || null };
}
let gate: Gate | null = readDiskFullFaultGate(typeof process === "undefined" ? undefined : process.env);
if (gate) process.stderr.write(`[disk-full-fault] armed from ${gate.controlPath} pid=${process.pid}\n`);
function armed(current: Gate, ids: readonly string[]): boolean {
let text: string;
try { text = readFileSync(current.controlPath, "utf8"); } catch { return false; }
const lines = new Set(text.split("\n").map(line => line.trim()).filter(Boolean));
return ids.some(id => lines.has(id));
}
/**
* The target a workspace save captures through. Disarmed, or for any boundary other than the
* final copy-back, it is `target` itself.
*/
export function sandboxDiskFullFaultTarget<T extends RemoteWorkspaceTarget>(target: T,
save: { boundary: string; issueId: string | null; runId: string }): T {
if (gate === null) return target;
if (save.boundary !== "final" || !armed(gate, [save.runId, ...(save.issueId ? [save.issueId] : [])])) return target;
const receipt: DiskFullFaultReceipt = { issueId: save.issueId, runId: save.runId, boundary: save.boundary,
pid: process.pid, at: new Date().toISOString() };
if (gate.receiptPath) {
try { appendFileSync(gate.receiptPath, `${JSON.stringify(receipt)}\n`); } catch { /* the log line still records it */ }
}
process.stderr.write(`[disk-full-fault] save ${JSON.stringify(receipt)}\n`);
const runner = target.runner!;
const full = Object.create(runner) as typeof runner;
full.execute = async () => ({ exitCode: 1, signal: null, timedOut: false, stdout: "",
stderr: REMOTE_WORKSPACE_SANDBOX_DISK_FULL, pid: null, startedAt: null });
if (runner.syncOut) full.syncOut = async () => { throw new Error(DISK_FULL_FAULT_SYNC_OUT_MESSAGE); };
return { ...target, runner: full };
}
/** Replaces the gate inside a test process; returns a function that restores the previous one. */
export function setDiskFullFaultGateForTest(next: Gate | null): () => void {
const previous = gate;
gate = next;
return () => { gate = previous; };
}