mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:19:23 +01:00
test(chaos): add DF1, a sandbox disk-full fault on the final workspace save
A test-only server hook, armed only by PAPERCLIP_DISK_FULL_FAULT_CONTROL and never under NODE_ENV=production, makes an armed task's final workspace saves get what a full sandbox returns. The harness arms DF1 tasks at launch, clears them at the first rest, and grades the sandbox_disk_full rest, the offered Retry and whether that Retry finished the task. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
556b249897
commit
57e8ebe355
14 files changed
+548
-13
No files matched your search
@@ -9,6 +9,7 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { and, eq, inArray, isNull, or, sql } from "drizzle-orm";
|
||||
import { crashPoint } from "@paperclipai/shared/crash-points";
|
||||
import { sandboxDiskFullFaultTarget } from "./sandbox-disk-full-fault.js";
|
||||
import {
|
||||
companies, environmentLeases, executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issues, workspaceOperations, type Db,
|
||||
} from "@paperclipai/db";
|
||||
@@ -912,7 +913,9 @@ export async function saveRemoteWorkspace(input: {
|
||||
const existingUpload = snapshot.status === "uploaded" ? await getWorkspaceSnapshot(input.db, { companyId: input.workspace.companyId,
|
||||
workspaceId: input.workspace.id, snapshotId: snapshot.id }) : null;
|
||||
if (existingUpload) timing.reusedUpload = true;
|
||||
captured = existingUpload ? null : await captureRemoteWorkspaceArchive({ target, remoteCwd: expected.location.remoteCwd,
|
||||
captured = existingUpload ? null : await captureRemoteWorkspaceArchive({ remoteCwd: expected.location.remoteCwd,
|
||||
// Test-only DF1 hook (sandbox-disk-full-fault.ts): the target itself unless a harness armed this save.
|
||||
target: sandboxDiskFullFaultTarget(target, { boundary, issueId: operation.issueId, runId: input.runId }),
|
||||
operationId: operation.id, timing });
|
||||
await input.assertOwned();
|
||||
const uploadStarted = performance.now();
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL, captureRemoteWorkspaceArchive, type RemoteWorkspaceTarget } from "./remote-workspace-transport.js";
|
||||
import {
|
||||
DISK_FULL_FAULT_CONTROL_ENV, DISK_FULL_FAULT_RECEIPT_ENV, DISK_FULL_FAULT_SYNC_OUT_MESSAGE,
|
||||
readDiskFullFaultGate, sandboxDiskFullFaultTarget, setDiskFullFaultGateForTest,
|
||||
} from "./sandbox-disk-full-fault.js";
|
||||
|
||||
const ISSUE = "11111111-1111-4111-8111-111111111111";
|
||||
const RUN = "22222222-2222-4222-8222-222222222222";
|
||||
const OPERATION = "33333333-3333-4333-8333-333333333333";
|
||||
|
||||
function sandboxTarget(calls: string[]): RemoteWorkspaceTarget {
|
||||
return {
|
||||
kind: "remote", transport: "sandbox", remoteCwd: "/workspace", providerKey: "test",
|
||||
runner: {
|
||||
execute: async () => { calls.push("execute"); return { exitCode: 0, signal: null, timedOut: false, stdout: "{}", stderr: "", pid: 1, startedAt: null }; },
|
||||
syncIn: async () => { calls.push("syncIn"); return {} as never; },
|
||||
syncOut: async () => { calls.push("syncOut"); return {} as never; },
|
||||
},
|
||||
} as unknown as RemoteWorkspaceTarget;
|
||||
}
|
||||
|
||||
describe("sandbox disk-full fault gate", () => {
|
||||
let restore: (() => void) | null = null;
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => {
|
||||
restore?.(); restore = null;
|
||||
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
const scratch = () => { const dir = mkdtempSync(path.join(os.tmpdir(), "disk-full-fault-")); dirs.push(dir); return dir; };
|
||||
|
||||
it("is off unless the harness switch is set, and never on in production", () => {
|
||||
expect(readDiskFullFaultGate({})).toBeNull();
|
||||
expect(readDiskFullFaultGate({ [DISK_FULL_FAULT_CONTROL_ENV]: " " })).toBeNull();
|
||||
expect(readDiskFullFaultGate({ NODE_ENV: "production", [DISK_FULL_FAULT_CONTROL_ENV]: "/tmp/control" })).toBeNull();
|
||||
expect(readDiskFullFaultGate({ NODE_ENV: "test", [DISK_FULL_FAULT_CONTROL_ENV]: "/tmp/control", [DISK_FULL_FAULT_RECEIPT_ENV]: "/tmp/receipt" }))
|
||||
.toEqual({ controlPath: "/tmp/control", receiptPath: "/tmp/receipt" });
|
||||
});
|
||||
|
||||
it("returns the save's own target when the process was started without the switch", () => {
|
||||
const target = sandboxTarget([]);
|
||||
expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: ISSUE, runId: RUN })).toBe(target);
|
||||
});
|
||||
|
||||
it("faults only a final save whose issue or run is listed, and stops when cleared", async () => {
|
||||
const dir = scratch();
|
||||
const control = path.join(dir, "control"), receipt = path.join(dir, "receipts.jsonl");
|
||||
restore = setDiskFullFaultGateForTest({ controlPath: control, receiptPath: receipt });
|
||||
const calls: string[] = [];
|
||||
const target = sandboxTarget(calls);
|
||||
// No control file yet: disarmed.
|
||||
expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: ISSUE, runId: RUN })).toBe(target);
|
||||
writeFileSync(control, `other-issue\n${ISSUE}\n`);
|
||||
expect(sandboxDiskFullFaultTarget(target, { boundary: "initial", issueId: ISSUE, runId: RUN })).toBe(target);
|
||||
expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: "other", runId: "other-run" })).toBe(target);
|
||||
|
||||
const faulted = sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: ISSUE, runId: RUN });
|
||||
expect(faulted).not.toBe(target);
|
||||
expect(await faulted.runner!.execute({ command: "node" })).toMatchObject({ exitCode: 1, stderr: REMOTE_WORKSPACE_SANDBOX_DISK_FULL });
|
||||
await expect(faulted.runner!.syncOut!([])).rejects.toThrow(DISK_FULL_FAULT_SYNC_OUT_MESSAGE);
|
||||
await faulted.runner!.syncIn!([]);
|
||||
expect(calls).toEqual(["syncIn"]);
|
||||
expect(JSON.parse(readFileSync(receipt, "utf8").trim())).toMatchObject({ issueId: ISSUE, runId: RUN, boundary: "final" });
|
||||
|
||||
// The save's own capture path turns the sandbox's answer into the named disk-full failure.
|
||||
await expect(captureRemoteWorkspaceArchive({ target: faulted, remoteCwd: "/workspace", operationId: OPERATION }))
|
||||
.rejects.toThrow(REMOTE_WORKSPACE_SANDBOX_DISK_FULL);
|
||||
|
||||
// A run id arms it too; clearing the file disarms the next save.
|
||||
writeFileSync(control, `${RUN}\n`);
|
||||
expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: null, runId: RUN })).not.toBe(target);
|
||||
writeFileSync(control, "");
|
||||
expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: ISSUE, runId: RUN })).toBe(target);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
/**
|
||||
* Test-only sandbox disk-full fault (restart-chaos DF1, tests/restart-chaos/disk-full-faults.ts).
|
||||
*
|
||||
* A real full sandbox disk cannot be made on demand, locally or on Daytona. This hook makes a
|
||||
* final workspace save see what a full sandbox returns, and leaves every decision after that to
|
||||
* the product: the save's own failure handling, finalization, saved-work recovery and Retry.
|
||||
*
|
||||
* Arming (test mode only), in the same style as `@paperclipai/shared/crash-points`:
|
||||
*
|
||||
* PAPERCLIP_DISK_FULL_FAULT_CONTROL=<file> each final save reads this file. A line that equals
|
||||
* the save's issue id or run id arms that save. No file,
|
||||
* or no matching line, means disarmed.
|
||||
* PAPERCLIP_DISK_FULL_FAULT_RECEIPT=<file> optional; one JSON line is appended per faulted save.
|
||||
*
|
||||
* While a save is armed, every sandbox command its capture runs exits 1 with the exact message the
|
||||
* in-sandbox capture script writes on ENOSPC, and the provider's file download fails with tar's
|
||||
* "No space left on device". Restore and other operations are untouched.
|
||||
*
|
||||
* The variable is read once, when this module is first imported, and is ignored when
|
||||
* `NODE_ENV=production`. Disarmed, the hook is one comparison against a module constant and
|
||||
* returns the save's own target. Only the chaos harness sets the variable, on the server it starts.
|
||||
*
|
||||
* This is test instrumentation, not Telemetry or Observability: nothing is recorded or sent
|
||||
* anywhere unless a test arms a save.
|
||||
*/
|
||||
import { appendFileSync, readFileSync } from "node:fs";
|
||||
import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL, type RemoteWorkspaceTarget } from "./remote-workspace-transport.js";
|
||||
|
||||
export const DISK_FULL_FAULT_CONTROL_ENV = "PAPERCLIP_DISK_FULL_FAULT_CONTROL";
|
||||
export const DISK_FULL_FAULT_RECEIPT_ENV = "PAPERCLIP_DISK_FULL_FAULT_RECEIPT";
|
||||
/** The provider download's failure on a full disk: tar cannot write its archive. */
|
||||
export const DISK_FULL_FAULT_SYNC_OUT_MESSAGE = "tar: workspace archive: Cannot write: No space left on device";
|
||||
|
||||
export type DiskFullFaultReceipt = { issueId: string | null; runId: string; boundary: string; pid: number; at: string };
|
||||
|
||||
type Gate = { controlPath: string; receiptPath: string | null };
|
||||
|
||||
export function readDiskFullFaultGate(env: NodeJS.ProcessEnv | undefined): Gate | null {
|
||||
if (!env || env.NODE_ENV === "production") return null;
|
||||
const controlPath = env[DISK_FULL_FAULT_CONTROL_ENV]?.trim();
|
||||
if (!controlPath) return null;
|
||||
return { controlPath, receiptPath: env[DISK_FULL_FAULT_RECEIPT_ENV]?.trim() || null };
|
||||
}
|
||||
|
||||
let gate: Gate | null = readDiskFullFaultGate(typeof process === "undefined" ? undefined : process.env);
|
||||
if (gate) process.stderr.write(`[disk-full-fault] armed from ${gate.controlPath} pid=${process.pid}\n`);
|
||||
|
||||
function armed(current: Gate, ids: readonly string[]): boolean {
|
||||
let text: string;
|
||||
try { text = readFileSync(current.controlPath, "utf8"); } catch { return false; }
|
||||
const lines = new Set(text.split("\n").map(line => line.trim()).filter(Boolean));
|
||||
return ids.some(id => lines.has(id));
|
||||
}
|
||||
|
||||
/**
|
||||
* The target a workspace save captures through. Disarmed, or for any boundary other than the
|
||||
* final copy-back, it is `target` itself.
|
||||
*/
|
||||
export function sandboxDiskFullFaultTarget<T extends RemoteWorkspaceTarget>(target: T,
|
||||
save: { boundary: string; issueId: string | null; runId: string }): T {
|
||||
if (gate === null) return target;
|
||||
if (save.boundary !== "final" || !armed(gate, [save.runId, ...(save.issueId ? [save.issueId] : [])])) return target;
|
||||
const receipt: DiskFullFaultReceipt = { issueId: save.issueId, runId: save.runId, boundary: save.boundary,
|
||||
pid: process.pid, at: new Date().toISOString() };
|
||||
if (gate.receiptPath) {
|
||||
try { appendFileSync(gate.receiptPath, `${JSON.stringify(receipt)}\n`); } catch { /* the log line still records it */ }
|
||||
}
|
||||
process.stderr.write(`[disk-full-fault] save ${JSON.stringify(receipt)}\n`);
|
||||
const runner = target.runner!;
|
||||
const full = Object.create(runner) as typeof runner;
|
||||
full.execute = async () => ({ exitCode: 1, signal: null, timedOut: false, stdout: "",
|
||||
stderr: REMOTE_WORKSPACE_SANDBOX_DISK_FULL, pid: null, startedAt: null });
|
||||
if (runner.syncOut) full.syncOut = async () => { throw new Error(DISK_FULL_FAULT_SYNC_OUT_MESSAGE); };
|
||||
return { ...target, runner: full };
|
||||
}
|
||||
|
||||
/** Replaces the gate inside a test process; returns a function that restores the previous one. */
|
||||
export function setDiskFullFaultGateForTest(next: Gate | null): () => void {
|
||||
const previous = gate;
|
||||
gate = next;
|
||||
return () => { gate = previous; };
|
||||
}
|
||||
Reference in new issue
Block a user