diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index dceb36391cd..1731fc5df0e 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2377,6 +2377,15 @@ runnerd recovery, and durable control-plane readers. These paths synchronously read and parse the full JSON file, so memory use and parse time grow with file size. Remote checkpoint archive and expanded-size limits remain 64 MiB. +Remote provider checkpoints normally apply exclusions through a filtered copy +inside the sandbox. If that copy fails with a full-disk diagnostic, the controller +removes the partial copy and passes the exclusions to the provider's sync-out. +That transfer can stream without another sandbox archive. Provider exclusions can +also omit a nested entry with the same name; credentials remain excluded. Other +copy failures and timeouts still fail. Workspace capture itself still needs free +sandbox space. Its failed save operation names `remote_workspace_sandbox_disk_full` +when that capacity is exhausted, without changing retry or ownership rules. + Native assigned app tools save governed responses above 500,000 encoded bytes before returning a bounded reference through PRP. `read_tool_result` retrieves the saved JSON in UTF-8 pages under the same task, agent, and responsible-user access checks. diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index a139347bed6..109b25df894 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -74864,7 +74864,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { } }); - it("settles responses on different endpoints for one principal without retrying its own identity lock", async () => { + it.each(["initial", "rescoped"] as const)("settles responses on different endpoints for one principal without retrying its own identity lock: %s", async (mode) => { const fixture = await seedCompany(); const first = await configuredSlackEndpoint(fixture); const second = await configuredSlackEndpoint(fixture); @@ -74882,10 +74882,36 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const functionName = `hold_identity_${randomUUID().replaceAll("-", "")}`; await db.execute(sql.raw(`create function ${functionName}() returns trigger language plpgsql as $$ begin perform pg_sleep(0.1); return NEW; end $$`)); await db.execute(sql.raw(`create trigger ${functionName} before insert on issue_comments for each row execute function ${functionName}()`)); + const scopedWork = await import("../services/scoped-deadline-work.js"); + const register = scopedWork.registerScopedDeadlineWork; + let invalidated = false; + let invalidateSource: (() => void) | undefined; + const registration = vi.spyOn(scopedWork, "registerScopedDeadlineWork").mockImplementation((owner, options, canRun) => + register(owner, { ...options, load: async (scope, signal) => { + const entries = await options.load(scope, signal); + // An authority registration can invalidate discovery while its read is + // in flight. The real worker retains another pass under its deadline. + if (mode === "rescoped" && !invalidated) { + invalidated = true; + invalidateSource!(); + } + return entries; + } }, canRun)); const worker = nativeResponsePresentationWork(db).register(coordinator, () => true); + invalidateSource = () => worker.invalidate(); + registration.mockRestore(); try { await worker.ready; await expect.poll(() => idleWorkSnapshot().active, { timeout: 10_000 }).toBe(0); + if (mode === "rescoped") { + expect(invalidated).toBe(true); + expect(coordinator.nextWakeAt()).not.toBeNull(); + } + await expect.poll(async () => Promise.all(contexts.map(async context => + (await db.select({ id: issueComments.id }).from(issueComments) + .where(eq(issueComments.createdByRunId, context.runId))).length)), { timeout: 10_000 }).toEqual([1, 1]); + await expect.poll(() => ({ active: idleWorkSnapshot().active, nextWakeAt: coordinator.nextWakeAt() }), + { timeout: 10_000 }).toEqual({ active: 0, nextWakeAt: null }); expect(errors).toEqual([]); for (const context of contexts) { const comments = await db.select().from(issueComments).where(eq(issueComments.createdByRunId, context.runId)); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 57b549bffd1..1ee386fce17 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -37,7 +37,7 @@ import { import { execFile, execFileSync, spawn, spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import { inspect } from "node:util"; -import { basename, dirname, join } from "node:path"; +import { basename, dirname, join, relative } from "node:path"; import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; import { activityLog, @@ -3421,6 +3421,86 @@ describe("remote provider checkpoint snapshots", () => { expect(syncOut).toHaveBeenCalledOnce(); }); + it("lets the provider apply the exclusions when the sandbox disk cannot hold the snapshot", async () => { + const root = await mkdtemp(join(tmpdir(), "provider-cache-checkpoint-full-")); + try { + const live = join(root, "live", "acpx"); + const backup = join(root, "backup", "acpx"); + const sessionPath = `acpx/${acpxRuntimeSessionDirectoryName("session")}`; + const files: Record = { + "pi-home/sessions/session.jsonl": "history", + "data/pnpm/config.json": "{}", + "home/.npm/_cacache/index": "npm-cache", + "data/pnpm/store/v10/files/00/blob": "pnpm-store", + }; + for (const [path, contents] of Object.entries(files)) { + await mkdir(join(live, sessionPath, path, ".."), { recursive: true }); + await writeFile(join(live, sessionPath, path), contents); + } + const excludeEntries = resolveNativeHarnessPersistenceProfile({ + provider: { kind: "acpx", agent: "pi" }, + session: { normalizedSessionId: "session", driverKind: "acpx_runtime" }, + } as unknown as NativeExecutionInputV1).directories.find(entry => entry.name === "acpx")!.excludeEntries; + const commands: string[] = []; + const execute = vi.fn(async ({ command, args }: { command: string; args: string[] }) => { + const script = String(args[1]); + commands.push(script); + if (script.includes(" -cf ")) { + // Leave a partial copy behind, as tar does when the disk fills. + const snapshot = /mkdir -p '([^']+)'/.exec(script)![1]!; + await mkdir(join(snapshot, sessionPath), { recursive: true }); + return { exitCode: 2, timedOut: false, stdout: "", + stderr: "tar: ./acpx/session/pi-home/sessions/session.jsonl: Wrote only 4096 of 10240 bytes\n" }; + } + return { exitCode: 0, timedOut: false, stdout: execFileSync(command, args, { encoding: "utf8" }), stderr: "" }; + }); + // Provider exclusions match after any `/`, like GNU tar --exclude. + const syncOut = vi.fn(async (operations: Array<{ files: Array<{ sourcePath: string; targetPath: string; exclude?: string[] }> }>) => { + for (const operation of operations) for (const file of operation.files) { + await cp(file.sourcePath, file.targetPath, { recursive: true, filter: (path) => { + const entry = relative(file.sourcePath, path); + return !(file.exclude ?? []).some(pattern => entry === pattern || entry.endsWith(`/${pattern}`)); + } }); + } + }); + + await syncRemoteRunnerDirectoryOut({ runner: { execute, syncOut } as never, + sourcePath: live, targetPath: backup, mode: 0o700, excludeEntries }); + + expect(syncOut).toHaveBeenCalledOnce(); + expect(syncOut.mock.calls[0]![0][0]!.files[0]).toMatchObject({ sourcePath: live, exclude: [...excludeEntries] }); + expect(await readFile(join(backup, sessionPath, "pi-home/sessions/session.jsonl"), "utf8")).toBe("history"); + expect(await readFile(join(backup, sessionPath, "data/pnpm/config.json"), "utf8")).toBe("{}"); + for (const path of ["home/.npm", "data/pnpm/store"]) + await expect(access(join(backup, sessionPath, path))).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(join(root, "live"))).toEqual(["acpx"]); + expect(commands.filter(script => script.startsWith("rm -rf -- "))).toHaveLength(1); + } finally { await rm(root, { recursive: true, force: true }); } + }); + + it.each([ + { name: "a tar failure that is not a full disk", stderr: "tar: ./auth.json: Cannot open: Permission denied\n", timedOut: false }, + { name: "a timeout", stderr: "No space left on device\n", timedOut: true }, + ])("removes the partial snapshot and does not widen the copy after $name", async ({ stderr, timedOut }) => { + const execute = vi.fn() + .mockResolvedValueOnce({ exitCode: 0, timedOut: false, stdout: "", stderr: "" }) + .mockResolvedValueOnce({ exitCode: 2, timedOut, stdout: "", stderr }) + .mockResolvedValue({ exitCode: 0, timedOut: false, stdout: "", stderr: "" }); + const syncOut = vi.fn(async () => undefined); + await expect(syncRemoteRunnerDirectoryOut({ + runner: { execute, syncOut } as never, + sourcePath: "/remote/session/filesystem/codex-home", + targetPath: "/tmp/paperclip-checkpoint-test-codex-home", + mode: 0o700, + excludeEntries: ["tmp", "auth.json"], + })).rejects.toThrow("runner_remote_checkpoint_snapshot_failed"); + expect(syncOut).not.toHaveBeenCalled(); + expect(execute).toHaveBeenCalledTimes(3); + expect(String(execute.mock.calls[2]?.[0]?.args?.[1])).toMatch( + /^rm -rf -- '\/remote\/session\/filesystem\/\.paperclip-checkpoint-/, + ); + }); + it("rejects unsafe relative checkpoint exclusions", async () => { const execute = vi.fn().mockResolvedValue({ exitCode: 0, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index f6df58888bd..da9b2b858cf 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -11205,6 +11205,25 @@ function archiveExcludeArgs(entries: readonly string[]): string[] { return entries.map((entry) => `--exclude=./${entry}`); } +// The provider sync-out recognizes the same tar diagnostics for a full disk. +const SANDBOX_DISK_FULL_PATTERN = + /No space left on device|Disk quota exceeded|File\s?size limit exceeded|Wrote only \d+ of \d+ bytes/i; + +async function removeRemoteCheckpointSnapshot( + runner: CommandManagedRuntimeRunner, + snapshotPath: string, +): Promise { + const escapedSnapshot = snapshotPath.replaceAll("'", "'\\''"); + await runner + .execute({ + command: "sh", + args: ["-c", `rm -rf -- '${escapedSnapshot}'`], + bypassSession: true, + timeoutMs: 30_000, + }) + .catch(() => undefined); +} + export async function stageRemoteRunnerDirectory(input: { target: Extract; runner: CommandManagedRuntimeRunner; @@ -11540,6 +11559,14 @@ function assertSafeExtractedCheckpoint(root: string): void { * provider supplies a native syncOut implementation, create a sibling snapshot * first so the live runtime remains untouched and the provider's archive safety * checks still apply to every persisted entry. + * + * The snapshot needs free sandbox disk. When the disk is full, the provider + * applies the exclusions itself: its transfer can stream the directory without + * writing an archive into the sandbox. Provider exclusions are not anchored to + * the directory root, so this fallback also leaves out a nested entry that has + * an excluded name (for example a nested `tmp`). It never copies more than the + * snapshot would, and a checkpoint without those entries is more useful than + * no checkpoint. */ export async function syncRemoteRunnerDirectoryOut(input: { runner: CommandManagedRuntimeRunner; @@ -11564,6 +11591,7 @@ export async function syncRemoteRunnerDirectoryOut(input: { if (input.runner.syncOut) { let syncSourcePath = input.sourcePath; let snapshotPath: string | null = null; + let providerExclude: string[] | null = null; if (excluded.length > 0) { const checkpointId = randomUUID(); snapshotPath = posix.join( @@ -11587,9 +11615,21 @@ export async function syncRemoteRunnerDirectoryOut(input: { timeoutMs: 120_000, }); if (snapshotResult.exitCode !== 0 || snapshotResult.timedOut) { - throw new Error("runner_remote_checkpoint_snapshot_failed"); + // A partial copy would hold sandbox disk that the next save needs. + await removeRemoteCheckpointSnapshot(input.runner, snapshotPath); + snapshotPath = null; + if ( + snapshotResult.timedOut || + !SANDBOX_DISK_FULL_PATTERN.test( + `${snapshotResult.stderr}\n${snapshotResult.stdout}`, + ) + ) { + throw new Error("runner_remote_checkpoint_snapshot_failed"); + } + providerExclude = [...excluded]; + } else { + syncSourcePath = snapshotPath; } - syncSourcePath = snapshotPath; } try { await input.runner.syncOut([ @@ -11601,21 +11641,14 @@ export async function syncRemoteRunnerDirectoryOut(input: { targetPath: input.targetPath, kind: "directory", mode: input.mode, + ...(providerExclude ? { exclude: providerExclude } : {}), }, ], }, ]); } finally { if (snapshotPath) { - const escapedSnapshot = snapshotPath.replaceAll("'", "'\\''"); - await input.runner - .execute({ - command: "sh", - args: ["-c", `rm -rf -- '${escapedSnapshot}'`], - bypassSession: true, - timeoutMs: 30_000, - }) - .catch(() => undefined); + await removeRemoteCheckpointSnapshot(input.runner, snapshotPath); } } return; diff --git a/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts b/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts index 46d6c7cdde4..d5f9cb6e829 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts @@ -2,7 +2,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { GitShallowMetadataError } from "@paperclipai/adapter-utils/git-workspace-sync"; import type { Db } from "@paperclipai/db"; -const sync = vi.hoisted(() => ({ resume: vi.fn() })); +const sync = vi.hoisted(() => ({ resume: vi.fn(), save: vi.fn() })); vi.mock("./native-workspace-finalization-ownership.js", () => ({ withNativeWorkspaceFinalizationOwnership: async (_input: unknown, action: (owner: unknown) => unknown) => ({ acquired: true, value: await action({ token: "fixture-owner", assertHeld: async () => {} }) }), @@ -11,8 +11,13 @@ vi.mock("./native-workspace-sync.js", () => ({ readNativeWorkspaceSyncReference: () => ({ leaseId: "lease", providerLeaseId: "sandbox", workspaceId: "workspace" }), resumeNativeWorkspaceSync: sync.resume, })); +vi.mock("../remote-workspaces.js", () => ({ + readRemoteWorkspaceAuthority: (metadata: { remoteWorkspace?: unknown } | null) => metadata?.remoteWorkspace ?? null, + saveRemoteWorkspace: sync.save, +})); vi.mock("../environments.js", () => ({ environmentService: () => ({ - getLeaseById: async () => ({ id: "lease", companyId: "company", environmentId: "environment", providerLeaseId: "sandbox", status: "active" }), + getLeaseById: async () => ({ id: "lease", companyId: "company", environmentId: "environment", providerLeaseId: "sandbox", status: "active", + heartbeatRunId: "run", executionWorkspaceId: "workspace" }), getById: async () => ({ id: "environment", companyId: "company" }), }) })); vi.mock("../environment-execution-target.js", () => ({ resolveEnvironmentExecutionTarget: async () => ({ kind: "remote" }) })); @@ -20,12 +25,13 @@ vi.mock("../workspace-operations.js", () => ({ workspaceOperationService: () => createRecorder: () => ({ recordOperation: async (input: { run: () => Promise }) => input.run() }), }) })); import { resumeNativeWorkspaceFinalization } from "./native-workspace-finalizer.js"; +import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL } from "../remote-workspace-transport.js"; -function fixtureDb(): Db { +function fixtureDb(workspace?: Record): Db { const responses = [[{ - companyId: "company", runtimeMode: "native", issueId: "issue", resultId: "result", + companyId: "company", runtimeMode: "native", issueId: "issue", resultId: "result", issueWorkspaceId: workspace?.id, runnerProfileJson: { nativeWorkspaceSync: {}, nativeExecutionInput: { binding: {} } }, - }], [{ phase: "result_accepted", nextAttemptAt: null, resultId: "result" }], []]; + }], [{ phase: "result_accepted", nextAttemptAt: null, resultId: "result" }], [], [], ...(workspace ? [[workspace]] : [])]; return { select: () => { const query = { from: () => query, innerJoin: () => query, where: () => query, orderBy: () => query, @@ -70,3 +76,23 @@ describe("native workspace finalization without a manual export repair", () => { expect(result).toMatchObject({ status: "failed", exitCode: 1, stderr: "workspace_sync_out_unrecoverable\n" }); }); }); + +describe("native remote workspace save failures", () => { + const workspace = { id: "workspace", companyId: "company", cwd: "/workspace", + metadata: { remoteWorkspace: { location: { leaseId: "lease", providerLeaseId: "sandbox", environmentId: "environment" } } } }; + beforeEach(() => sync.save.mockReset()); + + it("names a full sandbox disk on the failed operation without changing the retry code", async () => { + sync.save.mockRejectedValueOnce(new Error(REMOTE_WORKSPACE_SANDBOX_DISK_FULL)); + const result = await resumeNativeWorkspaceFinalization({ db: fixtureDb(workspace), runId: "run", environmentRuntime: {} as never }); + expect(sync.save).toHaveBeenCalledOnce(); + expect(result).toMatchObject({ status: "failed", exitCode: 1, + stderr: `workspace_sync_out_failed\n${REMOTE_WORKSPACE_SANDBOX_DISK_FULL}\n`, metadata: { workspaceSync: { code: "workspace_sync_out_failed" } } }); + }); + + it("keeps other save failures to the bare code", async () => { + sync.save.mockRejectedValueOnce(new Error("remote_workspace_operation_failed")); + const result = await resumeNativeWorkspaceFinalization({ db: fixtureDb(workspace), runId: "run", environmentRuntime: {} as never }); + expect(result).toMatchObject({ status: "failed", exitCode: 1, stderr: "workspace_sync_out_failed\n" }); + }); +}); diff --git a/server/src/services/native-runtime/native-workspace-finalizer.ts b/server/src/services/native-runtime/native-workspace-finalizer.ts index 1094de249b4..f7528ce82a2 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.ts @@ -1,5 +1,6 @@ import { classifyNativeWorkspaceFailure } from "./native-workspace-failure.js"; import { readRemoteWorkspaceAuthority, saveRemoteWorkspace } from "../remote-workspaces.js"; +import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL } from "../remote-workspace-transport.js"; import { recordNativeFinalizationFailure } from "./native-run-finalizer.js"; import { inspectPluginWorkspaceForFinalization } from "./native-plugin-workspace-finalization.js"; import type { WorkspaceWorkers } from "../plugin-workspaces.js"; @@ -38,11 +39,12 @@ function readString(value: unknown) { function workspaceSyncFailure( code: "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable", + cause?: string, ) { return { status: "failed" as const, exitCode: 1, - stderr: `${code}\n`, + stderr: cause ? `${code}\n${cause}\n` : `${code}\n`, metadata: { workspaceSync: { code } }, }; } @@ -190,8 +192,11 @@ export async function attemptNativeWorkspaceFinalization(input: { metadata: { remoteWorkspace: saved } }; } catch (error) { await ownership.assertHeld(); + // The code keeps the retry policy. A full sandbox disk is named on the + // operation, because no retry can save the workspace until space is freed. + const diskFull = error instanceof Error && error.message === REMOTE_WORKSPACE_SANDBOX_DISK_FULL; return workspaceSyncFailure(classifyNativeWorkspaceFailure(error).code === "workspace_sync_out_unrecoverable" - ? "workspace_sync_out_unrecoverable" : "workspace_sync_out_failed"); + ? "workspace_sync_out_unrecoverable" : "workspace_sync_out_failed", diskFull ? REMOTE_WORKSPACE_SANDBOX_DISK_FULL : undefined); } } if (nativeWorkspaceSync) { diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index be74e5fefbd..a78612d2832 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -311,6 +311,42 @@ describe("remote workspace transport safety with real sandbox commands", () => { .rejects.toThrow(/^remote_workspace_operation_failed$/); }); + it.each([ + { step: "helper install", failing: String.raw`/archive\.cjs\.[^/]+\.tmp$/` }, + { step: "archive write", failing: String.raw`/workspace\.pcws$/` }, + ])("names a full sandbox disk when the $step runs out of space", async ({ failing }) => { + await fs.writeFile(path.join(workspace, "result.txt"), "synthetic saved result\n"); + // Every sandbox Node process inherits this preload. A matching write keeps + // one byte and then fails with ENOSPC, as a real full disk does. + const preload = path.join(scratch, "enospc.cjs"); + await fs.writeFile(preload, String.raw`const fs=require('node:fs/promises'),failing=${failing}; + const enospc=file=>Object.assign(new Error("ENOSPC: no space left on device, write '"+file+"'"),{code:'ENOSPC',errno:-28,syscall:'write'}); + const writeFile=fs.writeFile,open=fs.open; + fs.writeFile=async(file,data,...rest)=>{if(!failing.test(String(file)))return writeFile(file,data,...rest); + await writeFile(file,Buffer.from(data).subarray(0,1),...rest);throw enospc(file);}; + fs.open=async(file,...rest)=>{const handle=await open(file,...rest);if(failing.test(String(file))){const write=handle.writeFile.bind(handle); + handle.writeFile=async data=>{await write(Buffer.from(data).subarray(0,1));throw enospc(file);};}return handle;}; + require('node:module').syncBuiltinESMExports();`); + const { remote, execute } = target(); + const run = execute.getMockImplementation()!; + execute.mockImplementation(input => run({ ...input, env: { ...input.env, NODE_OPTIONS: `--require ${preload}` } })); + const operationId = randomUUID(); + await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId })) + .rejects.toThrow(/^remote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace$/); + const directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId); + const leftovers = await fs.readdir(directory).catch(() => []); + expect(leftovers.filter(name => name !== "archive.cjs")).toEqual([]); + }); + + it("keeps the full-disk diagnostic exact", async () => { + const { remote, execute } = target(); + execute.mockResolvedValueOnce({ exitCode: 1, stdout: "", timedOut: false, signal: null, pid: process.pid, + startedAt: new Date().toISOString(), + stderr: "remote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace\nSYNTHETIC_PRIVATE_PATH" }); + await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) + .rejects.toThrow(/^remote_workspace_operation_failed$/); + }); + it("never follows a preexisting snapshot receipt temporary symlink", async () => { await fs.writeFile(path.join(workspace, "result.txt"), "synthetic saved result\n"); const operationId = randomUUID(); diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 45a4a2417d5..ab4b17dbbd0 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -16,12 +16,19 @@ export function requireRemoteWorkspaceTarget(target: AdapterExecutionTarget | nu return target; } +/** The workspace archive is built inside the sandbox, so a full sandbox disk stops the save. */ +export const REMOTE_WORKSPACE_SANDBOX_DISK_FULL = + "remote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace"; +// Node reports ENOSPC or EDQUOT. Git and tar child processes report the text. +const DISK_FULL_CHECK = String.raw`const diskFull=e=>['ENOSPC','EDQUOT'].includes(e?.code)||/No space left on device|Disk quota exceeded/i.test(String(e?.stderr??'')+String(e?.message??''));`; + async function command(target: RemoteWorkspaceTarget, args: string[], env?: Record, timeoutMs = 300_000) { const result = await target.runner!.execute({ command: "node", args, cwd: target.remoteCwd, env: { NODE_OPTIONS: "", NODE_PATH: "", ...env }, timeoutMs, bypassSession: true }); if (result.exitCode !== 0 || result.timedOut) { const gitOperation = "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace"; - throw new Error(!result.timedOut && result.stderr.trim() === gitOperation ? gitOperation : "remote_workspace_operation_failed"); + const reported = result.timedOut ? "" : result.stderr.trim(); + throw new Error([gitOperation, REMOTE_WORKSPACE_SANDBOX_DISK_FULL].includes(reported) ? reported : "remote_workspace_operation_failed"); } return result.stdout; } @@ -153,13 +160,16 @@ async function installArchiveHelper(target: RemoteWorkspaceTarget, remoteCwd: st if (!/^[a-f0-9-]{36}$/.test(operationId)) throw new Error("remote_workspace_operation_identity_invalid"); const directory = path.posix.join(remoteCwd, ".paperclip-runtime", "workspace-snapshots", operationId); const helper = path.posix.join(directory, "archive.cjs"); - const install = String.raw`const fs=require('node:fs/promises'),path=require('node:path');(async()=>{ + const install = String.raw`const fs=require('node:fs/promises'),path=require('node:path');${DISK_FULL_CHECK}(async()=>{ const [root,dir,file,body]=process.argv.slice(1);let p='/'; for(const s of root.split('/').filter(Boolean)){p=path.join(p,s);await fs.mkdir(p,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e});const st=await fs.lstat(p);if(st.isSymbolicLink()||!st.isDirectory())throw Error('unsafe');} for(const s of path.relative(root,dir).split('/')){p=path.join(p,s);await fs.mkdir(p,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e});const st=await fs.lstat(p);if(st.isSymbolicLink()||!st.isDirectory())throw Error('unsafe');} const existing=await fs.lstat(file).catch(e=>{if(e.code==='ENOENT')return null;throw e});if(existing&&(!existing.isFile()||existing.isSymbolicLink()))throw Error('unsafe'); - const temp=file+'.'+require('node:crypto').randomUUID()+'.tmp';await fs.writeFile(temp,Buffer.from(body,'base64'),{mode:0o600,flag:'wx'});await fs.rename(temp,file); - })().catch(()=>{process.exitCode=1});`; + const temp=file+'.'+require('node:crypto').randomUUID()+'.tmp'; + // A failed write can leave partial bytes. Remove only the temporary file this run created. + try{await fs.writeFile(temp,Buffer.from(body,'base64'),{mode:0o600,flag:'wx'});await fs.rename(temp,file);} + catch(e){if(e?.code!=='EEXIST')await fs.rm(temp,{force:true}).catch(()=>{});throw e;} + })().catch(error=>{if(diskFull(error))process.stderr.write(${JSON.stringify(REMOTE_WORKSPACE_SANDBOX_DISK_FULL)});process.exitCode=1});`; await command(target, ["-e", install, remoteCwd, directory, helper, Buffer.from(remoteWorkspaceArchiveScript()).toString("base64")]); return { directory, helper, archive: path.posix.join(directory, "workspace.pcws"), receipt: path.posix.join(directory, "receipt.json") }; } @@ -186,7 +196,7 @@ export async function captureRemoteWorkspaceArchive(input: { const staged = await installArchiveHelper(input.target, input.remoteCwd, input.operationId); // Publish the archive and receipt as one immutable directory. Concurrent // invocations own separate attempts; no invocation unlinks another's archive. - const capture = String.raw`const fs=require('node:fs/promises'),path=require('node:path'),cp=require('node:child_process');const {promisify}=require('node:util');(async()=>{ + const capture = String.raw`const fs=require('node:fs/promises'),path=require('node:path'),cp=require('node:child_process');const {promisify}=require('node:util');${DISK_FULL_CHECK}(async()=>{ const [helper,root,directory]=process.argv.slice(1),canonical=path.join(directory,'captured'); const readPair=async(dir,layout,required)=>{ const st=await fs.lstat(dir).catch(e=>{if(e.code==='ENOENT')return null;throw e});if(!st)return null; @@ -211,7 +221,8 @@ export async function captureRemoteWorkspaceArchive(input: { }process.stdout.write(JSON.stringify(result));}finally{if(attempt)await fs.rm(attempt,{recursive:true,force:true});} })().catch(error=>{ const gitOperation='remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace'; - process.stderr.write(typeof error?.stderr==='string'&&error.stderr.trim()===gitOperation?gitOperation:'remote_workspace_capture_failed');process.exitCode=1; + process.stderr.write(typeof error?.stderr==='string'&&error.stderr.trim()===gitOperation?gitOperation + :diskFull(error)?${JSON.stringify(REMOTE_WORKSPACE_SANDBOX_DISK_FULL)}:'remote_workspace_capture_failed');process.exitCode=1; });`; const { layout, ...receipt } = JSON.parse(await command(input.target, ["-e", capture, staged.helper, input.remoteCwd, staged.directory])) as { manifestDigest: string; sha256: string; byteSize: number; entries: number; layout: "legacy" | "captured";