diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index d58586a675b..b788e06ff84 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1449,6 +1449,22 @@ folder), or an exhausted budget leaves a board-owned recovery action. Stranded-w recovery names this failure for the statuses it scans; for other statuses, such as a `blocked` task woken by a reply, the failure path records the action directly. +A restore needs about one workspace of sandbox disk, not two. The uploaded +archive is a disposable copy of a saved version that control-plane storage +already holds, and the sandbox checks every file against that version's +manifest. Extraction writes the files from the last to the first and shortens the +archive behind each one, so the archive and the restored tree together need the +workspace size plus the largest file. Before the upload, a reclaim step takes the +root's restore lock and checks the epoch fence. It removes what an earlier attempt +on that root left when it died: a `.restoring-*` partial tree, an archive it +was uploading or consuming, and archive staging that names the same root. Each +attempt names its staging before its upload starts. Staging that an earlier +release left without that name is kept. A sandbox that still cannot hold one copy +fails with `reason=sandbox_disk_full` at the stage that ran out of space (`helper`, +`upload`, or a restore script stage). A retry on the same disk would fail again, +so it is not retried automatically. The board recovery action's Retry works once +the sandbox has room for the workspace. + A retained sandbox can still hold a workspace's earlier root after the workspace moved to another allocation, for example when child tasks that an older release bound to a parent's workspace saved newer work on their own sandboxes. When the parent wakes on its @@ -1458,7 +1474,12 @@ recorded use of that root on this sandbox is an earlier epoch that ended in a quiesced, committed final save, and the folder carries a Paperclip preparation or restore receipt. Its files are then already part of the saved lineage being restored. Any other occupant (no receipt, a later unsaved use, or a sandbox that -never saved that root) is still refused with `reason=occupied_root`. The replaced +never saved that root) is still refused with `reason=occupied_root`, and none of +its files are removed. After that proof, the reclaim step removes the root's files +before the saved archive is uploaded, so the old copy never shares the disk with +the new one. Its receipt and `.paperclip-runtime` stay until the restored root is +published, so a restore interrupted after the removal proves the same root again +and finishes from the saved version. The replaced root's `.paperclip-runtime` state, such as a retained runner session, is never in a snapshot and belongs to the sandbox, so it moves into the restored root. Only the old preparation or restore receipt and export staging are dropped. A retained diff --git a/server/src/services/remote-workspace-snapshot-archive.ts b/server/src/services/remote-workspace-snapshot-archive.ts index 990d92f1fc8..57339a6fbaa 100644 --- a/server/src/services/remote-workspace-snapshot-archive.ts +++ b/server/src/services/remote-workspace-snapshot-archive.ts @@ -26,6 +26,14 @@ export async function workspaceArchive(input: { maxEntries?: number; maxManifestBytes?: number; singlePass?: boolean; + /** + * Restore only: give the archive's space back while extracting. Files are + * written from the last body to the first and the archive is truncated + * behind each one, so the archive and the restored tree together need about + * one workspace of space plus the largest file. The archive is unusable + * afterwards; use this only for a disposable copy of a durably saved archive. + */ + consumeArchive?: boolean; }): Promise<{ schema: "paperclip.remote-workspace-archive.v1"; manifestDigest: string; byteSize: number; sha256: string; entries: number }> { const fs = await import("node:fs/promises"); const { createReadStream } = await import("node:fs"); @@ -633,7 +641,7 @@ export async function workspaceArchive(input: { if (!input.targetDir || !/^[a-f0-9]{64}$/.test(input.expectedManifestDigest ?? "")) fail("restore_binding_invalid"); if (input.maxEntries !== undefined && (!Number.isSafeInteger(input.maxEntries) || input.maxEntries < 0)) fail("restore_binding_invalid"); if (input.maxManifestBytes !== undefined && (!Number.isSafeInteger(input.maxManifestBytes) || input.maxManifestBytes < 0)) fail("restore_binding_invalid"); - const archive = await fs.open(input.archivePath, "r"); + const archive = await fs.open(input.archivePath, input.consumeArchive === true ? "r+" : "r"); let staging: string | undefined; const stagedDirectories: string[] = []; try { @@ -663,19 +671,29 @@ export async function workspaceArchive(input: { kinds.set(entry.path, entry.type); } if (!Number.isSafeInteger(bodySize) || (await archive.stat()).size !== prefix.length + manifestSize + bodySize) fail("size_mismatch"); + const archiveSize = prefix.length + manifestSize + bodySize; const parent = await fs.realpath(path.dirname(input.targetDir!)); const existing = await fs.lstat(input.targetDir!).catch((error: NodeJS.ErrnoException) => { if (error.code !== "ENOENT") throw error; return null; }); if (existing && (!existing.isDirectory() || existing.isSymbolicLink() || (await fs.readdir(input.targetDir!)).length)) fail("target_not_empty"); check(); staging = await fs.mkdtemp(path.join(parent, ".workspace-restore-")); - let offset = prefix.length + manifestSize; + // Each file body starts where the previous one ends. A consuming restore + // writes the bodies last to first and truncates the archive behind each. + const offsets = new Map(); + let next = prefix.length + manifestSize; + for (const entry of manifest.entries) if (entry.type === "file") { offsets.set(entry, next); next += entry.size!; } + const consume = input.consumeArchive === true; + const ordered = consume + ? [...manifest.entries.filter(entry => entry.type !== "file"), ...manifest.entries.filter(entry => entry.type === "file").reverse()] + : manifest.entries; const buffer = Buffer.alloc(1024 * 1024); - for (const entry of manifest.entries) { + for (const entry of ordered) { check(); const target = path.join(staging, entry.path); if (entry.type === "directory") { await fs.mkdir(target, { mode: 0o700 }); stagedDirectories.push(target); } else if (entry.type === "symlink") await fs.symlink(entry.target!, target); else { const output = await fs.open(target, "wx", 0o600), digest = crypto.createHash("sha256"); + let offset = offsets.get(entry)!; try { let remaining = entry.size!; while (remaining) { @@ -687,12 +705,15 @@ export async function workspaceArchive(input: { if (digest.digest("hex") !== entry.sha256) fail("file_hash_mismatch"); } finally { await output.close(); } await fs.chmod(target, entry.mode!); + if (consume) await archive.truncate(offsets.get(entry)!); } } for (const entry of manifest.entries.filter(entry => entry.type === "directory").reverse()) { check(); await fs.chmod(path.join(staging, entry.path), entry.mode!); } - const content = await hashFile(input.archivePath); + // Every body was checked against the manifest, whose digest was checked + // against the expected one; a consumed archive has no bytes left to hash. + const content = consume ? { sha256: "", byteSize: archiveSize } : await hashFile(input.archivePath); check(); await fs.rename(staging, input.targetDir!); staging = undefined; return { schema, manifestDigest: input.expectedManifestDigest!, ...content, entries: manifest.entries.length }; @@ -865,5 +886,5 @@ export function remoteWorkspaceFailureDetail(error: unknown): string { export function remoteWorkspaceArchiveScript(): string { // tsx's function-name preservation helper may appear in Function#toString. const source = workspaceArchive.toString().replaceAll("__vite_ssr_dynamic_import__", "import"); - return `const __name = (value) => value;\nconst captureFailure = ${remoteWorkspaceCaptureFailureMessage.toString()};\nconst failureDetail = ${remoteWorkspaceFailureDetail.toString()};\nconst archive = ${source};\nconst [kind, first, second, digest] = process.argv.slice(2);\narchive(kind === 'capture' ? {kind,cwd:first,archivePath:second} : {kind,archivePath:first,targetDir:second,expectedManifestDigest:digest}).then(value=>process.stdout.write(JSON.stringify(value))).catch(error=>{const detail = failureDetail(error);process.stderr.write(kind === 'capture' ? captureFailure(error) ?? ('remote_workspace_archive_failed' + (detail ? ': ' + detail : '')) : error.message);process.exitCode=1;});\n`; + return `const __name = (value) => value;\nconst captureFailure = ${remoteWorkspaceCaptureFailureMessage.toString()};\nconst failureDetail = ${remoteWorkspaceFailureDetail.toString()};\nconst archive = ${source};\nconst [kind, first, second, digest] = process.argv.slice(2), mode = process.argv[6];\narchive(kind === 'capture' ? {kind,cwd:first,archivePath:second} : {kind,archivePath:first,targetDir:second,expectedManifestDigest:digest,consumeArchive:mode === 'consume'}).then(value=>process.stdout.write(JSON.stringify(value))).catch(error=>{const detail = failureDetail(error);process.stderr.write(kind === 'capture' ? captureFailure(error) ?? ('remote_workspace_archive_failed' + (detail ? ': ' + detail : '')) : error.message);process.exitCode=1;});\n`; } diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index 1c46c97259e..8197db8ec2c 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -285,7 +285,8 @@ describe("remote workspace transport safety with real sandbox commands", () => { let lost = false; execute.mockImplementation(async input => { const result = await original(input); - if (!lost && input.args?.[0] === "-e" && input.args[1]?.includes("workspace-restored.json") && result.exitCode === 0) { + if (!lost && input.args?.[0] === "-e" && input.args[1]?.includes("workspace-restored.json") && input.args.at(-1) === "extract" + && result.exitCode === 0) { lost = true; throw new Error("synthetic lost restore reply"); } return result; @@ -333,7 +334,7 @@ describe("remote workspace transport safety with real sandbox commands", () => { if (scenario === "interrupted") { const run = execute.getMockImplementation()!; let interrupt = true; execute.mockImplementation(async command => { - if (interrupt && command.args?.[1]?.includes("remote_workspace_restore_failed")) { + if (interrupt && command.args?.[1]?.includes("remote_workspace_restore_failed") && command.args.at(-1) === "extract") { interrupt = false; return run({ ...command, args: command.args.map((arg, index) => index === 1 ? arg.replace("try{await fs.rename(staged,root);}", "throw Error('interrupted');try{await fs.rename(staged,root);}") : arg) }); @@ -388,6 +389,7 @@ fs.unlinkSync(${JSON.stringify(postPublish)});throw Error('published_interrupted const run = execute.getMockImplementation()!; let interrupt = scenario === "interrupted"; execute.mockImplementation(async command => { if (!command.args?.[1]?.includes("remote_workspace_restore_failed")) return run(command); + if (command.args.at(-1) !== "extract") return run({ ...command, args: ["--require", preload, ...command.args] }); const args = interrupt ? command.args.map((arg, index) => index === 1 ? arg.replace("await fs.rename(path.join(staged,'.paperclip-runtime','workspace-restored.json')", "throw Error('interrupted');await fs.rename(path.join(staged,'.paperclip-runtime','workspace-restored.json')") : arg) : command.args; interrupt = false; @@ -720,6 +722,292 @@ fs.unlinkSync(${JSON.stringify(kill)});process.kill(process.pid,'SIGKILL');}retu } finally { await captured.cleanup(); } }); + describe("restoring into a sandbox disk that cannot hold two workspace copies", () => { + const MiB = 1024 * 1024; + // The sandbox disk is a folder with a byte capacity. Every write the restore + // commands make through a file handle counts what the folder already holds. + async function boundedDisk(capacity: number, kill?: { afterWrites: number }) { + const disk = path.join(scratch, "disk"), preload = path.join(scratch, `disk-${randomUUID()}.cjs`); + await fs.mkdir(disk, { recursive: true }); + await fs.writeFile(preload, `const fs=require('node:fs'),path=require('node:path'); +const disk=${JSON.stringify(disk)},capacity=${capacity},killAfter=${kill?.afterWrites ?? 0}; +const used=()=>{let total=0;const walk=d=>{for(const e of fs.readdirSync(d,{withFileTypes:true})){const p=path.join(d,e.name); + if(e.isDirectory())walk(p);else if(e.isFile())total+=fs.lstatSync(p).size;}};walk(disk);return total;}; +let writes=0;const helper=process.argv.includes('restore');const open=fs.promises.open; +fs.promises.open=async(...args)=>{const handle=await open.apply(fs.promises,args); + if(!String(args[0]).startsWith(disk+path.sep))return handle;const writeFile=handle.writeFile.bind(handle); + handle.writeFile=async(data,...rest)=>{ + if(used()+Buffer.byteLength(data)>capacity)throw Object.assign(Error('ENOSPC: no space left on device, write'),{code:'ENOSPC',errno:-28,syscall:'write'}); + const result=await writeFile(data,...rest); + if(helper&&killAfter&&++writes>=killAfter){process.kill(process.ppid,'SIGKILL');process.kill(process.pid,'SIGKILL');} + return result;}; + return handle;}; +const writeFile=fs.promises.writeFile; +fs.promises.writeFile=async(file,data,...rest)=>{ + if(String(file).startsWith(disk+path.sep)&&used()+Buffer.byteLength(data)>capacity) + throw Object.assign(Error('ENOSPC: no space left on device, write'),{code:'ENOSPC',errno:-28,syscall:'write'}); + return writeFile.call(fs.promises,file,data,...rest);}; +require('node:module').syncBuiltinESMExports();`); + const { remote, execute } = target(); + const actual = execute.getMockImplementation()!; + // Every sandbox command, including the helper install, writes to this disk. + execute.mockImplementation(command => actual({ ...command, env: { ...command.env, NODE_OPTIONS: `--require ${preload}` } })); + const used = async () => { + let total = 0; + const walk = async (directory: string): Promise => { + for (const entry of await fs.readdir(directory, { withFileTypes: true })) { + if (entry.isDirectory()) await walk(path.join(directory, entry.name)); + else if (entry.isFile()) total += (await fs.lstat(path.join(directory, entry.name))).size; + } + }; + await walk(disk); + return total; + }; + // The upload lands on the same disk, so it must fit beside what is already there. + const syncIn = remote.runner!.syncIn!; + remote.runner!.syncIn = async (operations, options) => { + let incoming = 0; + for (const operation of operations) for (const file of operation.files) { + incoming += (await fs.stat(file.sourcePath)).size; + incoming -= (await fs.stat(file.targetPath).catch(() => null))?.size ?? 0; + } + if (await used() + incoming > capacity) throw Object.assign(new Error("upload failed: ENOSPC: no space left on device"), { code: "ENOSPC" }); + return syncIn(operations, options); + }; + return { disk, remote, execute, used, root: path.join(disk, "workspace-root") }; + } + async function savedWorkspace() { + const files = Array.from({ length: 4 }, (_, index) => ({ name: `part-${index}.bin`, bytes: randomBytes(MiB) })); + await fs.mkdir(path.join(workspace, "nested")); + for (const file of files) await fs.writeFile(path.join(workspace, "nested", file.name), file.bytes); + const { remote } = target(); + const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }); + return { files, captured, archiveSize: (await fs.stat(captured.archivePath)).size }; + } + async function expectRestored(root: string, files: Array<{ name: string; bytes: Buffer }>) { + for (const file of files) expect((await fs.readFile(path.join(root, "nested", file.name))).equals(file.bytes)).toBe(true); + } + + it("restores a saved workspace when the archive and the extracted tree do not both fit", async () => { + const { files, captured, archiveSize } = await savedWorkspace(); + try { + // Room for the uploaded archive and one file, not for a second copy. + const { remote, root, used, disk } = await boundedDisk(archiveSize + 1.5 * MiB); + await restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: root, archivePath: captured.archivePath, + manifestDigest: captured.manifestDigest, restorationIdentity: "1".repeat(64), restorationEpoch: 1 }); + await expectRestored(root, files); + // The consumed archive is gone; nothing but the restored files holds space. + expect(await used()).toBeLessThan(4 * MiB + 256 * 1024); + expect((await fs.readdir(disk)).filter(name => name.includes(".restoring-") || name.startsWith(".workspace-restore-"))).toEqual([]); + } finally { await captured.cleanup(); } + }); + + it("names a disk that cannot take the upload so it is not retried automatically", async () => { + const { captured, archiveSize } = await savedWorkspace(); + try { + const { remote, root } = await boundedDisk(archiveSize / 2); + await expect(restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: root, archivePath: captured.archivePath, + manifestDigest: captured.manifestDigest, restorationIdentity: "9".repeat(64), restorationEpoch: 1 })).rejects.toThrow( + /^remote_workspace_operation_failed: operation=upload; remote_workspace_restore_failed: stage=upload reason=sandbox_disk_full$/); + } finally { await captured.cleanup(); } + }); + + it("names a disk that cannot take the archive helper so it is not retried automatically", async () => { + const { remote, execute } = target(); + // The reclaim phase runs first; then the helper install finds the disk full. + execute.mockResolvedValueOnce({ exitCode: 0, stdout: "", stderr: "", timedOut: false, signal: null, + pid: process.pid, startedAt: new Date().toISOString() }); + execute.mockResolvedValueOnce({ exitCode: 1, stdout: "", stderr: REMOTE_WORKSPACE_SANDBOX_DISK_FULL, timedOut: false, signal: null, + pid: process.pid, startedAt: new Date().toISOString() }); + const archivePath = path.join(scratch, "restore.pcws"); await fs.writeFile(archivePath, "synthetic archive"); + await expect(restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: path.join(scratch, "restored"), archivePath, + manifestDigest: "a".repeat(64), restorationIdentity: "b".repeat(64) })).rejects.toThrow( + /^remote_workspace_operation_failed: operation=archive_helper; remote_workspace_restore_failed: stage=helper reason=sandbox_disk_full$/); + }); + + it("names a disk too small for one workspace copy so it is not retried automatically, and restores once space exists", async () => { + const { files, captured, archiveSize } = await savedWorkspace(); + try { + const small = await boundedDisk(archiveSize + MiB / 2); + const input = { target: small.remote, remoteCwd: small.root, archivePath: captured.archivePath, + manifestDigest: captured.manifestDigest, restorationIdentity: "2".repeat(64), restorationEpoch: 1 }; + await expect(restoreRemoteWorkspaceDirectory(input)).rejects.toThrow( + /^remote_workspace_operation_failed: operation=restore exit=1; remote_workspace_restore_failed: stage=extract reason=sandbox_disk_full$/); + expect(await fs.lstat(small.root).catch(() => null)).toBeNull(); + // The failed attempt leaves no partial tree or consumed archive behind. + expect(await small.used()).toBeLessThan(256 * 1024); + const larger = await boundedDisk(archiveSize + 1.5 * MiB); + await restoreRemoteWorkspaceDirectory({ ...input, target: larger.remote }); + await expectRestored(small.root, files); + } finally { await captured.cleanup(); } + }); + + it.each(["same", "new"] as const)("reclaims a restore that died mid-extraction before retrying with the %s identity", async identity => { + const { files, captured, archiveSize } = await savedWorkspace(); + try { + const dying = await boundedDisk(archiveSize + 1.5 * MiB, { afterWrites: 2 }); + const input = { target: dying.remote, remoteCwd: dying.root, archivePath: captured.archivePath, + manifestDigest: captured.manifestDigest, restorationIdentity: "3".repeat(64), restorationEpoch: 1 }; + await expect(restoreRemoteWorkspaceDirectory(input)).rejects.toThrow(/^remote_workspace_operation_failed: operation=restore/); + // Both processes died: a partial tree and the unconsumed archive remain. + expect((await fs.readdir(dying.disk)).some(name => name.startsWith("workspace-root.restoring-"))).toBe(true); + expect(await fs.lstat(dying.root).catch(() => null)).toBeNull(); + const retry = await boundedDisk(archiveSize + 1.5 * MiB); + await restoreRemoteWorkspaceDirectory(identity === "same" ? { ...input, target: retry.remote } + : { ...input, target: retry.remote, restorationIdentity: "4".repeat(64), restorationEpoch: 2 }); + await expectRestored(dying.root, files); + expect((await fs.readdir(dying.disk)).filter(name => name.includes(".restoring-"))).toEqual([]); + } finally { await captured.cleanup(); } + }); + + it("reclaims an earlier release's partial extraction and leaves its unbound archive staging alone", async () => { + const { files, captured, archiveSize } = await savedWorkspace(); + try { + const { remote, root, disk } = await boundedDisk(archiveSize + 1.5 * MiB); + // An earlier release's restore died here: its partial tree carries this + // root's staging name, and its archive staging names no root. + const partial = path.join(disk, "workspace-root.restoring-Ab12Cd"); + await fs.mkdir(path.join(partial, "tree", "nested"), { recursive: true }); + await fs.writeFile(path.join(partial, "tree", "nested", "part-0.bin"), randomBytes(MiB)); + await fs.chmod(path.join(partial, "tree", "nested"), 0o555); + const unbound = path.join(disk, ".paperclip-runtime", "workspace-snapshots", randomUUID()); + await fs.mkdir(unbound, { recursive: true }); + await fs.writeFile(path.join(unbound, "workspace.pcws"), "earlier release archive"); + await restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: root, archivePath: captured.archivePath, + manifestDigest: captured.manifestDigest, restorationIdentity: "8".repeat(64), restorationEpoch: 1 }); + await expectRestored(root, files); + expect(await fs.lstat(partial).catch(() => null)).toBeNull(); + expect(await fs.readFile(path.join(unbound, "workspace.pcws"), "utf8")).toBe("earlier release archive"); + } finally { await captured.cleanup(); } + }); + + it("reclaims a full disk before the helper install needs space", async () => { + const { files, captured, archiveSize } = await savedWorkspace(); + try { + const capacity = archiveSize + 1.5 * MiB; + const { remote, root, disk, used } = await boundedDisk(capacity); + // A dead attempt of this root left a partial tree that fills the disk. + const partial = path.join(disk, "workspace-root.restoring-Zz98Yy"); + await fs.mkdir(path.join(partial, "tree"), { recursive: true }); + await fs.writeFile(path.join(partial, "tree", "filler.bin"), Buffer.alloc(Math.floor(capacity - 1024))); + expect(capacity - await used()).toBeLessThan(2048); + await restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: root, archivePath: captured.archivePath, + manifestDigest: captured.manifestDigest, restorationIdentity: "a1".repeat(32), restorationEpoch: 1 }); + await expectRestored(root, files); + expect(await fs.lstat(partial).catch(() => null)).toBeNull(); + } finally { await captured.cleanup(); } + }); + + it("keeps a later attempt's upload when an earlier attempt's reclaim with the same identity runs late", async () => { + const { files, captured, archiveSize } = await savedWorkspace(); + try { + const { remote, execute, root } = await boundedDisk(archiveSize + 1.5 * MiB); + const bounded = execute.getMockImplementation()!; + let delayed = false; + execute.mockImplementation(async command => { + if (!delayed && command.args?.at(-1) === "extract") { + delayed = true; + // The earlier attempt's reclaim reached the sandbox only now. It names its own upload. + expect(command.args[3]).toMatch(/workspace-[0-9]{16}-[a-f0-9-]{36}\.pcws$/); + const late = command.args.map((arg, index) => index === 3 + ? path.join(path.dirname(arg), `workspace-${String(Date.now() - 60_000).padStart(16, "0")}-${randomUUID()}.pcws`) : index === command.args!.length - 1 ? "reclaim" : arg); + expect((await bounded({ ...command, args: late })).exitCode).toBe(0); + } + return bounded(command); + }); + await restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: root, archivePath: captured.archivePath, + manifestDigest: captured.manifestDigest, restorationIdentity: "b2".repeat(32), restorationEpoch: 1 }); + expect(delayed).toBe(true); + await expectRestored(root, files); + } finally { await captured.cleanup(); } + }); + + it("reclaims an abandoned upload of the same identity before uploading again", async () => { + const { files, captured, archiveSize } = await savedWorkspace(); + try { + const { remote, root, disk } = await boundedDisk(archiveSize + 1.5 * MiB); + const identity = "c3".repeat(32); + const operationId = `${identity.slice(0, 8)}-${identity.slice(8, 12)}-${identity.slice(12, 16)}-${identity.slice(16, 20)}-${identity.slice(20, 32)}`; + // An earlier attempt finished its upload, then the server stopped before extraction. + const staging = path.join(disk, ".paperclip-runtime", "workspace-snapshots", operationId); + await fs.mkdir(staging, { recursive: true }); + const abandoned = path.join(staging, `workspace-${String(Date.now() - 60_000).padStart(16, "0")}-${randomUUID()}.pcws`); + await fs.copyFile(captured.archivePath, abandoned); + await restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: root, archivePath: captured.archivePath, + manifestDigest: captured.manifestDigest, restorationIdentity: identity, restorationEpoch: 1 }); + await expectRestored(root, files); + expect(await fs.lstat(abandoned).catch(() => null)).toBeNull(); + } finally { await captured.cleanup(); } + }); + + async function retainedRoot(root: string) { + const runtime = path.join(root, ".paperclip-runtime"); + await fs.mkdir(path.join(runtime, "paperclip-runner"), { recursive: true }); + await fs.writeFile(path.join(runtime, "paperclip-runner", "runner-state.json"), "{\"lifecycle\":\"suspended\"}"); + await fs.writeFile(path.join(runtime, "workspace-ready.json"), JSON.stringify({ preparationIdentity: "f".repeat(64) })); + await fs.mkdir(path.join(root, "nested")); + for (let index = 0; index < 4; index++) await fs.writeFile(path.join(root, "nested", `old-${index}.bin`), randomBytes(MiB)); + } + + it("removes a retained root whose save the server proved before extracting into its space", async () => { + const { files, captured, archiveSize } = await savedWorkspace(); + try { + // The old copy, the uploaded archive, and one file fit; a third copy does not. + const { remote, root } = await boundedDisk(4 * MiB + archiveSize + 1.5 * MiB); + await retainedRoot(root); + await restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: root, archivePath: captured.archivePath, + manifestDigest: captured.manifestDigest, restorationIdentity: "5".repeat(64), restorationEpoch: 1, replaceRetiredRoot: true }); + await expectRestored(root, files); + expect(await fs.lstat(path.join(root, "nested", "old-0.bin")).catch(() => null)).toBeNull(); + expect(await fs.readFile(path.join(root, ".paperclip-runtime", "paperclip-runner", "runner-state.json"), "utf8")) + .toBe("{\"lifecycle\":\"suspended\"}"); + } finally { await captured.cleanup(); } + }); + + it.each(["unproven", "no_receipt"] as const)("never removes a retained root whose save is %s", async scenario => { + const { captured, archiveSize } = await savedWorkspace(); + try { + const { remote, root } = await boundedDisk(4 * MiB + archiveSize + 1.5 * MiB); + await retainedRoot(root); + if (scenario === "no_receipt") await fs.rm(path.join(root, ".paperclip-runtime", "workspace-ready.json")); + const before = await fs.readdir(path.join(root, "nested")); + await expect(restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: root, archivePath: captured.archivePath, + manifestDigest: captured.manifestDigest, restorationIdentity: "6".repeat(64), restorationEpoch: 1, + replaceRetiredRoot: scenario !== "unproven" })) + .rejects.toThrow(/; remote_workspace_restore_failed: stage=inspect_root reason=occupied_root$/); + expect(await fs.readdir(path.join(root, "nested"))).toEqual(before); + } finally { await captured.cleanup(); } + }); + + it("finishes the removal and restore after a proven retained root's removal is interrupted", async () => { + const { files, captured, archiveSize } = await savedWorkspace(); + try { + const { remote, execute, root, disk } = await boundedDisk(4 * MiB + archiveSize + 1.5 * MiB); + await retainedRoot(root); + const flag = path.join(scratch, "interrupt-removal"), interrupt = path.join(scratch, "interrupt-removal.cjs"); + await fs.writeFile(flag, "once"); + // The command dies right after deleting the old files, as a restart would leave it. + await fs.writeFile(interrupt, `const fs=require('node:fs'),rm=fs.promises.rm; +fs.promises.rm=async(file,options)=>{const result=await rm.call(fs.promises,file,options); +if(String(file)===${JSON.stringify(path.join(root, "nested"))}&&fs.existsSync(${JSON.stringify(flag)})){fs.unlinkSync(${JSON.stringify(flag)});process.kill(process.pid,'SIGKILL');} +return result};`); + const bounded = execute.getMockImplementation()!; + execute.mockImplementation(command => bounded(command.args?.[1]?.includes("remote_workspace_restore_failed") + ? { ...command, args: ["--require", interrupt, ...command.args!] } : command)); + const input = { target: remote, remoteCwd: root, archivePath: captured.archivePath, manifestDigest: captured.manifestDigest, + restorationIdentity: "7".repeat(64), restorationEpoch: 1, replaceRetiredRoot: true }; + await expect(restoreRemoteWorkspaceDirectory(input)).rejects.toThrow(/^remote_workspace_operation_failed: operation=restore/); + expect(await fs.lstat(path.join(root, "nested")).catch(() => null)).toBeNull(); + expect((await fs.lstat(path.join(root, ".paperclip-runtime", "workspace-ready.json"))).isFile()).toBe(true); + await restoreRemoteWorkspaceDirectory(input); + await expectRestored(root, files); + expect(await fs.readFile(path.join(root, ".paperclip-runtime", "paperclip-runner", "runner-state.json"), "utf8")) + .toBe("{\"lifecycle\":\"suspended\"}"); + expect((await fs.readdir(disk)).filter(name => name.includes(".restoring-") || name.startsWith("workspace-root.retired-"))).toEqual([]); + } finally { await captured.cleanup(); } + }); + }); + it("names the preparation step that could not select a missing branch", async () => { const origin = await repository(), { remote } = target(), root = path.join(workspace, "missing-branch-cause"); await expect(prepareRemoteWorkspaceDirectory({ target: remote, remoteCwd: root, @@ -740,8 +1028,9 @@ fs.unlinkSync(${JSON.stringify(kill)});process.kill(process.pid,'SIGKILL');}retu it.each(["prepare", "repository", "restore"] as const)("drops %s output that is outside the closed cause grammar", async operation => { const { remote, execute } = target(); const forged = `remote_workspace_${operation}_failed: stage=extract reason=/home/secret`; - if (operation === "restore") execute.mockResolvedValueOnce({ exitCode: 0, stdout: "", stderr: "", timedOut: false, - signal: null, pid: process.pid, startedAt: new Date().toISOString() }); + // A restore runs its reclaim phase and installs its helper before extracting. + if (operation === "restore") for (let call = 0; call < 2; call++) execute.mockResolvedValueOnce({ exitCode: 0, stdout: "", stderr: "", + timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() }); execute.mockResolvedValueOnce({ exitCode: 1, stdout: "", stderr: forged, timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() }); const request = operation === "prepare" @@ -1352,7 +1641,8 @@ fs.unlinkSync(${JSON.stringify(kill)});process.kill(process.pid,'SIGKILL');}retu const { remote, execute } = target(); const failed = { exitCode: 1, stdout: "remote_workspace_archive_fs_enoent", stderr: "", timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() }; - if (operation === "restore") execute.mockResolvedValueOnce({ ...failed, exitCode: 0, stdout: "" }); + // A restore runs its reclaim phase and installs its helper before extracting. + if (operation === "restore") execute.mockResolvedValueOnce({ ...failed, exitCode: 0, stdout: "" }).mockResolvedValueOnce({ ...failed, exitCode: 0, stdout: "" }); execute.mockResolvedValueOnce(failed); const request = operation === "prepare" ? prepareRemoteWorkspaceDirectory({ target: remote, remoteCwd: workspace, source: { kind: "empty" }, @@ -1367,7 +1657,7 @@ fs.unlinkSync(${JSON.stringify(kill)});process.kill(process.pid,'SIGKILL');}retu manifestDigest: "a".repeat(64), restorationIdentity: "b".repeat(64) }); })(); await expect(request).rejects.toThrow(/^remote_workspace_operation_failed: operation=[a-z_]+ exit=1$/); - expect(execute).toHaveBeenCalledTimes(operation === "restore" ? 2 : 1); + expect(execute).toHaveBeenCalledTimes(operation === "restore" ? 3 : 1); }); it("keeps the existing disk diagnostic ahead of a new capture diagnostic", async () => { diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 42ecdfd82bd..79e2fac753f 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -639,6 +639,22 @@ async function captureSandboxWorkspaceArchive(input: { } } +/** A full sandbox disk repeats on every retry of the same restore, so it names a refusal reason. */ +function restoreDiskFull(operation: RemoteWorkspaceOperation, stage: "helper" | "upload") { + return new RemoteWorkspaceOperationError( + `remote_workspace_operation_failed: operation=${operation}; remote_workspace_restore_failed: stage=${stage} reason=sandbox_disk_full`); +} + +/** Provider text only selects this answer; it is never copied. */ +function sandboxDiskFull(error: unknown) { + for (let cause = error, depth = 0; cause && depth < 4; cause = (cause as { cause?: unknown }).cause, depth++) { + const { code, message } = cause as { code?: unknown; message?: unknown }; + if (code === "ENOSPC" || code === "EDQUOT" + || (typeof message === "string" && /\bENOSPC\b|\bEDQUOT\b|No space left on device|Disk quota exceeded/i.test(message))) return true; + } + return false; +} + export async function restoreRemoteWorkspaceDirectory(input: { target: RemoteWorkspaceTarget; remoteCwd: string; archivePath: string; manifestDigest: string; restorationIdentity: string; restorationEpoch?: number; allowEmptyRoot?: boolean; allocationFiles?: Array<{ relativePath: string; sha256: string }>; @@ -654,15 +670,14 @@ export async function restoreRemoteWorkspaceDirectory(input: { const operationId = `${id.slice(0, 8)}-${id.slice(8, 12)}-${id.slice(12, 16)}-${id.slice(16, 20)}-${id.slice(20, 32)}`; // Stage outside the destination; the restore validator requires an empty target. const parent = path.posix.dirname(input.remoteCwd); - const staged = await installArchiveHelper(input.target, parent, operationId); - await measureRunPhase("workspace.snapshot.sandbox_upload", () => stat(input.archivePath).then(archive => transferArchive(input.target, "syncIn", [{ operationId, - files: [{ kind: "file", sourcePath: input.archivePath, targetPath: staged.archive }], - }], archive.size))).catch((error: unknown) => { noteProviderRefusal("upload", error); throw error; }); // Publish the restored folder and its identity together. A lost successful // reply can then adopt the same files without repeating a destructive restore. const restore = String.raw`const fs=require('node:fs/promises'),path=require('node:path'),cp=require('node:child_process'),crypto=require('node:crypto'); ${DISK_FULL_CHECK}${CAPTURE_FAILURE_CHECK}${SETUP_FAILURE_CHECK}let stage='validate'; - (async()=>{const [helper,archive,root,digest,identity,allowEmpty,rawFiles,replaceRetired,rawEpoch]=process.argv.slice(1),files=JSON.parse(rawFiles); + (async()=>{const [helper,archive,root,digest,identity,allowEmpty,rawFiles,replaceRetired,rawEpoch,phase]=process.argv.slice(1),files=JSON.parse(rawFiles); + // The reclaim phase runs before the upload: it frees what this root's + // earlier attempts and proven retired files hold, then stops before extraction. + const reclaimOnly=phase==='reclaim'; const validEpoch=value=>Number.isSafeInteger(value)&&value>0,restorationEpoch=rawEpoch===undefined||rawEpoch===''?null:Number(rawEpoch); if(restorationEpoch!==null&&!validEpoch(restorationEpoch))throw Error('invalid_epoch'); const completedReceipt=(id,manifestDigest,epoch)=>epoch===null?{identity:id,manifestDigest} @@ -736,6 +751,12 @@ export async function restoreRemoteWorkspaceDirectory(input: { await fs.writeFile(receiptFile,JSON.stringify(receipt),{mode:0o600}); await fs.rename(path.join(staged,'.paperclip-runtime','workspace-restored.json'),path.join(runtime,'workspace-restored.json')); await finishInPlace(receipt);}; + // The reclaim phase runs before the helper install, so it creates the + // operation's staging folder itself without following a symlink. + if(reclaimOnly){stage='create_parent';let current='/'; + for(const part of path.dirname(archive).split('/').filter(Boolean)){current=path.join(current,part); + await fs.mkdir(current,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e}); + const s=await fs.lstat(current);if(!s.isDirectory()||s.isSymbolicLink())throw Error('unsafe_root');}} stage='restore_lock';await directory(path.dirname(root)); const lockRoot=path.join(path.dirname(root),'.paperclip-runtime','workspace-restore-locks'); await directory(path.dirname(lockRoot)); @@ -782,6 +803,33 @@ export async function restoreRemoteWorkspaceDirectory(input: { await fs.rename(pending,fencePath);const directory=await fs.open(lockRoot,'r');try{await directory.sync();}finally{await directory.close();}} finally{await fs.rm(pending,{force:true});} } + // Only a restore of this root, under this lock, creates its staging folders + // and binds its archive staging to the root. While the lock is held, any + // such leftover is from an attempt that died, and its bytes are disposable: + // the saved version in control-plane storage is the source of truth. + stage='reclaim';const stagingRoot=path.dirname(archive),snapshotsRoot=path.dirname(stagingRoot); + const discard=async p=>{await open(p);await fs.rm(p,{recursive:true,force:true});}; + // Uploads are named by the attempt's start time. An older attempt's upload is + // abandoned once a newer attempt holds the lock; a newer one is left alone. + const attemptOf=name=>name==='workspace.pcws'?0:Number(/^workspace-([0-9]{16})-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}\.pcws$/.exec(name)?.[1]??NaN); + const ownAttempt=attemptOf(path.basename(archive));if(!Number.isSafeInteger(ownAttempt)||ownAttempt<1)throw Error('invalid_attempt'); + for(const name of await fs.readdir(stagingRoot)) + if(/\.pcws\.[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}\.consuming$/.test(name)||attemptOf(name)16384)continue; + const value=await readJson(path.join(directory,'restore-target.json'),'unsafe_staging').catch(()=>null); + if(value?.schema==='paperclip.workspace-restore-staging.v1'&&value.root===root)await discard(directory);} + // Bind this attempt's staging only after the space is reclaimed. + const binding=path.join(stagingRoot,'restore-target.json'),bindingTemp=binding+'.'+crypto.randomUUID()+'.tmp'; + try{await fs.writeFile(bindingTemp,JSON.stringify({schema:'paperclip.workspace-restore-staging.v1',root}),{flag:'wx',mode:0o600}); + await fs.rename(bindingTemp,binding);}finally{await fs.rm(bindingTemp,{force:true});} stage='allocation';if(await stat(parked)){if(allowEmpty!=='true')throw Error('unexpected_allocation');await allocation(parked);if(!await stat(root))await fs.rename(parked,root);} stage='inspect_root';const prior=await stat(root);if(prior)await directory(root); const marker=path.join(root,'.paperclip-runtime','workspace-restored.json'); @@ -807,9 +855,19 @@ export async function restoreRemoteWorkspaceDirectory(input: { if(prior&&!retire&&allowEmpty!=='true')throw Error('occupied_root'); stage='allocation';const retained=prior&&!retire?await allocation(root):[]; if(!prior&&files.length)throw Error('allocation_missing'); - stage='extract';const staged=await fs.mkdtemp(root+'.restoring-'); + // The server proved this root's files are already in the saved lineage being + // restored, so they are removed before extraction instead of after it. Its + // receipt and runtime state stay, so an interrupted restore repeats this step. + if(retire){stage='reclaim_retired';const s=await fs.lstat(root);await fs.chmod(root,(s.mode&0o7777)|0o700); + for(const name of await fs.readdir(root))if(name!=='.paperclip-runtime')await discard(path.join(root,name)); + if(await stat(path.join(root,'.paperclip-runtime','workspace-snapshots')))await discard(path.join(root,'.paperclip-runtime','workspace-snapshots'));} + if(reclaimOnly)return; + stage='extract';const staging=await fs.mkdtemp(root+'.restoring-'),staged=path.join(staging,'tree'); + // A private name keeps a later upload to the same path out of this extraction. + const consuming=archive+'.'+crypto.randomUUID()+'.consuming'; try{ - await require('node:util').promisify(cp.execFile)(process.execPath,[helper,'restore',archive,staged,digest],{maxBuffer:1048576}); + await fs.rename(archive,consuming); + await require('node:util').promisify(cp.execFile)(process.execPath,[helper,'restore',consuming,staged,digest,'consume'],{maxBuffer:1048576}); stage='write_receipt';const runtime=path.join(staged,'.paperclip-runtime');await fs.mkdir(runtime,{mode:0o700}); for(const file of retained)await fs.writeFile(path.join(runtime,file.name),file.content,{flag:'wx',mode:0o600}); await fs.writeFile(path.join(runtime,'workspace-restored.json'),JSON.stringify(completedReceipt(identity,digest,restorationEpoch)),{flag:'wx',mode:0o600}); @@ -821,12 +879,29 @@ export async function restoreRemoteWorkspaceDirectory(input: { try{await fs.rename(staged,root);}catch(e){if(prior&&!await stat(root))await fs.rename(holding,root);throw e;} stage='cleanup_parked';if(prior&&!retire){await allocation(parked);await fs.rm(parked,{recursive:true});} await removeRetired(); - }finally{await fs.rm(staged,{recursive:true,force:true});} + }finally{await fs.rm(consuming,{force:true});await open(staging).catch(()=>{});await fs.rm(staging,{recursive:true,force:true});} }finally{lock.close();} - })().catch(error=>{process.stderr.write(setupFailure('remote_workspace_restore_failed',stage,error));process.exitCode=1});`; - await measureRunPhase("workspace.snapshot.sandbox_extract", () => command(input.target, "restore", ["-e", restore, staged.helper, staged.archive, input.remoteCwd, + })().catch(error=>{process.stderr.write(setupFailure('remote_workspace_restore_failed',stage,diskFull(error)?Error('sandbox_disk_full'):error));process.exitCode=1});`; + // Each attempt uploads to its own name, so a delayed command of an earlier + // attempt with the same identity never removes or consumes this upload. + const directory = path.posix.join(parent, ".paperclip-runtime", "workspace-snapshots", operationId); + const archive = path.posix.join(directory, `workspace-${String(Date.now()).padStart(16, "0")}-${randomUUID()}.pcws`); + const args = (phase: "reclaim" | "extract") => ["-e", restore, path.posix.join(directory, "archive.cjs"), archive, input.remoteCwd, input.manifestDigest, input.restorationIdentity, String(input.allowEmptyRoot === true), JSON.stringify(input.allocationFiles ?? []), - String(input.replaceRetiredRoot === true), input.restorationEpoch === undefined ? "" : String(input.restorationEpoch)])); + String(input.replaceRetiredRoot === true), input.restorationEpoch === undefined ? "" : String(input.restorationEpoch), phase]; + // Free what this root's earlier attempts left before anything new needs space. + await command(input.target, "restore", args("reclaim")); + const staged = await installArchiveHelper(input.target, parent, operationId).catch((error: unknown) => { + throw error instanceof Error && error.message === REMOTE_WORKSPACE_SANDBOX_DISK_FULL ? restoreDiskFull("archive_helper", "helper") : error; + }); + if (staged.directory !== directory) throw new Error("remote_workspace_operation_identity_invalid"); + await measureRunPhase("workspace.snapshot.sandbox_upload", () => stat(input.archivePath).then(source => transferArchive(input.target, "syncIn", [{ operationId, + files: [{ kind: "file", sourcePath: input.archivePath, targetPath: archive }], + }], source.size))).catch((error: unknown) => { + if (sandboxDiskFull(error)) throw restoreDiskFull("upload", "upload"); + noteProviderRefusal("upload", error); throw error; + }); + await measureRunPhase("workspace.snapshot.sandbox_extract", () => command(input.target, "restore", args("extract"))); // Publish the restored location before cleanup can fail, stall, or lose its // reply. The owner journals this receipt and then reclaims disposable staging. return { operationId, stagingRoot: parent, cleanupPending: true }; diff --git a/server/src/services/remote-workspaces.integration.test.ts b/server/src/services/remote-workspaces.integration.test.ts index a990f11d667..88cb8cf0a8c 100644 --- a/server/src/services/remote-workspaces.integration.test.ts +++ b/server/src/services/remote-workspaces.integration.test.ts @@ -407,7 +407,7 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem } }, 60_000); - it.each(["same root", "different root", "destroyed allocation", "occupied root", "missing object", "stalled download", "transient restore", "restore lock busy"] as const)( + it.each(["same root", "different root", "destroyed allocation", "occupied root", "missing object", "stalled download", "transient restore", "full sandbox disk", "restore lock busy"] as const)( "preserves the published workspace path and files on a cold restore (%s)", async mode => { const reserved = await reserve({ kind: "empty" }), first = await target("sandbox-A"); const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: first }); @@ -464,6 +464,26 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem expect(await fs.lstat(expectedRemoteRoot).catch(() => null)).toBeNull(); vi.mocked(second.runner!.execute).mockRestore(); } + if (mode === "full sandbox disk") { + // The sandbox cannot hold even one copy: every restored file write fails + // with ENOSPC. That repeats on every automatic retry, so it is named and + // goes to the board, whose Retry works once the sandbox has space. + const preload = path.join(root, "full-disk.cjs"); + await fs.writeFile(preload, `const fs=require('node:fs'),open=fs.promises.open; +fs.promises.open=async(...args)=>{const handle=await open.apply(fs.promises,args); + if(process.argv.includes('restore'))handle.writeFile=async()=>{throw Object.assign(Error('ENOSPC: no space left on device, write'),{code:'ENOSPC'})}; + return handle;}; +require('node:module').syncBuiltinESMExports();`); + const run = second.runner!.execute.bind(second.runner); + vi.spyOn(second.runner!, "execute").mockImplementation(async command => run(command.args?.[1]?.includes("remote_workspace_restore_failed") + ? { ...command, env: { ...command.env, NODE_OPTIONS: `--require ${preload}` } } : command)); + const full = await prepareRemoteWorkspace({ db, storage, workspace: reservedAgain, runId: nextRun, lease: newLease, target: second }) + .then(() => null, (error: unknown) => error as RemoteWorkspaceRestoreError); + expect(full).toMatchObject({ step: "sandbox_restore", retryable: false, message: "remote_workspace_restore_failed: step=sandbox_restore; " + + "remote_workspace_operation_failed: operation=restore exit=1; remote_workspace_restore_failed: stage=extract reason=sandbox_disk_full" }); + expect(await fs.lstat(expectedRemoteRoot).catch(() => null)).toBeNull(); + vi.mocked(second.runner!.execute).mockRestore(); + } if (mode === "restore lock busy") { const lockDirectory = path.join(path.dirname(expectedRemoteRoot), ".paperclip-runtime", "workspace-restore-locks"); await fs.mkdir(lockDirectory, { recursive: true, mode: 0o700 });