diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 7672ca6362f..b418585f432 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1235,6 +1235,45 @@ a snapshot and belongs to the sandbox, so it moves into the restored root. Only old preparation or restore receipt and export staging are dropped. A retained native continuation then reads its runner state at the same path. +A sandbox resumed from its provider's saved image keeps earlier folders in a +lower overlay layer. Without the overlay `redirect_dir` feature, the kernel +refuses to rename such a directory with `EXDEV`, so the retired root cannot move +aside. The restore then replaces the root in place: it removes the old saved +files and export staging, moves the restored files in, and leaves the runtime +state where it is without a copy. + +Restore commands take a sandbox-side SQLite OS lock keyed by the canonical root, +outside the directories being replaced. The private lock file is permanent; it +is never unlinked to recover ownership. A lost reply does not release it. The OS +releases it when the command exits or dies. Contenders wait only for this active +operation, for at most 120 seconds, without blocking the command event loop. +This reuses the `node:sqlite` locking dependency of the qualified Node runtime. +An expired lock wait remains retryable only while the caller still owns the same +clean epoch and saved snapshot. It does not clear the predecessor's lock. A corrupt +fence or receipt instead reports a closed refusal reason and is not retried. + +The caller supplies its existing workspace authority epoch. While holding the +operation lock, the command atomically writes and syncs a private epoch fence +before changing the root. That fence is outside SQLite's rollback transaction, +so killing the process cannot erase its newer authority. Lower epochs, changed +identities or digests within one epoch, and missing epochs for already fenced +roots are refused. Opaque identity strings do not establish ordering. Legacy +unfenced receipts remain readable, but destructive retired-root replacement +requires an epoch; unknown or malformed authority records fail closed. + +An in-place replacement first publishes a versioned pending marker bound to the +root, epoch, identity, saved digest and final directory mode. It has no completed +`identity`, so an older reader cannot adopt it. Under the same lock, a matching +retry finishes stale preparation-receipt removal and mode cleanup, then atomically +publishes the ordinary completed receipt. An authorized higher-epoch replacement +can reconcile an earlier pending root before restoring its selected save; a +conflicting identity in the same epoch remains held. Completed adoption preserves +later file and permission edits. + +These fences protect participating commands. A legacy command that never took +the lock still needs actual stop evidence from the existing recovery owner before +hosted recovery proceeds; a timeout alone never supplies that evidence. + The provider's own idle auto-stop must not coincide with the warm runner idle window. The host forwards the run's effective warm idle window (environment `runnerIdleTimeoutMs`, else the agent's `idleTimeoutMs`) to the sandbox provider diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index c6fca2d4217..65c7e406dfc 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -239,6 +239,314 @@ describe("remote workspace transport safety with real sandbox commands", () => { } finally { await captured.cleanup(); } }); + it.each(["once", "interrupted", "published_interrupted"] as const)("replaces a retired root that the sandbox's overlay cannot rename (%s)", async scenario => { + await fs.writeFile(path.join(workspace, "saved.txt"), "newer saved work\n"); + const lower = path.join(scratch, "lower-layer.json"), preload = path.join(scratch, "overlay-lower-layer.cjs"); + const postPublish = path.join(scratch, "interrupt-published"); + if (scenario === "published_interrupted") await fs.writeFile(postPublish, "once"); + await fs.writeFile(preload, `const fs=require('node:fs'),lower=new Set(JSON.parse(fs.readFileSync(${JSON.stringify(lower)},'utf8'))); +const rename=fs.promises.rename;fs.promises.rename=async(from,to)=>{const s=await fs.promises.lstat(from).catch(()=>null); +if(s?.isDirectory()&&lower.has(String(s.ino)))throw Object.assign(new Error('EXDEV: cross-device link not permitted, rename'),{code:'EXDEV',errno:-18,syscall:'rename'}); +const result=await rename.call(fs.promises,from,to); +if(String(to)===${JSON.stringify(path.join(scratch, "retired", ".paperclip-runtime", "workspace-restored.json"))}&&fs.existsSync(${JSON.stringify(postPublish)})){ +fs.unlinkSync(${JSON.stringify(postPublish)});throw Error('published_interrupted');}return result};`); + const { remote, execute } = target(); + const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }); + try { + const root = path.join(scratch, "retired"), runtime = path.join(root, ".paperclip-runtime"); + const session = path.join(runtime, "paperclip-runner", "sessions", "parent", "runner"); + await fs.mkdir(session, { recursive: true }); + await fs.writeFile(path.join(session, "runner-state.json"), "{\"lifecycle\":\"suspended\"}"); + await fs.writeFile(path.join(runtime, "workspace-ready.json"), JSON.stringify({ preparationIdentity: "f".repeat(64) })); + await fs.mkdir(path.join(runtime, "workspace-snapshots", "old-export"), { recursive: true }); + await fs.writeFile(path.join(root, "stale.md"), "old\n"); + await fs.mkdir(path.join(root, "locked", "inner"), { recursive: true }); + await fs.writeFile(path.join(root, "locked", "inner", "file.txt"), "old\n"); + await fs.chmod(path.join(root, "locked", "inner"), 0o555); await fs.chmod(path.join(root, "locked"), 0o555); + const directories: string[] = []; + const walk = async (directory: string): Promise => { + directories.push(String((await fs.lstat(directory)).ino)); + for (const entry of await fs.readdir(directory, { withFileTypes: true })) if (entry.isDirectory()) await walk(path.join(directory, entry.name)); + }; + await walk(root); + await fs.writeFile(lower, JSON.stringify(directories)); + const input = { target: remote, remoteCwd: root, archivePath: captured.archivePath, manifestDigest: captured.manifestDigest, + restorationIdentity: "9".repeat(64), restorationEpoch: 1, replaceRetiredRoot: true }; + const run = execute.getMockImplementation()!; let interrupt = scenario === "interrupted"; + execute.mockImplementation(async command => { + if (!command.args?.[1]?.includes("remote_workspace_restore_failed")) return run(command); + const args = interrupt ? command.args.map((arg, index) => index === 1 + ? arg.replace("await fs.rename(path.join(staged,'.paperclip-runtime','workspace-restored.json')", "throw Error('interrupted');await fs.rename(path.join(staged,'.paperclip-runtime','workspace-restored.json')") : arg) : command.args; + interrupt = false; + return run({ ...command, args: ["--require", preload, ...args] }); + }); + if (scenario === "interrupted") { + await expect(restoreRemoteWorkspaceDirectory(input)).rejects.toThrow(/stage=replace_in_place reason=interrupted$/); + expect(JSON.parse(await fs.readFile(path.join(runtime, "workspace-ready.json"), "utf8"))).toEqual({ preparationIdentity: "f".repeat(64) }); + expect(await fs.lstat(path.join(runtime, "workspace-restored.json")).catch(() => null)).toBeNull(); + } + if (scenario === "published_interrupted") { + await expect(restoreRemoteWorkspaceDirectory(input)).rejects.toThrow(/reason=published_interrupted$/); + const pending = JSON.parse(await fs.readFile(path.join(runtime, "workspace-restored.json"), "utf8")); + expect(pending.identity).toBeUndefined(); expect(pending.pendingIdentity).toBe(input.restorationIdentity); + await fs.writeFile(path.join(root, "later.txt"), "preserved after publication"); + } + await restoreRemoteWorkspaceDirectory(input); + if (scenario === "published_interrupted") expect(await fs.readFile(path.join(root, "later.txt"), "utf8")).toBe("preserved after publication"); + expect((await fs.stat(root)).mode & 0o777).toBe(0o700); + expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("newer saved work\n"); + expect(await fs.lstat(path.join(root, "stale.md")).catch(() => null)).toBeNull(); + expect(await fs.lstat(path.join(root, "locked")).catch(() => null)).toBeNull(); + expect(await fs.readFile(path.join(session, "runner-state.json"), "utf8")).toBe("{\"lifecycle\":\"suspended\"}"); + expect(await fs.readdir(runtime)).toEqual(expect.arrayContaining(["paperclip-runner", "workspace-restored.json"])); + expect(await fs.lstat(path.join(runtime, "workspace-ready.json")).catch(() => null)).toBeNull(); + expect(await fs.lstat(path.join(runtime, "workspace-snapshots")).catch(() => null)).toBeNull(); + await fs.chmod(root, 0o750); + await restoreRemoteWorkspaceDirectory(input); + expect((await fs.stat(root)).mode & 0o777).toBe(0o750); + expect((await fs.readdir(scratch)).filter(name => name.startsWith("retired"))).toEqual(["retired"]); + } finally { + await captured.cleanup(); + await exec("chmod", ["-R", "u+w", scratch]).catch(() => undefined); + } + }); + + it.each(["same", "different"] as const)("reconciles interrupted in-place cleanup for a %s authorized identity", async identityMode => { + await fs.writeFile(path.join(workspace, "saved.txt"), "verified saved work"); + const { remote, execute } = target(), captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }); + try { + const root = path.join(scratch, "pending-root"), runtime = path.join(root, ".paperclip-runtime"); + const flag = path.join(scratch, "cleanup-interrupt"), preload = path.join(scratch, "cleanup-interrupt.cjs"); + await fs.mkdir(runtime, { recursive: true }); + await fs.writeFile(path.join(runtime, "workspace-ready.json"), JSON.stringify({ preparationIdentity: "f".repeat(64) })); + await fs.writeFile(path.join(root, "stale.txt"), "old retired work"); await fs.writeFile(flag, "once"); + await fs.writeFile(preload, `const fs=require('node:fs'),rename=fs.promises.rename,rm=fs.promises.rm; +fs.promises.rename=async(from,to)=>{if(String(from)===${JSON.stringify(root)})throw Object.assign(Error('EXDEV'),{code:'EXDEV'});return rename.call(fs.promises,from,to)}; +fs.promises.rm=async(file,options)=>{const result=await rm.call(fs.promises,file,options); +if(String(file)===${JSON.stringify(path.join(runtime, "workspace-ready.json"))}&&fs.existsSync(${JSON.stringify(flag)})){ +fs.unlinkSync(${JSON.stringify(flag)});throw Error('cleanup_interrupted');}return result};`); + const actual = execute.getMockImplementation()!; + execute.mockImplementation(command => actual(command.args?.[1]?.includes("remote_workspace_restore_failed") + ? { ...command, args: ["--require", preload, ...command.args] } : command)); + const input = { target: remote, remoteCwd: root, archivePath: captured.archivePath, manifestDigest: captured.manifestDigest, + restorationIdentity: "5".repeat(64), restorationEpoch: 1, replaceRetiredRoot: true }; + await expect(restoreRemoteWorkspaceDirectory(input)).rejects.toThrow(/reason=cleanup_interrupted$/); + await expect(fs.access(path.join(runtime, "workspace-ready.json"))).rejects.toMatchObject({ code: "ENOENT" }); + const interrupted = JSON.parse(await fs.readFile(path.join(runtime, "workspace-restored.json"), "utf8")); + expect(interrupted.identity).toBeUndefined(); + expect(interrupted).toMatchObject({ schema: "paperclip.workspace-restore-pending.v1", root, + pendingIdentity: input.restorationIdentity, manifestDigest: captured.manifestDigest, inPlaceMode: 0o700 }); + if (identityMode === "same") { + await fs.writeFile(path.join(runtime, "workspace-restored.json"), JSON.stringify({ ...interrupted, root: root + "-foreign" })); + await expect(restoreRemoteWorkspaceDirectory(input)).rejects.toThrow(/reason=unsafe_restore_fence$/); + expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("verified saved work"); + await fs.writeFile(path.join(runtime, "workspace-restored.json"), JSON.stringify(interrupted)); + } + const next = identityMode === "same" ? input : { ...input, restorationIdentity: "4".repeat(64), restorationEpoch: 2 }; + if (identityMode === "different") { + await expect(restoreRemoteWorkspaceDirectory({ ...next, replaceRetiredRoot: false })).rejects.toThrow(/reason=workspace_owner_changed$/); + expect(JSON.parse(await fs.readFile(path.join(runtime, "workspace-restored.json"), "utf8"))).toEqual(interrupted); + } + await restoreRemoteWorkspaceDirectory(next); + expect(JSON.parse(await fs.readFile(path.join(runtime, "workspace-restored.json"), "utf8"))) + .toEqual({ schema: "paperclip.workspace-restored.v1", root, restorationEpoch: next.restorationEpoch, identity: next.restorationIdentity, manifestDigest: captured.manifestDigest }); + expect((await fs.stat(root)).mode & 0o777).toBe(0o700); + expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("verified saved work"); + await fs.writeFile(path.join(root, "saved.txt"), "new governed work"); await restoreRemoteWorkspaceDirectory(next); + expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("new governed work"); + } finally { await captured.cleanup(); } + }); + + it.each(["reply_lost", "new_identity", "process_died"] as const)("keeps an overlapping retired restore exclusive after %s", async scenario => { + await fs.writeFile(path.join(workspace, "saved.txt"), "canonical saved work"); + const { remote, execute } = target(); + const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }); + const root = path.join(scratch, "overlap"), entered = path.join(scratch, "entered"), released = path.join(scratch, "released"); + const contended = path.join(scratch, "contended"), preload = path.join(scratch, "overlap.cjs"); + await fs.mkdir(path.join(root, ".paperclip-runtime"), { recursive: true }); + await fs.writeFile(path.join(root, ".paperclip-runtime", "workspace-ready.json"), JSON.stringify({ preparationIdentity: "f".repeat(64) })); + await fs.writeFile(path.join(root, "stale.txt"), "retired work"); + await fs.writeFile(preload, `const fs=require('node:fs'); +const rename=fs.promises.rename,rm=fs.promises.rm; +fs.promises.rename=async(from,to)=>{if(String(from)===${JSON.stringify(root)})throw Object.assign(Error('EXDEV'),{code:'EXDEV'});return rename.call(fs.promises,from,to)}; +fs.promises.rm=async(file,options)=>{if(String(file)===${JSON.stringify(path.join(root, "stale.txt"))}&&!fs.existsSync(${JSON.stringify(entered)})){ +fs.writeFileSync(${JSON.stringify(entered)},String(process.pid));while(!fs.existsSync(${JSON.stringify(released)}))await new Promise(r=>setTimeout(r,10));}return rm.call(fs.promises,file,options)}; +const {DatabaseSync}=require('node:sqlite'),exec=DatabaseSync.prototype.exec; +DatabaseSync.prototype.exec=function(sql){try{return exec.call(this,sql)}catch(e){if((e.errcode&255)===5)fs.writeFileSync(${JSON.stringify(contended)},'busy');throw e}};`); + const input = { target: remote, remoteCwd: root, archivePath: captured.archivePath, manifestDigest: captured.manifestDigest, + restorationIdentity: "8".repeat(64), restorationEpoch: 1, replaceRetiredRoot: true }; + const actual = execute.getMockImplementation()!; + let first = true, firstResult: ReturnType | undefined, firstPid: number | undefined; + let ownerReleased = false, publishedWhileActive = false, observing = true; + execute.mockImplementation(async command => { + if (!command.args?.[1]?.includes("remote_workspace_restore_failed")) return actual(command); + const operation = actual({ ...command, args: ["--require", preload, ...command.args] }); + if (!first) return operation; + first = false; firstResult = operation; void operation.catch(() => undefined); + await vi.waitFor(async () => { firstPid = Number(await fs.readFile(entered, "utf8")); expect(firstPid).toBeGreaterThan(0); }, { timeout: 5000 }); + return { stdout: "", stderr: "", exitCode: 1, timedOut: true, commandExitConfirmed: false, signal: null, + pid: firstPid!, startedAt: new Date().toISOString() }; + }); + let retry: Promise | undefined; + try { + await expect(restoreRemoteWorkspaceDirectory(input)).rejects.toThrow("remote_workspace_operation_timed_out: restore"); + if (scenario === "reply_lost") { + const unrelated = path.join(scratch, "unrelated-root"); + await restoreRemoteWorkspaceDirectory({ ...input, remoteCwd: unrelated, restorationIdentity: "2".repeat(64), replaceRetiredRoot: false }); + expect(await fs.readFile(path.join(unrelated, "saved.txt"), "utf8")).toBe("canonical saved work"); + } + const retryInput = scenario === "new_identity" ? { ...input, restorationIdentity: "7".repeat(64), restorationEpoch: 2 } : input; + retry = restoreRemoteWorkspaceDirectory(retryInput).then(async result => { + publishedWhileActive = !ownerReleased; + await fs.writeFile(path.join(root, "saved.txt"), "new provider work after admitted retry"); + return result; + }); + void retry.catch(() => undefined); + const contention = (async () => { + for (let attempt = 0; attempt < 500 && observing; attempt++) { + if (await fs.stat(contended).catch(() => null)) return; + await new Promise(resolve => setTimeout(resolve, 10)); + } + if (observing) throw Error("no restore outcome or contention observed"); + })(); + try { await Promise.race([retry, contention]); } + finally { observing = false; await contention; } + ownerReleased = true; + if (scenario === "process_died") process.kill(firstPid!, "SIGKILL"); + else await fs.writeFile(released, "release"); + await firstResult; await retry; + expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("new provider work after admitted retry"); + expect(publishedWhileActive).toBe(false); + const lockPath = path.join(scratch, ".paperclip-runtime", "workspace-restore-locks", createHash("sha256").update(root).digest("hex") + ".sqlite"); + const inode = (await fs.stat(lockPath)).ino; + await restoreRemoteWorkspaceDirectory(retryInput); + expect((await fs.stat(lockPath)).ino).toBe(inode); + expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("new provider work after admitted retry"); + } finally { + observing = false; await fs.writeFile(released, "release"); + await firstResult?.catch(() => undefined); await retry?.catch(() => undefined); await captured.cleanup(); + } + }); + + it.each(["completed", "killed"] as const)("fences a delayed old command after the newer epoch is %s", async outcome => { + await fs.writeFile(path.join(workspace, "saved.txt"), "canonical snapshot bytes"); + const { remote, execute } = target(), captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }); + const root = path.join(scratch, "epoch-root"), runtime = path.join(root, ".paperclip-runtime"); + const preload = path.join(scratch, "epoch-overlay.cjs"), kill = path.join(scratch, "kill-newer-once"); + await fs.mkdir(runtime, { recursive: true }); await fs.writeFile(path.join(root, "stale.txt"), "retired bytes"); + await fs.writeFile(path.join(runtime, "workspace-ready.json"), JSON.stringify({ preparationIdentity: "f".repeat(64) })); + if (outcome === "killed") await fs.writeFile(kill, "once"); + await fs.writeFile(preload, `const fs=require('node:fs'),rename=fs.promises.rename,chmod=fs.promises.chmod; +fs.promises.rename=async(from,to)=>{if(String(from)===${JSON.stringify(root)})throw Object.assign(Error('EXDEV'),{code:'EXDEV'});return rename.call(fs.promises,from,to)}; +fs.promises.chmod=async(file,mode)=>{if(String(file)===${JSON.stringify(root)}&&fs.existsSync(${JSON.stringify(kill)})){ +fs.unlinkSync(${JSON.stringify(kill)});process.kill(process.pid,'SIGKILL');}return chmod.call(fs.promises,file,mode)};`); + const actual = execute.getMockImplementation()!; + let release!: () => void, delayed: ReturnType | undefined, queued = false; + execute.mockImplementation(command => { + if (!command.args?.[1]?.includes("remote_workspace_restore_failed")) return actual(command); + const request = { ...command, args: ["--require", preload, ...command.args] }; + if (queued) return actual(request); + queued = true; + delayed = new Promise(resolve => { release = resolve; }).then(() => actual(request)); + return Promise.resolve({ stdout: "", stderr: "", exitCode: 1, timedOut: true, commandExitConfirmed: false, + signal: null, pid: process.pid, startedAt: new Date().toISOString() }); + }); + const old = { target: remote, remoteCwd: root, archivePath: captured.archivePath, manifestDigest: captured.manifestDigest, + restorationIdentity: "e".repeat(64), restorationEpoch: 1, replaceRetiredRoot: true }; + const current = { ...old, restorationIdentity: "a".repeat(64), restorationEpoch: 2 }; + try { + await expect(restoreRemoteWorkspaceDirectory(old)).rejects.toThrow("remote_workspace_operation_timed_out: restore"); + if (outcome === "killed") await expect(restoreRemoteWorkspaceDirectory(current)).rejects.toThrow("signal=SIGKILL"); + else { await restoreRemoteWorkspaceDirectory(current); await fs.writeFile(path.join(root, "saved.txt"), "new provider edits"); } + release(); const late = await delayed!; + expect(late.exitCode).not.toBe(0); + const fence = path.join(scratch, ".paperclip-runtime", "workspace-restore-locks", createHash("sha256").update(root).digest("hex") + ".fence.json"); + expect(JSON.parse(await fs.readFile(fence, "utf8"))).toMatchObject({ root, epoch: 2, identity: current.restorationIdentity }); + if (outcome === "killed") { + expect(await fs.readFile(path.join(root, "stale.txt"), "utf8")).toBe("retired bytes"); + await restoreRemoteWorkspaceDirectory(current); await fs.writeFile(path.join(root, "saved.txt"), "new provider edits"); + } + expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("new provider edits"); + if (outcome === "completed") { + await fs.unlink(fence); + const receiptPath = path.join(runtime, "workspace-restored.json"), receipt = JSON.parse(await fs.readFile(receiptPath, "utf8")); + await fs.writeFile(receiptPath, JSON.stringify({ ...receipt, restorationEpoch: undefined })); + await expect(restoreRemoteWorkspaceDirectory({ ...current, restorationEpoch: undefined, replaceRetiredRoot: false })) + .rejects.toThrow(/reason=unsafe_restore_fence$/); + await fs.writeFile(receiptPath, JSON.stringify(receipt)); + } + await expect(restoreRemoteWorkspaceDirectory({ ...current, restorationIdentity: "f".repeat(64) })).rejects.toThrow(/reason=workspace_owner_changed$/); + await expect(restoreRemoteWorkspaceDirectory({ ...current, restorationEpoch: undefined, replaceRetiredRoot: false })).rejects.toThrow(/reason=restore_fence_required$/); + await restoreRemoteWorkspaceDirectory(current); + expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("new provider edits"); + } finally { release?.(); await delayed?.catch(() => undefined); await captured.cleanup(); } + }); + + it.each(["fence", "receipt"] as const)("refuses malformed restore %s JSON with a closed diagnosis", async record => { + await fs.writeFile(path.join(workspace, "saved.txt"), "retained saved work"); + const { remote } = target(), captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }); + try { + const root = path.join(scratch, "corrupt-record-root"); + const input = { target: remote, remoteCwd: root, archivePath: captured.archivePath, manifestDigest: captured.manifestDigest, + restorationIdentity: "4".repeat(64), restorationEpoch: 1 }; + await restoreRemoteWorkspaceDirectory(input); + const file = record === "receipt" ? path.join(root, ".paperclip-runtime", "workspace-restored.json") + : path.join(scratch, ".paperclip-runtime", "workspace-restore-locks", createHash("sha256").update(root).digest("hex") + ".fence.json"); + await fs.writeFile(file, "{"); + await expect(restoreRemoteWorkspaceDirectory(input)).rejects.toThrow(/reason=unsafe_restore_fence$/); + expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("retained saved work"); + expect(await fs.readFile(file, "utf8")).toBe("{"); + } finally { await captured.cleanup(); } + }); + + it("does not poison the authority fence after malformed restore input", async () => { + await fs.writeFile(path.join(workspace, "saved.txt"), "valid saved work"); + const { remote } = target(), captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }); + try { + const root = path.join(scratch, "invalid-input-root"); await fs.mkdir(path.join(root, ".paperclip-runtime"), { recursive: true }); + await fs.writeFile(path.join(root, ".paperclip-runtime", "workspace-ready.json"), JSON.stringify({ preparationIdentity: "f".repeat(64) })); + await fs.writeFile(path.join(root, "sentinel.txt"), "untouched before valid restore"); + const input = { target: remote, remoteCwd: root, archivePath: captured.archivePath, manifestDigest: captured.manifestDigest, + restorationIdentity: "3".repeat(64), restorationEpoch: 1, replaceRetiredRoot: true }; + await expect(restoreRemoteWorkspaceDirectory({ ...input, manifestDigest: "invalid" })).rejects.toThrow(); + expect(await fs.readFile(path.join(root, "sentinel.txt"), "utf8")).toBe("untouched before valid restore"); + await expect(restoreRemoteWorkspaceDirectory(input)).resolves.toBeDefined(); + expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("valid saved work"); + } finally { await captured.cleanup(); } + }); + + it.each(["directory_symlink", "directory_shared", "parent_shared", "database_symlink", "database_hardlink", "database_oversized"] as const)( + "rejects an unsafe restore lock without changing workspace files: %s", async scenario => { + await fs.writeFile(path.join(workspace, "saved.txt"), "canonical safe work"); + const { remote } = target(), captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }); + try { + const root = path.join(scratch, "unsafe-lock"), directory = path.join(scratch, ".paperclip-runtime", "workspace-restore-locks"); + await fs.mkdir(path.join(root, ".paperclip-runtime"), { recursive: true }); + await fs.writeFile(path.join(root, ".paperclip-runtime", "workspace-ready.json"), JSON.stringify({ preparationIdentity: "f".repeat(64) })); + await fs.writeFile(path.join(root, "sentinel.txt"), "must remain"); + const outside = path.join(scratch, "outside-lock"); + await fs.mkdir(path.dirname(directory), { recursive: true, mode: 0o700 }); + if (scenario === "directory_symlink") { await fs.mkdir(outside); await fs.symlink(outside, directory); } + else if (scenario === "parent_shared") await fs.chmod(path.dirname(directory), 0o777); + else { + await fs.mkdir(directory, { mode: scenario === "directory_shared" ? 0o755 : 0o700 }); + if (scenario !== "directory_shared") { + await fs.writeFile(outside, "unrelated bytes"); + const database = path.join(directory, createHash("sha256").update(root).digest("hex") + ".sqlite"); + if (scenario === "database_symlink") await fs.symlink(outside, database); + else if (scenario === "database_hardlink") await fs.link(outside, database); + else await fs.writeFile(database, Buffer.alloc(65537)); + } + } + await expect(restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: root, archivePath: captured.archivePath, + manifestDigest: captured.manifestDigest, restorationIdentity: "6".repeat(64), restorationEpoch: 1, replaceRetiredRoot: true })) + .rejects.toThrow(scenario === "directory_symlink" ? /reason=unsafe_root$/ : /reason=unsafe_restore_lock$/); + expect(await fs.readFile(path.join(root, "sentinel.txt"), "utf8")).toBe("must remain"); + await expect(fs.access(path.join(root, "saved.txt"))).rejects.toMatchObject({ code: "ENOENT" }); + if (scenario === "database_symlink" || scenario === "database_hardlink") expect(await fs.readFile(outside, "utf8")).toBe("unrelated bytes"); + } finally { await captured.cleanup(); } + }); + it("keeps a completed restore available when the staging cleanup reply is lost", async () => { await fs.writeFile(path.join(workspace, "saved.txt"), "durable synthetic files\n"); const { remote, execute } = target(); diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 7377ccc8d0c..a4a3ec68f51 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -586,13 +586,16 @@ async function captureSandboxWorkspaceArchive(input: { } export async function restoreRemoteWorkspaceDirectory(input: { - target: RemoteWorkspaceTarget; remoteCwd: string; archivePath: string; manifestDigest: string; restorationIdentity: string; allowEmptyRoot?: boolean; + target: RemoteWorkspaceTarget; remoteCwd: string; archivePath: string; manifestDigest: string; restorationIdentity: string; restorationEpoch?: number; allowEmptyRoot?: boolean; allocationFiles?: Array<{ relativePath: string; sha256: string }>; /** The server proved this sandbox's root is the workspace's own earlier allocation, fully saved. */ replaceRetiredRoot?: boolean; }) { if (input.replaceRetiredRoot && input.allowEmptyRoot) throw new Error("remote_workspace_restore_mode_invalid"); + if (input.restorationEpoch !== undefined && (!Number.isSafeInteger(input.restorationEpoch) || input.restorationEpoch < 1)) + throw new Error("remote_workspace_restore_epoch_invalid"); if (!/^[a-f0-9]{64}$/.test(input.restorationIdentity)) throw new Error("remote_workspace_restore_identity_invalid"); + if (!/^[a-f0-9]{64}$/.test(input.manifestDigest)) throw new Error("remote_workspace_restore_manifest_invalid"); const id = input.restorationIdentity; const operationId = `${id.slice(0, 8)}-${id.slice(8, 12)}-${id.slice(12, 16)}-${id.slice(16, 20)}-${id.slice(20, 32)}`; // Stage outside the destination; the restore validator requires an empty target. @@ -605,13 +608,18 @@ export async function restoreRemoteWorkspaceDirectory(input: { // reply can then adopt the same files without repeating a destructive restore. const restore = String.raw`const fs=require('node:fs/promises'),path=require('node:path'),cp=require('node:child_process'),crypto=require('node:crypto'); ${DISK_FULL_CHECK}${CAPTURE_FAILURE_CHECK}${SETUP_FAILURE_CHECK}let stage='validate'; - (async()=>{const [helper,archive,root,digest,identity,allowEmpty,rawFiles,replaceRetired]=process.argv.slice(1),files=JSON.parse(rawFiles); - if(!/^[a-f0-9]{64}$/.test(identity)||!Array.isArray(files)||files.length>8)throw Error('invalid_identity'); + (async()=>{const [helper,archive,root,digest,identity,allowEmpty,rawFiles,replaceRetired,rawEpoch]=process.argv.slice(1),files=JSON.parse(rawFiles); + const validEpoch=value=>Number.isSafeInteger(value)&&value>0,restorationEpoch=rawEpoch===undefined||rawEpoch===''?null:Number(rawEpoch); + if(restorationEpoch!==null&&!validEpoch(restorationEpoch))throw Error('invalid_epoch'); + const completedReceipt=(id,manifestDigest,epoch)=>epoch===null?{identity:id,manifestDigest} + :{schema:'paperclip.workspace-restored.v1',root,restorationEpoch:epoch,identity:id,manifestDigest}; + if(!/^[a-f0-9]{64}$/.test(identity)||!/^[a-f0-9]{64}$/.test(digest)||!Array.isArray(files)||files.length>8)throw Error('invalid_identity'); for(const file of files)if(!/^\.paperclip-runtime\/[a-zA-Z0-9_-]+\.json$/.test(file.relativePath) ||file.relativePath==='.paperclip-runtime/workspace-restored.json'||!/^[a-f0-9]{64}$/.test(file.sha256))throw Error('invalid_allocation'); if(new Set(files.map(file=>file.relativePath)).size!==files.length)throw Error('invalid_allocation'); const stat=async p=>fs.lstat(p).catch(e=>{if(e.code==='ENOENT')return null;throw e}); const directory=async p=>{const s=await fs.lstat(p);if(!s.isDirectory()||s.isSymbolicLink()||await fs.realpath(p)!==p)throw Error('unsafe_root')}; + const readJson=async(file,reason)=>{const body=await fs.readFile(file,'utf8');try{return JSON.parse(body)}catch{throw Error(reason)}}; const allocation=async p=>{ await directory(p);const names=await fs.readdir(p); if(names.length===0&&files.length===0)return []; @@ -630,7 +638,7 @@ export async function restoreRemoteWorkspaceDirectory(input: { for(const [name,key] of [['workspace-restored.json','identity'],['workspace-ready.json','preparationIdentity']]){ const file=path.join(runtime,name),s=await stat(file);if(!s)continue; if(!s.isFile()||s.isSymbolicLink()||s.nlink!==1||s.size>16384)throw Error('unsafe_receipt'); - if(/^[a-f0-9]{64}$/.test(String(JSON.parse(await fs.readFile(file,'utf8'))?.[key])))return true;} + if(/^[a-f0-9]{64}$/.test(String((await readJson(file,'unsafe_receipt'))?.[key])))return true;} return false; }; const parked=root+'.allocation-'+identity,retired=root+'.retired-'+identity; @@ -641,22 +649,97 @@ export async function restoreRemoteWorkspaceDirectory(input: { // of the same restore finishes an interrupted carry without a new proof. // Saved directory modes can be read-only. Make the parked tree's own directories // owner-writable without following symlinks, then remove it. + const open=async d=>{const s=await fs.lstat(d);if(!s.isDirectory()||s.isSymbolicLink())return; + await fs.chmod(d,(s.mode&0o7777)|0o700);for(const name of await fs.readdir(d))await open(path.join(d,name));}; const removeRetired=async()=>{if(!await stat(retired))return; stage='carry_runtime';const from=path.join(retired,'.paperclip-runtime'),to=path.join(root,'.paperclip-runtime'); if(await stat(from)){await directory(from);await directory(to); for(const name of await fs.readdir(from)){ if(['workspace-ready.json','workspace-restored.json','workspace-snapshots'].includes(name)||await stat(path.join(to,name)))continue; await fs.rename(path.join(from,name),path.join(to,name));}} - stage='cleanup_parked';const open=async d=>{const s=await fs.lstat(d);if(!s.isDirectory()||s.isSymbolicLink())return; - await fs.chmod(d,(s.mode&0o7777)|0o700);for(const name of await fs.readdir(d))await open(path.join(d,name));}; - await open(retired);await fs.rm(retired,{recursive:true,force:true});}; + stage='cleanup_parked';await open(retired);await fs.rm(retired,{recursive:true,force:true});}; + const pendingInPlace=receipt=>receipt.schema==='paperclip.workspace-restore-pending.v1'&&receipt.root===root&&validEpoch(receipt.restorationEpoch) + &&typeof receipt.pendingIdentity==='string'&&/^[a-f0-9]{64}$/.test(receipt.pendingIdentity) + &&typeof receipt.manifestDigest==='string'&&/^[a-f0-9]{64}$/.test(receipt.manifestDigest) + &&Number.isSafeInteger(receipt.inPlaceMode)&&receipt.inPlaceMode>=0&&receipt.inPlaceMode<=0o7777; + const finishInPlace=async receipt=>{ + if(!pendingInPlace(receipt))throw Error('unsafe_receipt'); + stage='cleanup_replaced';await fs.rm(path.join(root,'.paperclip-runtime','workspace-ready.json'),{force:true}); + await fs.chmod(root,receipt.inPlaceMode); + stage='commit_replaced';const marker=path.join(root,'.paperclip-runtime','workspace-restored.json'),pending=marker+'.'+crypto.randomUUID()+'.tmp'; + const completed=completedReceipt(receipt.pendingIdentity,receipt.manifestDigest,receipt.restorationEpoch); + try{await fs.writeFile(pending,JSON.stringify(completed),{flag:'wx',mode:0o600});await fs.rename(pending,marker);return completed;} + finally{await fs.rm(pending,{force:true});} + }; + const replaceInPlace=async staged=>{stage='replace_in_place';const runtime=path.join(root,'.paperclip-runtime'); + const s=await fs.lstat(root);await fs.chmod(root,(s.mode&0o7777)|0o700); + for(const name of await fs.readdir(root))if(name!=='.paperclip-runtime'){await open(path.join(root,name));await fs.rm(path.join(root,name),{recursive:true,force:true});} + if(await stat(path.join(runtime,'workspace-snapshots'))){await open(path.join(runtime,'workspace-snapshots')); + await fs.rm(path.join(runtime,'workspace-snapshots'),{recursive:true,force:true});} + for(const name of await fs.readdir(staged))if(name!=='.paperclip-runtime')await fs.rename(path.join(staged,name),path.join(root,name)); + const receiptFile=path.join(staged,'.paperclip-runtime','workspace-restored.json'),receipt={schema:'paperclip.workspace-restore-pending.v1', + root,pendingIdentity:identity,manifestDigest:digest,restorationEpoch,inPlaceMode:(await fs.lstat(staged)).mode&0o7777}; + await fs.writeFile(receiptFile,JSON.stringify(receipt),{mode:0o600}); + await fs.rename(path.join(staged,'.paperclip-runtime','workspace-restored.json'),path.join(runtime,'workspace-restored.json')); + await finishInPlace(receipt);}; + stage='restore_lock';await directory(path.dirname(root)); + const lockRoot=path.join(path.dirname(root),'.paperclip-runtime','workspace-restore-locks'); + await directory(path.dirname(lockRoot)); + for(const parent of [path.dirname(root),path.dirname(lockRoot)]){const s=await fs.lstat(parent); + if(s.uid!==process.getuid()||(s.mode&0o022)!==0)throw Error('unsafe_restore_lock');} + await fs.mkdir(lockRoot,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e}); + await directory(lockRoot);const lockDirectory=await fs.lstat(lockRoot); + if(lockDirectory.uid!==process.getuid()||(lockDirectory.mode&0o077)!==0)throw Error('unsafe_restore_lock'); + const lockPath=path.join(lockRoot,crypto.createHash('sha256').update(root).digest('hex')+'.sqlite'),lockStat=await stat(lockPath); + if(lockStat&&(!lockStat.isFile()||lockStat.isSymbolicLink()||lockStat.nlink!==1||lockStat.uid!==process.getuid()||lockStat.size>65536))throw Error('unsafe_restore_lock'); + const lock=new(require('node:sqlite').DatabaseSync)(lockPath,{allowExtension:false}); + try{ + await fs.chmod(lockPath,0o600);lock.exec('PRAGMA busy_timeout=0');const deadline=performance.now()+120000; + while(true){try{lock.exec('BEGIN IMMEDIATE');break}catch(e){if(typeof e.errcode!=='number'||(e.errcode&255)!==5)throw e; + if(performance.now()>=deadline)throw Error('restore_lock_busy');await new Promise(resolve=>setTimeout(resolve,50));}} + stage='restore_fence';const fencePath=lockPath.slice(0,-7)+'.fence.json'; + const checkedJson=async file=>{const s=await stat(file);if(!s)return null; + if(!s.isFile()||s.isSymbolicLink()||s.nlink!==1||s.uid!==process.getuid()||s.size>16384)throw Error('unsafe_restore_fence'); + const value=await readJson(file,'unsafe_restore_fence');if(!value||typeof value!=='object'||Array.isArray(value))throw Error('unsafe_restore_fence');return value;}; + const validFence=value=>value.schema==='paperclip.workspace-restore-fence.v1'&&value.root===root&&validEpoch(value.epoch) + &&typeof value.identity==='string'&&/^[a-f0-9]{64}$/.test(value.identity) + &&typeof value.manifestDigest==='string'&&/^[a-f0-9]{64}$/.test(value.manifestDigest); + const persisted=await checkedJson(fencePath);if(persisted&&!validFence(persisted))throw Error('unsafe_restore_fence'); + let latest=persisted; + for(const location of [root,parked,retired]){ + if(!await stat(location))continue;await directory(location);const runtime=path.join(location,'.paperclip-runtime'); + if(!await stat(runtime))continue;await directory(runtime);const receipt=await checkedJson(path.join(runtime,'workspace-restored.json')); + if(!receipt)continue; + if(receipt.restorationEpoch===undefined){if(receipt.schema!==undefined)throw Error('unsafe_restore_fence');continue;} + const pending=receipt.schema==='paperclip.workspace-restore-pending.v1'; + if(receipt.root!==root||!validEpoch(receipt.restorationEpoch)||!(pending?pendingInPlace(receipt) + :receipt.schema==='paperclip.workspace-restored.v1'&&typeof receipt.identity==='string'&&/^[a-f0-9]{64}$/.test(receipt.identity) + &&typeof receipt.manifestDigest==='string'&&/^[a-f0-9]{64}$/.test(receipt.manifestDigest)))throw Error('unsafe_restore_fence'); + const observed={schema:'paperclip.workspace-restore-fence.v1',root,epoch:receipt.restorationEpoch, + identity:pending?receipt.pendingIdentity:receipt.identity,manifestDigest:receipt.manifestDigest}; + if(latest&&observed.epoch===latest.epoch&&(observed.identity!==latest.identity||observed.manifestDigest!==latest.manifestDigest))throw Error('unsafe_restore_fence'); + if(!latest||observed.epoch>latest.epoch)latest=observed; + } + if(latest&&restorationEpoch===null)throw Error('restore_fence_required'); + if(latest&&(restorationEpoch16384)throw Error('unsafe_receipt'); - const receipt=JSON.parse(await fs.readFile(marker,'utf8')); + let receipt=await readJson(marker,'unsafe_receipt'); + if(receipt.schema==='paperclip.workspace-restore-pending.v1'){ + if(!pendingInPlace(receipt))throw Error('unsafe_receipt'); + if(receipt.pendingIdentity===identity?receipt.manifestDigest!==digest:replaceRetired!=='true')throw Error('workspace_owner_changed'); + receipt=await finishInPlace(receipt); + } if(receipt.identity===identity&&receipt.manifestDigest===digest){ if(await stat(parked)){await allocation(parked);await fs.rm(parked,{recursive:true});} await removeRetired();return;} @@ -666,6 +749,7 @@ export async function restoreRemoteWorkspaceDirectory(input: { // Replacing is allowed only for a root the server proved is this workspace's // own earlier allocation whose last use ended in a completed final save. const retire=Boolean(prior)&&replaceRetired==='true'&&await ownReceipt(root); + if(retire&&restorationEpoch===null)throw Error('restore_fence_required'); if(prior&&!retire&&allowEmpty!=='true')throw Error('occupied_root'); stage='allocation';const retained=prior&&!retire?await allocation(root):[]; if(!prior&&files.length)throw Error('allocation_missing'); @@ -674,19 +758,21 @@ export async function restoreRemoteWorkspaceDirectory(input: { await require('node:util').promisify(cp.execFile)(process.execPath,[helper,'restore',archive,staged,digest],{maxBuffer:1048576}); stage='write_receipt';const runtime=path.join(staged,'.paperclip-runtime');await fs.mkdir(runtime,{mode:0o700}); for(const file of retained)await fs.writeFile(path.join(runtime,file.name),file.content,{flag:'wx',mode:0o600}); - await fs.writeFile(path.join(runtime,'workspace-restored.json'),JSON.stringify({identity,manifestDigest:digest}),{flag:'wx',mode:0o600}); + await fs.writeFile(path.join(runtime,'workspace-restored.json'),JSON.stringify(completedReceipt(identity,digest,restorationEpoch)),{flag:'wx',mode:0o600}); stage='publish';if(await fs.realpath(path.dirname(root))!==path.dirname(root))throw Error('unsafe_root'); const holding=retire?retired:parked; if(prior){if(retire){if(!await ownReceipt(root))throw Error('occupied_root');}else await allocation(root); - if(await stat(holding))throw Error('occupied_parking');await fs.rename(root,holding);} + if(await stat(holding))throw Error('occupied_parking'); + try{await fs.rename(root,holding);}catch(e){if(!retire||e?.code!=='EXDEV')throw e;await replaceInPlace(staged);return;}} try{await fs.rename(staged,root);}catch(e){if(prior&&!await stat(root))await fs.rename(holding,root);throw e;} stage='cleanup_parked';if(prior&&!retire){await allocation(parked);await fs.rm(parked,{recursive:true});} await removeRetired(); }finally{await fs.rm(staged,{recursive:true,force:true});} + }finally{lock.close();} })().catch(error=>{process.stderr.write(setupFailure('remote_workspace_restore_failed',stage,error));process.exitCode=1});`; await command(input.target, "restore", ["-e", restore, staged.helper, staged.archive, input.remoteCwd, input.manifestDigest, input.restorationIdentity, String(input.allowEmptyRoot === true), JSON.stringify(input.allocationFiles ?? []), - String(input.replaceRetiredRoot === true)]); + String(input.replaceRetiredRoot === true), input.restorationEpoch === undefined ? "" : String(input.restorationEpoch)]); // Publish the restored location before cleanup can fail, stall, or lose its // reply. The owner journals this receipt and then reclaims disposable staging. return { operationId, stagingRoot: parent, cleanupPending: true }; diff --git a/server/src/services/remote-workspaces.integration.test.ts b/server/src/services/remote-workspaces.integration.test.ts index 9e45c05470b..1a2990f9458 100644 --- a/server/src/services/remote-workspaces.integration.test.ts +++ b/server/src/services/remote-workspaces.integration.test.ts @@ -406,7 +406,7 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem } }, 60_000); - it.each(["same root", "different root", "destroyed allocation", "occupied root", "missing object", "transient restore"] as const)( + it.each(["same root", "different root", "destroyed allocation", "occupied root", "missing object", "transient restore", "restore lock busy"] as const)( "preserves the published workspace path and files on a cold restore (%s)", async mode => { const reserved = await reserve({ kind: "empty" }), first = await target("sandbox-A"); const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: first }); @@ -463,6 +463,45 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem expect(await fs.lstat(expectedRemoteRoot).catch(() => null)).toBeNull(); vi.mocked(second.runner!.execute).mockRestore(); } + if (mode === "restore lock busy") { + const lockDirectory = path.join(path.dirname(expectedRemoteRoot), ".paperclip-runtime", "workspace-restore-locks"); + await fs.mkdir(lockDirectory, { recursive: true, mode: 0o700 }); + const lockPath = path.join(lockDirectory, createHash("sha256").update(expectedRemoteRoot).digest("hex") + ".sqlite"); + const preload = path.join(root, "advance-lock-deadline.cjs"); + await fs.writeFile(preload, "let ticks=0;Object.defineProperty(performance,'now',{value:()=>ticks++*120001});"); + const owner = spawn(process.execPath, ["-e", "const {DatabaseSync}=require('node:sqlite');const db=new DatabaseSync(process.argv[1]);db.exec('BEGIN IMMEDIATE');process.stdout.write('ready');setInterval(()=>{},1000);", lockPath], + { stdio: ["ignore", "pipe", "pipe"] }); + let ready = false; owner.stdout.on("data", data => { ready ||= String(data).includes("ready"); }); owner.stderr.resume(); + const stopped = new Promise(resolve => owner.once("close", () => resolve())); + const run = second.runner!.execute.bind(second.runner); + let afterBusy: (() => Promise) | undefined; + const spy = vi.spyOn(second.runner!, "execute").mockImplementation(async command => { + if (!command.args?.[1]?.includes("remote_workspace_restore_failed")) return run(command); + const result = await run({ ...command, args: ["--require", preload, ...command.args] }); + await afterBusy?.(); return result; + }); + try { + await vi.waitFor(() => expect(ready).toBe(true), { timeout: 5000 }); + const attempt = () => prepareRemoteWorkspace({ db, storage, workspace: reservedAgain, runId: nextRun, lease: newLease, target: second }) + .then(() => null, (error: unknown) => error as RemoteWorkspaceRestoreError); + expect(await attempt()).toMatchObject({ step: "sandbox_restore", retryable: true, + message: expect.stringMatching(/stage=restore_lock reason=restore_lock_busy$/) }); + const authority = readRemoteWorkspaceAuthority(reservedAgain.metadata)!; + for (const changed of [{ dirty: true }, { epoch: authority.epoch + 1 }, { writerRunId: randomUUID() }, { snapshotId: randomUUID() }]) { + afterBusy = async () => { await db.update(executionWorkspaces).set({ metadata: { ...reservedAgain.metadata, + remoteWorkspace: { ...authority, ...changed } } }).where(eq(executionWorkspaces.id, reservedAgain.id)); }; + try { expect(await attempt()).toMatchObject({ step: "sandbox_restore", retryable: false }); } + finally { await db.update(executionWorkspaces).set({ metadata: reservedAgain.metadata }).where(eq(executionWorkspaces.id, reservedAgain.id)); } + } + expect(await fs.lstat(expectedRemoteRoot).catch(() => null)).toBeNull(); + } finally { spy.mockRestore(); owner.kill("SIGTERM"); await stopped; } + const fence = lockPath.slice(0, -7) + ".fence.json"; await fs.writeFile(fence, "{"); + const corrupt = await prepareRemoteWorkspace({ db, storage, workspace: reservedAgain, runId: nextRun, lease: newLease, target: second }) + .then(() => null, (error: unknown) => error as RemoteWorkspaceRestoreError); + expect(corrupt).toMatchObject({ step: "sandbox_restore", retryable: false, + message: expect.stringMatching(/reason=unsafe_restore_fence$/) }); + expect(await fs.readFile(fence, "utf8")).toBe("{"); await fs.unlink(fence); + } if (mode === "missing object") { // Storage reports a lost saved object as 404; retrying cannot bring it back. const lost: StorageService = { ...storage, getObject: async () => { throw notFound("Object not found"); } }; @@ -582,7 +621,9 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem // Only the new restore receipt identifies the root; the retired preparation receipt is gone. expect(await fs.lstat(path.join(restored.remoteRoot, ".paperclip-runtime", "workspace-ready.json")).catch(() => null)).toBeNull(); expect(JSON.parse(await fs.readFile(path.join(restored.remoteRoot, ".paperclip-runtime", "workspace-restored.json"), "utf8"))) - .toMatchObject({ identity: expect.stringMatching(/^[a-f0-9]{64}$/) }); + .toMatchObject({ schema: "paperclip.workspace-restored.v1", root: restored.remoteRoot, + restorationEpoch: readRemoteWorkspaceAuthority(restored.workspace.metadata)!.epoch, + identity: expect.stringMatching(/^[a-f0-9]{64}$/) }); // A lost reply repeats the same preparation without replacing the restored root again. await fs.writeFile(path.join(restored.remoteRoot, "parent-continued.md"), "unsaved parent edits\n"); // An attempt interrupted after publish left its retired root; the repeat keeps its runtime state. diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index 3bbdcd1c3fe..19b9ac31c44 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -422,7 +422,7 @@ export async function prepareRemoteWorkspace(input: { await loadWorkspaceSnapshotArchive({ storage: input.storage ?? getStorageService(), reference: snapshot, targetPath: archivePath }); step = "sandbox_restore"; restoredExport = await restoreRemoteWorkspaceDirectory({ target, remoteCwd: remoteRoot, archivePath, manifestDigest: snapshot.manifestDigest, - restorationIdentity: preparationIdentity, allowEmptyRoot: migration !== null, + restorationIdentity: preparationIdentity, restorationEpoch: initial.epoch, allowEmptyRoot: migration !== null, allocationFiles: migration ? workspaceAllocationFiles(input.lease, remoteRoot) : [], replaceRetiredRoot }); } finally { await rm(scratch, { recursive: true, force: true }); } @@ -438,7 +438,8 @@ export async function prepareRemoteWorkspace(input: { // A sandbox refusal that names its reason (an occupied folder, a changed // owner, a rejected archive) repeats on retry, so it goes to the board. let retryable = false; - const refused = error instanceof Error && / (?:reason|archive)=[a-z_]+/.test(error.message); + const refused = error instanceof Error && / (?:reason|archive)=[a-z_]+/.test(error.message) + && !(step === "sandbox_restore" && /; remote_workspace_restore_failed: stage=restore_lock reason=restore_lock_busy$/.test(error.message)); if (step !== "snapshot_lookup" && !refused) { const [latest] = await input.db.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces) .where(and(eq(executionWorkspaces.companyId, input.workspace.companyId), eq(executionWorkspaces.id, input.workspace.id)));