diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 641ff3ab1bd..1f1386445c8 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1057,6 +1057,17 @@ folder), or an exhausted budget leaves a board-owned recovery action. Stranded-w recovery names this failure for the statuses it scans; for other statuses, such as a `blocked` task woken by a reply, the failure path records the action directly. +A retained sandbox can still hold a workspace's earlier root after the workspace +moved to another allocation, for example when child tasks inherit a parent's +workspace and save newer work on their own sandboxes. When the parent wakes on its +retained sandbox, the restore may replace that occupied root only when the server +proves it is the workspace's own earlier allocation on this sandbox: the last +recorded use of that root on this sandbox is an earlier epoch that ended in a +quiesced, committed final save, and the folder carries a Paperclip preparation or +restore receipt. Its files are then already part of the saved lineage being +restored. Any other occupant (no receipt, a later unsaved use, or a sandbox that +never saved that root) is still refused with `reason=occupied_root`. + The provider's own idle auto-stop must not coincide with the warm runner idle window. The host forwards the run's effective warm idle window (environment `runnerIdleTimeoutMs`, else the agent's `idleTimeoutMs`) to the sandbox provider diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index c56122e2430..5271c5e0c6e 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -454,7 +454,10 @@ export async function captureRemoteWorkspaceArchive(input: { export async function restoreRemoteWorkspaceDirectory(input: { target: RemoteWorkspaceTarget; remoteCwd: string; archivePath: string; manifestDigest: string; restorationIdentity: string; allowEmptyRoot?: boolean; allocationFiles?: Array<{ relativePath: string; sha256: string }>; + /** The server proved this sandbox's root is the workspace's own earlier allocation, fully saved. */ + replaceRetiredRoot?: boolean; }) { + if (input.replaceRetiredRoot && input.allowEmptyRoot) throw new Error("remote_workspace_restore_mode_invalid"); if (!/^[a-f0-9]{64}$/.test(input.restorationIdentity)) throw new Error("remote_workspace_restore_identity_invalid"); const id = input.restorationIdentity; const operationId = `${id.slice(0, 8)}-${id.slice(8, 12)}-${id.slice(12, 16)}-${id.slice(16, 20)}-${id.slice(20, 32)}`; @@ -468,7 +471,7 @@ export async function restoreRemoteWorkspaceDirectory(input: { // reply can then adopt the same files without repeating a destructive restore. const restore = String.raw`const fs=require('node:fs/promises'),path=require('node:path'),cp=require('node:child_process'),crypto=require('node:crypto'); ${DISK_FULL_CHECK}${CAPTURE_FAILURE_CHECK}${SETUP_FAILURE_CHECK}let stage='validate'; - (async()=>{const [helper,archive,root,digest,identity,allowEmpty,rawFiles]=process.argv.slice(1),files=JSON.parse(rawFiles); + (async()=>{const [helper,archive,root,digest,identity,allowEmpty,rawFiles,replaceRetired]=process.argv.slice(1),files=JSON.parse(rawFiles); if(!/^[a-f0-9]{64}$/.test(identity)||!Array.isArray(files)||files.length>8)throw Error('invalid_identity'); for(const file of files)if(!/^\.paperclip-runtime\/[a-zA-Z0-9_-]+\.json$/.test(file.relativePath) ||file.relativePath==='.paperclip-runtime/workspace-restored.json'||!/^[a-f0-9]{64}$/.test(file.sha256))throw Error('invalid_allocation'); @@ -487,7 +490,22 @@ export async function restoreRemoteWorkspaceDirectory(input: { const content=await fs.readFile(name);if(crypto.createHash('sha256').update(content).digest('hex')!==file.sha256)throw Error('allocation_changed'); return {name:path.basename(name),content};})); }; - const parked=root+'.allocation-'+identity; + // A retired root holds a Paperclip preparation or restore receipt; anything else is unknown. + const ownReceipt=async p=>{ + const runtime=path.join(p,'.paperclip-runtime');if(!await stat(runtime))return false;await directory(runtime); + for(const [name,key] of [['workspace-restored.json','identity'],['workspace-ready.json','preparationIdentity']]){ + const file=path.join(runtime,name),s=await stat(file);if(!s)continue; + if(!s.isFile()||s.isSymbolicLink()||s.nlink!==1||s.size>16384)throw Error('unsafe_receipt'); + if(/^[a-f0-9]{64}$/.test(String(JSON.parse(await fs.readFile(file,'utf8'))?.[key])))return true;} + return false; + }; + const parked=root+'.allocation-'+identity,retired=root+'.retired-'+identity; + // Saved directory modes can be read-only. Make the parked tree's own directories + // owner-writable without following symlinks, then remove it. + const removeRetired=async()=>{if(!await stat(retired))return; + const open=async d=>{const s=await fs.lstat(d);if(!s.isDirectory()||s.isSymbolicLink())return; + await fs.chmod(d,(s.mode&0o7777)|0o700);for(const name of await fs.readdir(d))await open(path.join(d,name));}; + await open(retired);await fs.rm(retired,{recursive:true,force:true});}; stage='allocation';if(await stat(parked)){if(allowEmpty!=='true')throw Error('unexpected_allocation');await allocation(parked);if(!await stat(root))await fs.rename(parked,root);} stage='inspect_root';const prior=await stat(root);if(prior)await directory(root); const marker=path.join(root,'.paperclip-runtime','workspace-restored.json'); @@ -495,11 +513,17 @@ export async function restoreRemoteWorkspaceDirectory(input: { stage='existing_receipt';await directory(path.dirname(marker));const file=await fs.lstat(marker); if(!file.isFile()||file.isSymbolicLink()||file.nlink!==1||file.size>16384)throw Error('unsafe_receipt'); const receipt=JSON.parse(await fs.readFile(marker,'utf8')); - if(receipt.identity!==identity||receipt.manifestDigest!==digest)throw Error('workspace_owner_changed'); - if(await stat(parked)){await allocation(parked);await fs.rm(parked,{recursive:true});}return; + if(receipt.identity===identity&&receipt.manifestDigest===digest){ + if(await stat(parked)){await allocation(parked);await fs.rm(parked,{recursive:true});} + if(replaceRetired==='true')await removeRetired();return;} + if(replaceRetired!=='true')throw Error('workspace_owner_changed'); } - if(prior&&allowEmpty!=='true')throw Error('occupied_root'); - stage='allocation';const retained=prior?await allocation(root):[]; + stage='inspect_root'; + // Replacing is allowed only for a root the server proved is this workspace's + // own earlier allocation whose last use ended in a completed final save. + const retire=Boolean(prior)&&replaceRetired==='true'&&await ownReceipt(root); + if(prior&&!retire&&allowEmpty!=='true')throw Error('occupied_root'); + stage='allocation';const retained=prior&&!retire?await allocation(root):[]; if(!prior&&files.length)throw Error('allocation_missing'); stage='extract';const staged=await fs.mkdtemp(root+'.restoring-'); try{ @@ -508,13 +532,17 @@ export async function restoreRemoteWorkspaceDirectory(input: { for(const file of retained)await fs.writeFile(path.join(runtime,file.name),file.content,{flag:'wx',mode:0o600}); await fs.writeFile(path.join(runtime,'workspace-restored.json'),JSON.stringify({identity,manifestDigest:digest}),{flag:'wx',mode:0o600}); stage='publish';if(await fs.realpath(path.dirname(root))!==path.dirname(root))throw Error('unsafe_root'); - if(prior){await allocation(root);if(await stat(parked))throw Error('occupied_parking');await fs.rename(root,parked);} - try{await fs.rename(staged,root);}catch(e){if(prior&&!await stat(root))await fs.rename(parked,root);throw e;} - stage='cleanup_parked';if(prior){await allocation(parked);await fs.rm(parked,{recursive:true});} + const holding=retire?retired:parked; + if(prior){if(retire){if(!await ownReceipt(root))throw Error('occupied_root');}else await allocation(root); + if(await stat(holding))throw Error('occupied_parking');await fs.rename(root,holding);} + try{await fs.rename(staged,root);}catch(e){if(prior&&!await stat(root))await fs.rename(holding,root);throw e;} + stage='cleanup_parked';if(prior&&!retire){await allocation(parked);await fs.rm(parked,{recursive:true});} + if(replaceRetired==='true')await removeRetired(); }finally{await fs.rm(staged,{recursive:true,force:true});} })().catch(error=>{process.stderr.write(setupFailure('remote_workspace_restore_failed',stage,error));process.exitCode=1});`; await command(input.target, "restore", ["-e", restore, staged.helper, staged.archive, input.remoteCwd, - input.manifestDigest, input.restorationIdentity, String(input.allowEmptyRoot === true), JSON.stringify(input.allocationFiles ?? [])]); + input.manifestDigest, input.restorationIdentity, String(input.allowEmptyRoot === true), JSON.stringify(input.allocationFiles ?? []), + String(input.replaceRetiredRoot === true)]); // Publish the restored location before cleanup can fail, stall, or lose its // reply. The owner journals this receipt and then reclaims disposable staging. return { operationId, stagingRoot: parent, cleanupPending: true }; diff --git a/server/src/services/remote-workspaces.integration.test.ts b/server/src/services/remote-workspaces.integration.test.ts index 31399554086..3c509b1bdbf 100644 --- a/server/src/services/remote-workspaces.integration.test.ts +++ b/server/src/services/remote-workspaces.integration.test.ts @@ -470,6 +470,100 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem try { expect(await fs.readFile(path.join(browse.root, "report.md"), "utf8")).toContain("useful saved result"); } finally { await browse.cleanup(); } }, 60_000); + it.each(["own retired root", "own restored root", "read-only folder", "receipt missing", "forged receipt", "later allocation", + "unsettled save", "newer checkpoint", "never saved here"] as const)( + "restores a newer save over this workspace's own retired root on a reused sandbox (%s)", async mode => { + // A parent prepares on its retained sandbox, then children inherit the + // workspace on other sandboxes and save newer work. When the parent wakes on + // the same sandbox, its old root is still present. + const reserved = await reserve({ kind: "empty" }), parentSandbox = await target("sandbox-A"); + const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: parentSandbox }); + await fs.writeFile(path.join(prepared.remoteRoot, "plan.md"), "parent plan\n"); + await saveRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId: ids.run, + lease: prepared.lease, target: parentSandbox, assertOwned: async () => undefined }); + const writer = async (providerLeaseId: string) => { + const runId = randomUUID(), leaseId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: ids.company, agentId: ids.agent, nativeIssueId: ids.issue, runtimeMode: "native", status: "running" }); + await db.insert(environmentLeases).values({ id: leaseId, companyId: ids.company, environmentId: ids.environment, + heartbeatRunId: runId, issueId: ids.issue, provider: "daytona", providerLeaseId, status: "active" }); + const workspace = await reserveRemoteWorkspace({ db, storage, companyId: ids.company, issueId: ids.issue, runId, + agentId: ids.agent, environmentId: ids.environment, projectId: null, projectWorkspaceId: null, existingWorkspaceId: reserved.id, source: { kind: "empty" } }); + const leased = (await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)))[0]! as EnvironmentLease; + return { runId, workspace, lease: leased }; + }; + const child = await writer("allocation-child"), childSandbox = await target("sandbox-B"); + const childPrepared = await prepareRemoteWorkspace({ db, storage, workspace: child.workspace, runId: child.runId, lease: child.lease, target: childSandbox }); + await fs.writeFile(path.join(childPrepared.remoteRoot, "child-result.md"), "child result\n"); + const childSaved = await saveRemoteWorkspace({ db, storage, workspace: childPrepared.workspace, runId: child.runId, + lease: childPrepared.lease, target: childSandbox, assertOwned: async () => undefined }); + if (mode === "receipt missing") { + // An occupant without a Paperclip preparation or restore receipt is unknown. + await fs.rm(path.join(prepared.remoteRoot, ".paperclip-runtime"), { recursive: true, force: true }); + } + if (mode === "own restored root" || mode === "forged receipt") { + // The retired root was itself restored by an earlier preparation of this workspace. + const runtime = path.join(prepared.remoteRoot, ".paperclip-runtime"); + await fs.rm(path.join(runtime, "workspace-ready.json")); + await fs.writeFile(path.join(runtime, "workspace-restored.json"), JSON.stringify(mode === "forged receipt" + ? { identity: "not-an-identity" } : { identity: "a".repeat(64), manifestDigest: "b".repeat(64) })); + } + if (mode === "read-only folder") { + // A non-root sandbox user must still be able to remove the retired tree. + const locked = path.join(prepared.remoteRoot, "locked"); + await fs.mkdir(path.join(locked, "inner"), { recursive: true }); + await fs.writeFile(path.join(locked, "inner", "file.txt"), "old\n"); + await fs.chmod(path.join(locked, "inner"), 0o555); await fs.chmod(locked, 0o555); + } + if (mode === "unsettled save" || mode === "newer checkpoint") { + // The last use of this root ended in a save that is not a settled final save. + await db.insert(workspaceOperations).values({ companyId: ids.company, executionWorkspaceId: reserved.id, heartbeatRunId: ids.run, + issueId: ids.issue, phase: "remote_workspace_snapshot", status: mode === "unsettled save" ? "running" : "succeeded", + metadata: { epoch: 1, boundary: "checkpoint", generation: 99, + saveWork: { schema: "paperclip.remote-workspace-save-work.v1", activeIds: [], unknown: false, quiescedAt: new Date().toISOString() }, + exportCleanup: { ...readRemoteWorkspaceAuthority(prepared.workspace.metadata)!.location, state: "cleaned" } } }); + } + if (mode === "later allocation") { + // A later writer used this root on the parent's sandbox after its last final save. + await db.insert(workspaceOperations).values({ companyId: ids.company, executionWorkspaceId: reserved.id, heartbeatRunId: child.runId, + issueId: ids.issue, phase: "remote_workspace_restore", status: "succeeded", + metadata: { epoch: 2, exportCleanup: { ...readRemoteWorkspaceAuthority(prepared.workspace.metadata)!.location, state: "cleaned" } } }); + } + // Preparation narrows a target to its root; a later turn reattaches to the sandbox. + const reattach = () => ({ ...parentSandbox, remoteCwd: path.join(root, "sandbox-A") }); + let parentTarget = reattach(); + if (mode === "never saved here") { + // A different sandbox that holds an unknown folder at this path. + parentTarget = await target("sandbox-C"); + await fs.cp(prepared.remoteRoot, path.join(parentTarget.remoteCwd, ".paperclip-workspaces", path.basename(prepared.remoteRoot)), { recursive: true }); + } + const parent = await writer(mode === "never saved here" ? "allocation-unrelated" : prepared.lease.providerLeaseId!); + const restoring = prepareRemoteWorkspace({ db, storage, workspace: parent.workspace, runId: parent.runId, lease: parent.lease, target: parentTarget }); + if (mode !== "own retired root" && mode !== "own restored root" && mode !== "read-only folder") { + const refused = await restoring.then(() => null, (error: unknown) => error as RemoteWorkspaceRestoreError); + expect(refused).toBeInstanceOf(RemoteWorkspaceRestoreError); + expect(refused).toMatchObject({ step: "sandbox_restore", retryable: false }); + expect(refused!.message).toMatch(/stage=inspect_root reason=occupied_root$/); + expect(await fs.readFile(path.join(prepared.remoteRoot, "plan.md"), "utf8")).toBe("parent plan\n"); + expect(await fs.lstat(path.join(prepared.remoteRoot, "child-result.md")).catch(() => null)).toBeNull(); + expect(readRemoteWorkspaceAuthority((await workspace(reserved.id)).metadata)).toEqual(readRemoteWorkspaceAuthority(parent.workspace.metadata)); + return; + } + const restored = await restoring; + expect(restored.remoteRoot).toBe(prepared.remoteRoot); + expect(await fs.readFile(path.join(restored.remoteRoot, "plan.md"), "utf8")).toBe("parent plan\n"); + expect(await fs.readFile(path.join(restored.remoteRoot, "child-result.md"), "utf8")).toBe("child result\n"); + // The replaced root and the restore staging are gone; only saved files were replaced. + expect((await fs.readdir(path.dirname(restored.remoteRoot))).filter(name => name.startsWith(path.basename(restored.remoteRoot)))) + .toEqual([path.basename(restored.remoteRoot)]); + expect(readRemoteWorkspaceAuthority(restored.workspace.metadata)).toMatchObject({ snapshotId: childSaved.snapshotId, dirty: true, + writerRunId: parent.runId, location: { providerLeaseId: prepared.lease.providerLeaseId, remoteCwd: prepared.remoteRoot } }); + // A lost reply repeats the same preparation without replacing the restored root again. + await fs.writeFile(path.join(restored.remoteRoot, "parent-continued.md"), "unsaved parent edits\n"); + const repeated = await prepareRemoteWorkspace({ db, storage, workspace: parent.workspace, runId: parent.runId, lease: parent.lease, target: reattach() }); + expect(repeated.remoteRoot).toBe(prepared.remoteRoot); + expect(await fs.readFile(path.join(restored.remoteRoot, "parent-continued.md"), "utf8")).toBe("unsaved parent edits\n"); + }, 60_000); + it.each(["per_turn", "warm", "occupied", "foreign-target", "foreign-allocation", "forged-receipt"])("preserves retained native workspace identity through migration: %s", async scenario => { const original = path.join(root, "old-controller", "workspace"); await fs.mkdir(original, { recursive: true }); await fs.writeFile(path.join(original, "saved.txt"), "retained work"); diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index 298e1eed7ec..835c67ae8c3 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -370,6 +370,41 @@ function publishedWorkspaceDirectory(workspace: Workspace & Pick'exportCleanup'->>'environmentId' = ${lease.environmentId}`, + sql`${workspaceOperations.metadata}->'exportCleanup'->>'providerLeaseId' = ${lease.providerLeaseId}`, + sql`${workspaceOperations.metadata}->'exportCleanup'->>'remoteCwd' = ${remoteRoot}`)); + const epochOf = (metadata: Record | null) => Number(metadata?.epoch); + if (!rows.length || rows.some(row => !Number.isSafeInteger(epochOf(row.metadata)))) return false; + // Every later writer must have used another allocation. + const last = Math.max(...rows.map(row => epochOf(row.metadata))); + if (last >= epoch) return false; + const lastUse = rows.filter(row => epochOf(row.metadata) === last); + const saves = lastUse.filter(row => row.phase === "remote_workspace_snapshot" && object(row.metadata?.retiredSave).schema !== RETIRED_SAVE_SCHEMA); + if (saves.some(row => row.status !== "succeeded" && row.status !== "failed")) return false; + const generation = (row: typeof rows[number]) => Number(row.metadata?.generation); + const final = saves.filter(row => row.status === "succeeded").sort((a, b) => generation(b) - generation(a))[0]; + const work = readSaveWork(final?.metadata); + if (!final || final.metadata?.boundary !== "final" || !work || work.unknown || work.activeIds.length + || saves.some(row => row !== final && !(generation(row) < generation(final)))) return false; + const [snapshot] = await db.select({ status: executionWorkspaceSnapshots.status }).from(executionWorkspaceSnapshots).where(and( + eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id), + eq(executionWorkspaceSnapshots.sourceOperationId, final.id))); + return Boolean(snapshot && !["pending", "uploaded"].includes(snapshot.status)); +} + export async function prepareRemoteWorkspace(input: { db: Db; workspace: Workspace & Pick; runId: string; lease: EnvironmentLease; target: AdapterExecutionTarget | null; @@ -405,6 +440,8 @@ export async function prepareRemoteWorkspace(input: { try { const snapshot = await getWorkspaceSnapshot(input.db, { companyId: input.workspace.companyId, workspaceId: input.workspace.id, snapshotId: initial.snapshotId }); if (!snapshot) throw new Error("remote_workspace_snapshot_missing"); + const replaceRetiredRoot = migration === null && !initial.dirty + && await isRetiredOwnAllocation(input.db, input.workspace, input.lease, remoteRoot, initial.epoch); const scratch = await mkdtemp(path.join(os.tmpdir(), "pc-workspace-restore-")); try { const archivePath = path.join(scratch, "workspace.pcws"); @@ -413,7 +450,8 @@ export async function prepareRemoteWorkspace(input: { step = "sandbox_restore"; restoredExport = await restoreRemoteWorkspaceDirectory({ target, remoteCwd: remoteRoot, archivePath, manifestDigest: snapshot.manifestDigest, restorationIdentity: preparationIdentity, allowEmptyRoot: migration !== null, - allocationFiles: migration ? workspaceAllocationFiles(input.lease, remoteRoot) : [] }); + allocationFiles: migration ? workspaceAllocationFiles(input.lease, remoteRoot) : [], + replaceRetiredRoot }); } finally { await rm(scratch, { recursive: true, force: true }); } } catch (error) { // Cancellation keeps its own handling, as do lookup refusals.