diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index d1ceb36cb3c..35ced98709a 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1066,7 +1066,11 @@ recorded use of that root on this sandbox is an earlier epoch that ended in a quiesced, committed final save, and the folder carries a Paperclip preparation or restore receipt. Its files are then already part of the saved lineage being restored. Any other occupant (no receipt, a later unsaved use, or a sandbox that -never saved that root) is still refused with `reason=occupied_root`. +never saved that root) is still refused with `reason=occupied_root`. The replaced +root's `.paperclip-runtime` state, such as a retained runner session, is never in +a snapshot and belongs to the sandbox, so it moves into the restored root. Only the +old preparation or restore receipt and export staging are dropped. A retained +native continuation then reads its runner state at the same path. The provider's own idle auto-stop must not coincide with the warm runner idle window. The host forwards the run's effective warm idle window (environment diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 5271c5e0c6e..0e01f8c016a 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -500,10 +500,20 @@ export async function restoreRemoteWorkspaceDirectory(input: { return false; }; const parked=root+'.allocation-'+identity,retired=root+'.retired-'+identity; + // Runtime state beside the files (for example a retained runner session) is + // never saved; it belongs to this sandbox. Move it into the restored root + // before removing the retired one. Its old receipts and export staging stay behind. + // Only an authorized replace with this identity creates that path, so a repeat + // of the same restore finishes an interrupted carry without a new proof. // Saved directory modes can be read-only. Make the parked tree's own directories // owner-writable without following symlinks, then remove it. const removeRetired=async()=>{if(!await stat(retired))return; - const open=async d=>{const s=await fs.lstat(d);if(!s.isDirectory()||s.isSymbolicLink())return; + stage='carry_runtime';const from=path.join(retired,'.paperclip-runtime'),to=path.join(root,'.paperclip-runtime'); + if(await stat(from)){await directory(from);await directory(to); + for(const name of await fs.readdir(from)){ + if(['workspace-ready.json','workspace-restored.json','workspace-snapshots'].includes(name)||await stat(path.join(to,name)))continue; + await fs.rename(path.join(from,name),path.join(to,name));}} + stage='cleanup_parked';const open=async d=>{const s=await fs.lstat(d);if(!s.isDirectory()||s.isSymbolicLink())return; await fs.chmod(d,(s.mode&0o7777)|0o700);for(const name of await fs.readdir(d))await open(path.join(d,name));}; await open(retired);await fs.rm(retired,{recursive:true,force:true});}; stage='allocation';if(await stat(parked)){if(allowEmpty!=='true')throw Error('unexpected_allocation');await allocation(parked);if(!await stat(root))await fs.rename(parked,root);} @@ -515,7 +525,7 @@ export async function restoreRemoteWorkspaceDirectory(input: { const receipt=JSON.parse(await fs.readFile(marker,'utf8')); if(receipt.identity===identity&&receipt.manifestDigest===digest){ if(await stat(parked)){await allocation(parked);await fs.rm(parked,{recursive:true});} - if(replaceRetired==='true')await removeRetired();return;} + await removeRetired();return;} if(replaceRetired!=='true')throw Error('workspace_owner_changed'); } stage='inspect_root'; @@ -537,7 +547,7 @@ export async function restoreRemoteWorkspaceDirectory(input: { if(await stat(holding))throw Error('occupied_parking');await fs.rename(root,holding);} try{await fs.rename(staged,root);}catch(e){if(prior&&!await stat(root))await fs.rename(holding,root);throw e;} stage='cleanup_parked';if(prior&&!retire){await allocation(parked);await fs.rm(parked,{recursive:true});} - if(replaceRetired==='true')await removeRetired(); + await removeRetired(); }finally{await fs.rm(staged,{recursive:true,force:true});} })().catch(error=>{process.stderr.write(setupFailure('remote_workspace_restore_failed',stage,error));process.exitCode=1});`; await command(input.target, "restore", ["-e", restore, staged.helper, staged.archive, input.remoteCwd, diff --git a/server/src/services/remote-workspaces.integration.test.ts b/server/src/services/remote-workspaces.integration.test.ts index 3c509b1bdbf..e298568efae 100644 --- a/server/src/services/remote-workspaces.integration.test.ts +++ b/server/src/services/remote-workspaces.integration.test.ts @@ -27,6 +27,7 @@ import type { createGitRemoteAuthProvider } from "./git-credentials.js"; import { reconcileSafeNativeReplacements } from "./native-runtime/native-safe-replacement.js"; import { remoteTerminationReceipt } from "./remote-execution-termination.js"; import { RemoteWorkspaceRestoreError } from "./remote-workspace-transport.js"; +import { readRemoteRunnerState } from "./native-runtime/native-session-executor.js"; import { notFound } from "../errors.js"; import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; import { createRemoteWorkspaceFiles } from "./remote-workspace-files.js"; @@ -479,6 +480,13 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem const reserved = await reserve({ kind: "empty" }), parentSandbox = await target("sandbox-A"); const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: parentSandbox }); await fs.writeFile(path.join(prepared.remoteRoot, "plan.md"), "parent plan\n"); + // The parent's retained native session keeps its runner state beside the + // workspace files. Snapshots exclude it; it belongs to this sandbox. + const runnerStateDirectory = path.join(prepared.remoteRoot, ".paperclip-runtime", "paperclip-runner", "sessions", + createHash("sha256").update("parent-session").digest("hex"), "runner"); + const runnerState = { lifecycle: "suspended", outbox: [], pendingTerminalDelivery: null, runId: ids.run }; + await fs.mkdir(runnerStateDirectory, { recursive: true }); + await fs.writeFile(path.join(runnerStateDirectory, "runner-state.json"), JSON.stringify(runnerState)); await saveRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId: ids.run, lease: prepared.lease, target: parentSandbox, assertOwned: async () => undefined }); const writer = async (providerLeaseId: string) => { @@ -557,11 +565,27 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem .toEqual([path.basename(restored.remoteRoot)]); expect(readRemoteWorkspaceAuthority(restored.workspace.metadata)).toMatchObject({ snapshotId: childSaved.snapshotId, dirty: true, writerRunId: parent.runId, location: { providerLeaseId: prepared.lease.providerLeaseId, remoteCwd: prepared.remoteRoot } }); + // The parent's continuation reads its retained runner state from the restored root. + expect(await readRemoteRunnerState({ runner: parentSandbox.runner!, stateDirectory: runnerStateDirectory })).toEqual(runnerState); + // Only the new restore receipt identifies the root; the retired preparation receipt is gone. + expect(await fs.lstat(path.join(restored.remoteRoot, ".paperclip-runtime", "workspace-ready.json")).catch(() => null)).toBeNull(); + expect(JSON.parse(await fs.readFile(path.join(restored.remoteRoot, ".paperclip-runtime", "workspace-restored.json"), "utf8"))) + .toMatchObject({ identity: expect.stringMatching(/^[a-f0-9]{64}$/) }); // A lost reply repeats the same preparation without replacing the restored root again. await fs.writeFile(path.join(restored.remoteRoot, "parent-continued.md"), "unsaved parent edits\n"); + // An attempt interrupted after publish left its retired root; the repeat keeps its runtime state. + const { identity } = JSON.parse(await fs.readFile(path.join(restored.remoteRoot, ".paperclip-runtime", "workspace-restored.json"), "utf8")); + const interrupted = `${restored.remoteRoot}.retired-${identity}`; + await fs.mkdir(path.join(interrupted, ".paperclip-runtime", "other-session"), { recursive: true }); + await fs.writeFile(path.join(interrupted, ".paperclip-runtime", "other-session", "state.json"), "{}"); + await fs.writeFile(path.join(interrupted, "stale.md"), "old\n"); const repeated = await prepareRemoteWorkspace({ db, storage, workspace: parent.workspace, runId: parent.runId, lease: parent.lease, target: reattach() }); expect(repeated.remoteRoot).toBe(prepared.remoteRoot); expect(await fs.readFile(path.join(restored.remoteRoot, "parent-continued.md"), "utf8")).toBe("unsaved parent edits\n"); + expect(await fs.readFile(path.join(restored.remoteRoot, ".paperclip-runtime", "other-session", "state.json"), "utf8")).toBe("{}"); + expect(await fs.lstat(interrupted).catch(() => null)).toBeNull(); + expect(await fs.lstat(path.join(restored.remoteRoot, "stale.md")).catch(() => null)).toBeNull(); + expect(await readRemoteRunnerState({ runner: parentSandbox.runner!, stateDirectory: runnerStateDirectory })).toEqual(runnerState); }, 60_000); it.each(["per_turn", "warm", "occupied", "foreign-target", "foreign-allocation", "forged-receipt"])("preserves retained native workspace identity through migration: %s", async scenario => {