From b5b0dbb03fc6272e7f761d63d794f424a8920e26 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 00:31:57 -0600 Subject: [PATCH] Preserve published workspace paths across cold restores Co-Authored-By: Paperclip --- .../remote-workspaces.integration.test.ts | 30 +++++++++++++++---- server/src/services/remote-workspaces.ts | 18 +++++++++-- tests/runner-e2e/README.md | 5 ++++ 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/server/src/services/remote-workspaces.integration.test.ts b/server/src/services/remote-workspaces.integration.test.ts index 0ef9e7ccdf3..566934be7dd 100644 --- a/server/src/services/remote-workspaces.integration.test.ts +++ b/server/src/services/remote-workspaces.integration.test.ts @@ -387,7 +387,8 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem } }, 60_000); - it("prepares, saves and restores task files without a controller checkout or original allocation", async () => { + it.each(["same root", "different root", "destroyed allocation", "occupied root"] as const)( + "preserves the published workspace path and files on a cold restore (%s)", async mode => { const reserved = await reserve({ kind: "empty" }), first = await target("sandbox-A"); const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: first }); expect(readRemoteWorkspaceAuthority(prepared.workspace.metadata)?.snapshotId).toBeTruthy(); @@ -399,8 +400,16 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem lease: prepared.lease, target: first, assertOwned: async () => undefined }); const current = await workspace(reserved.id); expect(readRemoteWorkspaceAuthority(current.metadata)).toMatchObject({ dirty: false, writerRunId: null, snapshotId: saved.snapshotId }); - await fs.rm(path.dirname(prepared.remoteRoot), { recursive: true, force: true }); - const second = await target("sandbox-B"); + if (mode === "destroyed allocation") { + const destroyed: EnvironmentLease = { ...prepared.lease, status: "released", releasedAt: new Date(), cleanupStatus: "success", + metadata: { ...prepared.lease.metadata, remoteExecutionTermination: remoteTerminationReceipt(prepared.lease, + { providerLeaseId: prepared.lease.providerLeaseId, state: "destroyed" }) } }; + await db.transaction(tx => clearRemoteWorkspaceLocationForReleasedLease({ db: tx, lease: destroyed })); + expect(readRemoteWorkspaceAuthority((await workspace(reserved.id)).metadata)?.location).toBeNull(); + } + await fs.rm(path.join(root, "sandbox-A"), { recursive: true, force: true }); + const second = await target(mode === "different root" ? "sandbox-B" : "sandbox-A"); + const expectedRemoteRoot = path.join(second.remoteCwd, ".paperclip-workspaces", path.basename(prepared.remoteRoot)); const nextRun = randomUUID(), nextLease = randomUUID(); await db.insert(heartbeatRuns).values({ id: nextRun, companyId: ids.company, agentId: ids.agent, nativeIssueId: ids.issue, runtimeMode: "native", status: "running" }); await db.insert(environmentLeases).values({ id: nextLease, companyId: ids.company, environmentId: ids.environment, @@ -408,7 +417,19 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem const reservedAgain = await reserveRemoteWorkspace({ db, storage, companyId: ids.company, issueId: ids.issue, runId: nextRun, agentId: ids.agent, environmentId: ids.environment, projectId: null, projectWorkspaceId: null, existingWorkspaceId: reserved.id, source: { kind: "empty" } }); const newLease = (await db.select().from(environmentLeases).where(eq(environmentLeases.id, nextLease)))[0]! as EnvironmentLease; + if (mode === "occupied root") { + await fs.mkdir(expectedRemoteRoot, { recursive: true }); + await fs.writeFile(path.join(expectedRemoteRoot, "unpublished.txt"), "preserve unknown preparation\n"); + await expect(prepareRemoteWorkspace({ db, storage, workspace: reservedAgain, runId: nextRun, lease: newLease, target: second })) + .rejects.toThrow("remote_workspace_operation_failed"); + expect(await fs.readFile(path.join(expectedRemoteRoot, "unpublished.txt"), "utf8")).toBe("preserve unknown preparation\n"); + expect(readRemoteWorkspaceAuthority((await workspace(reserved.id)).metadata)) + .toEqual(readRemoteWorkspaceAuthority(reservedAgain.metadata)); + await fs.rename(expectedRemoteRoot, path.join(root, "retained-unknown-preparation")); + } const restored = await prepareRemoteWorkspace({ db, storage, workspace: reservedAgain, runId: nextRun, lease: newLease, target: second }); + expect(restored.remoteRoot).toBe(expectedRemoteRoot); + if (mode !== "different root") expect(restored.remoteRoot).toBe(prepared.remoteRoot); expect(await fs.readFile(path.join(restored.remoteRoot, "report.md"), "utf8")).toBe("useful saved result\n"); expect(await fs.readFile(path.join(restored.remoteRoot, "binary.bin"))).toEqual(Buffer.from([0, 1, 255])); expect((await fs.stat(path.join(restored.remoteRoot, "empty-folder"))).isDirectory()).toBe(true); @@ -619,8 +640,7 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem existingWorkspaceId: reserved.id, source: { kind: "empty" } }); const nextTarget = await target("restore-cleanup-destination"); const sandbox = nextTarget as Extract; - const expectedRemoteRoot = path.join(sandbox.remoteCwd, ".paperclip-workspaces", - `${reserved.id}-${readRemoteWorkspaceAuthority(nextWorkspace.metadata)!.epoch}-${nextLease}`); + const expectedRemoteRoot = path.join(sandbox.remoteCwd, ".paperclip-workspaces", path.basename(prepared.remoteRoot)); const execute = sandbox.runner!.execute.bind(sandbox.runner!); let failCleanup = true, cleanupAttempts = 0; sandbox.runner!.execute = async input => { diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index 521b6c365a0..d651a716096 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -321,8 +321,22 @@ function isSavedWorkspaceRepository(workspace: Workspace, authority: RemoteWorks && (saved.readOnly ?? false) === (repository.readOnly ?? false)); } +function publishedWorkspaceDirectory(workspace: Workspace & Pick, + authority: RemoteWorkspaceAuthority) { + const publishedRoot = authority.snapshotId && (authority.location?.remoteCwd ?? workspace.providerRef); + if (!publishedRoot) return null; + const directory = path.posix.basename(publishedRoot); + const suffix = directory.slice(workspace.id.length); + if (!path.posix.isAbsolute(publishedRoot) || path.posix.basename(path.posix.dirname(publishedRoot)) !== ".paperclip-workspaces" + || !directory.startsWith(workspace.id) || (suffix !== "" && !/^-[1-9][0-9]*-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(suffix))) { + throw new Error("remote_workspace_published_path_invalid"); + } + return directory; +} + export async function prepareRemoteWorkspace(input: { - db: Db; workspace: Workspace; runId: string; lease: EnvironmentLease; target: AdapterExecutionTarget | null; + db: Db; workspace: Workspace & Pick; + runId: string; lease: EnvironmentLease; target: AdapterExecutionTarget | null; source?: RemoteWorkspaceSource; assertOwned?: () => Promise; provisionCommand?: string | null; runtimeProvisionCommand?: string | null; storage?: StorageService; }) { @@ -339,7 +353,7 @@ export async function prepareRemoteWorkspace(input: { && initial.location?.environmentId === input.lease.environmentId; if (initial.location && !sameAllocation && initial.dirty) throw conflict("The prior workspace has unsaved work; recover it before replacing its allocation."); const remoteRoot = sameAllocation ? initial.location!.remoteCwd : path.posix.join(target.remoteCwd, ".paperclip-workspaces", - `${input.workspace.id}-${initial.epoch}-${input.lease.id}`); + publishedWorkspaceDirectory(input.workspace, initial) ?? `${input.workspace.id}-${initial.epoch}-${input.lease.id}`); const preparationIdentity = createHash("sha256").update(JSON.stringify([input.workspace.companyId, input.workspace.id, input.runId, initial.epoch, input.lease.id, input.lease.providerLeaseId, remoteRoot, initial.source])).digest("hex"); let restoredExport: Awaited> | null = null; diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 147cae9f871..bc76e5bf606 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -116,6 +116,11 @@ The observer rechecks the original live owner immediately before switching traffic and fails if that turn already settled. A completed turn cannot count as active takeover. Per-turn file inspection requires the exact committed saved snapshot; the warm lane still requires the live workspace. +Cold restores retain the published workspace directory name, including after +the old allocation is destroyed, because provider recovery binds the working +directory into session identity. Unpublished preparations retain distinct +epoch/lease directories, and a restore refuses to overwrite an occupied path +owned by another preparation. The ordinary Everyday matrix retains its 125-second active outage default. The two explicit continuity suites require 300 seconds and reject a shorter override.