diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 474bdd9aec6..516d2888527 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1445,7 +1445,11 @@ authorize an empty replacement or replay completed work. Read-only imports can use a superseded immutable snapshot with the exact saved operation receipt. They do not establish current workspace execution authority. Restored sidecars keep user edits, and only paths present with their recorded -provenance in the current prepared root are published to the provider. Older +provenance in the current prepared root are published to the provider. +A remote sidecar records the run it was last shown to. Once that run +succeeds, the next preparation removes the sidecar from the workspace and +keeps only a retired record, so later saves stop carrying it and it is never +attached or shown again. The source run's own saved snapshot is unaffected. Older attachment records rebuild this proof from saved receipts; reusing a live allocation never replaces their files. For a related task, retention requires the selected workspace’s committed migration history and matching source marker, diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index cce5290dce8..d42489f5649 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -1485,7 +1485,8 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace } if (remoteAuthority && persistedExecutionWorkspace?.metadata?.remoteWorkspaceRecoveredFiles) { const recorded = persistedExecutionWorkspace?.metadata?.remoteWorkspaceRecoveredFiles; - const recoveredFiles = (Array.isArray(recorded) ? recorded : []).map(value => { + // Retired copies were removed from the workspace; they are kept only as records. + const recoveredFiles = (Array.isArray(recorded) ? recorded : []).filter(value => parseObject(value).retired !== true).map(value => { const file = parseObject(value); if (typeof file.operationId !== "string" || typeof file.sourceRunId !== "string" || file.path !== `.paperclip-recovered/${file.operationId}` || !/^[a-f0-9-]{36}$/.test(file.operationId)) { diff --git a/server/src/services/remote-workspace-recovery-attachments.ts b/server/src/services/remote-workspace-recovery-attachments.ts index 3f3bea5335f..9ee0360a040 100644 --- a/server/src/services/remote-workspace-recovery-attachments.ts +++ b/server/src/services/remote-workspace-recovery-attachments.ts @@ -2,7 +2,7 @@ import { lstat, mkdir, mkdtemp, readFile, realpath, rename, rm, writeFile } from import path from "node:path"; import os from "node:os"; import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; -import { hasRemoteRecoveryAttachment } from "./remote-workspace-transport.js"; +import { hasRemoteRecoveryAttachment, removeRemoteRecoveryAttachment } from "./remote-workspace-transport.js"; import { removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js"; import { LEGACY_WORKSPACE_RECOVERY_MIGRATION } from "./legacy-workspace-recovery-migration.js"; import { and, eq, inArray, sql } from "drizzle-orm"; @@ -21,7 +21,7 @@ import { getStorageService, type StorageService } from "../storage/index.js"; * its own files stay intact; this does not grant execution or replay authority. */ export async function attachPreservedRemoteWorkspaceSnapshots(input: { db: Db; companyId: string; issueId: string; runId: string; workspaceId: string; - sourceRunIds: string[]; cwd: string; storage?: StorageService; + sourceRunIds: string[]; cwd: string; storage?: StorageService; retiredOperationIds?: ReadonlySet; }) { const [writer] = await input.db.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId), @@ -61,6 +61,7 @@ export async function attachPreservedRemoteWorkspaceSnapshots(input: { continue; } const operationId = proof.operationId; + if (input.retiredOperationIds?.has(operationId)) continue; await assertExecutionWorkspaceRecordAccess(input.db, { type: "agent", source: "agent_jwt", companyId: input.companyId, agentId: writer.agentId, runId: writer.id, onBehalfOfUserId: writer.responsibleUserId === "local-board" ? null : writer.responsibleUserId }, @@ -152,6 +153,25 @@ async function retainedRecoveryAttachmentProvenance(input: { return provenance && (historyAllowed || await input.verifySelectedSnapshot(input.operationId, provenance)) ? provenance : null; } +/** Recovered copies to remove: those already retired, and those shown to a + * run other than this one that then succeeded, which had its turn to use them. */ +export async function retiredRecoveryAttachments(db: Db, + workspace: Pick, runId: string) { + const retired = new Set(); + const recorded = workspace.metadata?.remoteWorkspaceRecoveredFiles; + for (const value of Array.isArray(recorded) ? recorded : []) { + const item = record(value); + if (typeof item.operationId !== "string") continue; + if (item.retired === true) { retired.add(item.operationId); continue; } + const shownTo = item.presentedToRunId; + if (typeof shownTo !== "string" || shownTo === runId || !/^[a-f0-9-]{36}$/.test(shownTo)) continue; + const [run] = await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, workspace.companyId), eq(heartbeatRuns.id, shownTo))); + if (run?.status === "succeeded") retired.add(item.operationId); + } + return retired; +} + function record(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; } @@ -161,8 +181,8 @@ function record(value: unknown): Record { export async function retainRemoteRecoveryAttachments(input: { db: Db; workspace: Pick; runId: string; remoteRoot: string; target: AdapterSandboxExecutionTarget; assertOwned?: () => Promise; - recoveredFiles: Array<{ operationId: string; sourceRunId: string; path: string; provenance: string }>; - storage?: StorageService; + recoveredFiles: Array<{ operationId: string; sourceRunId: string; path: string; provenance: string; retired?: true }>; + retiredOperationIds: ReadonlySet; storage?: StorageService; }) { let snapshotScratch: string | null = null, snapshotFiles: string | null = null, snapshotLoaded = false; const verifySelectedSnapshot = async (operationId: string, provenance: string) => { @@ -211,7 +231,13 @@ export async function retainRemoteRecoveryAttachments(input: { if (!provenance) throw new Error("remote_workspace_recovery_receipt_invalid"); } await input.assertOwned?.(); - if (await hasRemoteRecoveryAttachment({ target: input.target, remoteCwd: input.remoteRoot, operationId: item.operationId, provenance })) + const location = { target: input.target, remoteCwd: input.remoteRoot, operationId: item.operationId, provenance }; + if (input.retiredOperationIds.has(item.operationId)) { + await removeRemoteRecoveryAttachment(location); + input.recoveredFiles.push({ operationId: item.operationId, sourceRunId: item.sourceRunId, path: item.path, provenance, retired: true }); + continue; + } + if (await hasRemoteRecoveryAttachment(location)) input.recoveredFiles.push({ operationId: item.operationId, sourceRunId: item.sourceRunId, path: item.path, provenance }); } } finally { if (snapshotScratch) await removeTaskDirectoryTree(snapshotScratch); } diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index cfaa5eb7581..d98970321c6 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -16,6 +16,7 @@ import { prepareRemoteRepository, prepareRemoteWorkspaceDirectory, remoteWorkspaceFailureCause, + removeRemoteRecoveryAttachment, restoreRemoteWorkspaceDirectory, type RemoteWorkspaceTarget, } from "./remote-workspace-transport.js"; @@ -59,6 +60,73 @@ describe("remote workspace transport safety with real sandbox commands", () => { if (state !== "missing") expect(await fs.readFile(path.join(directory, "user.txt"), "utf8")).toBe("preserved user edit\n"); }); + it.each(["missing", "present", "foreign", "directory_symlink"])( + "removes a retired recovery attachment only with its exact provenance: %s", async state => { + const operationId = randomUUID(), parent = path.join(workspace, ".paperclip-recovered"); + const directory = path.join(parent, operationId), provenance = JSON.stringify({ operationId, sourceRunId: randomUUID() }); + const sibling = path.join(parent, randomUUID()); + if (state !== "missing") { + // A read-only directory, as a restored dependency cache can be, must still go. + await fs.mkdir(path.join(directory, "files", "locked"), { recursive: true }); + await fs.writeFile(path.join(directory, "files", "locked", "cache.bin"), "cached\n"); + await fs.chmod(path.join(directory, "files", "locked"), 0o555); + await fs.writeFile(path.join(directory, "provenance.json"), state === "foreign" ? "foreign proof" : provenance); + await fs.mkdir(sibling); + } + let outside = ""; + if (state === "directory_symlink") { + outside = path.join(scratch, "outside-attachment"); + await fs.rename(directory, outside); await fs.symlink(outside, directory); + } + const removal = removeRemoteRecoveryAttachment({ target: target().remote, remoteCwd: workspace, operationId, provenance }); + if (state === "missing" || state === "present") await expect(removal).resolves.toBe(state === "present"); + else await expect(removal).rejects.toThrow("remote_workspace_operation_failed"); + if (state === "present") { + expect(await fs.lstat(directory).catch(() => null)).toBeNull(); + expect((await fs.lstat(sibling)).isDirectory()).toBe(true); + } + if (state === "foreign") expect(await fs.readFile(path.join(directory, "files", "locked", "cache.bin"), "utf8")).toBe("cached\n"); + if (state === "directory_symlink") expect(await fs.readFile(path.join(outside, "files", "locked", "cache.bin"), "utf8")).toBe("cached\n"); + for (const kept of [directory, outside].filter(Boolean)) await fs.chmod(path.join(kept, "files", "locked"), 0o755).catch(() => undefined); + }); + + it("finishes a removal that was interrupted after its proof was deleted", async () => { + const operationId = randomUUID(), parent = path.join(workspace, ".paperclip-recovered"); + const provenance = JSON.stringify({ operationId, sourceRunId: randomUUID() }); + // An earlier removal renamed the verified attachment, then stopped after + // deleting its proof and part of its files. + const retiring = path.join(parent, `.retiring-${operationId}`); + await fs.mkdir(path.join(retiring, "files", "locked"), { recursive: true }); + await fs.writeFile(path.join(retiring, "files", "locked", "left.txt"), "left behind\n"); + await fs.chmod(path.join(retiring, "files", "locked"), 0o555); + const probe = { target: target().remote, remoteCwd: workspace, operationId, provenance }; + await expect(hasRemoteRecoveryAttachment(probe)).resolves.toBe(false); + await expect(removeRemoteRecoveryAttachment(probe)).resolves.toBe(true); + expect(await fs.readdir(workspace)).toEqual([]); + await expect(removeRemoteRecoveryAttachment(probe)).resolves.toBe(false); + }); + + it("finishes an interrupted removal before removing a present attachment", async () => { + const operationId = randomUUID(), parent = path.join(workspace, ".paperclip-recovered"); + const provenance = JSON.stringify({ operationId, sourceRunId: randomUUID() }); + await fs.mkdir(path.join(parent, `.retiring-${operationId}`, "files"), { recursive: true }); + await fs.mkdir(path.join(parent, operationId), { recursive: true }); + await fs.writeFile(path.join(parent, operationId, "provenance.json"), provenance); + await expect(removeRemoteRecoveryAttachment({ target: target().remote, remoteCwd: workspace, operationId, provenance })) + .resolves.toBe(true); + expect(await fs.readdir(workspace)).toEqual([]); + }); + + it("removes the empty recovery folder with its last attachment", async () => { + const operationId = randomUUID(), parent = path.join(workspace, ".paperclip-recovered"); + const provenance = JSON.stringify({ operationId, sourceRunId: randomUUID() }); + await fs.mkdir(path.join(parent, operationId), { recursive: true }); + await fs.writeFile(path.join(parent, operationId, "provenance.json"), provenance); + await expect(removeRemoteRecoveryAttachment({ target: target().remote, remoteCwd: workspace, operationId, provenance })) + .resolves.toBe(true); + expect(await fs.readdir(workspace)).toEqual([]); + }); + async function git(cwd: string, ...args: string[]) { return exec("git", ["-C", cwd, "-c", "user.name=Synthetic fixture", "-c", "user.email=fixture@example.invalid", "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", ...args], { diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 02bf739623b..d9b09835014 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -247,11 +247,23 @@ const fs=require('node:fs/promises'),path=require('node:path'); if(!s.isDirectory()||s.isSymbolicLink()||await fs.realpath(p)!==p)throw Error('unsafe_attachment');return true}; if(!await safeDirectory(root))throw Error('missing_workspace'); const parent=path.join(root,'.paperclip-recovered'),destination=path.join(parent,input.operationId); - if(!await safeDirectory(parent)||!await safeDirectory(destination)){process.stdout.write('missing');return} + // Removal first renames a verified attachment here, so an interrupted delete + // never leaves a proofless attachment behind; the next removal finishes it. + const retiring=path.join(parent,'.retiring-'+input.operationId); + const unlock=async p=>{const s=await fs.lstat(p);if(!s.isDirectory()||s.isSymbolicLink())return; + await fs.chmod(p,0o700);for(const name of await fs.readdir(p))await unlock(path.join(p,name))}; + const discard=async()=>{await unlock(retiring);await fs.rm(retiring,{recursive:true}); + await fs.rmdir(parent).catch(e=>{if(e.code!=='ENOTEMPTY'&&e.code!=='EEXIST')throw e})}; + if(!await safeDirectory(parent)){process.stdout.write('missing');return} + let resumed=false; + if(input.remove===true&&await safeDirectory(retiring)){await discard();resumed=true} + if(!await safeDirectory(destination)){process.stdout.write(resumed?'removed':'missing');return} const proof=path.join(destination,'provenance.json'),s=await inspect(proof); if(!s?.isFile()||s.isSymbolicLink()||s.nlink!==1||s.size>16384||await fs.readFile(proof,'utf8')!==input.provenance) throw Error('attachment_proof_changed'); - process.stdout.write('present'); + if(input.remove!==true){process.stdout.write('present');return} + await fs.rename(destination,retiring);await discard(); + process.stdout.write('removed'); })().catch(()=>{process.stderr.write('remote_workspace_recovery_attachment_invalid');process.exitCode=1});`; /** Reuse a restored attachment only with its exact provenance. User edits to @@ -265,6 +277,17 @@ export async function hasRemoteRecoveryAttachment(input: { return output === "present"; } +/** Remove a retired attachment, but only while its exact provenance is still + * in place; anything else at that path is left alone. */ +export async function removeRemoteRecoveryAttachment(input: { + target: RemoteWorkspaceTarget; remoteCwd: string; operationId: string; provenance: string; +}) { + const output = await command(input.target, "recovery_attachment", ["-e", RECOVERY_ATTACHMENT, + input.remoteCwd, JSON.stringify({ operationId: input.operationId, provenance: input.provenance, remove: true })]); + if (output !== "removed" && output !== "missing") throw new Error("remote_workspace_recovery_attachment_invalid"); + return output === "removed"; +} + const PREPARE = String.raw` const fs = require('node:fs/promises'), path = require('node:path'), cp = require('node:child_process'); const {promisify}=require('node:util'); const exec=promisify(cp.execFile); diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index bdc1a54498e..e45cbb99b55 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -38,7 +38,7 @@ import { import { restoreRemoteWorkspaceArchive, readRemoteWorkspaceArchiveDirectories } from "./remote-workspace-snapshot-archive.js"; import { removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js"; import { attachPreservedWorkspaceSnapshots } from "./workspace-recovery-snapshot.js"; -import { attachPreservedRemoteWorkspaceSnapshots, retainRemoteRecoveryAttachments } from "./remote-workspace-recovery-attachments.js"; +import { attachPreservedRemoteWorkspaceSnapshots, retainRemoteRecoveryAttachments, retiredRecoveryAttachments } from "./remote-workspace-recovery-attachments.js"; import { notifyRemoteWorkspaceSnapshots } from "./remote-workspace-snapshot-retention.js"; import { notifyTerminalWorkspaceWork } from "./terminal-workspace-work.js"; import { preserveRemoteWorkspacePreparationFailure } from "./native-runtime/native-workspace-finalization-state.js"; @@ -495,7 +495,9 @@ export async function prepareRemoteWorkspace(input: { savedArchive: isSavedWorkspaceRepository(input.workspace, initial, repository), resolveGitAuth: await createRemoteWorkspaceGitAuth(input.db, input.workspace.companyId, input.runId) }); } - const recoveredFiles: Array<{ operationId: string; sourceRunId: string; path: string; provenance: string }> = []; + const recoveredFiles: Array<{ operationId: string; sourceRunId: string; path: string; provenance: string; retired?: true }> = []; + // A copy that was shown to a run that then succeeded is removed and never attached again. + const retiredOperationIds = await retiredRecoveryAttachments(input.db, input.workspace, input.runId); if (!sameAllocation) { // Shared and inherited workspaces retain their original source task. The // current writer's task owns its recovery attachments, including a child's @@ -515,7 +517,7 @@ export async function prepareRemoteWorkspace(input: { const scratch = await mkdtemp(path.join(os.tmpdir(), "pc-workspace-recovery-import-")); try { const scope = { db: input.db, companyId: input.workspace.companyId, issueId, sourceRunIds: recoverySourceRunIds, - cwd: scratch, storage: input.storage }; + cwd: scratch, storage: input.storage, retiredOperationIds }; const recovered = [ ...await attachPreservedWorkspaceSnapshots(scope), ...await attachPreservedRemoteWorkspaceSnapshots({ ...scope, runId: input.runId, workspaceId: input.workspace.id }), @@ -535,7 +537,7 @@ export async function prepareRemoteWorkspace(input: { } } await retainRemoteRecoveryAttachments({ db: input.db, workspace: input.workspace, runId: input.runId, - remoteRoot, target, recoveredFiles, assertOwned: input.assertOwned, storage: input.storage }); + remoteRoot, target, recoveredFiles, retiredOperationIds, assertOwned: input.assertOwned, storage: input.storage }); await input.assertOwned?.(); const prepared = await input.db.transaction(async tx => { await lockWorkspaceRunParents(tx, input.workspace.companyId, input.runId); @@ -558,8 +560,12 @@ export async function prepareRemoteWorkspace(input: { .where(eq(environmentLeases.id, lease.id)).returning(); const [savedWorkspace] = await tx.update(executionWorkspaces).set({ cwd: remoteRoot, providerRef: remoteRoot, metadata: { ...workspace.metadata, remoteWorkspace: authority, + // A live copy names the run it is shown to; the next preparation retires + // it once that run has succeeded. Retired records stay so a sandbox + // restored from an older save is cleaned up too. ...(recoveredFiles.length || workspace.metadata?.remoteWorkspaceRecoveredFiles !== undefined - ? { remoteWorkspaceRecoveredFiles: recoveredFiles } : {}) }, updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id)).returning(); + ? { remoteWorkspaceRecoveredFiles: recoveredFiles.map(item => item.retired ? item : { ...item, presentedToRunId: input.runId }) } : {}) }, + updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id)).returning(); const runReference = reference(workspace.id, authority, "prepared"); const issueId = await persistRunReference(tx, workspace.companyId, input.runId, runReference); if (restoredExport) { diff --git a/server/src/services/workspace-recovery-preparation.test.ts b/server/src/services/workspace-recovery-preparation.test.ts index ae7a5a987aa..1327cf5194b 100644 --- a/server/src/services/workspace-recovery-preparation.test.ts +++ b/server/src/services/workspace-recovery-preparation.test.ts @@ -102,7 +102,7 @@ describe("automatic retained workspace recovery", () => { finally { await saved.cleanup(); } }, 30_000); - it.each(["retained_history", "ordinary_followup"])("restores the newest recovered tick bytes after a later workspace selection: %s", async kind => { + it.each(["retained_history", "ordinary_followup", "retired_followup"])("restores the newest recovered tick bytes after a later workspace selection: %s", async kind => { vi.stubEnv("DATABASE_URL", connectionString); vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "62".repeat(32)); try { @@ -393,8 +393,9 @@ describe("automatic retained workspace recovery", () => { .rejects.toThrow("fixture: selected snapshot unavailable"); expect(await readFile(path.join(snapshotRetained.remoteRoot, newest!.path, "files", "ticks.log"), "utf8")).toBe(editedLog); } - if (kind === "ordinary_followup") { - await rm(path.join(cold.remoteRoot, newest!.path), { recursive: true }); + if (kind === "ordinary_followup" || kind === "retired_followup") { + // The agent removes the copy itself, or leaves it for Paperclip to retire. + if (kind === "ordinary_followup") await rm(path.join(cold.remoteRoot, newest!.path), { recursive: true }); await saveRemoteWorkspace({ db, storage, workspace: cold.workspace, runId: successor.id, lease: cold.lease, target: { ...target, leaseId: coldLease.id, remoteCwd: cold.remoteRoot }, assertOwned: async () => {} }); await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, successor.id)); @@ -411,7 +412,32 @@ describe("automatic retained workspace recovery", () => { const ordinary = await prepareRemoteWorkspace({ db, storage, workspace: ordinaryWorkspace, runId: normal.id, lease: ordinaryLease as EnvironmentLease, target: { ...target, leaseId: ordinaryLease.id, remoteCwd: ordinaryAllocation } }); expect(await lstat(path.join(ordinary.remoteRoot, newest!.path)).then(() => true, () => false)).toBe(false); - expect(ordinary.workspace.metadata?.remoteWorkspaceRecoveredFiles ?? []).toEqual([]); + // The successor was shown the copy and succeeded, so this preparation retired it, whether + // or not the agent had already removed it. Only a record remains, never shown to an agent. + expect(ordinary.workspace.metadata?.remoteWorkspaceRecoveredFiles).toEqual([expect.objectContaining({ + operationId: path.basename(newest!.path), path: newest!.path, retired: true })]); + expect(JSON.stringify(ordinary.workspace.metadata?.remoteWorkspaceRecoveredFiles)).not.toContain("presentedToRunId"); + if (kind === "retired_followup") { + // The retired record outlives a failed run, so the copy is never attached or shown again. + await saveRemoteWorkspace({ db, storage, workspace: ordinary.workspace, runId: normal.id, + lease: ordinary.lease, target: { ...target, leaseId: ordinaryLease.id, remoteCwd: ordinary.remoteRoot }, assertOwned: async () => {} }); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, normal.id)); + await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success" }) + .where(eq(environmentLeases.id, ordinaryLease.id)); + const [later] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent.id, issueId: f.issueId, + status: "running", runtimeMode: "legacy", contextSnapshot: { issueId: f.issueId } }).returning(); + const laterWorkspace = await reserveRemoteWorkspace({ db, storage, companyId: f.companyId, issueId: f.issueId, + runId: later.id, agentId: agent.id, environmentId: f.lease.environmentId!, projectId: null, + projectWorkspaceId: null, existingWorkspaceId: placeholder.id, source: { kind: "empty" } }); + const laterAllocation = await realpath(await mkdtemp(path.join(root, "later-"))); + const [laterLease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, + heartbeatRunId: later.id, environmentId: f.lease.environmentId, provider: "daytona", providerLeaseId: randomUUID(), status: "active" }).returning(); + const laterPrepared = await prepareRemoteWorkspace({ db, storage, workspace: laterWorkspace, runId: later.id, + lease: laterLease as EnvironmentLease, target: { ...target, leaseId: laterLease.id, remoteCwd: laterAllocation } }); + expect(await lstat(path.join(laterPrepared.remoteRoot, newest!.path)).then(() => true, () => false)).toBe(false); + expect(laterPrepared.workspace.metadata?.remoteWorkspaceRecoveredFiles).toEqual([expect.objectContaining({ + operationId: path.basename(newest!.path), retired: true })]); + } } } finally { acquire.mockRestore(); storageRead.mockRestore(); } expect((await run(f.runId)).error).toBe("native_remote_recovery_lease_mismatch"); @@ -433,6 +459,11 @@ describe("automatic retained workspace recovery", () => { expect(await readFile(path.join(attached[0].path, "files", "saved.txt"), "utf8")).toBe("finished work\n"); expect(injected.getObject).toHaveBeenCalled(); expect(wrongBackend).not.toHaveBeenCalled(); + // A retired copy is never attached again, even to an empty destination. + const fresh = await mkdtemp(path.join(root, "retired-destination-")); + expect(await attachPreservedRemoteWorkspaceSnapshots({ ...scope, cwd: fresh, + retiredOperationIds: new Set([attached[0].operationId]) })).toEqual([]); + expect(await readdir(fresh)).toEqual([]); } finally { wrongBackend.mockRestore(); } }, 30_000); diff --git a/server/src/services/workspace-recovery-snapshot.ts b/server/src/services/workspace-recovery-snapshot.ts index 819136d17c8..52602be8168 100644 --- a/server/src/services/workspace-recovery-snapshot.ts +++ b/server/src/services/workspace-recovery-snapshot.ts @@ -162,7 +162,7 @@ export async function preserveWorkspaceSnapshot(input: { /** Recovered files are an attachment to a fresh attempt, never an overlay. * Atomically publish each immutable snapshot directory after verification. */ export async function attachPreservedWorkspaceSnapshots(input: { - db: Db; companyId: string; issueId: string; sourceRunIds: string[]; cwd: string; + db: Db; companyId: string; issueId: string; sourceRunIds: string[]; cwd: string; retiredOperationIds?: ReadonlySet; }) { const storage = getStorageService(); const results: Array<{ operationId: string; path: string; sourceRunId: string }> = []; @@ -173,6 +173,7 @@ export async function attachPreservedWorkspaceSnapshots(input: { eq(workspaceOperations.heartbeatRunId, sourceRunId), eq(workspaceOperations.phase, "workspace_recovery_snapshot"), eq(workspaceOperations.status, "succeeded"))); for (const operation of rows) { + if (input.retiredOperationIds?.has(operation.id)) continue; const saved = readSnapshot(operation.metadata, input.companyId, sourceRunId); const parent = path.join(await realpath(input.cwd), ".paperclip-recovered"); await mkdir(parent, { mode: 0o700 }).catch((error: NodeJS.ErrnoException) => { if (error.code !== "EEXIST") throw error; });