diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 84141d4f0d5..0fad16bcc3e 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1294,6 +1294,23 @@ must bind the company, task, run, allocation, descriptor and final file digest. The original accepted result stays unchanged. A coordinator that is still committing, an active owner, or an unverified stop keeps its existing hold. +If Retry retains a later selected workspace, preparation attaches the verified +canonical snapshot from its admitted predecessor under `.paperclip-recovered/`. +It preserves that workspace's own files and older recovery attachments. The +provider receives the verified relative paths in its continuation context. +Missing or invalid canonical recovery evidence blocks the import; it does not +authorize an empty replacement or replay completed work. +Read-only imports can use a superseded immutable snapshot with the exact saved +operation receipt. They do not establish current workspace execution authority. +Restored sidecars keep user edits, and only paths present with their recorded +provenance in the current prepared root are published to the provider. Older +attachment records rebuild this proof from saved receipts; reusing a live +allocation never replaces their files. For a related task, retention requires +the selected workspace’s committed migration history and matching source marker, +or the exact provenance inside its verified committed saved snapshot. The latter +uses one temporary read-only archive verification per preparation when needed. +Task relationships alone do not authorize importing or retaining files. + When a legacy turn admitted by an explicit user message fails or times out, a bounded transient retry needs its own durable continuation authorization. Initial admission binds the message's exact body digest and revision. Scheduling requires that original binding and revalidates the human author, source task, assignee, and execution ownership, then records the successor's receipt in the same transaction as its run. Historical receipts without a message binding cannot authorize automatic retries and are not backfilled. The parent receipt remains intact. Dispatch checks the exact successor, retry parent, and unchanged message binding again. Deleted or edited input, changed actors or ownership, superseded work, and duplicate successors cannot renew authority. One-run Retry and Interrupt intents retain their separate admission contracts; they are not renewable user-message authority. Other retry policies cannot borrow this receipt: their scheduling fails closed. Successful explicit turns do not create a prose-only missing-comment follow-up with spent authority. Explicit retry admission checks the local parent adapter controller and verified environment cleanup. The existing queue-first release policy may retain only its own terminal execution claim while that cleanup is pending; finalization and the periodic stale-lock sweep reconsider the same policy after cleanup. Generic terminal-lock cleanup and stale checkout adoption leave these claims for that settlement. New user input, reassignment, and a successor's claim take precedence. Exhausted retries and revoked or missing authorization release the terminal claim without requesting another retry. Pending or failed environment cleanup cannot mint a successor receipt. diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index 06d5f019937..125cce63165 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -1425,7 +1425,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace projectRepositoryPaths.push(...repositories.map(repository => repository.relativePath)); persistedExecutionWorkspace = await executionWorkspacesSvc.getStoredById(agent.companyId, persistedExecutionWorkspace.id); } - if (remoteFresh?.mode === "fresh_isolated_workspace") { + if (remoteAuthority && persistedExecutionWorkspace?.metadata?.remoteWorkspaceRecoveredFiles) { const recorded = persistedExecutionWorkspace?.metadata?.remoteWorkspaceRecoveredFiles; const recoveredFiles = (Array.isArray(recorded) ? recorded : []).map(value => { const file = parseObject(value); diff --git a/server/src/services/remote-workspace-recovery-attachments.ts b/server/src/services/remote-workspace-recovery-attachments.ts new file mode 100644 index 00000000000..3f3bea5335f --- /dev/null +++ b/server/src/services/remote-workspace-recovery-attachments.ts @@ -0,0 +1,218 @@ +import { lstat, mkdir, mkdtemp, readFile, realpath, rename, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { hasRemoteRecoveryAttachment } from "./remote-workspace-transport.js"; +import { removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js"; +import { LEGACY_WORKSPACE_RECOVERY_MIGRATION } from "./legacy-workspace-recovery-migration.js"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { executionWorkspaceSnapshots, executionWorkspaces, heartbeatRuns, workspaceOperations, type Db } from "@paperclipai/db"; +import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared"; +import { readSnapshot, verifiedWorkspaceSnapshotReceipt } from "./workspace-recovery-snapshot.js"; +import { recoveryRunAncestry } from "./recovery-run-ancestry.js"; +import { immutableRemoteWorkspaceRecoveryAttachment } from "./workspace-recovery-receipt.js"; +import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; +import { assertExecutionWorkspaceRecordAccess } from "./task-workspace-access.js"; +import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "./remote-workspace-snapshot-storage.js"; +import { restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; +import { getStorageService, type StorageService } from "../storage/index.js"; + +/** Import verified canonical recovery as a sidecar. The selected workspace and + * its own files stay intact; this does not grant execution or replay authority. */ +export async function attachPreservedRemoteWorkspaceSnapshots(input: { + db: Db; companyId: string; issueId: string; runId: string; workspaceId: string; + sourceRunIds: string[]; cwd: string; storage?: StorageService; +}) { + const [writer] = await input.db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId), + eq(heartbeatRuns.status, "running"), + sql`coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.issueId}::text, + ${heartbeatRuns.contextSnapshot}->>'issueId') = ${input.issueId}`, + )); + if (!writer) throw new Error("remote_workspace_recovery_writer_changed"); + const ancestry: Array<{ id: string }> = await recoveryRunAncestry(input); + const sources = ancestry.length ? await input.db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.companyId), inArray(heartbeatRuns.id, ancestry.map(row => row.id)), + )) : []; + const canonicalReceipts = sources.length ? await input.db.select({ runId: workspaceOperations.heartbeatRunId }) + .from(workspaceOperations).where(and(eq(workspaceOperations.companyId, input.companyId), + eq(workspaceOperations.issueId, input.issueId), inArray(workspaceOperations.heartbeatRunId, sources.map(source => source.id)), + eq(workspaceOperations.phase, "workspace_recovery"), sql`${workspaceOperations.metadata}->>'storage' = 'remote_workspace'`, + )) : []; + const canonicalSources = new Set(canonicalReceipts.map(receipt => receipt.runId)); + const result: Array<{ operationId: string; path: string; sourceRunId: string }> = []; + for (const source of sources) { + if (source.runnerProfileJson?.remoteWorkspace == null) { + if (canonicalSources.has(source.id)) throw new Error("remote_workspace_recovery_receipt_invalid"); + continue; + } + const parsed = remoteWorkspaceRunReferenceSchema.safeParse(source.runnerProfileJson.remoteWorkspace); + if (!parsed.success) { + if (hasRequiredWorkspaceRecovery(source.resultJson)) throw new Error("remote_workspace_recovery_receipt_invalid"); + continue; + } + const remote = parsed.data; + if (canonicalSources.has(source.id) && (remote.state !== "saved" || !remote.snapshotId)) + throw new Error("remote_workspace_recovery_receipt_invalid"); + if (remote.workspaceId === input.workspaceId || remote.state !== "saved" || !remote.snapshotId) continue; + const proof = await immutableRemoteWorkspaceRecoveryAttachment(input.db, source); + if (!proof) { + if (hasRequiredWorkspaceRecovery(source.resultJson)) throw new Error("remote_workspace_recovery_receipt_invalid"); + continue; + } + const operationId = proof.operationId; + await assertExecutionWorkspaceRecordAccess(input.db, { type: "agent", source: "agent_jwt", + companyId: input.companyId, agentId: writer.agentId, runId: writer.id, + onBehalfOfUserId: writer.responsibleUserId === "local-board" ? null : writer.responsibleUserId }, + input.companyId, remote.workspaceId); + const saved = await getWorkspaceSnapshot(input.db, { companyId: input.companyId, + workspaceId: remote.workspaceId, snapshotId: remote.snapshotId }); + if (!saved || saved.sha256 !== proof.sha256 || saved.generation !== proof.generation) + throw new Error("remote_workspace_recovery_snapshot_missing"); + const provenance = JSON.stringify({ schema: "paperclip.remote-workspace-recovery-attachment.v1", + operationId, sourceRunId: source.id, workspaceId: remote.workspaceId, + snapshotId: remote.snapshotId, sha256: saved.sha256 }); + const parent = path.join(await realpath(input.cwd), ".paperclip-recovered"); + await mkdir(parent, { mode: 0o700 }).catch((error: NodeJS.ErrnoException) => { if (error.code !== "EEXIST") throw error; }); + if ((await lstat(parent)).isSymbolicLink() || await realpath(parent) !== parent) + throw new Error("remote_workspace_recovery_attachment_invalid"); + const destination = path.join(parent, operationId); + const prior = await lstat(destination).catch((error: NodeJS.ErrnoException) => { if (error.code !== "ENOENT") throw error; return null; }); + if (prior) { + const proofPath = path.join(destination, "provenance.json"); + const proof = await lstat(proofPath).catch(() => null); + if (!prior.isDirectory() || prior.isSymbolicLink() || await realpath(destination) !== destination + || !proof?.isFile() || proof.isSymbolicLink() || await readFile(proofPath, "utf8") !== provenance) + throw new Error("remote_workspace_recovery_attachment_invalid"); + } else { + const staging = await mkdtemp(path.join(parent, ".preparing-")); + try { + const archivePath = path.join(staging, "archive.pcws"); + await loadWorkspaceSnapshotArchive({ storage: input.storage ?? getStorageService(), reference: saved, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir: path.join(staging, "files"), expectedManifestDigest: saved.manifestDigest }); + await rm(archivePath); + await writeFile(path.join(staging, "provenance.json"), provenance, { flag: "wx", mode: 0o600 }); + await rename(staging, destination); + } finally { await removeTaskDirectoryTree(staging); } + } + result.push({ operationId, path: destination, sourceRunId: source.id }); + } + return result; +} + +/** Verify a sidecar already inside an authorized shared workspace. This only + * reconstructs its immutable identity; it cannot import files or admit work. */ +async function retainedRecoveryAttachmentProvenance(input: { + db: Db; companyId: string; issueIds: string[]; sourceRunId: string; operationId: string; + workspace: Pick; + verifySelectedSnapshot: (operationId: string, provenance: string) => Promise; +}): Promise { + const [source] = await input.db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.sourceRunId))); + const issueId = source?.nativeIssueId ?? source?.issueId ?? source?.contextSnapshot?.issueId; + if (!source || typeof issueId !== "string") return null; + let historyAllowed = input.issueIds.includes(issueId); + if (!historyAllowed) { + // Committed migration history is a workspace-scoped proof, independent of + // task relationships. A verified saved workspace can prove the same path. + const marker = record(input.workspace.metadata?.legacyWorkspaceRecoveryMigration); + const sourceMarker = record(source.runnerProfileJson?.legacyWorkspaceRecoveryMigration + ?? source.runnerProfileJson?.finalizedWorkspaceRecoveryMigration); + if (marker.schema === LEGACY_WORKSPACE_RECOVERY_MIGRATION && typeof marker.operationId === "string" + && sourceMarker.schema === marker.schema && sourceMarker.operationId === marker.operationId + && sourceMarker.workspaceId === input.workspace.id) { + const [migration] = await input.db.select().from(workspaceOperations).where(and( + eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.executionWorkspaceId, input.workspace.id), + eq(workspaceOperations.id, marker.operationId), eq(workspaceOperations.phase, "remote_workspace_migration"), + eq(workspaceOperations.status, "succeeded"))); + const history = migration?.metadata?.legacyRecoverySources; + historyAllowed = Array.isArray(history) && history.some(value => { + const entry = record(value); return entry.runId === source.id && entry.issueId === issueId; + }); + } + } + const [operation] = await input.db.select().from(workspaceOperations).where(and( + eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.issueId, issueId), + eq(workspaceOperations.heartbeatRunId, source.id), eq(workspaceOperations.id, input.operationId), + eq(workspaceOperations.status, "succeeded"))); + let provenance: string | null = null; + if (operation?.phase === "workspace_recovery_snapshot") { + const saved = readSnapshot(operation.metadata, input.companyId, source.id); + const proof = await verifiedWorkspaceSnapshotReceipt(input.db, { companyId: input.companyId, issueId, + runId: source.id, actionId: saved.recoveryActionId, operationIds: [operation.id], + leaseId: saved.leaseId, providerLeaseId: saved.providerLeaseId }); + provenance = proof ? JSON.stringify({ schema: saved.schema, operationId: operation.id, sourceRunId: source.id, + leaseId: saved.leaseId, sha256: saved.archive.sha256 }) : null; + } else if (operation?.phase === "workspace_recovery") { + const proof = await immutableRemoteWorkspaceRecoveryAttachment(input.db, source); + provenance = proof?.operationId === operation.id ? JSON.stringify({ schema: "paperclip.remote-workspace-recovery-attachment.v1", + operationId: proof.operationId, sourceRunId: source.id, workspaceId: proof.workspaceId, + snapshotId: proof.snapshotId, sha256: proof.sha256 }) : null; + } + return provenance && (historyAllowed || await input.verifySelectedSnapshot(input.operationId, provenance)) ? provenance : null; +} + +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +/** Publish only existing sidecars with exact host-recorded or receipt-backed + * provenance. This checks the selected root; it never imports or replaces files. */ +export async function retainRemoteRecoveryAttachments(input: { + db: Db; workspace: Pick; + runId: string; remoteRoot: string; target: AdapterSandboxExecutionTarget; assertOwned?: () => Promise; + recoveredFiles: Array<{ operationId: string; sourceRunId: string; path: string; provenance: string }>; + storage?: StorageService; +}) { + let snapshotScratch: string | null = null, snapshotFiles: string | null = null, snapshotLoaded = false; + const verifySelectedSnapshot = async (operationId: string, provenance: string) => { + if (!snapshotLoaded) { + snapshotLoaded = true; + const authority = readRemoteWorkspaceAuthority(input.workspace.metadata); + if (!authority?.snapshotId) return false; + const [receipt] = await input.db.select({ status: executionWorkspaceSnapshots.status }).from(executionWorkspaceSnapshots).where(and( + eq(executionWorkspaceSnapshots.companyId, input.workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, input.workspace.id), + eq(executionWorkspaceSnapshots.id, authority.snapshotId))); + if (!receipt || !["committed", "superseded"].includes(receipt.status)) return false; + const saved = await getWorkspaceSnapshot(input.db, { companyId: input.workspace.companyId, + workspaceId: input.workspace.id, snapshotId: authority.snapshotId }); + if (!saved) return false; + await input.assertOwned?.(); + snapshotScratch = await realpath(await mkdtemp(path.join(os.tmpdir(), "pc-retained-recovery-proof-"))); + const archivePath = path.join(snapshotScratch, "archive.pcws"); + await loadWorkspaceSnapshotArchive({ storage: input.storage ?? getStorageService(), reference: saved, targetPath: archivePath }); + snapshotFiles = path.join(snapshotScratch, "files"); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir: snapshotFiles, expectedManifestDigest: saved.manifestDigest }); + await input.assertOwned?.(); + } + if (!snapshotFiles) return false; + const directory = path.join(snapshotFiles, ".paperclip-recovered", operationId); + if (await realpath(directory).catch(() => null) !== directory) return false; + const file = path.join(directory, "provenance.json"), stat = await lstat(file).catch(() => null); + return Boolean(stat?.isFile() && !stat.isSymbolicLink() && stat.nlink === 1 && stat.size <= 16_384 + && await readFile(file, "utf8") === provenance); + }; + try { + const retained = input.workspace.metadata?.remoteWorkspaceRecoveredFiles; + for (const value of Array.isArray(retained) ? retained : []) { + const item = record(value); + if (typeof item.operationId !== "string" || !/^[a-f0-9-]{36}$/.test(item.operationId) + || typeof item.sourceRunId !== "string" || item.path !== `.paperclip-recovered/${item.operationId}`) + throw new Error("remote_workspace_recovered_files_invalid"); + if (input.recoveredFiles.some(current => current.operationId === item.operationId)) continue; + let provenance = typeof item.provenance === "string" ? item.provenance : null; + if (!provenance) { + const [writer] = await input.db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.workspace.companyId), eq(heartbeatRuns.id, input.runId))); + const issueId = writer?.nativeIssueId ?? writer?.issueId ?? writer?.contextSnapshot?.issueId; + provenance = await retainedRecoveryAttachmentProvenance({ db: input.db, companyId: input.workspace.companyId, + workspace: input.workspace, issueIds: [issueId, input.workspace.sourceIssueId].filter((id): id is string => typeof id === "string"), + sourceRunId: item.sourceRunId, operationId: item.operationId, verifySelectedSnapshot }); + if (!provenance) throw new Error("remote_workspace_recovery_receipt_invalid"); + } + await input.assertOwned?.(); + if (await hasRemoteRecoveryAttachment({ target: input.target, remoteCwd: input.remoteRoot, operationId: item.operationId, provenance })) + input.recoveredFiles.push({ operationId: item.operationId, sourceRunId: item.sourceRunId, path: item.path, provenance }); + } + } finally { if (snapshotScratch) await removeTaskDirectoryTree(snapshotScratch); } +} diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index cdab59635ca..c6fca2d4217 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -12,6 +12,7 @@ import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/com import { captureRemoteWorkspaceArchive, cleanupRemoteWorkspaceArchive, + hasRemoteRecoveryAttachment, prepareRemoteRepository, prepareRemoteWorkspaceDirectory, remoteWorkspaceFailureCause, @@ -34,6 +35,30 @@ describe("remote workspace transport safety with real sandbox commands", () => { }); afterEach(async () => { await fs.rm(scratch, { recursive: true, force: true }); }); + it.each(["missing", "present", "foreign", "proof_symlink", "proof_hardlink", "directory_symlink"])( + "checks existing recovery attachment provenance without changing files: %s", async state => { + const operationId = randomUUID(), parent = path.join(workspace, ".paperclip-recovered"); + const directory = path.join(parent, operationId), provenance = JSON.stringify({ operationId, sourceRunId: randomUUID() }); + if (state !== "missing") { + await fs.mkdir(directory, { recursive: true }); + await fs.writeFile(path.join(directory, "user.txt"), "preserved user edit\n"); + await fs.writeFile(path.join(directory, "provenance.json"), state === "foreign" ? "foreign proof" : provenance); + } + if (state === "proof_symlink" || state === "proof_hardlink") { + const proof = path.join(directory, "provenance.json"), outside = path.join(scratch, "outside-proof"); + await fs.rename(proof, outside); + if (state === "proof_symlink") await fs.symlink(outside, proof); else await fs.link(outside, proof); + } + if (state === "directory_symlink") { + const outside = path.join(scratch, "outside-attachment"); + await fs.rename(directory, outside); await fs.symlink(outside, directory); + } + const probe = hasRemoteRecoveryAttachment({ target: target().remote, remoteCwd: workspace, operationId, provenance }); + if (state === "missing" || state === "present") await expect(probe).resolves.toBe(state === "present"); + else await expect(probe).rejects.toThrow("remote_workspace_operation_failed"); + if (state !== "missing") expect(await fs.readFile(path.join(directory, "user.txt"), "utf8")).toBe("preserved user edit\n"); + }); + async function git(cwd: string, ...args: string[]) { return exec("git", ["-C", cwd, "-c", "user.name=Synthetic fixture", "-c", "user.email=fixture@example.invalid", "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", ...args], { diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index b1e9e1ec32f..7377ccc8d0c 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -62,7 +62,7 @@ const SETUP_FAILURE_CHECK = String.raw`const setupFailure=(prefix,stage,error)=> return prefix+': '+fields.join(' ');};`; const SANDBOX_SETUP_CAUSE = /^remote_workspace_(prepare|repository|restore)_failed: stage=[a-z_]{1,32}(?: (?:reason|archive|disk|git|code|exit|signal|syscall|killed|program|name)=[A-Za-z0-9_]{1,64}){0,16}$/; -type RemoteWorkspaceOperation = "prepare" | "repository" | "archive_helper" | "capture" | "download" | "upload" | "restore" | "cleanup"; +type RemoteWorkspaceOperation = "prepare" | "repository" | "archive_helper" | "capture" | "download" | "upload" | "restore" | "cleanup" | "recovery_attachment"; export class RemoteWorkspaceOperationTimeoutError extends Error { constructor(readonly operation: RemoteWorkspaceOperation, readonly timeoutMs: number) { @@ -233,6 +233,35 @@ async function command(target: RemoteWorkspaceTarget, operation: RemoteWorkspace return result.stdout; } +const RECOVERY_ATTACHMENT = String.raw` +const fs=require('node:fs/promises'),path=require('node:path'); +(async()=>{ + const [root,raw]=process.argv.slice(1),input=JSON.parse(raw); + if(!path.isAbsolute(root)||!/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(input.operationId) + ||typeof input.provenance!=='string'||Buffer.byteLength(input.provenance)>16384)throw Error('invalid_attachment'); + const inspect=async p=>{try{return await fs.lstat(p)}catch(e){if(e.code==='ENOENT')return null;throw e}}; + const safeDirectory=async p=>{const s=await inspect(p);if(!s)return false; + if(!s.isDirectory()||s.isSymbolicLink()||await fs.realpath(p)!==p)throw Error('unsafe_attachment');return true}; + if(!await safeDirectory(root))throw Error('missing_workspace'); + const parent=path.join(root,'.paperclip-recovered'),destination=path.join(parent,input.operationId); + if(!await safeDirectory(parent)||!await safeDirectory(destination)){process.stdout.write('missing');return} + const proof=path.join(destination,'provenance.json'),s=await inspect(proof); + if(!s?.isFile()||s.isSymbolicLink()||s.nlink!==1||s.size>16384||await fs.readFile(proof,'utf8')!==input.provenance) + throw Error('attachment_proof_changed'); + process.stdout.write('present'); +})().catch(()=>{process.stderr.write('remote_workspace_recovery_attachment_invalid');process.exitCode=1});`; + +/** Reuse a restored attachment only with its exact provenance. User edits to + * its files remain intact; unknown destinations are never overwritten. */ +export async function hasRemoteRecoveryAttachment(input: { + target: RemoteWorkspaceTarget; remoteCwd: string; operationId: string; provenance: string; +}) { + const output = await command(input.target, "recovery_attachment", ["-e", RECOVERY_ATTACHMENT, + input.remoteCwd, JSON.stringify({ operationId: input.operationId, provenance: input.provenance })], undefined, 30_000); + if (output !== "present" && output !== "missing") throw new Error("remote_workspace_recovery_attachment_invalid"); + return output === "present"; +} + const PREPARE = String.raw` const fs = require('node:fs/promises'), path = require('node:path'), cp = require('node:child_process'); const {promisify}=require('node:util'); const exec=promisify(cp.execFile); diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index fbdd45e1e64..3bbdcd1c3fe 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -3,7 +3,7 @@ import { workspaceAllocationFiles } from "./remote-workspace-allocation.js"; import { readNativeWorkspaceMigration } from "./native-runtime/native-workspace-migration.js"; import { createHash, randomUUID } from "node:crypto"; import { existsSync } from "node:fs"; -import { mkdtemp, rm, statfs } from "node:fs/promises"; +import { mkdtemp, readFile, rm, statfs } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { and, eq, inArray, sql } from "drizzle-orm"; @@ -27,7 +27,7 @@ import { readRemoteWorkspaceWriterHold, type RemoteWorkspaceWriter } from "./rem import { persistActivity } from "./activity-log.js"; import { captureRemoteWorkspaceArchive, cleanupRemoteWorkspaceArchive, prepareRemoteWorkspaceDirectory, prepareRemoteRepository, remoteWorkspaceRepositoryPath, - RemoteWorkspaceOperationTimeoutError, RemoteWorkspacePreparationTimeoutError, remoteWorkspaceFailureCause, + RemoteWorkspaceOperationTimeoutError, RemoteWorkspacePreparationTimeoutError, remoteWorkspaceFailureCause, hasRemoteRecoveryAttachment, requireRemoteWorkspaceTarget, restoreRemoteWorkspaceDirectory, RemoteWorkspaceRestoreError, type RemoteWorkspaceRestoreStep, type RemoteWorkspaceSavePhaseTiming, } from "./remote-workspace-transport.js"; @@ -37,6 +37,7 @@ import { import { restoreRemoteWorkspaceArchive, readRemoteWorkspaceArchiveDirectories } from "./remote-workspace-snapshot-archive.js"; import { removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js"; import { attachPreservedWorkspaceSnapshots } from "./workspace-recovery-snapshot.js"; +import { attachPreservedRemoteWorkspaceSnapshots, retainRemoteRecoveryAttachments } from "./remote-workspace-recovery-attachments.js"; import { notifyRemoteWorkspaceSnapshots } from "./remote-workspace-snapshot-retention.js"; import { notifyTerminalWorkspaceWork } from "./terminal-workspace-work.js"; import { preserveRemoteWorkspacePreparationFailure } from "./native-runtime/native-workspace-finalization-state.js"; @@ -477,30 +478,47 @@ export async function prepareRemoteWorkspace(input: { savedArchive: isSavedWorkspaceRepository(input.workspace, initial, repository), resolveGitAuth: await createRemoteWorkspaceGitAuth(input.db, input.workspace.companyId, input.runId) }); } - const recoveredFiles: Array<{ operationId: string; sourceRunId: string; path: string }> = []; - const recoverySourceRunIds = input.workspace.metadata?.remoteWorkspaceRecoverySourceRunIds; - if (!sameAllocation && Array.isArray(recoverySourceRunIds) && recoverySourceRunIds.length) { + const recoveredFiles: Array<{ operationId: string; sourceRunId: string; path: string; provenance: string }> = []; + if (!sameAllocation) { // Shared and inherited workspaces retain their original source task. The // current writer's task owns its recovery attachments, including a child's // preserved files recorded during migration of an ancestor's workspace. const [run] = await input.db.select({ nativeIssueId: heartbeatRuns.nativeIssueId, issueId: heartbeatRuns.issueId, - contextIssueId: sql`${heartbeatRuns.contextSnapshot}->'issueId'` }).from(heartbeatRuns) + context: heartbeatRuns.contextSnapshot }).from(heartbeatRuns) .where(and(eq(heartbeatRuns.companyId, input.workspace.companyId), eq(heartbeatRuns.id, input.runId))); - const issueId = run?.nativeIssueId ?? run?.issueId ?? run?.contextIssueId; + const retainedSources = input.workspace.metadata?.remoteWorkspaceRecoverySourceRunIds; + const recoverySourceRunIds = [...new Set([ + ...(Array.isArray(retainedSources) ? retainedSources.filter((id): id is string => typeof id === "string") : []), + ...(typeof run?.context?.taskRetryRequestId === "string" || run?.context?.explicitUserContinuation + ? [input.runId] : []), + ])]; + const issueId = run?.nativeIssueId ?? run?.issueId ?? run?.context?.issueId; if (typeof issueId !== "string") throw new Error("remote_workspace_recovery_task_missing"); - const scratch = await mkdtemp(path.join(os.tmpdir(), "pc-workspace-recovery-import-")); - try { - const recovered = await attachPreservedWorkspaceSnapshots({ db: input.db, companyId: input.workspace.companyId, - issueId, sourceRunIds: recoverySourceRunIds as string[], cwd: scratch }); - for (const item of recovered) { - const relative = `.paperclip-recovered/${item.operationId}`; - await target.runner!.syncIn!([{ operationId: randomUUID(), files: [{ - kind: "directory", sourcePath: item.path, targetPath: path.posix.join(remoteRoot, relative), - }] }]); - recoveredFiles.push({ operationId: item.operationId, sourceRunId: item.sourceRunId, path: relative }); - } - } finally { await rm(scratch, { recursive: true, force: true }); } + if (recoverySourceRunIds.length) { + const scratch = await mkdtemp(path.join(os.tmpdir(), "pc-workspace-recovery-import-")); + try { + const scope = { db: input.db, companyId: input.workspace.companyId, issueId, sourceRunIds: recoverySourceRunIds, + cwd: scratch, storage: input.storage }; + const recovered = [ + ...await attachPreservedWorkspaceSnapshots(scope), + ...await attachPreservedRemoteWorkspaceSnapshots({ ...scope, runId: input.runId, workspaceId: input.workspace.id }), + ]; + for (const item of recovered) { + const relative = `.paperclip-recovered/${item.operationId}`; + await input.assertOwned?.(); + const provenance = await readFile(path.join(item.path, "provenance.json"), "utf8"); + const exists = await hasRemoteRecoveryAttachment({ target, remoteCwd: remoteRoot, operationId: item.operationId, + provenance }); + if (!exists) await target.runner!.syncIn!([{ operationId: randomUUID(), files: [{ + kind: "directory", sourcePath: item.path, targetPath: path.posix.join(remoteRoot, relative), + }] }]); + recoveredFiles.push({ operationId: item.operationId, sourceRunId: item.sourceRunId, path: relative, provenance }); + } + } finally { await removeTaskDirectoryTree(scratch); } + } } + await retainRemoteRecoveryAttachments({ db: input.db, workspace: input.workspace, runId: input.runId, + remoteRoot, target, recoveredFiles, assertOwned: input.assertOwned, storage: input.storage }); await input.assertOwned?.(); const prepared = await input.db.transaction(async tx => { await lockWorkspaceRunParents(tx, input.workspace.companyId, input.runId); @@ -523,7 +541,8 @@ export async function prepareRemoteWorkspace(input: { .where(eq(environmentLeases.id, lease.id)).returning(); const [savedWorkspace] = await tx.update(executionWorkspaces).set({ cwd: remoteRoot, providerRef: remoteRoot, metadata: { ...workspace.metadata, remoteWorkspace: authority, - ...(recoveredFiles.length ? { remoteWorkspaceRecoveredFiles: recoveredFiles } : {}) }, updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id)).returning(); + ...(recoveredFiles.length || workspace.metadata?.remoteWorkspaceRecoveredFiles !== undefined + ? { remoteWorkspaceRecoveredFiles: recoveredFiles } : {}) }, updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id)).returning(); const runReference = reference(workspace.id, authority, "prepared"); const issueId = await persistRunReference(tx, workspace.companyId, input.runId, runReference); if (restoredExport) { diff --git a/server/src/services/workspace-recovery-receipt.ts b/server/src/services/workspace-recovery-receipt.ts index 33e44007914..f0dda4daff1 100644 --- a/server/src/services/workspace-recovery-receipt.ts +++ b/server/src/services/workspace-recovery-receipt.ts @@ -1,4 +1,4 @@ -import { and, desc, eq, sql, type AnyColumn, type SQL } from "drizzle-orm"; +import { and, desc, eq, inArray, sql, type AnyColumn, type SQL } from "drizzle-orm"; import { executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issueRecoveryActions, workspaceOperations, type Db } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema, type ExecutionBlocker } from "@paperclipai/shared"; import { z } from "zod"; @@ -168,3 +168,38 @@ export async function verifiedWorkspaceRepairReceipt(db: Db, run: Pick) { return (await verifiedWorkspaceRepairReceipt(...args)) !== null; } + +/** Read-only historical bytes have no execution authority. Their immutable + * receipt stays valid after a later writer advances the source workspace. + * Admission must continue to use verifiedWorkspaceRepairReceipt instead. */ +export async function immutableRemoteWorkspaceRecoveryAttachment(db: Db, run: Pick): Promise<{ + kind: "immutable_remote_workspace_attachment"; operationId: string; workspaceId: string; + snapshotId: string; sha256: string; generation: number; + } | null> { + const issueId = run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId; + const parsed = remoteWorkspaceRunReferenceSchema.safeParse(run.runnerProfileJson?.remoteWorkspace); + if (typeof issueId !== "string" || !parsed.success || parsed.data.state !== "saved") return null; + const remote = parsed.data, snapshotId = remote.snapshotId; + if (!snapshotId) return null; + const [saved] = await db.select({ operationId: workspaceOperations.id, metadata: workspaceOperations.metadata, + evidence: issueRecoveryActions.evidence, sha256: executionWorkspaceSnapshots.sha256, + generation: executionWorkspaceSnapshots.generation }).from(workspaceOperations) + .innerJoin(issueRecoveryActions, and(eq(issueRecoveryActions.companyId, workspaceOperations.companyId), + eq(issueRecoveryActions.sourceIssueId, workspaceOperations.issueId), + sql`${issueRecoveryActions.id}::text = ${workspaceOperations.metadata}->>'recoveryActionId'`)) + .innerJoin(executionWorkspaceSnapshots, and(eq(executionWorkspaceSnapshots.companyId, run.companyId), + eq(executionWorkspaceSnapshots.executionWorkspaceId, remote.workspaceId), + eq(executionWorkspaceSnapshots.id, snapshotId), eq(executionWorkspaceSnapshots.sourceRunId, run.id), + eq(executionWorkspaceSnapshots.epoch, remote.epoch), inArray(executionWorkspaceSnapshots.status, ["committed", "superseded"]))) + .where(and(eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.issueId, issueId), + eq(workspaceOperations.executionWorkspaceId, remote.workspaceId), eq(workspaceOperations.heartbeatRunId, run.id), + eq(workspaceOperations.phase, "workspace_recovery"), eq(workspaceOperations.status, "succeeded"), + sql`${workspaceOperations.metadata}->>'snapshotId' = ${snapshotId}`)).limit(1); + const proof = saved?.metadata, state = readWorkspaceRecovery(saved?.evidence ?? null); + if (!saved || !proof || state?.runId !== run.id || state.status !== "recovered" + || proof.storage !== "remote_workspace" || proof.leaseId !== remote.leaseId || proof.providerLeaseId !== remote.providerLeaseId + || proof.epoch !== remote.epoch || proof.sha256 !== saved.sha256 || proof.generation !== saved.generation || !saved.sha256) return null; + return { kind: "immutable_remote_workspace_attachment" as const, operationId: saved.operationId, + workspaceId: remote.workspaceId, snapshotId, sha256: saved.sha256, generation: saved.generation }; +} diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index 6aec8b9be18..8e5dad9af60 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -22,6 +22,11 @@ import { adapterExecutionControls, createAdapterExecutionControl } from "./adapt import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js"; import { DELIVERY_QUEUES, subscribeDeliveryWork } from "./delivery-work-notifications.js"; import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js"; +import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js"; +import { attachPreservedRemoteWorkspaceSnapshots } from "./remote-workspace-recovery-attachments.js"; +import { Readable } from "node:stream"; +import { renderPaperclipWakePrompt } from "@paperclipai/adapter-utils/server-utils"; +import { buildEnvironmentLeaseContext } from "./environment-runtime.js"; import { createHeartbeatWorkspaceResolver, resolveExecutionWorkspaceReuseRequestForIssue } from "./heartbeat/workspaces.js"; import { environmentService } from "./environments.js"; import * as agentStarts from "./agent-start-lock.js"; @@ -40,7 +45,7 @@ import { adoptSystemRecovery, updateSystemRecovery } from "./system-recovery.js" import { getStorageService } from "../storage/index.js"; import * as storageBackend from "../storage/index.js"; import { migrateRemoteWorkspace } from "./remote-workspace-migration.js"; -import { attachPreservedWorkspaceSnapshots, verifiedWorkspaceSnapshotReceipt } from "./workspace-recovery-snapshot.js"; +import { attachPreservedWorkspaceSnapshots, preserveWorkspaceSnapshot, verifiedWorkspaceSnapshotReceipt } from "./workspace-recovery-snapshot.js"; import { adoptHistoricalWorkspaceRecoveries, historicalRecoveryCounts } from "./workspace-recovery-backfill.js"; import { createSystemRecovery, failSystemRecovery, reserveSystemRecoveryAttempt, SystemRecoveryFailure } from "./system-recovery-state.js"; import { requestTaskRetry } from "./task-retry.js"; @@ -1705,6 +1710,466 @@ describe("automatic retained workspace recovery", () => { finally { await saved.cleanup(); } }, 30_000); + it.each(["retained_history", "ordinary_followup"])("restores the newest recovered tick bytes after a later workspace selection: %s", async kind => { + vi.stubEnv("DATABASE_URL", connectionString); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "62".repeat(32)); + try { + const f = await failedRemotePreparation(); + const tickLog = Array.from({ length: 12 }, (_, i) => `2026-11-06T07:${String(21 + i).padStart(2, "0")}:00Z\n`).join(""); + await writeFile(path.join(f.remote, "ticks.log"), tickLog); + if (kind === "retained_history") { + const readonlySource = path.join(f.remote, "source-assets"); + await mkdir(readonlySource); await writeFile(path.join(readonlySource, "asset.txt"), "Preserved source asset\n"); await chmod(readonlySource, 0o555); + } + await recoverRetainedWorkspace(f.input); + const initial = await run(f.runId), oldRunId = randomUUID(), oldActionId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: oldRunId, companyId: f.companyId, issueId: f.issueId, + agentId: initial.agentId, status: "failed", contextSnapshot: { issueId: f.issueId }, + createdAt: new Date(initial.createdAt.getTime() - 60_000), finishedAt: new Date(initial.createdAt.getTime() - 30_000) }); + await db.insert(issueRecoveryActions).values({ id: oldActionId, companyId: f.companyId, sourceIssueId: f.issueId, + kind: "active_run_watchdog", status: "resolved", ownerType: "board", cause: "native_continuation_requires_reconciliation", + fingerprint: oldRunId, nextAction: "Historical preserved files", evidence: { runId: oldRunId } }); + const oldRoot = path.join(root, oldRunId); await mkdir(oldRoot); + await writeFile(path.join(oldRoot, "ticks.log"), "2026-11-06T07:21:00Z\n"); + const oldProviderId = randomUUID(); + const [oldLease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, + heartbeatRunId: oldRunId, environmentId: f.lease.environmentId, provider: "daytona", providerLeaseId: oldProviderId, + status: "released", cleanupStatus: "success", releasedAt: new Date() }).returning(); + await preserveWorkspaceSnapshot({ db, companyId: f.companyId, issueId: f.issueId, runId: oldRunId, + actionId: oldActionId, leaseId: oldLease.id, providerLeaseId: oldProviderId, + target: { ...targets.get(f.lease.id)!, leaseId: oldLease.id, remoteCwd: oldRoot }, assertHeld: async () => {} }); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issueId, + error: "native_remote_recovery_lease_mismatch", errorCode: "setup_failed", sessionIdBefore: randomUUID(), + }).where(eq(heartbeatRuns.id, f.runId)); + await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issueId, + runId: f.runId, phase: "terminal_failure" }); + const [placeholder] = await db.insert(executionWorkspaces).values({ companyId: f.companyId, + sourceIssueId: f.issueId, name: "Later empty placeholder", providerType: "cloud_sandbox", + mode: "isolated_workspace", strategyType: "project_primary", cwd: `/workspaces/${randomUUID()}`, + metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, + snapshotId: null, source: { kind: "empty" }, writerRunId: f.runId, dirty: false, location: null }, + ...(kind === "retained_history" ? { remoteWorkspaceRecoverySourceRunIds: [oldRunId] } : {}), + remoteWorkspaceSnapshotRepositories: { snapshotId: randomUUID(), repositories: [] } }, + }).returning(); + await db.update(issues).set({ executionWorkspaceId: placeholder.id }).where(eq(issues.id, f.issueId)); + const source = await run(f.runId); + expect(await hasVerifiedWorkspaceRepair(db, source)).toBe(true); + const sourceReference = source.runnerProfileJson!.remoteWorkspace as { snapshotId: string }; + const savedSource = await createSavedWorkspaceLoader(db)({ companyId: f.companyId, + workspaceId: f.workspaceId, snapshotId: sourceReference.snapshotId }); + try { expect(await readFile(path.join(savedSource.root, "ticks.log"), "utf8")).toBe(tickLog); } + finally { await savedSource.cleanup(); } + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, status: "running" }); + const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, + actorId: "local-board", actorSource: "local_implicit", observedRunId: f.runId, idempotencyKey: randomUUID() }); + expect(receipt.accepted).toBe(true); + await heartbeatService(db).resumeTaskRetry(f.companyId, receipt.requestId!); + const [successor] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`)); + expect(successor).toBeDefined(); + const readiness = await readFreshWorkspaceReadiness(db, successor); + expect(readiness).toMatchObject({ mode: "resume_preserved_workspace" }); + const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + const [agent] = await db.select().from(agents).where(eq(agents.id, source.agentId)); + const selection = await selectHeartbeatEnvironment(db, { + run: successor, agent, + task: { issueRef: issue, issueContext: issue, issueId: f.issueId, context: successor.contextSnapshot!, + executionProjectId: issue.projectId, workspaceProjectContext: null, boundSourceWorkspace: null, + freshWorkspace: readiness, hasSelectedWorkspaceSource: false, taskWorkspaceIntent: null, responsibleUserId: "local-board" }, + policy: { trustPreset: { kind: "standard", preset: "standard", boundary: null, sourcePresets: {} }, + isolatedWorkspacesEnabled: true, requestedExecutionWorkspaceMode: "isolated_workspace", + projectExecutionWorkspacePolicy: null, issueExecutionWorkspaceSettings: null }, + config: { config: {}, sourceWorkspaceConfig: {}, issueAssigneeOverrides: null }, + services: { environmentsSvc: environmentService(db), executionWorkspacesSvc: executionWorkspaceService(db), + instanceSettings: instanceSettingsService(db), findSharedWorkspaceHolder: async () => null, + findExecutionWorkspaceHolder: async () => null }, + }); + await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, successor.id)); + const storage = { ...getStorageService(), provider: "s3" as const }; + const reserved = await reserveRemoteWorkspace({ db, storage, companyId: f.companyId, issueId: f.issueId, + runId: successor.id, agentId: source.agentId, environmentId: f.lease.environmentId!, + projectId: null, projectWorkspaceId: null, existingWorkspaceId: selection.existingExecutionWorkspace?.id ?? null, + source: { kind: "empty" } }); + const allocationPath = path.join(root, successor.id); await mkdir(allocationPath); + const allocation = await realpath(allocationPath); + const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, + heartbeatRunId: successor.id, environmentId: f.lease.environmentId, provider: "daytona", + providerLeaseId: randomUUID(), status: "active" }).returning(); + const target = { ...targets.get(f.lease.id)!, leaseId: lease.id, remoteCwd: allocation, + runner: { ...targets.get(f.lease.id)!.runner!, syncIn: async operations => { + for (const operation of operations) for (const file of operation.files) { + await mkdir(path.dirname(file.targetPath), { recursive: true }); + await cp(file.sourcePath, file.targetPath, { recursive: true, dereference: false }); + } + return { filesTransferred: 1, bytesTransferred: 0, operations: [] }; + } } } satisfies AdapterSandboxExecutionTarget; + const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: successor.id, + lease: lease as EnvironmentLease, target }); + const recovered = prepared.workspace.metadata?.remoteWorkspaceRecoveredFiles as Array<{ path: string }> | undefined; + const contents = await Promise.all(["", ...(recovered ?? []).map(item => `${item.path}/files`)] + .map(relative => readFile(path.join(prepared.remoteRoot, relative, "ticks.log"), "utf8").catch(() => null))); + if (kind === "retained_history") expect(contents).toContain("2026-11-06T07:21:00Z\n"); + expect(contents).toContain(tickLog); + if (kind === "retained_history") { + const currentSidecar = (recovered as Array<{ path: string; sourceRunId: string }>).find(item => item.sourceRunId === source.id)!; + expect(await readFile(path.join(prepared.remoteRoot, currentSidecar.path, "files", "source-assets", "asset.txt"), "utf8")) + .toBe("Preserved source asset\n"); + expect((await lstat(path.join(prepared.remoteRoot, currentSidecar.path, "files", "source-assets"))).mode & 0o777).toBe(0o555); + } + const continuation = await buildExecutionContinuation({ db, companyId: f.companyId, issueId: f.issueId, + agentId: agent.id, runId: successor.id, context: successor.contextSnapshot!, summary: null, exposeLowTrustRaw: false }); + const providerContext: Record = { ...successor.contextSnapshot, + executionContinuation: continuation, paperclipWake: { issue: { id: f.issueId }, executionContinuation: continuation } }; + const workspaces = executionWorkspaceService(db); + const current = (await workspaces.getStoredById(f.companyId, placeholder.id))!; + const currentIssue = (await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]; + const environment = (await environmentService(db).getById(f.lease.environmentId!))!; + const orchestrator = environmentRunOrchestrator(db, { environmentRuntime: f.environmentRuntime }); + const acquire = vi.spyOn(orchestrator, "acquireForRun").mockResolvedValue({ environment, + lease: prepared.lease, leaseContext: buildEnvironmentLeaseContext({ persistedExecutionWorkspace: current }), executionTransport: null }); + const storageRead = vi.spyOn(storageBackend, "getStorageService").mockReturnValue(storage); + targets.set(lease.id, { ...target, remoteCwd: prepared.remoteRoot }); + const localEnvironment = await environmentService(db).ensureLocalEnvironment(f.companyId); + try { + const boundary = await prepareHeartbeatWorkspace(db, { run: await run(successor.id), agent, + task: { issueRef: currentIssue, issueId: f.issueId, wasIssueAssignee: true, + context: providerContext, responsibleUserId: "local-board", previousSessionParams: null, freshWorkspace: readiness }, + policy: { trustPreset: { kind: "standard", preset: "standard", boundary: null, sourcePresets: {} }, + isolatedWorkspacesEnabled: true, effectiveExecutionWorkspaceMode: "isolated_workspace", + requestedExecutionWorkspaceMode: "isolated_workspace", useIsolatedTaskDirectory: true, + nativeChatWorkspaceScope: null, projectExecutionWorkspacePolicy: null, issueExecutionWorkspaceSettings: null, + environmentExecutionWorkspaceSettings: null, executionPolicy: { executionMode: "any" } }, + environment: { selectedEnvironmentId: environment.id, localEnvironment, selectedEnvironmentForConfig: environment, + environmentResolution: { environmentId: environment.id, source: "agent" }, resolvedInstanceSettings: await instanceSettingsService(db).get() }, + config: { mergedConfig: {}, configSnapshot: null, secretManifest: [] }, + reuse: { ...selection, existingExecutionWorkspace: current, reusableExistingExecutionWorkspace: current, + requestedShouldReuseExisting: true, workspaceReuseRequest: resolveExecutionWorkspaceReuseRequestForIssue({ + issueExecutionWorkspaceId: current.id, issueExecutionWorkspacePreference: "reuse_existing", existingExecutionWorkspaceStatus: current.status }), + runOptions: {} }, + services: { envOrchestrator: orchestrator, executionWorkspacesSvc: workspaces, workspaceOperationsSvc: workspaceOperationService(db), + ...createHeartbeatWorkspaceResolver(db), appendRunEvent: vi.fn() }, + controllerLease: { assertOwned: vi.fn(async () => {}) }, nativeRunnerPreparationSpans: [], + }); + const paths = providerContext.recoveredWorkspaceFiles as Array<{ sourceRunId: string; path: string }>; + const newest = paths?.find(item => item.sourceRunId === source.id); + expect(newest).toBeDefined(); + const prompt = renderPaperclipWakePrompt(providerContext.paperclipWake); + expect(prompt).toContain(newest!.path); + const readback = await new Promise((resolve, reject) => execFile(process.execPath, ["-e", + "const fs=require('node:fs');const p=JSON.parse(process.argv[1]);const text=fs.readFileSync(require('node:path').join(p.cwd,p.path,'files/ticks.log'),'utf8');const lines=text.trimEnd().split('\\n');process.stdout.write(JSON.stringify({count:lines.length,first:lines[0],last:lines.at(-1)}));", + JSON.stringify({ cwd: boundary.executionWorkspace.cwd, path: newest!.path })], + (error, stdout) => error ? reject(error) : resolve(stdout))); + expect(JSON.parse(readback)).toEqual({ count: 12, first: "2026-11-06T07:21:00Z", last: "2026-11-06T07:32:00Z" }); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].executionWorkspaceId).toBe(placeholder.id); + const editedLog = `${tickLog}User preserved annotation\n`; + await writeFile(path.join(boundary.executionWorkspace.cwd, newest!.path, "files", "ticks.log"), editedLog); + let [selectedWorkspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, placeholder.id)); + if (kind === "retained_history") { + const olderPaths = selectedWorkspace.metadata!.remoteWorkspaceRecoveredFiles as Array<{ operationId: string; sourceRunId: string; path: string }>; + const [legacy] = await db.update(executionWorkspaces).set({ metadata: { ...selectedWorkspace.metadata, + remoteWorkspaceRecoveredFiles: olderPaths.map(({ operationId, sourceRunId, path }) => ({ operationId, sourceRunId, path })) } }) + .where(eq(executionWorkspaces.id, selectedWorkspace.id)).returning(); + const upgraded = await prepareRemoteWorkspace({ db, storage, workspace: legacy, runId: successor.id, + lease: boundary.activeEnvironmentLease!.lease, target: boundary.executionTarget }); + expect(upgraded.workspace.metadata?.remoteWorkspaceRecoveredFiles).toEqual(expect.arrayContaining(olderPaths.map(item => + expect.objectContaining({ operationId: item.operationId, sourceRunId: item.sourceRunId, path: item.path })))); + expect(await readFile(path.join(upgraded.remoteRoot, newest!.path, "files", "ticks.log"), "utf8")).toBe(editedLog); + selectedWorkspace = upgraded.workspace; + } + await saveRemoteWorkspace({ db, storage, workspace: selectedWorkspace, runId: successor.id, + lease: boundary.activeEnvironmentLease!.lease, target: boundary.executionTarget, assertOwned: async () => {} }); + await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success" }) + .where(eq(environmentLeases.id, lease.id)); + const coldAllocationPath = path.join(root, randomUUID()); await mkdir(coldAllocationPath); + const [coldLease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, + heartbeatRunId: successor.id, environmentId: f.lease.environmentId, provider: "daytona", + providerLeaseId: randomUUID(), status: "active" }).returning(); + const coldWorkspace = await reserveRemoteWorkspace({ db, storage, companyId: f.companyId, issueId: f.issueId, + runId: successor.id, agentId: source.agentId, environmentId: f.lease.environmentId!, projectId: null, + projectWorkspaceId: null, existingWorkspaceId: placeholder.id, source: { kind: "empty" } }); + const cold = await prepareRemoteWorkspace({ db, storage, workspace: coldWorkspace, runId: successor.id, + lease: coldLease as EnvironmentLease, target: { ...target, leaseId: coldLease.id, remoteCwd: await realpath(coldAllocationPath) } }); + expect(await readFile(path.join(cold.remoteRoot, newest!.path, "files", "ticks.log"), "utf8")).toBe(editedLog); + if (kind === "retained_history") { + await saveRemoteWorkspace({ db, storage, workspace: cold.workspace, runId: successor.id, + lease: cold.lease, target: { ...target, leaseId: coldLease.id, remoteCwd: cold.remoteRoot }, assertOwned: async () => {} }); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, successor.id)); + await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success" }).where(eq(environmentLeases.id, coldLease.id)); + const [child] = await db.insert(issues).values({ companyId: f.companyId, parentId: f.issueId, + assigneeAgentId: agent.id, title: "Shared workspace continuation", status: "in_progress", executionWorkspaceId: placeholder.id }).returning(); + const [childRun] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent.id, issueId: child.id, + status: "running", contextSnapshot: { issueId: child.id } }).returning(); + const [saved] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, placeholder.id)); + const olderPaths = saved.metadata!.remoteWorkspaceRecoveredFiles as Array<{ operationId: string; sourceRunId: string; path: string }>; + const [earlierChild] = await db.insert(issues).values({ companyId: f.companyId, parentId: f.issueId, + title: "Earlier related recovery source", status: "done", executionWorkspaceId: placeholder.id }).returning(); + const [related] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, issueId: earlierChild.id, + agentId: agent.id, status: "failed", contextSnapshot: { issueId: earlierChild.id } }).returning(); + const [relatedAction] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: earlierChild.id, + kind: "active_run_watchdog", status: "resolved", ownerType: "board", cause: "native_continuation_requires_reconciliation", + fingerprint: related.id, nextAction: "Saved related files", evidence: { runId: related.id } }).returning(); + const [relatedLease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: earlierChild.id, + heartbeatRunId: related.id, environmentId: f.lease.environmentId, provider: "daytona", providerLeaseId: randomUUID(), + status: "released", cleanupStatus: "success", releasedAt: new Date() }).returning(); + await preserveWorkspaceSnapshot({ db, companyId: f.companyId, issueId: earlierChild.id, runId: related.id, + actionId: relatedAction.id, leaseId: relatedLease.id, providerLeaseId: relatedLease.providerLeaseId!, + target: { ...targets.get(f.lease.id)!, leaseId: relatedLease.id, remoteCwd: oldRoot }, assertHeld: async () => {} }); + const [attachedRelated] = await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: earlierChild.id, + sourceRunIds: [related.id], cwd: cold.remoteRoot }); + olderPaths.push({ operationId: attachedRelated.operationId, sourceRunId: related.id, + path: `.paperclip-recovered/${attachedRelated.operationId}` }); + const [migration] = await db.insert(workspaceOperations).values({ companyId: f.companyId, issueId: f.issueId, + executionWorkspaceId: placeholder.id, phase: "remote_workspace_migration", status: "succeeded", + metadata: { legacyRecoverySources: [{ runId: related.id, issueId: earlierChild.id }] } }).returning(); + const marker = { schema: "paperclip.legacy-workspace-recovery-migration.v1", operationId: migration.id }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { legacyWorkspaceRecoveryMigration: + { ...marker, workspaceId: placeholder.id } } }).where(eq(heartbeatRuns.id, related.id)); + await db.update(executionWorkspaces).set({ metadata: { ...saved.metadata, legacyWorkspaceRecoveryMigration: marker, + remoteWorkspaceRecoveredFiles: + olderPaths.map(({ operationId, sourceRunId, path }) => ({ operationId, sourceRunId, path })) } }).where(eq(executionWorkspaces.id, saved.id)); + const shared = await reserveRemoteWorkspace({ db, storage, companyId: f.companyId, issueId: child.id, + runId: childRun.id, agentId: agent.id, environmentId: f.lease.environmentId!, projectId: null, + projectWorkspaceId: null, existingWorkspaceId: placeholder.id, source: { kind: "empty" } }); + const [childLease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: child.id, + heartbeatRunId: childRun.id, environmentId: f.lease.environmentId, provider: "daytona", providerLeaseId: coldLease.providerLeaseId, status: "active" }).returning(); + const inherited = await prepareRemoteWorkspace({ db, storage, workspace: shared, runId: childRun.id, + lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: cold.remoteRoot } }); + expect(inherited.workspace.metadata?.remoteWorkspaceRecoveredFiles).toEqual(expect.arrayContaining(olderPaths.map(item => + expect.objectContaining({ operationId: item.operationId, sourceRunId: item.sourceRunId, path: item.path })))); + expect(await readFile(path.join(inherited.remoteRoot, newest!.path, "files", "ticks.log"), "utf8")).toBe(editedLog); + const [withoutProof] = await db.update(executionWorkspaces).set({ metadata: { ...inherited.workspace.metadata, + remoteWorkspaceRecoveredFiles: olderPaths.map(({ operationId, sourceRunId, path }) => ({ operationId, sourceRunId, path })) } }) + .where(eq(executionWorkspaces.id, inherited.workspace.id)).returning(); + for (const fault of ["migration_status", "source_marker_workspace", "history_issue"]) { + await db.update(workspaceOperations).set({ status: fault === "migration_status" ? "failed" : "succeeded", + metadata: { legacyRecoverySources: [{ runId: related.id, issueId: fault === "history_issue" ? randomUUID() : earlierChild.id }] } }) + .where(eq(workspaceOperations.id, migration.id)); + await db.update(heartbeatRuns).set({ runnerProfileJson: { legacyWorkspaceRecoveryMigration: + { ...marker, workspaceId: fault === "source_marker_workspace" ? randomUUID() : placeholder.id } } }) + .where(eq(heartbeatRuns.id, related.id)); + await expect(prepareRemoteWorkspace({ db, storage, workspace: withoutProof, runId: childRun.id, + lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: cold.remoteRoot } })) + .rejects.toThrow("remote_workspace_recovery_receipt_invalid"); + } + await db.update(workspaceOperations).set({ status: "succeeded", metadata: { + legacyRecoverySources: [{ runId: related.id, issueId: earlierChild.id }] } }).where(eq(workspaceOperations.id, migration.id)); + await db.update(heartbeatRuns).set({ runnerProfileJson: { legacyWorkspaceRecoveryMigration: + { ...marker, workspaceId: placeholder.id } } }).where(eq(heartbeatRuns.id, related.id)); + await db.update(workspaceOperations).set({ status: "failed" }).where(eq(workspaceOperations.id, + (olderPaths.find(item => item.sourceRunId === source.id))!.operationId)); + await expect(prepareRemoteWorkspace({ db, storage, workspace: withoutProof, runId: childRun.id, + lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: cold.remoteRoot } })) + .rejects.toThrow("remote_workspace_recovery_receipt_invalid"); + expect(await readFile(path.join(inherited.remoteRoot, newest!.path, "files", "ticks.log"), "utf8")).toBe(editedLog); + await db.update(workspaceOperations).set({ status: "succeeded" }).where(eq(workspaceOperations.id, + (olderPaths.find(item => item.sourceRunId === source.id))!.operationId)); + const immutableAssets = path.join(inherited.remoteRoot, "readonly-assets"); + await mkdir(immutableAssets); await writeFile(path.join(immutableAssets, "asset.txt"), "Read-only workspace asset\n"); + await chmod(immutableAssets, 0o555); + await saveRemoteWorkspace({ db, storage, workspace: inherited.workspace, runId: childRun.id, + lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: inherited.remoteRoot }, + assertOwned: async () => {} }); + const [selectedSaved] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, inherited.workspace.id)); + const [snapshotOnly] = await db.update(executionWorkspaces).set({ metadata: { + ...Object.fromEntries(Object.entries(selectedSaved.metadata!).filter(([key]) => key !== "legacyWorkspaceRecoveryMigration")), + remoteWorkspaceRecoveredFiles: olderPaths.map(({ operationId, sourceRunId, path }) => ({ operationId, sourceRunId, path })) } }) + .where(eq(executionWorkspaces.id, inherited.workspace.id)).returning(); + const snapshotReserved = await reserveRemoteWorkspace({ db, storage, companyId: f.companyId, issueId: child.id, + runId: childRun.id, agentId: agent.id, environmentId: f.lease.environmentId!, projectId: null, + projectWorkspaceId: null, existingWorkspaceId: snapshotOnly.id, source: { kind: "empty" } }); + const snapshotRetained = await prepareRemoteWorkspace({ db, storage, workspace: snapshotReserved, runId: childRun.id, + lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: inherited.remoteRoot } }); + expect(snapshotRetained.workspace.metadata?.remoteWorkspaceRecoveredFiles).toHaveLength(olderPaths.length); + expect((await lstat(immutableAssets)).mode & 0o777).toBe(0o555); + expect(await readFile(path.join(immutableAssets, "asset.txt"), "utf8")).toBe("Read-only workspace asset\n"); + expect(await readFile(path.join(snapshotRetained.remoteRoot, newest!.path, "files", "ticks.log"), "utf8")).toBe(editedLog); + const legacySnapshot = { ...snapshotRetained.workspace, metadata: { ...snapshotRetained.workspace.metadata, + remoteWorkspaceRecoveredFiles: olderPaths.map(({ operationId, sourceRunId, path }) => ({ operationId, sourceRunId, path })) } }; + const authority = readRemoteWorkspaceAuthority(legacySnapshot.metadata)!; + await db.update(executionWorkspaceSnapshots).set({ status: "uploaded" }).where(eq(executionWorkspaceSnapshots.id, authority.snapshotId!)); + await expect(prepareRemoteWorkspace({ db, storage, workspace: legacySnapshot, runId: childRun.id, + lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: inherited.remoteRoot } })) + .rejects.toThrow("remote_workspace_recovery_receipt_invalid"); + await db.update(executionWorkspaceSnapshots).set({ status: "committed" }).where(eq(executionWorkspaceSnapshots.id, authority.snapshotId!)); + await expect(prepareRemoteWorkspace({ db, storage, workspace: { ...legacySnapshot, metadata: { ...legacySnapshot.metadata, + remoteWorkspace: { ...authority, snapshotId: sourceReference.snapshotId } } }, runId: childRun.id, + lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: inherited.remoteRoot } })) + .rejects.toThrow("remote_workspace_recovery_receipt_invalid"); + const unreadable = { ...storage, getObject: vi.fn().mockRejectedValue(new Error("fixture: selected snapshot unavailable")) }; + await expect(prepareRemoteWorkspace({ db, storage: unreadable, workspace: legacySnapshot, runId: childRun.id, + lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: inherited.remoteRoot } })) + .rejects.toThrow("fixture: selected snapshot unavailable"); + expect(await readFile(path.join(snapshotRetained.remoteRoot, newest!.path, "files", "ticks.log"), "utf8")).toBe(editedLog); + } + if (kind === "ordinary_followup") { + await rm(path.join(cold.remoteRoot, newest!.path), { recursive: true }); + await saveRemoteWorkspace({ db, storage, workspace: cold.workspace, runId: successor.id, + lease: cold.lease, target: { ...target, leaseId: coldLease.id, remoteCwd: cold.remoteRoot }, assertOwned: async () => {} }); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, successor.id)); + await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success" }) + .where(eq(environmentLeases.id, coldLease.id)); + const [normal] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent.id, issueId: f.issueId, + status: "running", runtimeMode: "legacy", contextSnapshot: { issueId: f.issueId } }).returning(); + const ordinaryWorkspace = await reserveRemoteWorkspace({ db, storage, companyId: f.companyId, issueId: f.issueId, + runId: normal.id, agentId: agent.id, environmentId: f.lease.environmentId!, projectId: null, + projectWorkspaceId: null, existingWorkspaceId: placeholder.id, source: { kind: "empty" } }); + const ordinaryAllocation = await realpath(await mkdtemp(path.join(root, "ordinary-"))); + const [ordinaryLease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, + heartbeatRunId: normal.id, environmentId: f.lease.environmentId, provider: "daytona", providerLeaseId: randomUUID(), status: "active" }).returning(); + const ordinary = await prepareRemoteWorkspace({ db, storage, workspace: ordinaryWorkspace, runId: normal.id, + lease: ordinaryLease as EnvironmentLease, target: { ...target, leaseId: ordinaryLease.id, remoteCwd: ordinaryAllocation } }); + expect(await lstat(path.join(ordinary.remoteRoot, newest!.path)).then(() => true, () => false)).toBe(false); + expect(ordinary.workspace.metadata?.remoteWorkspaceRecoveredFiles ?? []).toEqual([]); + } + } finally { acquire.mockRestore(); storageRead.mockRestore(); } + expect((await run(f.runId)).error).toBe("native_remote_recovery_lease_mismatch"); + } finally { vi.unstubAllEnvs(); } + }, 30_000); + + it("reads canonical attachments from the caller's storage backend", async () => { + const f = await failedRemotePreparation(); await recoverRetainedWorkspace(f.input); + const source = await run(f.runId), destination = await mkdtemp(path.join(root, "injected-storage-")); + const [writer] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, + issueId: f.issueId, status: "running", responsibleUserId: "local-board", contextSnapshot: { issueId: f.issueId } }).returning(); + const global = getStorageService(), read = global.getObject.bind(global); + const injected = { ...global, getObject: vi.fn(read) }; + const wrongBackend = vi.spyOn(global, "getObject").mockRejectedValue(new Error("wrong storage backend")); + try { + const scope = { db, companyId: f.companyId, issueId: f.issueId, runId: writer.id, workspaceId: randomUUID(), + sourceRunIds: [source.id], cwd: destination, storage: injected }; + const attached = await attachPreservedRemoteWorkspaceSnapshots(scope); + expect(await readFile(path.join(attached[0].path, "files", "saved.txt"), "utf8")).toBe("finished work\n"); + expect(injected.getObject).toHaveBeenCalled(); + expect(wrongBackend).not.toHaveBeenCalled(); + } finally { wrongBackend.mockRestore(); } + }, 30_000); + + it("restores ordinary same-workspace Retry after reservation advances current authority", async () => { + vi.stubEnv("DATABASE_URL", connectionString); + try { + const f = await failedRemotePreparation(); await recoverRetainedWorkspace(f.input); + const source = await run(f.runId); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, status: "running" }); + const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: "local-board", + actorSource: "local_implicit", observedRunId: source.id, idempotencyKey: randomUUID() }); + await heartbeatService(db).resumeTaskRetry(f.companyId, receipt.requestId!); + const [successor] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`)); + expect(await readFreshWorkspaceReadiness(db, successor)).toMatchObject({ mode: "resume_preserved_workspace" }); + await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, successor.id)); + const storage = { ...getStorageService(), provider: "s3" as const }; + const reserved = await reserveRemoteWorkspace({ db, storage, companyId: f.companyId, issueId: f.issueId, + runId: successor.id, agentId: source.agentId, environmentId: f.lease.environmentId!, projectId: null, + projectWorkspaceId: null, existingWorkspaceId: f.workspaceId, source: { kind: "empty" } }); + expect(await hasVerifiedWorkspaceRepair(db, source)).toBe(false); + const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, + heartbeatRunId: successor.id, environmentId: f.lease.environmentId, provider: "daytona", providerLeaseId: randomUUID(), status: "active" }).returning(); + const target = { ...targets.get(f.lease.id)!, leaseId: lease.id, + remoteCwd: await realpath(await mkdtemp(path.join(root, "same-workspace-"))), + runner: { ...targets.get(f.lease.id)!.runner!, syncIn: async operations => { + for (const operation of operations) for (const file of operation.files) { + await mkdir(path.dirname(file.targetPath), { recursive: true }); await cp(file.sourcePath, file.targetPath, { recursive: true }); + } + return { filesTransferred: 1, bytesTransferred: 0, operations: [] }; + } } } satisfies AdapterSandboxExecutionTarget; + const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: successor.id, + lease: lease as EnvironmentLease, target }); + expect(prepared.workspace.id).toBe(f.workspaceId); + expect(await readFile(path.join(prepared.remoteRoot, "saved.txt"), "utf8")).toBe("finished work\n"); + expect(prepared.workspace.metadata?.remoteWorkspaceRecoveredFiles ?? []).toEqual([]); + } finally { vi.unstubAllEnvs(); } + }, 30_000); + + it.each(["missing_receipt", "missing_reference", "prepared_reference", "missing_snapshot_id", "uncommitted", "foreign_snapshot", "wrong_lease", "missing_object", "corrupt_object", "wrong_company", "wrong_issue"])( + "refuses canonical recovery attachments with %s evidence", async fault => { + const f = await failedRemotePreparation(); + await recoverRetainedWorkspace(f.input); + const source = await run(f.runId), remote = source.runnerProfileJson!.remoteWorkspace as { snapshotId: string }; + const [writer] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, + issueId: f.issueId, status: "running", responsibleUserId: "local-board", contextSnapshot: { issueId: f.issueId } }).returning(); + const destination = path.join(root, writer.id); await mkdir(destination); + const [receipt] = await db.select().from(workspaceOperations).where(and(eq(workspaceOperations.heartbeatRunId, f.runId), + eq(workspaceOperations.phase, "workspace_recovery"), eq(workspaceOperations.status, "succeeded"))); + if (fault === "missing_receipt") await db.update(workspaceOperations).set({ status: "failed" }).where(eq(workspaceOperations.id, receipt.id)); + if (fault === "missing_reference") await db.update(heartbeatRuns).set({ runnerProfileJson: + Object.fromEntries(Object.entries(source.runnerProfileJson!).filter(([key]) => key !== "remoteWorkspace")) }).where(eq(heartbeatRuns.id, source.id)); + if (fault === "prepared_reference" || fault === "missing_snapshot_id") await db.update(heartbeatRuns).set({ runnerProfileJson: { + ...source.runnerProfileJson, remoteWorkspace: { ...source.runnerProfileJson!.remoteWorkspace as object, + ...(fault === "prepared_reference" ? { state: "prepared" } : { snapshotId: null }) }, + } }).where(eq(heartbeatRuns.id, source.id)); + if (fault === "uncommitted") await db.update(executionWorkspaceSnapshots).set({ status: "uploaded" }).where(eq(executionWorkspaceSnapshots.id, remote.snapshotId)); + if (fault === "foreign_snapshot") await db.update(heartbeatRuns).set({ runnerProfileJson: { ...source.runnerProfileJson, + remoteWorkspace: { ...source.runnerProfileJson!.remoteWorkspace as object, snapshotId: randomUUID() } } }).where(eq(heartbeatRuns.id, source.id)); + if (fault === "wrong_lease") await db.update(workspaceOperations).set({ metadata: { ...receipt.metadata, leaseId: randomUUID() } }) + .where(eq(workspaceOperations.id, receipt.id)); + const storage = getStorageService(), getObject = storage.getObject.bind(storage); + const failingRead = ["missing_object", "corrupt_object"].includes(fault) ? vi.spyOn(storage, "getObject").mockImplementation(async (...args) => { + if (fault === "missing_object") throw new Error("fixture: object missing"); + const object = await getObject(...args); object.stream.destroy(); + return { ...object, stream: Readable.from([Buffer.from("corrupt")]) }; + }) : undefined; + try { + await expect(attachPreservedRemoteWorkspaceSnapshots({ db, + companyId: fault === "wrong_company" ? randomUUID() : f.companyId, + issueId: fault === "wrong_issue" ? randomUUID() : f.issueId, + runId: writer.id, workspaceId: randomUUID(), sourceRunIds: [source.id], cwd: destination })).rejects.toThrow(); + expect(await readdir(destination)).toEqual(fault === "missing_object" || fault === "corrupt_object" ? [".paperclip-recovered"] : []); + expect(await readFile(path.join(f.remote, "saved.txt"), "utf8")).toBe("finished work\n"); + } finally { failingRead?.mockRestore(); } + }, 30_000); + + it.each(["unchanged", "advanced"])("deduplicates immutable canonical sidecars when the source workspace is %s", async state => { + const f = await failedRemotePreparation(); await recoverRetainedWorkspace(f.input); + const source = await run(f.runId), destination = path.join(root, randomUUID()); await mkdir(destination); + await writeFile(path.join(destination, "later.txt"), "Later selected work\n"); + const [selected] = await db.insert(executionWorkspaces).values({ companyId: f.companyId, sourceIssueId: f.issueId, + name: "Later selected workspace", cwd: destination, mode: "isolated_workspace", strategyType: "project_primary" }).returning(); + await db.update(issues).set({ executionWorkspaceId: selected.id }).where(eq(issues.id, f.issueId)); + const [writer] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, + issueId: f.issueId, status: "running", responsibleUserId: "local-board", contextSnapshot: { issueId: f.issueId } }).returning(); + const scope = { db, companyId: f.companyId, issueId: f.issueId, runId: writer.id, + workspaceId: selected.id, sourceRunIds: [source.id, source.id], cwd: destination }; + const first = await attachPreservedRemoteWorkspaceSnapshots(scope); + expect(first).toHaveLength(1); + expect(await readFile(path.join(first[0].path, "files", "saved.txt"), "utf8")).toBe("finished work\n"); + await writeFile(path.join(first[0].path, "files", "saved.txt"), "User edited sidecar\n"); + if (state === "advanced") { + vi.stubEnv("DATABASE_URL", connectionString); + try { + const [child] = await db.insert(issues).values({ companyId: f.companyId, parentId: f.issueId, + assigneeAgentId: source.agentId, title: "Later legitimate workspace writer", status: "in_progress", executionWorkspaceId: f.workspaceId }).returning(); + const [later] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, + issueId: child.id, status: "running", responsibleUserId: "local-board", contextSnapshot: { issueId: child.id } }).returning(); + const storage = { ...getStorageService(), provider: "s3" as const }; + const reserved = await reserveRemoteWorkspace({ db, storage, companyId: f.companyId, issueId: child.id, + runId: later.id, agentId: source.agentId, environmentId: f.lease.environmentId!, projectId: null, + projectWorkspaceId: null, existingWorkspaceId: f.workspaceId, source: { kind: "empty" } }); + const allocation = await realpath(await mkdtemp(path.join(root, "later-writer-"))); + const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: child.id, + heartbeatRunId: later.id, environmentId: f.lease.environmentId, provider: "daytona", providerLeaseId: randomUUID(), status: "active" }).returning(); + const target = { ...targets.get(f.lease.id)!, leaseId: lease.id, remoteCwd: allocation, + runner: { ...targets.get(f.lease.id)!.runner!, syncIn: async operations => { + for (const operation of operations) for (const file of operation.files) { + await mkdir(path.dirname(file.targetPath), { recursive: true }); await cp(file.sourcePath, file.targetPath, { recursive: true }); + } + return { filesTransferred: 1, bytesTransferred: 0, operations: [] }; + } } } satisfies AdapterSandboxExecutionTarget; + const advanced = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: later.id, lease: lease as EnvironmentLease, target }); + await writeFile(path.join(advanced.remoteRoot, "later.txt"), "newer source workspace bytes\n"); + await saveRemoteWorkspace({ db, storage, workspace: advanced.workspace, runId: later.id, lease: advanced.lease, + target: { ...target, remoteCwd: advanced.remoteRoot }, assertOwned: async () => {} }); + expect(await hasVerifiedWorkspaceRepair(db, source)).toBe(false); + const old = source.runnerProfileJson!.remoteWorkspace as { snapshotId: string }; + expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, old.snapshotId)))[0].status).toBe("superseded"); + } finally { vi.unstubAllEnvs(); } + } + expect(await attachPreservedRemoteWorkspaceSnapshots(scope)).toEqual(first); + expect(await readFile(path.join(first[0].path, "files", "saved.txt"), "utf8")).toBe("User edited sidecar\n"); + expect(await readFile(path.join(destination, "later.txt"), "utf8")).toBe("Later selected work\n"); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].executionWorkspaceId).toBe(selected.id); + expect(await readdir(path.join(destination, ".paperclip-recovered"))).toEqual([first[0].operationId]); + }, 30_000); + it("admits one HTTP Retry after a failed native preparation has verified saved work", async () => { const f = await failedRemotePreparation(); await recoverRetainedWorkspace(f.input);