diff --git a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md index d92485eee42..0f028d6a037 100644 --- a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md +++ b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md @@ -76,6 +76,12 @@ use the retained sources and remote workspace. This is not successful recovery, provider termination evidence, or permission to delete the original allocation. After independently verified sandbox preservation, the task accepts the exact migrated snapshot receipt for continuation without requiring a fresh workspace. +For older runs without a synchronization descriptor, the migration journals +all declared recovery lease/allocation/directory identities under row locks. +Continuation requires the unchanged complete set and a verified snapshot for +every source. Missing or ambiguous source identity remains held; a partial +archive cannot certify recovery. The recovery journey covers two allocations, +removal of the old filesystem, and the files appearing in the next remote turn. Accepted results with unfinished copyback remain blockers. Once the original sync reference is finalized and its successful `workspace_finalize` operation receipt exists, assessment and status effects can continue from their diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 5a3493a0dbe..a153e4202b2 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -384,7 +384,7 @@ export async function admitExplicitNativeContinuation(input: { // before admitting new user input; never apply or replay the old result. const legacyReference = readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync); const migratedSource = coordinator && !coordinator.resultId && !coordinator.leaseOwner - && ["observed", "retryable_failure"].includes(coordinator.phase) && legacyReference + && ["observed", "retryable_failure"].includes(coordinator.phase) && await hasVerifiedWorkspaceRepair(db, run) ? await readMigratedLegacyWorkspaceSource(db, { companyId, runId: run.id, reference: legacyReference }) : null; // Offline migration fenced this exact unfinished turn and preserved its diff --git a/server/src/services/legacy-workspace-recovery-migration.ts b/server/src/services/legacy-workspace-recovery-migration.ts index 85c4596e5c7..02f6e633dbc 100644 --- a/server/src/services/legacy-workspace-recovery-migration.ts +++ b/server/src/services/legacy-workspace-recovery-migration.ts @@ -1,11 +1,12 @@ import { isDeepStrictEqual } from "node:util"; import { and, eq, sql } from "drizzle-orm"; -import { executionWorkspaces, heartbeatRuns, issues, workspaceOperations, type Db } from "@paperclipai/db"; +import { environmentLeases, executionWorkspaces, heartbeatRuns, issues, workspaceOperations, type Db } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; import type { NativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js"; export type LegacyWorkspaceRecoverySource = { runId: string; issueId: string; reference: NativeWorkspaceSyncReference | null; + recoveryAllocations?: { leaseId: string; providerLeaseId: string; remoteCwd: string }[] | null; unresolvedFinalization?: { phase: string; leaseOwner: string | null; leaseExpiresAt: string | null }; }; export const LEGACY_WORKSPACE_RECOVERY_MIGRATION = "paperclip.legacy-workspace-recovery-migration.v1"; @@ -34,34 +35,64 @@ export async function isLegacyWorkspaceTaskRelated(db: Pick, input return false; } +/** Record every declared source, or none: partial preservation cannot release + * the task's recovery hold. The recovery owner separately verifies provenance, + * provider termination and archive bytes before writing its receipts. */ +export async function readLegacyWorkspaceRecoveryAllocations(db: Pick, run: { + id: string; companyId: string; nativeIssueId: string | null; resultJson: Record | null; +}): Promise { + const recovery = run.resultJson?.workspaceRestoreRecovery as { schema?: unknown; leaseIds?: unknown } | undefined; + const ids = recovery?.leaseIds; + if (recovery?.schema !== "paperclip.workspace-restore-recovery.v1" || !Array.isArray(ids) || !ids.length + || ids.some(id => typeof id !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)) + || new Set(ids).size !== ids.length || !run.nativeIssueId) return null; + const allocations = []; + for (const id of [...ids].sort()) { + const [lease] = await db.select({ providerLeaseId: environmentLeases.providerLeaseId, metadata: environmentLeases.metadata }) + .from(environmentLeases).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.id, id), + eq(environmentLeases.heartbeatRunId, run.id), eq(environmentLeases.issueId, run.nativeIssueId))); + if (!lease?.providerLeaseId || typeof lease.metadata?.remoteCwd !== "string" || !lease.metadata.remoteCwd.startsWith("/")) return null; + allocations.push({ leaseId: id as string, providerLeaseId: lease.providerLeaseId, remoteCwd: lease.metadata.remoteCwd }); + } + return allocations; +} + /** This receipt changes where recovery may save files; it never certifies that * a provider stopped or that its files were recovered. The recovery owner must * still validate allocation provenance and termination before exporting. */ export async function readMigratedLegacyWorkspaceSource(db: Pick, input: { - companyId: string; runId: string; reference: NativeWorkspaceSyncReference; + companyId: string; runId: string; reference: NativeWorkspaceSyncReference | null; }): Promise { - if (input.reference.state !== "prepared") return null; + if (input.reference && input.reference.state !== "prepared") return null; + const [run] = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId), + )); + const runMarker = run?.runnerProfileJson?.legacyWorkspaceRecoveryMigration as { workspaceId?: unknown } | undefined; + const workspaceId = input.reference?.workspaceId ?? runMarker?.workspaceId; + if (typeof workspaceId !== "string") return null; const [workspace] = await db.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces).where(and( - eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.reference.workspaceId), + eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, workspaceId), )); if (!readRemoteWorkspaceAuthority(workspace?.metadata)) return null; const marker = workspace?.metadata?.legacyWorkspaceRecoveryMigration as { schema?: unknown; operationId?: unknown } | undefined; if (marker?.schema !== LEGACY_WORKSPACE_RECOVERY_MIGRATION || typeof marker.operationId !== "string") throw new Error("legacy_workspace_migration_receipt_invalid"); - const [run] = await db.select({ profile: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId), - )); - if (!isDeepStrictEqual(run?.profile?.legacyWorkspaceRecoveryMigration, { - schema: LEGACY_WORKSPACE_RECOVERY_MIGRATION, operationId: marker.operationId, workspaceId: input.reference.workspaceId, + if (!isDeepStrictEqual(runMarker, { + schema: LEGACY_WORKSPACE_RECOVERY_MIGRATION, operationId: marker.operationId, workspaceId, })) throw new Error("legacy_workspace_migration_receipt_invalid"); const [operation] = await db.select({ metadata: workspaceOperations.metadata }).from(workspaceOperations).where(and( - eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.executionWorkspaceId, input.reference.workspaceId), + eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.executionWorkspaceId, workspaceId), eq(workspaceOperations.id, marker.operationId), eq(workspaceOperations.phase, "remote_workspace_migration"), eq(workspaceOperations.status, "succeeded"), )); const sources = operation?.metadata?.legacyRecoverySources; const source = Array.isArray(sources) ? sources.find(source => source && typeof source === "object" && source.runId === input.runId && typeof source.issueId === "string" && isDeepStrictEqual(source.reference, input.reference)) : null; + if (source && source.issueId !== run?.nativeIssueId) throw new Error("legacy_workspace_migration_receipt_invalid"); + if (source && !input.reference && (run?.runnerProfileJson?.nativeWorkspaceSync != null + || run?.runnerProfileJson?.remoteWorkspace != null + || !isDeepStrictEqual(source.recoveryAllocations ?? null, run ? await readLegacyWorkspaceRecoveryAllocations(db, run) : null))) + throw new Error("legacy_workspace_migration_receipt_invalid"); if (!source) throw new Error("legacy_workspace_migration_receipt_invalid"); return source as LegacyWorkspaceRecoverySource; } diff --git a/server/src/services/remote-workspace-migration.test.ts b/server/src/services/remote-workspace-migration.test.ts index d2d013756dc..8b75b7be4e1 100644 --- a/server/src/services/remote-workspace-migration.test.ts +++ b/server/src/services/remote-workspace-migration.test.ts @@ -8,7 +8,7 @@ import { createServer, type Server } from "node:http"; import { pathToFileURL } from "node:url"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; -import { activityLog, agents, heartbeatRuns, nativeRunFinalizations, nativeRunResults, completionContracts, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots, +import { activityLog, agents, environmentLeases, heartbeatRuns, nativeRunFinalizations, nativeRunResults, completionContracts, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots, executionWorkspaceRuntimeLeases, executionWorkspaces, issues, projects, projectWorkspaces, workspaceOperations, workspaceRuntimeServices } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; @@ -267,6 +267,25 @@ describe.skipIf(!support.supported)("remote workspace migration", () => { await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: true })).rejects.toThrow("remote_workspace_migration_source_changed"); expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); }); + it("rejects a descriptor-free allocation changed during upload", async () => { + const f = await failedCopybackFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + await db.insert(environmentLeases).values({ id: f.reference.leaseId, companyId: f.companyId, + issueId: f.issueId, heartbeatRunId: f.runId, provider: "daytona", providerLeaseId: f.reference.providerLeaseId, + status: "released", metadata: { remoteCwd: f.reference.remoteCwd } }); + await db.update(heartbeatRuns).set({ runnerProfileJson: {}, resultJson: { + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.reference.leaseId] }, + } }).where(eq(heartbeatRuns.id, f.runId)); + await db.update(nativeRunFinalizations).set({ phase: "observed" }).where(eq(nativeRunFinalizations.runId, f.runId)); + const put = storage.putFile.bind(storage); + storage.putFile = async input => { + const result = await put(input); + await db.update(environmentLeases).set({ providerLeaseId: randomUUID() }).where(eq(environmentLeases.id, f.reference.leaseId)); + return result; + }; + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: true })) + .rejects.toThrow("remote_workspace_migration_source_changed"); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); + }); it("does not release a coordinator whose expired owner changed during upload", async () => { const f = await failedCopybackFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); await db.update(nativeRunFinalizations).set({ phase: "observed", leaseOwner: "journaled-owner", diff --git a/server/src/services/remote-workspace-migration.ts b/server/src/services/remote-workspace-migration.ts index 7fe1d147045..7ba8b29a79c 100644 --- a/server/src/services/remote-workspace-migration.ts +++ b/server/src/services/remote-workspace-migration.ts @@ -4,7 +4,7 @@ import os from "node:os"; import path from "node:path"; import { and, eq, inArray, ne, or, sql } from "drizzle-orm"; import { - executionWorkspaceRepositories, executionWorkspaceRuntimeLeases, executionWorkspaceSnapshots, executionWorkspaces, + environmentLeases, executionWorkspaceRepositories, executionWorkspaceRuntimeLeases, executionWorkspaceSnapshots, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, projects, workspaceOperations, workspaceRuntimeServices, type Db, } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority, remoteWorkspaceSourceSchema, type RemoteWorkspaceAuthority, type RemoteWorkspaceSource } from "@paperclipai/shared"; @@ -18,7 +18,7 @@ import { materializeTaskDirectorySnapshot, removeTaskDirectoryTree } from "./iso import { loadTaskDirectoryRestorePoint, mayBeIsolatedTaskDirectoryWorkspace, readTaskDirectoryRetention, validateTaskDirectoryRestorePointRef } from "./isolated-task-directory-retention.js"; import { inspectArchivedGitMigrationSource, materializeArchivedGitMigrationSource, type ArchivedGitMigrationSource } from "./remote-workspace-migration-git.js"; import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js"; -import { isLegacyWorkspaceTaskRelated, LEGACY_WORKSPACE_RECOVERY_MIGRATION, type LegacyWorkspaceRecoverySource } from "./legacy-workspace-recovery-migration.js"; +import { isLegacyWorkspaceTaskRelated, readLegacyWorkspaceRecoveryAllocations, LEGACY_WORKSPACE_RECOVERY_MIGRATION, type LegacyWorkspaceRecoverySource } from "./legacy-workspace-recovery-migration.js"; import { offlineWorkspaceMigrationBlockers, prepareNestedWorkspaceImports, validateImportedWorkspaceChildren, workspaceMigrationLayoutBlockers } from "./remote-workspace-migration-tree.js"; const SCHEMA = "paperclip.remote-workspace-migration.v1"; @@ -161,6 +161,7 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri if (stopped && row.run.runnerProfileJson?.nativeWorkspaceSync == null && row.run.runnerProfileJson?.remoteWorkspace == null && row.resultId === null && ["observed", "retryable_failure"].includes(row.phase)) { legacyRecoverySources.push({ runId: row.id, issueId: row.run.nativeIssueId!, reference: null, + recoveryAllocations: await readLegacyWorkspaceRecoveryAllocations(db, row.run), unresolvedFinalization: { phase: row.phase, leaseOwner: row.leaseOwner, leaseExpiresAt: row.leaseExpiresAt?.toISOString() ?? null } }); continue; @@ -317,6 +318,9 @@ export async function migrateRemoteWorkspace(input: { .where(and(eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, source.runId))).for("update"); await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns) .where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, source.runId))).for("update"); + for (const allocation of source.recoveryAllocations ?? []) await tx.select({ id: environmentLeases.id }) + .from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId), + eq(environmentLeases.id, allocation.leaseId))).for("update"); } const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update"); if (!workspace) throw new Error("remote_workspace_migration_workspace_missing"); diff --git a/server/src/services/workspace-recovery-receipt.ts b/server/src/services/workspace-recovery-receipt.ts index f45ab8f3abe..6ef2c7af4fb 100644 --- a/server/src/services/workspace-recovery-receipt.ts +++ b/server/src/services/workspace-recovery-receipt.ts @@ -3,7 +3,7 @@ import { executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issueR import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema, type ExecutionBlocker } from "@paperclipai/shared"; import { z } from "zod"; import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js"; -import { hasMigratedLegacyWorkspaceSource } from "./legacy-workspace-recovery-migration.js"; +import { readMigratedLegacyWorkspaceSource } from "./legacy-workspace-recovery-migration.js"; import { readSnapshot } from "./workspace-recovery-snapshot.js"; export const workspaceRecoverySchema = z.object({ @@ -45,29 +45,36 @@ export async function verifiedWorkspaceRepairReceipt(db: Db, run: Pick>'recoveryActionId'`, - )).where(and(eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.issueId, issueId), - eq(workspaceOperations.heartbeatRunId, run.id), eq(workspaceOperations.phase, "workspace_recovery_snapshot"), - eq(workspaceOperations.status, "succeeded"), - sql`${workspaceOperations.metadata}->>'leaseId' = ${reference.leaseId}`, - sql`${workspaceOperations.metadata}->>'providerLeaseId' = ${reference.providerLeaseId}`, - sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,runId}' = ${run.id}`, - sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,status}' = 'recovered'`, - )).orderBy(desc(workspaceOperations.createdAt)).limit(1); - if (!saved || readWorkspaceRecovery(saved.evidence)?.status !== "recovered") return null; - try { readSnapshot(saved.metadata, run.companyId, run.id); } catch { return null; } - return saved.id; + const allocations = reference ? [{ leaseId: reference.leaseId, providerLeaseId: reference.providerLeaseId }] + : migrated.recoveryAllocations; + if (!allocations?.length) return null; + let receiptId: string | null = null; + for (const allocation of allocations) { + // Migration retires the local merge destination. A verified immutable + // attachment is the repair receipt; the old copyback remains unresolved. + // Provider stop and current execution ownership are checked separately. + const [saved] = await db.select({ id: workspaceOperations.id, metadata: workspaceOperations.metadata, + evidence: issueRecoveryActions.evidence }).from(workspaceOperations).innerJoin(issueRecoveryActions, and( + eq(issueRecoveryActions.companyId, workspaceOperations.companyId), eq(issueRecoveryActions.sourceIssueId, workspaceOperations.issueId), + sql`${issueRecoveryActions.id}::text = ${workspaceOperations.metadata}->>'recoveryActionId'`, + )).where(and(eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.issueId, issueId), + eq(workspaceOperations.heartbeatRunId, run.id), eq(workspaceOperations.phase, "workspace_recovery_snapshot"), + eq(workspaceOperations.status, "succeeded"), + sql`${workspaceOperations.metadata}->>'leaseId' = ${allocation.leaseId}`, + sql`${workspaceOperations.metadata}->>'providerLeaseId' = ${allocation.providerLeaseId}`, + sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,runId}' = ${run.id}`, + sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,status}' = 'recovered'`, + )).orderBy(desc(workspaceOperations.createdAt)).limit(1); + if (!saved || readWorkspaceRecovery(saved.evidence)?.status !== "recovered") return null; + try { readSnapshot(saved.metadata, run.companyId, run.id); } catch { return null; } + receiptId = saved.id; + } + return receiptId; } } if (typeof issueId !== "string" || (!remote && (!reference || reference.state !== "finalized" || !reference.finalHostSha256))) return null; diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index b22ed0178b2..bdc0d58c81f 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -184,7 +184,7 @@ describe("automatic retained workspace recovery", () => { ownerType: "board", cause: mode === "native" ? "native_continuation_requires_reconciliation" : "legacy_execution_requires_reconciliation", fingerprint: runId, nextAction: "Manual repair required", evidence: { runId, workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery } }); const runtime = { resumeRunLease: vi.fn(async (_input: { lease: EnvironmentLease }) => ({ providerLeaseId, metadata: { remoteCwd: remote } })), - retryPendingSandboxTeardown: vi.fn(async () => ({ providerLeaseId, state: "stopped" })) }; + retryPendingSandboxTeardown: vi.fn(async (_input: { lease: EnvironmentLease }) => ({ providerLeaseId, state: "stopped" })) }; const environmentRuntime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", acquireRunLease: vi.fn(), releaseRunLease: vi.fn(), ...runtime, resolveCapabilities: async () => classifyEnvironmentCapabilities({ verifiedMethods: [], declared: null, narrowing: null }), @@ -195,10 +195,10 @@ describe("automatic retained workspace recovery", () => { const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!; const run = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!; - it.each(["retained", "removed", "child", "expired-owner"])("preserves an old sandbox after storage migration without writing its old host (%s)", async mode => { + it.each(["retained", "removed", "child", "expired-owner", "no-descriptor"])("preserves an old sandbox after storage migration without writing its old host (%s)", async mode => { vi.stubEnv("DATABASE_URL", connectionString); vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "63".repeat(32)); - const f = await seed(); + const f = await seed("native", mode !== "no-descriptor"); // The fixture's command runner supports native copyback; the provider // snapshot boundary separately needs its real-file syncOut capability. targets.get(f.lease.id)!.runner!.syncIn = async operations => { @@ -211,6 +211,25 @@ describe("automatic retained workspace recovery", () => { await cp(file.sourcePath, file.targetPath, { recursive: true, dereference: false }); return { filesTransferred: 2, bytesTransferred: 0, operations: [] }; }; + if (mode === "no-descriptor") { + const secondId = randomUUID(), secondProviderId = randomUUID(), secondRemote = path.join(root, secondId); + await cp(f.remote, secondRemote, { recursive: true }); + const second: typeof f.lease = { ...f.lease, id: secondId, providerLeaseId: secondProviderId, + metadata: { ...f.lease.metadata, remoteCwd: secondRemote } }; + second.metadata = { ...second.metadata, + remoteExecutionTermination: remoteTerminationReceipt(second, { providerLeaseId: secondProviderId, state: "stopped" }) }; + await db.insert(environmentLeases).values(second); + targets.set(secondId, { ...targets.get(f.lease.id)!, leaseId: secondId, remoteCwd: secondRemote }); + f.runtime.retryPendingSandboxTeardown.mockImplementation(async ({ lease }) => ({ + providerLeaseId: lease.providerLeaseId!, state: "stopped", + })); + f.runtime.resumeRunLease.mockImplementation(async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!, + metadata: { remoteCwd: lease.metadata!.remoteCwd as string } })); + const source = await run(f.runId); + await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson, + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id, secondId] }, + } }).where(eq(heartbeatRuns.id, f.runId)); + } const sourceIssueId = mode === "child" ? randomUUID() : f.issueId; if (mode === "child") { await db.insert(issues).values({ id: sourceIssueId, companyId: f.companyId, title: "Ancestor workspace owner" }); @@ -219,7 +238,7 @@ describe("automatic retained workspace recovery", () => { await db.insert(executionWorkspaces).values({ id: f.issueId, companyId: f.companyId, sourceIssueId, name: "Existing files", mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: f.host }); await db.update(issues).set({ executionWorkspaceId: f.issueId }).where(eq(issues.id, f.issueId)); - if (mode === "expired-owner") await db.update(nativeRunFinalizations).set({ phase: "observed", + if (mode === "expired-owner" || mode === "no-descriptor") await db.update(nativeRunFinalizations).set({ phase: "observed", leaseOwner: "stopped-old-controller", leaseExpiresAt: new Date(Date.now() - 60_000) }) .where(eq(nativeRunFinalizations.runId, f.runId)); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); @@ -231,17 +250,35 @@ describe("automatic retained workspace recovery", () => { await recoverRetainedWorkspace(f.input); expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete"); expect(await readFile(path.join(f.host, "saved.txt")).catch(() => null)).toBeNull(); - expect(await run(f.runId)).toMatchObject({ status: "failed", runnerProfileJson: { nativeWorkspaceSync: { state: "prepared" } } }); + expect(await run(f.runId)).toMatchObject({ status: "failed", runnerProfileJson: mode === "no-descriptor" ? {} : { nativeWorkspaceSync: { state: "prepared" } } }); expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true); expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ workspaceRepairRequired: false }); const restored = path.join(root, randomUUID()); await mkdir(restored); const copies = await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: f.issueId, sourceRunIds: [f.runId], cwd: restored }); - expect(copies).toHaveLength(1); + expect(copies).toHaveLength(mode === "no-descriptor" ? 2 : 1); expect(await readFile(path.join(copies[0]!.path, "files", "saved.txt"), "utf8")).toBe("finished work\n"); expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0); - if (mode === "child" || mode === "expired-owner") { + if (mode === "no-descriptor") { + const source = await run(f.runId); + await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson, + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id] }, + } }).where(eq(heartbeatRuns.id, f.runId)); + expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false); + await db.update(heartbeatRuns).set({ resultJson: source.resultJson }).where(eq(heartbeatRuns.id, f.runId)); + for (const copy of copies) { + await db.update(workspaceOperations).set({ status: "running" }).where(eq(workspaceOperations.id, copy.operationId)); + expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false); + await db.update(workspaceOperations).set({ status: "succeeded" }).where(eq(workspaceOperations.id, copy.operationId)); + } + const [allocation] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + await db.update(environmentLeases).set({ providerLeaseId: randomUUID() }).where(eq(environmentLeases.id, f.lease.id)); + expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false); + await db.update(environmentLeases).set({ providerLeaseId: allocation.providerLeaseId }).where(eq(environmentLeases.id, f.lease.id)); + expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true); + } + if (mode === "child" || mode === "expired-owner" || mode === "no-descriptor") { // The parent must not receive the child's task-scoped recovery copy. if (mode === "child") expect(await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: sourceIssueId, sourceRunIds: [f.runId], cwd: restored })).toHaveLength(0); @@ -252,7 +289,7 @@ describe("automatic retained workspace recovery", () => { const continuationInput = { db, companyId: f.companyId, issueId: f.issueId, agentId: sourceRun.agentId, actorType: "user", actorId: "operator", reason: "issue_commented", commentId: comment.id, successorRunId: nextRunId }; - if (mode === "expired-owner") { + if (mode === "expired-owner" || mode === "no-descriptor") { for (const changed of [{ phase: "retryable_failure", leaseOwner: null }, { phase: "observed", leaseOwner: "new-controller" }]) { await db.update(nativeRunFinalizations).set(changed).where(eq(nativeRunFinalizations.runId, f.runId)); expect(await admitExplicitNativeContinuation({ ...continuationInput, dryRun: true })).toBeNull(); @@ -281,7 +318,7 @@ describe("automatic retained workspace recovery", () => { expect(await readFile(path.join(prepared.remoteRoot, ".paperclip-recovered", copies[0]!.operationId, "files", "saved.txt"), "utf8")) .toBe("finished work\n"); } - if (mode === "retained") { + if (mode === "retained" || mode === "no-descriptor") { const [receipt] = await db.select().from(workspaceOperations).where(eq(workspaceOperations.id, copies[0]!.operationId)); for (const metadata of [{ ...receipt.metadata, leaseId: randomUUID() }, { ...receipt.metadata, providerLeaseId: randomUUID() }, { ...receipt.metadata, recoveryActionId: randomUUID() }, { ...receipt.metadata, archive: { broken: true } }]) {