From 1c172fc0579293775645fc00d674957cbef6519d Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 15:04:05 -0600 Subject: [PATCH] fix(workspaces): require durable finalization before migration --- ...1-03-existing-workspace-ephemeral-migration.md | 3 ++- .../services/remote-workspace-migration.test.ts | 13 +++++++++++-- server/src/services/remote-workspace-migration.ts | 15 ++++++++++++--- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md index 5c35130fd09..d92485eee42 100644 --- a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md +++ b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md @@ -77,7 +77,8 @@ provider termination evidence, or permission to delete the original allocation. After independently verified sandbox preservation, the task accepts the exact migrated snapshot receipt for continuation without requiring a fresh workspace. Accepted results with unfinished copyback remain blockers. Once the original -sync receipt is finalized, assessment and status effects can continue from their +sync reference is finalized and its successful `workspace_finalize` operation +receipt exists, assessment and status effects can continue from their existing database coordinator independently of the file storage migration. Offline maintenance stops writers; it does not silently change workspace diff --git a/server/src/services/remote-workspace-migration.test.ts b/server/src/services/remote-workspace-migration.test.ts index 96ac5ec5c09..d2d013756dc 100644 --- a/server/src/services/remote-workspace-migration.test.ts +++ b/server/src/services/remote-workspace-migration.test.ts @@ -28,6 +28,7 @@ import { workspaceOperationService } from "./workspace-operations.js"; import { resumeNativeWorkspaceSync } from "./native-runtime/native-workspace-sync.js"; import { claimNativeSessionResumptions } from "./native-runtime/native-finalization-reconciler.js"; import { claimNativeRestartRecoveries } from "./native-runtime/native-restart-recovery.js"; +import { attemptNativeWorkspaceFinalization } from "./native-runtime/native-workspace-finalizer.js"; import { managedProjectCheckoutName } from "./managed-project-checkout-name.js"; import { captureGitWorkspaceContents, readManagedCheckoutContents } from "./git-workspace-contents.js"; @@ -214,7 +215,8 @@ describe.skipIf(!support.supported)("remote workspace migration", () => { legacyWorkspaceRecoveryMigration: { workspaceId: f.workspaceId } }, }); }); - it.each(["prepared", "finalized"])("keeps accepted-result finalization independent of migrated file storage (%s)", async state => { + it.each(["prepared", "finalized", "missing-receipt", "failed-receipt"])("keeps accepted-result finalization independent of migrated file storage (%s)", async mode => { + const state = mode === "prepared" ? "prepared" : "finalized"; const f = await failedCopybackFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); const contractId = randomUUID(), resultId = randomUUID(); await db.insert(completionContracts).values({ id: contractId, companyId: f.companyId, issueId: f.issueId, revision: 1, @@ -226,14 +228,21 @@ describe.skipIf(!support.supported)("remote workspace migration", () => { await db.insert(nativeRunResults).values({ id: resultId, companyId: f.companyId, issueId: f.issueId, runId: f.runId, completionContractId: contractId, serverFingerprint: "saved-result", schemaStatus: "accepted", resultJson: { saved: true }, canonicalSha256: "34".repeat(32) }); await db.update(nativeRunFinalizations).set({ phase: "retryable_failure", resultId }).where(eq(nativeRunFinalizations.runId, f.runId)); + const operationId = randomUUID(); + if (mode === "finalized" || mode === "failed-receipt") await db.insert(workspaceOperations).values({ id: operationId, + companyId: f.companyId, executionWorkspaceId: f.workspaceId, heartbeatRunId: f.runId, issueId: f.issueId, + phase: "workspace_finalize", status: mode === "finalized" ? "succeeded" : "failed" }); const [before] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); const migrate = () => migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: true }); - if (state === "prepared") await expect(migrate()).rejects.toThrow("remote_copyback_unresolved"); + if (mode !== "finalized") await expect(migrate()).rejects.toThrow(state === "prepared" + ? "remote_copyback_unresolved" : "execution_finalization_unresolved"); else { await migrate(); expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)?.snapshotId).toBeTruthy(); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!.runnerProfileJson) .not.toHaveProperty("legacyWorkspaceRecoveryMigration"); + expect(await attemptNativeWorkspaceFinalization({ db, runId: f.runId })) + .toMatchObject({ kind: "completed", operation: { id: operationId } }); } expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(before); }); diff --git a/server/src/services/remote-workspace-migration.ts b/server/src/services/remote-workspace-migration.ts index c99256bb025..7fe1d147045 100644 --- a/server/src/services/remote-workspace-migration.ts +++ b/server/src/services/remote-workspace-migration.ts @@ -137,9 +137,18 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri && !(row.run.controllerLeaseExpiresAt && row.run.controllerLeaseExpiresAt > new Date()) && !(row.leaseOwner && (!row.leaseExpiresAt || row.leaseExpiresAt > new Date())); const reference = readNativeWorkspaceSyncReference(row.run.runnerProfileJson?.nativeWorkspaceSync); - // Finalized copyback has no outstanding local writer. Assessment/status - // effects stay with their existing durable coordinator after the move. - if (stopped && reference?.state === "finalized") continue; + // The finalizer retries from its operation receipt, not the sync descriptor. + // Without that receipt it would try exporting the newly migrated workspace, + // which intentionally has no live location. Keep the local authority until + // the existing owner records completion; then assessment/status can resume. + if (stopped && reference?.state === "finalized") { + const [finalized] = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and( + eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.heartbeatRunId, row.id), + eq(workspaceOperations.issueId, row.run.nativeIssueId!), eq(workspaceOperations.phase, "workspace_finalize"), + eq(workspaceOperations.status, "succeeded"), + )).limit(1); + if (finalized) continue; + } if (stopped && reference && reference.workspaceId !== workspace.id) { const [other] = await db.select({ providerType: executionWorkspaces.providerType, cwd: executionWorkspaces.cwd, sourceIssueId: executionWorkspaces.sourceIssueId, metadata: executionWorkspaces.metadata }).from(executionWorkspaces).where(and(