diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.md index e0ce2ac50c3..18bd5adc010 100644 --- a/doc/native-workspace-finalization-recovery.md +++ b/doc/native-workspace-finalization-recovery.md @@ -119,6 +119,9 @@ confirmed stopped receipt. No provider turn is created by this recovery. Cleanup and its readiness probe use the plugin ID recorded on that lease. Another plugin with the same provider name cannot take over; an unavailable original plugin defers cleanup without consuming an attempt. +New intents use schema v2 with an explicit plugin pin. A v1 intent created before +that field existed remains recoverable using only the plugin ID already recorded +on its exact lease; an explicit mismatched pin is still rejected. The opt-in `native-workspace-export-resume.live.test.ts` is a provider-boundary fault integration, separate from the browser Product E2E. After building the diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 40410cbcaf6..39fcb56145d 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -5037,11 +5037,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect(await db.select().from(environmentLeases)).toHaveLength(1); }); - it.each(["stopped", "unconfirmed", "foreign_marker", "competing_owner", "live_request", "late_receipt", "duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing"])("recovers a pending export resume with stop-only cleanup after restart: %s", async outcome => { + it.each(["stopped", "unconfirmed", "foreign_marker", "competing_owner", "live_request", "late_receipt", "duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing", "legacy_intent", "foreign_plugin_pin", "v2_missing_pin"])("recovers a pending export resume with stop-only cleanup after restart: %s", async outcome => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); const lease = seeded.reusableLease; const requestId = randomUUID(); - if (["duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) { + if (["duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing", "legacy_intent"].includes(outcome)) { const [owner] = await db.select().from(plugins).where(eq(plugins.id, seeded.pluginId)); await db.insert(plugins).values({ ...owner, id: randomUUID(), pluginKey: "other.same-driver", packageName: "@other/same-driver", installOrder: -1, manifestJson: { ...owner.manifestJson, id: "other.same-driver" } }); @@ -5050,8 +5050,8 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { } await db.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", metadata: { ...lease.metadata, - nativeWorkspaceExportResume: { schema: "paperclip.workspace-export-resume.v1", requestId, companyId: seeded.companyId, - pluginId: seeded.pluginId, + nativeWorkspaceExportResume: { schema: outcome === "v2_missing_pin" ? "paperclip.workspace-export-resume.v2" : "paperclip.workspace-export-resume.v1", requestId, companyId: seeded.companyId, + ...(["legacy_intent", "v2_missing_pin"].includes(outcome) ? {} : { pluginId: outcome === "foreign_plugin_pin" ? randomUUID() : seeded.pluginId }), runId: outcome === "foreign_marker" ? randomUUID() : lease.heartbeatRunId, leaseId: lease.id, provider: lease.provider, providerLeaseId: lease.providerLeaseId, resultId: randomUUID() }, pendingCleanupAttemptId: requestId, pendingCleanupInFlight: true, @@ -5072,7 +5072,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { await heartbeatService(db, { environmentRuntime: restarted }).sweepPendingCleanupLeases({ backoffMs: 0 }); expect(call.mock.calls.some((entry) => entry[1] === "environmentDestroyLease")).toBe(false); const persisted = (await environmentService(db).getLeaseById(lease.id))!; - if (["stopped", "duplicate_key"].includes(outcome)) { + if (["stopped", "duplicate_key", "legacy_intent"].includes(outcome)) { expect(call).toHaveBeenCalledWith(seeded.pluginId, "environmentReleaseLease", expect.objectContaining({ providerLeaseId: lease.providerLeaseId, cancelActiveWork: true }), expect.any(Number)); expect(persisted).toMatchObject({ status: "expired", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "stopped", runId: lease.heartbeatRunId } } }); expect(persisted.metadata?.nativeWorkspaceExportResume).toBeUndefined(); @@ -5081,7 +5081,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { } else { expect(persisted).toMatchObject({ status: "pending_cleanup", cleanupStatus: "failed" }); expect(persisted.metadata?.remoteExecutionTermination).toBeUndefined(); - if (["foreign_marker", "competing_owner", "live_request", "owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) expect(call).not.toHaveBeenCalled(); + if (["foreign_marker", "foreign_plugin_pin", "v2_missing_pin", "competing_owner", "live_request", "owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) expect(call).not.toHaveBeenCalled(); if (["owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) { expect(persisted.metadata?.pendingCleanupAttemptId).toBe(requestId); } diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index a757833b1ad..70aed47408a 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -2583,7 +2583,8 @@ function createSandboxEnvironmentDriver( )).limit(1); if (!current || current.status !== "pending_cleanup" || current.companyId !== input.lease.companyId || current.metadata?.pendingCleanupAttemptId !== input.lease.metadata?.pendingCleanupAttemptId - || readNativeWorkspaceExportResume(current)?.requestId !== resumeIntent.requestId || otherOwner) { + || readNativeWorkspaceExportResume(current)?.requestId !== resumeIntent.requestId + || readNativeWorkspaceExportResume(current)?.pluginId !== resumeIntent.pluginId || otherOwner) { throw new Error("Workspace export resume ownership changed before cleanup."); } }; diff --git a/server/src/services/native-runtime/native-workspace-export-resume.ts b/server/src/services/native-runtime/native-workspace-export-resume.ts index 727b991b148..f8c828224bf 100644 --- a/server/src/services/native-runtime/native-workspace-export-resume.ts +++ b/server/src/services/native-runtime/native-workspace-export-resume.ts @@ -20,14 +20,19 @@ export function readNativeWorkspaceExportResume(lease: Lease) { const value = lease.metadata?.[NATIVE_WORKSPACE_EXPORT_RESUME_KEY]; if (!value || typeof value !== "object" || Array.isArray(value)) return null; const marker = value as Record; - if (marker.schema !== "paperclip.workspace-export-resume.v1" + if (!["paperclip.workspace-export-resume.v1", "paperclip.workspace-export-resume.v2"].includes(String(marker.schema)) || marker.companyId !== lease.companyId || marker.runId !== lease.heartbeatRunId || !lease.heartbeatRunId || marker.leaseId !== lease.id || marker.provider !== lease.provider || !lease.provider || marker.providerLeaseId !== lease.providerLeaseId || !lease.providerLeaseId - || typeof marker.pluginId !== "string" || !marker.pluginId || marker.pluginId !== lease.metadata?.pluginId || typeof marker.requestId !== "string" || !marker.requestId || typeof marker.resultId !== "string" || !marker.resultId) return null; - return marker; + // v1 preceded the intent's explicit plugin pin. Its exact lease already + // recorded the acquiring plugin; never reconstruct that owner from a driver + // name, and never override an explicit (even invalid) intent pin. + const pluginId = marker.schema === "paperclip.workspace-export-resume.v1" && marker.pluginId === undefined + ? lease.metadata?.pluginId : marker.pluginId; + if (typeof pluginId !== "string" || !pluginId || pluginId !== lease.metadata?.pluginId) return null; + return { ...marker, requestId: marker.requestId, resultId: marker.resultId, pluginId }; } /** A late cleanup receipt cannot rewrite a rebound lease or a newer attempt. */ diff --git a/server/src/services/native-runtime/native-workspace-export-retry.ts b/server/src/services/native-runtime/native-workspace-export-retry.ts index 2cb07e41c55..faf9d00e89f 100644 --- a/server/src/services/native-runtime/native-workspace-export-retry.ts +++ b/server/src/services/native-runtime/native-workspace-export-retry.ts @@ -95,7 +95,7 @@ export async function retryNativeWorkspaceExport(input: { const [lease] = await tx.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", releasedAt: now, failureReason: "workspace_export_resume_pending", updatedAt: now, metadata: { ...current.lease.metadata, remoteExecutionTermination: undefined, - [NATIVE_WORKSPACE_EXPORT_RESUME_KEY]: { schema: "paperclip.workspace-export-resume.v1", requestId, + [NATIVE_WORKSPACE_EXPORT_RESUME_KEY]: { schema: "paperclip.workspace-export-resume.v2", requestId, companyId: input.companyId, runId: input.runId, leaseId: current.lease.id, pluginId: current.lease.metadata?.pluginId, provider: current.lease.provider, providerLeaseId: current.lease.providerLeaseId, resultId: current.result.id },