fix(runtime): recover earlier export cleanup intents safely

Version new plugin-pinned intents and recover v1 from the exact lease recorded plugin without driver-key fallback or accepting mismatched explicit pins.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-27 22:07:41 -05:00
1 parent 47d435b762
commit 2bedd0f23b
5 files changed
+20 -11

No files matched your search

@@ -119,6 +119,9 @@ confirmed stopped receipt. No provider turn is created by this recovery.
Cleanup and its readiness probe use the plugin ID recorded on that lease. Another
plugin with the same provider name cannot take over; an unavailable original
plugin defers cleanup without consuming an attempt.
New intents use schema v2 with an explicit plugin pin. A v1 intent created before
that field existed remains recoverable using only the plugin ID already recorded
on its exact lease; an explicit mismatched pin is still rejected.
The opt-in `native-workspace-export-resume.live.test.ts` is a provider-boundary
fault integration, separate from the browser Product E2E. After building the
@@ -5037,11 +5037,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
expect(await db.select().from(environmentLeases)).toHaveLength(1);
});
it.each(["stopped", "unconfirmed", "foreign_marker", "competing_owner", "live_request", "late_receipt", "duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing"])("recovers a pending export resume with stop-only cleanup after restart: %s", async outcome => {
it.each(["stopped", "unconfirmed", "foreign_marker", "competing_owner", "live_request", "late_receipt", "duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing", "legacy_intent", "foreign_plugin_pin", "v2_missing_pin"])("recovers a pending export resume with stop-only cleanup after restart: %s", async outcome => {
const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
const lease = seeded.reusableLease;
const requestId = randomUUID();
if (["duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) {
if (["duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing", "legacy_intent"].includes(outcome)) {
const [owner] = await db.select().from(plugins).where(eq(plugins.id, seeded.pluginId));
await db.insert(plugins).values({ ...owner, id: randomUUID(), pluginKey: "other.same-driver", packageName: "@other/same-driver",
installOrder: -1, manifestJson: { ...owner.manifestJson, id: "other.same-driver" } });
@@ -5050,8 +5050,8 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
}
await db.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", metadata: {
...lease.metadata,
nativeWorkspaceExportResume: { schema: "paperclip.workspace-export-resume.v1", requestId, companyId: seeded.companyId,
pluginId: seeded.pluginId,
nativeWorkspaceExportResume: { schema: outcome === "v2_missing_pin" ? "paperclip.workspace-export-resume.v2" : "paperclip.workspace-export-resume.v1", requestId, companyId: seeded.companyId,
...(["legacy_intent", "v2_missing_pin"].includes(outcome) ? {} : { pluginId: outcome === "foreign_plugin_pin" ? randomUUID() : seeded.pluginId }),
runId: outcome === "foreign_marker" ? randomUUID() : lease.heartbeatRunId, leaseId: lease.id,
provider: lease.provider, providerLeaseId: lease.providerLeaseId, resultId: randomUUID() },
pendingCleanupAttemptId: requestId, pendingCleanupInFlight: true,
@@ -5072,7 +5072,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
await heartbeatService(db, { environmentRuntime: restarted }).sweepPendingCleanupLeases({ backoffMs: 0 });
expect(call.mock.calls.some((entry) => entry[1] === "environmentDestroyLease")).toBe(false);
const persisted = (await environmentService(db).getLeaseById(lease.id))!;
if (["stopped", "duplicate_key"].includes(outcome)) {
if (["stopped", "duplicate_key", "legacy_intent"].includes(outcome)) {
expect(call).toHaveBeenCalledWith(seeded.pluginId, "environmentReleaseLease", expect.objectContaining({ providerLeaseId: lease.providerLeaseId, cancelActiveWork: true }), expect.any(Number));
expect(persisted).toMatchObject({ status: "expired", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "stopped", runId: lease.heartbeatRunId } } });
expect(persisted.metadata?.nativeWorkspaceExportResume).toBeUndefined();
@@ -5081,7 +5081,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
} else {
expect(persisted).toMatchObject({ status: "pending_cleanup", cleanupStatus: "failed" });
expect(persisted.metadata?.remoteExecutionTermination).toBeUndefined();
if (["foreign_marker", "competing_owner", "live_request", "owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) expect(call).not.toHaveBeenCalled();
if (["foreign_marker", "foreign_plugin_pin", "v2_missing_pin", "competing_owner", "live_request", "owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) expect(call).not.toHaveBeenCalled();
if (["owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) {
expect(persisted.metadata?.pendingCleanupAttemptId).toBe(requestId);
}
+2 -1
View File
@@ -2583,7 +2583,8 @@ function createSandboxEnvironmentDriver(
)).limit(1);
if (!current || current.status !== "pending_cleanup" || current.companyId !== input.lease.companyId
|| current.metadata?.pendingCleanupAttemptId !== input.lease.metadata?.pendingCleanupAttemptId
|| readNativeWorkspaceExportResume(current)?.requestId !== resumeIntent.requestId || otherOwner) {
|| readNativeWorkspaceExportResume(current)?.requestId !== resumeIntent.requestId
|| readNativeWorkspaceExportResume(current)?.pluginId !== resumeIntent.pluginId || otherOwner) {
throw new Error("Workspace export resume ownership changed before cleanup.");
}
};
@@ -20,14 +20,19 @@ export function readNativeWorkspaceExportResume(lease: Lease) {
const value = lease.metadata?.[NATIVE_WORKSPACE_EXPORT_RESUME_KEY];
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
const marker = value as Record<string, unknown>;
if (marker.schema !== "paperclip.workspace-export-resume.v1"
if (!["paperclip.workspace-export-resume.v1", "paperclip.workspace-export-resume.v2"].includes(String(marker.schema))
|| marker.companyId !== lease.companyId || marker.runId !== lease.heartbeatRunId || !lease.heartbeatRunId
|| marker.leaseId !== lease.id || marker.provider !== lease.provider || !lease.provider
|| marker.providerLeaseId !== lease.providerLeaseId || !lease.providerLeaseId
|| typeof marker.pluginId !== "string" || !marker.pluginId || marker.pluginId !== lease.metadata?.pluginId
|| typeof marker.requestId !== "string" || !marker.requestId
|| typeof marker.resultId !== "string" || !marker.resultId) return null;
return marker;
// v1 preceded the intent's explicit plugin pin. Its exact lease already
// recorded the acquiring plugin; never reconstruct that owner from a driver
// name, and never override an explicit (even invalid) intent pin.
const pluginId = marker.schema === "paperclip.workspace-export-resume.v1" && marker.pluginId === undefined
? lease.metadata?.pluginId : marker.pluginId;
if (typeof pluginId !== "string" || !pluginId || pluginId !== lease.metadata?.pluginId) return null;
return { ...marker, requestId: marker.requestId, resultId: marker.resultId, pluginId };
}
/** A late cleanup receipt cannot rewrite a rebound lease or a newer attempt. */
@@ -95,7 +95,7 @@ export async function retryNativeWorkspaceExport(input: {
const [lease] = await tx.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed",
releasedAt: now, failureReason: "workspace_export_resume_pending", updatedAt: now,
metadata: { ...current.lease.metadata, remoteExecutionTermination: undefined,
[NATIVE_WORKSPACE_EXPORT_RESUME_KEY]: { schema: "paperclip.workspace-export-resume.v1", requestId,
[NATIVE_WORKSPACE_EXPORT_RESUME_KEY]: { schema: "paperclip.workspace-export-resume.v2", requestId,
companyId: input.companyId, runId: input.runId, leaseId: current.lease.id,
pluginId: current.lease.metadata?.pluginId,
provider: current.lease.provider, providerLeaseId: current.lease.providerLeaseId, resultId: current.result.id },