mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-24 18:19:05 +02:00
fix(runtime): recover earlier export cleanup intents safely
Version new plugin-pinned intents and recover v1 from the exact lease recorded plugin without driver-key fallback or accepting mismatched explicit pins. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
47d435b762
commit
2bedd0f23b
5 files changed
+20
-11
No files matched your search
@@ -119,6 +119,9 @@ confirmed stopped receipt. No provider turn is created by this recovery.
|
||||
Cleanup and its readiness probe use the plugin ID recorded on that lease. Another
|
||||
plugin with the same provider name cannot take over; an unavailable original
|
||||
plugin defers cleanup without consuming an attempt.
|
||||
New intents use schema v2 with an explicit plugin pin. A v1 intent created before
|
||||
that field existed remains recoverable using only the plugin ID already recorded
|
||||
on its exact lease; an explicit mismatched pin is still rejected.
|
||||
|
||||
The opt-in `native-workspace-export-resume.live.test.ts` is a provider-boundary
|
||||
fault integration, separate from the browser Product E2E. After building the
|
||||
|
||||
@@ -5037,11 +5037,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
expect(await db.select().from(environmentLeases)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each(["stopped", "unconfirmed", "foreign_marker", "competing_owner", "live_request", "late_receipt", "duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing"])("recovers a pending export resume with stop-only cleanup after restart: %s", async outcome => {
|
||||
it.each(["stopped", "unconfirmed", "foreign_marker", "competing_owner", "live_request", "late_receipt", "duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing", "legacy_intent", "foreign_plugin_pin", "v2_missing_pin"])("recovers a pending export resume with stop-only cleanup after restart: %s", async outcome => {
|
||||
const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
|
||||
const lease = seeded.reusableLease;
|
||||
const requestId = randomUUID();
|
||||
if (["duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) {
|
||||
if (["duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing", "legacy_intent"].includes(outcome)) {
|
||||
const [owner] = await db.select().from(plugins).where(eq(plugins.id, seeded.pluginId));
|
||||
await db.insert(plugins).values({ ...owner, id: randomUUID(), pluginKey: "other.same-driver", packageName: "@other/same-driver",
|
||||
installOrder: -1, manifestJson: { ...owner.manifestJson, id: "other.same-driver" } });
|
||||
@@ -5050,8 +5050,8 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
}
|
||||
await db.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", metadata: {
|
||||
...lease.metadata,
|
||||
nativeWorkspaceExportResume: { schema: "paperclip.workspace-export-resume.v1", requestId, companyId: seeded.companyId,
|
||||
pluginId: seeded.pluginId,
|
||||
nativeWorkspaceExportResume: { schema: outcome === "v2_missing_pin" ? "paperclip.workspace-export-resume.v2" : "paperclip.workspace-export-resume.v1", requestId, companyId: seeded.companyId,
|
||||
...(["legacy_intent", "v2_missing_pin"].includes(outcome) ? {} : { pluginId: outcome === "foreign_plugin_pin" ? randomUUID() : seeded.pluginId }),
|
||||
runId: outcome === "foreign_marker" ? randomUUID() : lease.heartbeatRunId, leaseId: lease.id,
|
||||
provider: lease.provider, providerLeaseId: lease.providerLeaseId, resultId: randomUUID() },
|
||||
pendingCleanupAttemptId: requestId, pendingCleanupInFlight: true,
|
||||
@@ -5072,7 +5072,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
await heartbeatService(db, { environmentRuntime: restarted }).sweepPendingCleanupLeases({ backoffMs: 0 });
|
||||
expect(call.mock.calls.some((entry) => entry[1] === "environmentDestroyLease")).toBe(false);
|
||||
const persisted = (await environmentService(db).getLeaseById(lease.id))!;
|
||||
if (["stopped", "duplicate_key"].includes(outcome)) {
|
||||
if (["stopped", "duplicate_key", "legacy_intent"].includes(outcome)) {
|
||||
expect(call).toHaveBeenCalledWith(seeded.pluginId, "environmentReleaseLease", expect.objectContaining({ providerLeaseId: lease.providerLeaseId, cancelActiveWork: true }), expect.any(Number));
|
||||
expect(persisted).toMatchObject({ status: "expired", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "stopped", runId: lease.heartbeatRunId } } });
|
||||
expect(persisted.metadata?.nativeWorkspaceExportResume).toBeUndefined();
|
||||
@@ -5081,7 +5081,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
} else {
|
||||
expect(persisted).toMatchObject({ status: "pending_cleanup", cleanupStatus: "failed" });
|
||||
expect(persisted.metadata?.remoteExecutionTermination).toBeUndefined();
|
||||
if (["foreign_marker", "competing_owner", "live_request", "owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) expect(call).not.toHaveBeenCalled();
|
||||
if (["foreign_marker", "foreign_plugin_pin", "v2_missing_pin", "competing_owner", "live_request", "owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) expect(call).not.toHaveBeenCalled();
|
||||
if (["owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) {
|
||||
expect(persisted.metadata?.pendingCleanupAttemptId).toBe(requestId);
|
||||
}
|
||||
|
||||
@@ -2583,7 +2583,8 @@ function createSandboxEnvironmentDriver(
|
||||
)).limit(1);
|
||||
if (!current || current.status !== "pending_cleanup" || current.companyId !== input.lease.companyId
|
||||
|| current.metadata?.pendingCleanupAttemptId !== input.lease.metadata?.pendingCleanupAttemptId
|
||||
|| readNativeWorkspaceExportResume(current)?.requestId !== resumeIntent.requestId || otherOwner) {
|
||||
|| readNativeWorkspaceExportResume(current)?.requestId !== resumeIntent.requestId
|
||||
|| readNativeWorkspaceExportResume(current)?.pluginId !== resumeIntent.pluginId || otherOwner) {
|
||||
throw new Error("Workspace export resume ownership changed before cleanup.");
|
||||
}
|
||||
};
|
||||
|
||||
@@ -20,14 +20,19 @@ export function readNativeWorkspaceExportResume(lease: Lease) {
|
||||
const value = lease.metadata?.[NATIVE_WORKSPACE_EXPORT_RESUME_KEY];
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
const marker = value as Record<string, unknown>;
|
||||
if (marker.schema !== "paperclip.workspace-export-resume.v1"
|
||||
if (!["paperclip.workspace-export-resume.v1", "paperclip.workspace-export-resume.v2"].includes(String(marker.schema))
|
||||
|| marker.companyId !== lease.companyId || marker.runId !== lease.heartbeatRunId || !lease.heartbeatRunId
|
||||
|| marker.leaseId !== lease.id || marker.provider !== lease.provider || !lease.provider
|
||||
|| marker.providerLeaseId !== lease.providerLeaseId || !lease.providerLeaseId
|
||||
|| typeof marker.pluginId !== "string" || !marker.pluginId || marker.pluginId !== lease.metadata?.pluginId
|
||||
|| typeof marker.requestId !== "string" || !marker.requestId
|
||||
|| typeof marker.resultId !== "string" || !marker.resultId) return null;
|
||||
return marker;
|
||||
// v1 preceded the intent's explicit plugin pin. Its exact lease already
|
||||
// recorded the acquiring plugin; never reconstruct that owner from a driver
|
||||
// name, and never override an explicit (even invalid) intent pin.
|
||||
const pluginId = marker.schema === "paperclip.workspace-export-resume.v1" && marker.pluginId === undefined
|
||||
? lease.metadata?.pluginId : marker.pluginId;
|
||||
if (typeof pluginId !== "string" || !pluginId || pluginId !== lease.metadata?.pluginId) return null;
|
||||
return { ...marker, requestId: marker.requestId, resultId: marker.resultId, pluginId };
|
||||
}
|
||||
|
||||
/** A late cleanup receipt cannot rewrite a rebound lease or a newer attempt. */
|
||||
|
||||
@@ -95,7 +95,7 @@ export async function retryNativeWorkspaceExport(input: {
|
||||
const [lease] = await tx.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed",
|
||||
releasedAt: now, failureReason: "workspace_export_resume_pending", updatedAt: now,
|
||||
metadata: { ...current.lease.metadata, remoteExecutionTermination: undefined,
|
||||
[NATIVE_WORKSPACE_EXPORT_RESUME_KEY]: { schema: "paperclip.workspace-export-resume.v1", requestId,
|
||||
[NATIVE_WORKSPACE_EXPORT_RESUME_KEY]: { schema: "paperclip.workspace-export-resume.v2", requestId,
|
||||
companyId: input.companyId, runId: input.runId, leaseId: current.lease.id,
|
||||
pluginId: current.lease.metadata?.pluginId,
|
||||
provider: current.lease.provider, providerLeaseId: current.lease.providerLeaseId, resultId: current.result.id },
|
||||
|
||||
Reference in new issue
Block a user