From 2d939a92b21dcaf5c77c88b54d96784d2ddd0699 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 24 Sep 2026 21:48:15 -0700 Subject: [PATCH] Journal observed sandbox identity before retry deletion Co-Authored-By: Paperclip --- .../sandbox-providers/daytona/README.md | 4 +- .../daytona/src/plugin.test.ts | 47 +++++++++++++- .../sandbox-providers/daytona/src/plugin.ts | 50 ++++++++++----- .../sdk/src/environment-creation-cleanup.ts | 4 ++ packages/plugins/sdk/src/worker-rpc-host.ts | 3 +- .../environment-creation-cleanup.test.ts | 9 +-- .../src/__tests__/environment-runtime.test.ts | 62 +++++++++++++++++++ server/src/services/environment-runtime.ts | 57 ++++++++++++++--- 8 files changed, 200 insertions(+), 36 deletions(-) diff --git a/packages/plugins/sandbox-providers/daytona/README.md b/packages/plugins/sandbox-providers/daytona/README.md index 74922b18d60..76edf25e51d 100644 --- a/packages/plugins/sandbox-providers/daytona/README.md +++ b/packages/plugins/sandbox-providers/daytona/README.md @@ -27,8 +27,8 @@ Notes: - The current published Daytona SDK package is `@daytonaio/sdk`. - The driver supports both `snapshot`-based and `image`-based sandbox creation. If both are set, validation rejects the config as ambiguous. -- Each cold create uses a unique provider name and ownership labels. If creation fails after Daytona has allocated a sandbox, the driver looks up that exact name, checks every ownership label, and waits for deletion. Failed, missing, or timed-out lookups and failed deletions report unconfirmed cleanup with the provider name; they never return a usable lease. A missing lookup after an uncertain create is not proof that a delayed provider request cannot create a resource. -- For plugin-backed sandbox lease acquisition, unresolved creation cleanup crosses the worker RPC as a validated ownership envelope. The host records `pending_cleanup` before retrying deletion, and its existing cleanup sweep and durable spool preserve retries across controller restarts and environment deletion. Provider exceptions and resolved credentials are excluded from that envelope. This requires the matching host and plugin SDK update; probe and custom-image interactive-setup calls still report unconfirmed immediate cleanup without this lease-recovery path. +- Each cold create uses a unique provider name and ownership labels. If creation fails after Daytona has allocated a sandbox, the driver looks up that exact name, checks every ownership label, and waits for deletion. Failed, missing, or timed-out lookups and failed deletions report unconfirmed cleanup with the provider name; they never return a usable lease. A missing name lookup after an uncertain create is not proof that a delayed provider request cannot create a resource. +- For plugin-backed sandbox lease acquisition, unresolved creation cleanup crosses the worker RPC as a validated ownership envelope. The host records `pending_cleanup` before retrying deletion. If only a creation name is known, the provider first returns an ownership-verified sandbox ID without deleting it. The host saves that observation before authorizing deletion. Its existing cleanup sweep and durable spool preserve retries across controller restarts and environment deletion. A missing observed ID confirms cleanup after a lost deletion reply or database update; a name that has never been observed remains unresolved. Provider exceptions and resolved credentials are excluded from that envelope. This requires the matching host and plugin SDK update; probe and custom-image interactive-setup calls still report unconfirmed immediate cleanup without this lease-recovery path. - Reusable leases map to Daytona stop/start semantics. Non-reusable leases are deleted on release. A provider-resolved `target` does not change the identity of an existing sandbox. Release closes the same scoped lease that a later sentinel-verified resume reopens. - A sandbox record can survive the loss of its underlying container. Resume treats it as expired only when a fresh provider read confirms the exact missing-container error for that sandbox and marks it unrecoverable. Unknown errors and failed confirmation reads preserve the lease. The host still requires a verified native-runner backup before replacement. diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index b3c24ebf224..86b4fbfd650 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -549,6 +549,15 @@ describe("Daytona sandbox provider plugin", () => { return { error, cleanup: cleanup! }; } + async function journalObservation(cleanup: NonNullable>) { + const error = await plugin.definition.onEnvironmentDestroyLease!({ ...params, + providerLeaseId: cleanup.providerLeaseId, leaseMetadata: { failedCreateCleanup: cleanup }, + }).catch(error => error); + const observed = readEnvironmentCreationCleanupError(error); + expect(observed?.observedProviderLeaseId).toBeTruthy(); + return observed!; + } + it("hands non-secret cleanup ownership to the host when creation is uncertain", async () => { const { error, cleanup } = await unresolvedCreation(); expect(cleanup).toMatchObject({ companyId: params.companyId, environmentId: params.environmentId, runId: params.runId, @@ -569,8 +578,10 @@ describe("Daytona sandbox provider plugin", () => { expect(cleanup.labels).not.toHaveProperty("code-toolbox-language"); const orphan = ownedSandbox(); mockGet.mockResolvedValue(orphan); + const observed = await journalObservation(cleanup); + expect(orphan.delete).not.toHaveBeenCalled(); await expect(plugin.definition.onEnvironmentDestroyLease!({ ...params, - providerLeaseId: cleanup.providerLeaseId, leaseMetadata: { failedCreateCleanup: cleanup }, + providerLeaseId: cleanup.providerLeaseId, leaseMetadata: { failedCreateCleanup: observed }, })).resolves.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }); expect(orphan.delete).toHaveBeenCalledWith(10, true); }); @@ -578,9 +589,11 @@ describe("Daytona sandbox provider plugin", () => { it("retries a late-visible failed creation using its persisted ownership envelope", async () => { const { cleanup } = await unresolvedCreation(); const orphan = ownedSandbox(); mockGet.mockResolvedValue(orphan); + const observed = await journalObservation(cleanup); + expect(orphan.delete).not.toHaveBeenCalled(); await expect(plugin.definition.onEnvironmentDestroyLease!({ ...params, providerLeaseId: cleanup.providerLeaseId, - leaseMetadata: { failedCreateCleanup: cleanup } })).resolves.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }); - expect(mockGet).toHaveBeenLastCalledWith(cleanup.providerLeaseId); + leaseMetadata: { failedCreateCleanup: observed } })).resolves.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }); + expect(mockGet).toHaveBeenLastCalledWith(orphan.id); expect(orphan.delete).toHaveBeenCalledWith(10, true); expect(orphan.process.executeCommand).not.toHaveBeenCalled(); }); @@ -591,6 +604,34 @@ describe("Daytona sandbox provider plugin", () => { leaseMetadata: { failedCreateCleanup: cleanup } })).rejects.toThrow(); }); + it("accepts absence only after a matching provider ID was observed", async () => { + const { cleanup } = await unresolvedCreation(); + const orphan = ownedSandbox(); mockGet.mockResolvedValue(orphan); + const observed = await journalObservation(cleanup); + expect(orphan.delete).not.toHaveBeenCalled(); + mockGet.mockRejectedValue(new MockDaytonaNotFoundError("already deleted")); + await expect(plugin.definition.onEnvironmentDestroyLease!({ ...params, + providerLeaseId: cleanup.providerLeaseId, leaseMetadata: { failedCreateCleanup: observed }, + })).resolves.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }); + expect(mockGet).toHaveBeenLastCalledWith(orphan.id); + }); + + it("retains the observed ID when immediate deletion loses its acknowledgement", async () => { + let orphan: ReturnType; + mockGet.mockImplementation(async () => { + orphan = ownedSandbox(); + orphan.delete.mockRejectedValue(new Error("delete response lost")); + return orphan; + }); + const error = await plugin.definition.onEnvironmentAcquireLease!(params).catch(error => error); + const cleanup = readEnvironmentCreationCleanupError(error)!; + expect(cleanup.observedProviderLeaseId).toBe(orphan!.id); + mockGet.mockRejectedValue(new MockDaytonaNotFoundError("already deleted")); + await expect(plugin.definition.onEnvironmentDestroyLease!({ ...params, + providerLeaseId: cleanup.providerLeaseId, leaseMetadata: { failedCreateCleanup: cleanup }, + })).resolves.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }); + }); + it.each(["company", "account", "labels"])("fences failed-creation retries by %s", async (mismatch) => { const { cleanup } = await unresolvedCreation(); const orphan = ownedSandbox(); mockGet.mockResolvedValue(orphan); diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 4acbc2f9a01..8e1b19866f4 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -963,31 +963,32 @@ async function createSandbox( timeout: toTimeoutSeconds(config.timeoutMs), }); } catch (createError) { + const cleanup: PluginEnvironmentCreationCleanup = { + providerLeaseId: name, companyId: params.companyId, environmentId: params.environmentId, + ...("runId" in params ? { runId: params.runId } : {}), + attemptId, labels, accountFingerprint: sandboxAccountDiscriminator(config), + }; try { // A not-found lookup after an uncertain create is not a deletion receipt: // the provider may still materialize the request. Keep the name in the // error so cleanup can be reconciled rather than declaring success. - await destroyFailedCreation(config, { - providerLeaseId: name, companyId: params.companyId, environmentId: params.environmentId, - ...("runId" in params ? { runId: params.runId } : {}), - attemptId, labels, accountFingerprint: sandboxAccountDiscriminator(config), - }); + await destroyFailedCreation(config, cleanup); } catch (cleanupError) { throw new PluginEnvironmentCreationCleanupError([createError, cleanupError], - `Daytona sandbox creation failed; cleanup could not be confirmed for ${name}`, { - providerLeaseId: name, companyId: params.companyId, environmentId: params.environmentId, - ...("runId" in params ? { runId: params.runId } : {}), - attemptId, labels, accountFingerprint: sandboxAccountDiscriminator(config), - }); + `Daytona sandbox creation failed; cleanup could not be confirmed for ${name}`, cleanup); } throw createError; } } // Resolve by the immutable creation name, not through the admitted-lease cache: -// the SDK's returned sandbox ID differs from this provisional name. A 404 is -// still uncertain and must leave the host's pending-cleanup row eligible to retry. -async function destroyFailedCreation(config: DaytonaDriverConfig, cleanup: PluginEnvironmentCreationCleanup): Promise { +// the SDK's returned sandbox ID differs from this provisional name. Persist a +// verified provider ID before host-driven deletion. Its later absence is then +// conclusive, even if the deletion receipt or database update was lost. +async function destroyFailedCreation( + config: DaytonaDriverConfig, cleanup: PluginEnvironmentCreationCleanup, + requireDurableObservation = false, +): Promise { if (cleanup.providerLeaseId !== `paperclip-create-${cleanup.attemptId}` || cleanup.accountFingerprint !== sandboxAccountDiscriminator(config) || cleanup.labels["paperclip-provider"] !== "daytona" || @@ -998,10 +999,27 @@ async function destroyFailedCreation(config: DaytonaDriverConfig, cleanup: Plugi throw new Error("Failed-create sandbox ownership does not match"); } const client = createDaytonaClient(config); - const orphan = await withLivenessTimeout("sandbox.failedCreateLookup", 10_000, () => client.get(cleanup.providerLeaseId)); - if (orphan.name !== cleanup.providerLeaseId || Object.entries(cleanup.labels).some(([key, value]) => orphan.labels?.[key] !== value)) { + let orphan: Sandbox; + try { + orphan = await withLivenessTimeout("sandbox.failedCreateLookup", 10_000, + () => client.get(cleanup.observedProviderLeaseId ?? cleanup.providerLeaseId)); + } catch (error) { + // A name that has never been observed may still materialize. An ID already + // observed with exact ownership cannot materialize as a new allocation. + if (cleanup.observedProviderLeaseId && error instanceof DaytonaNotFoundError) return; + throw error; + } + if ((cleanup.observedProviderLeaseId && orphan.id !== cleanup.observedProviderLeaseId) || + orphan.name !== cleanup.providerLeaseId || Object.entries(cleanup.labels).some(([key, value]) => orphan.labels?.[key] !== value)) { throw new Error("Failed-create sandbox ownership does not match"); } + if (!cleanup.observedProviderLeaseId) { + cleanup.observedProviderLeaseId = orphan.id; + if (requireDurableObservation) { + // No destructive call yet: the host journals the observation and retries. + throw new PluginEnvironmentCreationCleanupError([], "Persist observed sandbox identity before cleanup", cleanup); + } + } await withLivenessTimeout("sandbox.failedCreateDelete", 15_000, () => orphan.delete(10, true)); } @@ -2460,7 +2478,7 @@ const plugin = definePlugin({ cleanup.providerLeaseId !== params.providerLeaseId) { throw new Error("Failed-create sandbox cleanup scope does not match"); } - await destroyFailedCreation(config, cleanup); + await destroyFailedCreation(config, cleanup, true); return { providerLeaseId: params.providerLeaseId, state: "destroyed" }; } const scope: SandboxScope = { diff --git a/packages/plugins/sdk/src/environment-creation-cleanup.ts b/packages/plugins/sdk/src/environment-creation-cleanup.ts index 792386f5dbf..8dcdc1f768f 100644 --- a/packages/plugins/sdk/src/environment-creation-cleanup.ts +++ b/packages/plugins/sdk/src/environment-creation-cleanup.ts @@ -1,6 +1,8 @@ /** Non-secret ownership evidence for an allocation whose creation failed. */ export interface PluginEnvironmentCreationCleanup { providerLeaseId: string; + /** Provider ID observed with matching ownership, distinct from a provisional creation name. */ + observedProviderLeaseId?: string; companyId: string; environmentId: string; runId?: string; @@ -42,12 +44,14 @@ export function readEnvironmentCreationCleanupError(error: unknown): PluginEnvir const identifier = (value: unknown): value is string => typeof value === "string" && /^[a-zA-Z0-9_-]{1,200}$/.test(value); if (![row.providerLeaseId, row.companyId, row.environmentId, row.attemptId].every(identifier)) return null; if (row.runId !== undefined && !identifier(row.runId)) return null; + if (row.observedProviderLeaseId !== undefined && !identifier(row.observedProviderLeaseId)) return null; if (typeof row.accountFingerprint !== "string" || !/^[a-f0-9]{64}$/.test(row.accountFingerprint)) return null; if (!row.labels || typeof row.labels !== "object" || Array.isArray(row.labels)) return null; const labels = row.labels as Record; if (Object.keys(labels).length > 16 || Object.entries(labels).some(([key, value]) => !/^paperclip-[a-z-]+$/.test(key) || !identifier(value))) return null; return { providerLeaseId: row.providerLeaseId as string, + ...(typeof row.observedProviderLeaseId === "string" ? { observedProviderLeaseId: row.observedProviderLeaseId } : {}), companyId: row.companyId as string, environmentId: row.environmentId as string, ...(typeof row.runId === "string" ? { runId: row.runId } : {}), diff --git a/packages/plugins/sdk/src/worker-rpc-host.ts b/packages/plugins/sdk/src/worker-rpc-host.ts index 1e98a7a8267..edf9634901c 100644 --- a/packages/plugins/sdk/src/worker-rpc-host.ts +++ b/packages/plugins/sdk/src/worker-rpc-host.ts @@ -1573,7 +1573,8 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost : PLUGIN_RPC_ERROR_CODES.WORKER_ERROR; sendMessage(createErrorResponse(id, errorCode, errorMessage, - method === "environmentAcquireLease" ? environmentCreationCleanupErrorData(err) : undefined)); + method === "environmentAcquireLease" || method === "environmentDestroyLease" + ? environmentCreationCleanupErrorData(err) : undefined)); } } diff --git a/packages/plugins/sdk/tests/environment-creation-cleanup.test.ts b/packages/plugins/sdk/tests/environment-creation-cleanup.test.ts index b91486dfc15..e75f7877ee0 100644 --- a/packages/plugins/sdk/tests/environment-creation-cleanup.test.ts +++ b/packages/plugins/sdk/tests/environment-creation-cleanup.test.ts @@ -21,6 +21,7 @@ async function invoke(method: string, error: Error) { async setup() {}, async onEnvironmentAcquireLease() { throw error; }, async onEnvironmentProbe() { throw error; }, + async onEnvironmentDestroyLease() { throw error; }, }) }); try { const response = new Promise((resolve) => { @@ -37,17 +38,17 @@ async function invoke(method: string, error: Error) { } describe("failed environment creation ownership", () => { - it("crosses the actual worker RPC as allowlisted ownership only", async () => { + it.each(["environmentAcquireLease", "environmentDestroyLease"])("crosses %s RPC as allowlisted ownership only", async (method) => { const cleanup = { ...ownership, accessToken: "secret-token", config: { apiKey: "secret-key" } }; const error = new PluginEnvironmentCreationCleanupError([new Error("secret-provider-response")], "Cleanup required", cleanup); - const response = await invoke("environmentAcquireLease", error); + const response = await invoke(method, error); expect("error" in response && response.error).toMatchObject({ data: { schema, cleanup: ownership } }); if (!("error" in response) || !response.error) throw new Error("Expected RPC error"); expect(readEnvironmentCreationCleanupError(new JsonRpcCallError(response.error))).toEqual(ownership); expect(JSON.stringify(response)).not.toContain("secret-"); }); - it.each(["environmentAcquireLease", "environmentProbe"])("does not forward arbitrary error data for %s", async (method) => { + it.each(["environmentAcquireLease", "environmentDestroyLease", "environmentProbe"])("does not forward arbitrary error data for %s", async (method) => { const response = await invoke(method, Object.assign(new Error("provider failure"), { data: { apiKey: "secret-key", schema, cleanup: ownership } })); expect("error" in response && response.error).not.toHaveProperty("data"); }); @@ -59,7 +60,7 @@ describe("failed environment creation ownership", () => { it.each([ { providerLeaseId: "../other" }, { companyId: "" }, { environmentId: null }, { attemptId: "a".repeat(201) }, - { runId: "other/run" }, { accountFingerprint: "secret-key" }, { labels: { apiKey: "secret-key" } }, + { runId: "other/run" }, { observedProviderLeaseId: "../other" }, { accountFingerprint: "secret-key" }, { labels: { apiKey: "secret-key" } }, { labels: { "paperclip-provider": "Bearer secret-key" } }, { labels: [] }, { labels: Object.fromEntries(Array.from({ length: 17 }, (_, i) => [`paperclip-${String.fromCharCode(97 + i)}`, "x"])) }, ])("rejects malformed evidence %j", (invalid) => { diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index bb626e4ffd3..285d01c4dbe 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -712,6 +712,68 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { } }); + it.each(["deleted receipt lost", "observation response lost", "journal write failed", "foreign observation"])( + "journals observed creation identity before deletion and recovers: %s", async (fault) => { + const seeded = await seedReusablePluginSandboxLease(); + await environmentService(db).releaseLease(seeded.reusableLease.id, "expired"); + const cleanup = { + providerLeaseId: `paperclip-create-${randomUUID()}`, attemptId: randomUUID(), + companyId: seeded.companyId, environmentId: seeded.environment.id, runId: seeded.runId, + accountFingerprint: "a".repeat(64), labels: { "paperclip-provider": "fake-plugin" }, + }; + const observed = { ...cleanup, observedProviderLeaseId: "verified-provider-id" }; + const failure = Object.assign(new Error("Creation uncertain"), { + data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup }, + }); + let injected = false; + let deleted = false; + const call = vi.fn(async (_id: string, method: string, params: Record) => { + if (method === "environmentAcquireLease") throw failure; + if (method !== "environmentDestroyLease") throw new Error(`Unexpected ${method}`); + const evidence = (params.leaseMetadata as { failedCreateCleanup: typeof observed }).failedCreateCleanup; + if (!evidence.observedProviderLeaseId) { + expect(deleted).toBe(false); + if (!injected && fault === "observation response lost") { + injected = true; + throw new Error("Lost observation response"); + } + if (!injected && fault === "foreign observation") { + injected = true; + throw Object.assign(new Error("Foreign observation"), { + data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup: { ...observed, companyId: "another-company" } }, + }); + } + if (!injected && fault === "journal write failed") { + injected = true; + vi.spyOn(db, "update").mockImplementationOnce(() => { throw new Error("Database unavailable"); }); + } + throw Object.assign(new Error("Persist observed sandbox identity before cleanup"), { + data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup: observed }, + }); + } + const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, cleanup.providerLeaseId)); + expect(rows).toHaveLength(1); + expect(rows[0].metadata).toMatchObject({ failedCreateCleanup: observed }); + if (!deleted) { + deleted = true; + if (fault === "deleted receipt lost") throw new Error("Deletion succeeded but reply lost"); + } + return { providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }; + }); + const worker = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }), call } as unknown as PluginWorkerManager; + const runtime = environmentRuntimeService(db, { pluginWorkerManager: worker }); + await expect(runtime.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, + issueId: null, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: null })).rejects.toBe(failure); + const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, cleanup.providerLeaseId)); + expect(rows).toHaveLength(1); + expect(rows[0].status).toBe("pending_cleanup"); + expect(deleted).toBe(fault === "deleted receipt lost"); + const restarted = environmentRuntimeService(db, { pluginWorkerManager: worker }); + await expect(restarted.retryPendingSandboxTeardown({ environment: null, lease: rows[0]! })).resolves.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }); + expect(deleted).toBe(true); + }, + ); + it("resumes the same provider lease on the next per-turn run", async () => { const seeded = await seedReusablePluginSandboxLease(); const workerManager = { diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 64c7bb0beb7..e5956c1e393 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1556,6 +1556,37 @@ function createSandboxEnvironmentDriver( }); }; + // Resolve an uncertain creation to a verified provider ID, journal it, then + // authorize deletion. This makes a later missing-ID response conclusive even + // when the destructive RPC reply or the final lease update was lost. + const destroyWithCreationObservation = async (input: { + leaseId: string | null; + metadata: Record; + teardown: (metadata: Record) => Promise; + }): Promise => { + try { + return await input.teardown(input.metadata); + } catch (error) { + const previous = readEnvironmentCreationCleanupError({ data: { + schema: "paperclip/environment-creation-cleanup/v1", cleanup: input.metadata.failedCreateCleanup, + } }); + const observed = readEnvironmentCreationCleanupError(error); + const scopeKeys = ["providerLeaseId", "companyId", "environmentId", "runId", "attemptId", "accountFingerprint"] as const; + if (!input.leaseId || !previous || previous.observedProviderLeaseId || !observed?.observedProviderLeaseId || + scopeKeys.some((key) => previous[key] !== observed[key]) || + Object.keys(previous.labels).length !== Object.keys(observed.labels).length || + Object.entries(previous.labels).some(([key, value]) => observed.labels[key] !== value)) throw error; + const lease = await environmentsSvc.getLeaseById(input.leaseId); + if (!lease || lease.companyId !== previous.companyId || lease.providerLeaseId !== previous.providerLeaseId || + (lease.environmentId !== null && lease.environmentId !== previous.environmentId)) throw error; + const metadata = { ...lease.metadata, failedCreateCleanup: observed }; + // A failed write must stop here; no destructive call is permitted until + // the provider identity can survive a host or worker restart. + if (!await environmentsSvc.updateLeaseMetadata(lease.id, metadata)) throw error; + return await input.teardown(metadata); + } + }; + // Clean up an orphan after a rejected insert or uncertain provider creation. // The acquire provisioned a remote sandbox, then the insert rejected (a // foreign-company binding), so no lease row tracks the live sandbox. This @@ -1579,14 +1610,14 @@ function createSandboxEnvironmentDriver( record: DeferredOrphanCleanupRecord; cause: unknown; canTeardown: boolean; - teardown: () => Promise; + teardown: (pendingLeaseId: string | null) => Promise; }): Promise => { const durable = await tryWriteDurablePendingCleanup(input.record); let teardownFailed = !input.canTeardown; let receipt: unknown; if (!teardownFailed) { try { - receipt = await input.teardown(); + receipt = await input.teardown(durable.leaseId); } catch { teardownFailed = true; } @@ -2081,12 +2112,15 @@ function createSandboxEnvironmentDriver( }, cause: error, canTeardown: pluginWorkerManager.isRunning(pluginProvider.resolved.plugin.id), - teardown: () => pluginWorkerManager.call(pluginProvider.resolved.plugin.id, "environmentDestroyLease", { - driverKey: parsed.config.provider, companyId: input.companyId, - environmentId: input.environment.id, issueId: input.issueId, - config: workerConfig, providerLeaseId: cleanup.providerLeaseId, - leaseMetadata: { failedCreateCleanup: cleanup }, - }, resolvePluginSandboxRpcTimeoutMs(workerConfig)), + teardown: (pendingLeaseId) => destroyWithCreationObservation({ + leaseId: pendingLeaseId, metadata: cleanupMetadata, + teardown: (metadata) => pluginWorkerManager.call(pluginProvider.resolved.plugin.id, "environmentDestroyLease", { + driverKey: parsed.config.provider, companyId: input.companyId, + environmentId: input.environment.id, issueId: input.issueId, + config: workerConfig, providerLeaseId: cleanup.providerLeaseId, + leaseMetadata: metadata, + }, resolvePluginSandboxRpcTimeoutMs(workerConfig)), + }), }); if (cleanupConfirmed) { throw new Error("Sandbox creation failed; allocated sandbox cleanup was confirmed.", { cause: error }); @@ -2582,7 +2616,7 @@ function createSandboxEnvironmentDriver( (input.lease.environmentId !== null && failedCreation.environmentId !== input.lease.environmentId))) { throw new Error("Pending creation cleanup ownership does not match the lease"); } - return await pluginWorkerManager.call( + const teardown = (metadata: Record) => pluginWorkerManager.call( pluginProvider.resolved.plugin.id, "environmentDestroyLease", { @@ -2592,10 +2626,13 @@ function createSandboxEnvironmentDriver( issueId: input.lease.issueId, config: workerConfig, providerLeaseId: input.lease.providerLeaseId, - leaseMetadata: input.lease.metadata ?? undefined, + leaseMetadata: metadata, }, resolvePluginSandboxRpcTimeoutMs(workerConfig), ); + return failedCreation + ? await destroyWithCreationObservation({ leaseId: input.lease.id, metadata: input.lease.metadata ?? {}, teardown }) + : await teardown(input.lease.metadata ?? {}); } // Built-in provider path. Resolve the recorded config secrets through the