diff --git a/packages/plugins/sandbox-providers/daytona/README.md b/packages/plugins/sandbox-providers/daytona/README.md index e8dbbc436f8..74922b18d60 100644 --- a/packages/plugins/sandbox-providers/daytona/README.md +++ b/packages/plugins/sandbox-providers/daytona/README.md @@ -28,6 +28,7 @@ Notes: - The current published Daytona SDK package is `@daytonaio/sdk`. - The driver supports both `snapshot`-based and `image`-based sandbox creation. If both are set, validation rejects the config as ambiguous. - Each cold create uses a unique provider name and ownership labels. If creation fails after Daytona has allocated a sandbox, the driver looks up that exact name, checks every ownership label, and waits for deletion. Failed, missing, or timed-out lookups and failed deletions report unconfirmed cleanup with the provider name; they never return a usable lease. A missing lookup after an uncertain create is not proof that a delayed provider request cannot create a resource. +- For plugin-backed sandbox lease acquisition, unresolved creation cleanup crosses the worker RPC as a validated ownership envelope. The host records `pending_cleanup` before retrying deletion, and its existing cleanup sweep and durable spool preserve retries across controller restarts and environment deletion. Provider exceptions and resolved credentials are excluded from that envelope. This requires the matching host and plugin SDK update; probe and custom-image interactive-setup calls still report unconfirmed immediate cleanup without this lease-recovery path. - Reusable leases map to Daytona stop/start semantics. Non-reusable leases are deleted on release. A provider-resolved `target` does not change the identity of an existing sandbox. Release closes the same scoped lease that a later sentinel-verified resume reopens. - A sandbox record can survive the loss of its underlying container. Resume treats it as expired only when a fresh provider read confirms the exact missing-container error for that sandbox and marks it unrecoverable. Unknown errors and failed confirmation reads preserve the lease. The host still requires a verified native-runner backup before replacement. diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index 939ac0066fa..5380d120e2b 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -36,6 +36,7 @@ import plugin, { __setDaytonaPluginContextForTest, } from "./plugin.js"; import type { PluginContext } from "@paperclipai/plugin-sdk"; +import { environmentCreationCleanupErrorData, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk"; import manifest from "./manifest.js"; import { parseTarVerboseListingLine, splitLinkEntryOnce } from "./file-sync.js"; @@ -540,6 +541,50 @@ describe("Daytona sandbox provider plugin", () => { return { ...createMockSandbox(), name: requested.name, labels: { ...requested.labels } }; } + async function unresolvedCreation() { + mockGet.mockRejectedValue(new MockDaytonaNotFoundError("not visible yet")); + const error = await plugin.definition.onEnvironmentAcquireLease!(params).catch(error => error); + const cleanup = readEnvironmentCreationCleanupError(error); + expect(cleanup).not.toBeNull(); + return { error, cleanup: cleanup! }; + } + + it("hands non-secret cleanup ownership to the host when creation is uncertain", async () => { + const { error, cleanup } = await unresolvedCreation(); + expect(cleanup).toMatchObject({ companyId: params.companyId, environmentId: params.environmentId, runId: params.runId, + providerLeaseId: mockCreate.mock.calls[0][0].name, labels: mockCreate.mock.calls[0][0].labels }); + expect(cleanup.accountFingerprint).toMatch(/^[a-f0-9]{64}$/); + expect(JSON.stringify(environmentCreationCleanupErrorData(error))).not.toContain("host-key"); + expect(JSON.stringify(environmentCreationCleanupErrorData(error))).not.toContain(createError.message); + }); + + it("retries a late-visible failed creation using its persisted ownership envelope", async () => { + const { cleanup } = await unresolvedCreation(); + const orphan = ownedSandbox(); mockGet.mockResolvedValue(orphan); + await expect(plugin.definition.onEnvironmentDestroyLease!({ ...params, providerLeaseId: cleanup.providerLeaseId, + leaseMetadata: { failedCreateCleanup: cleanup } })).resolves.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }); + expect(mockGet).toHaveBeenLastCalledWith(cleanup.providerLeaseId); + expect(orphan.delete).toHaveBeenCalledWith(10, true); + expect(orphan.process.executeCommand).not.toHaveBeenCalled(); + }); + + it("keeps a failed-creation retry unresolved while the allocation is still not visible", async () => { + const { cleanup } = await unresolvedCreation(); + await expect(plugin.definition.onEnvironmentDestroyLease!({ ...params, providerLeaseId: cleanup.providerLeaseId, + leaseMetadata: { failedCreateCleanup: cleanup } })).rejects.toThrow(); + }); + + it.each(["company", "account", "labels"])("fences failed-creation retries by %s", async (mismatch) => { + const { cleanup } = await unresolvedCreation(); + const orphan = ownedSandbox(); mockGet.mockResolvedValue(orphan); + if (mismatch === "labels") orphan.labels["paperclip-run-id"] = "foreign-run"; + const retry = { ...params, providerLeaseId: cleanup.providerLeaseId, leaseMetadata: { failedCreateCleanup: cleanup } }; + if (mismatch === "company") retry.companyId = "foreign-company"; + if (mismatch === "account") process.env.DAYTONA_API_KEY = "another-account-key"; + await expect(plugin.definition.onEnvironmentDestroyLease!(retry)).rejects.toThrow(); + expect(orphan.delete).not.toHaveBeenCalled(); + }); + it("deletes the exact sandbox left behind when create times out, then preserves the create error", async () => { let orphan: ReturnType; mockGet.mockImplementation(async () => (orphan = ownedSandbox())); diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index b7714494536..b153ff935c5 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -9,9 +9,10 @@ import type { Resources, Sandbox, } from "@daytonaio/sdk"; -import { decodeChannelBytes, definePlugin, NOOP_PLUGIN_TRACER } from "@paperclipai/plugin-sdk"; +import { decodeChannelBytes, definePlugin, NOOP_PLUGIN_TRACER, PluginEnvironmentCreationCleanupError, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk"; import type { PluginContext, + PluginEnvironmentCreationCleanup, PluginTracer, PluginEnvironmentAcquireLeaseParams, PluginEnvironmentCancelInteractiveSetupParams, @@ -964,21 +965,44 @@ async function createSandbox( // A not-found lookup after an uncertain create is not a deletion receipt: // the provider may still materialize the request. Keep the name in the // error so cleanup can be reconciled rather than declaring success. - const orphan = await withLivenessTimeout("sandbox.failedCreateLookup", 10_000, () => client.get(name)); - if (orphan.name !== name || Object.entries(labels).some(([key, value]) => orphan.labels?.[key] !== value)) { - throw new Error("Failed-create sandbox ownership does not match"); - } - // Wait for provider deletion, rather than treating a deletion request as - // a cleanup receipt. Never run tools or install credentials in this sandbox. - await withLivenessTimeout("sandbox.failedCreateDelete", 15_000, () => orphan.delete(10, true)); + await destroyFailedCreation(config, { + providerLeaseId: name, companyId: params.companyId, environmentId: params.environmentId, + ...("runId" in params ? { runId: params.runId } : {}), + attemptId, labels, accountFingerprint: sandboxAccountDiscriminator(config), + }); } catch (cleanupError) { - throw new AggregateError([createError, cleanupError], - `Daytona sandbox creation failed; cleanup could not be confirmed for ${name}`); + throw new PluginEnvironmentCreationCleanupError([createError, cleanupError], + `Daytona sandbox creation failed; cleanup could not be confirmed for ${name}`, { + providerLeaseId: name, companyId: params.companyId, environmentId: params.environmentId, + ...("runId" in params ? { runId: params.runId } : {}), + attemptId, labels, accountFingerprint: sandboxAccountDiscriminator(config), + }); } throw createError; } } +// Resolve by the immutable creation name, not through the admitted-lease cache: +// the SDK's returned sandbox ID differs from this provisional name. A 404 is +// still uncertain and must leave the host's pending-cleanup row eligible to retry. +async function destroyFailedCreation(config: DaytonaDriverConfig, cleanup: PluginEnvironmentCreationCleanup): Promise { + if (cleanup.providerLeaseId !== `paperclip-create-${cleanup.attemptId}` || + cleanup.accountFingerprint !== sandboxAccountDiscriminator(config) || + cleanup.labels["paperclip-provider"] !== "daytona" || + cleanup.labels["paperclip-create-attempt"] !== cleanup.attemptId || + cleanup.labels["paperclip-company-id"] !== cleanup.companyId || + cleanup.labels["paperclip-environment-id"] !== cleanup.environmentId || + (cleanup.runId !== undefined && cleanup.labels["paperclip-run-id"] !== cleanup.runId)) { + throw new Error("Failed-create sandbox ownership does not match"); + } + const client = createDaytonaClient(config); + const orphan = await withLivenessTimeout("sandbox.failedCreateLookup", 10_000, () => client.get(cleanup.providerLeaseId)); + if (orphan.name !== cleanup.providerLeaseId || Object.entries(cleanup.labels).some(([key, value]) => orphan.labels?.[key] !== value)) { + throw new Error("Failed-create sandbox ownership does not match"); + } + await withLivenessTimeout("sandbox.failedCreateDelete", 15_000, () => orphan.delete(10, true)); +} + // ─── Per-lease started-sandbox handle cache ────────────────────────────────── // Memoize the started Daytona `Sandbox` handle so repeated exec/sync/resume/ // teardown calls on one lease skip the per-call `client.get(sandboxId)` REST @@ -2426,6 +2450,17 @@ const plugin = definePlugin({ ): Promise { if (!params.providerLeaseId) return; const config = parseDriverConfig(params.config); + if (params.leaseMetadata?.failedCreateCleanup !== undefined) { + const cleanup = readEnvironmentCreationCleanupError({ data: { + schema: "paperclip/environment-creation-cleanup/v1", cleanup: params.leaseMetadata.failedCreateCleanup, + } }); + if (!cleanup || cleanup.companyId !== params.companyId || cleanup.environmentId !== params.environmentId || + cleanup.providerLeaseId !== params.providerLeaseId) { + throw new Error("Failed-create sandbox cleanup scope does not match"); + } + await destroyFailedCreation(config, cleanup); + return { providerLeaseId: params.providerLeaseId, state: "destroyed" }; + } const scope: SandboxScope = { driverKey: params.driverKey, companyId: params.companyId, diff --git a/packages/plugins/sdk/src/environment-creation-cleanup.ts b/packages/plugins/sdk/src/environment-creation-cleanup.ts new file mode 100644 index 00000000000..792386f5dbf --- /dev/null +++ b/packages/plugins/sdk/src/environment-creation-cleanup.ts @@ -0,0 +1,58 @@ +/** Non-secret ownership evidence for an allocation whose creation failed. */ +export interface PluginEnvironmentCreationCleanup { + providerLeaseId: string; + companyId: string; + environmentId: string; + runId?: string; + attemptId: string; + accountFingerprint: string; + labels: Record; +} + +const SCHEMA = "paperclip/environment-creation-cleanup/v1"; + +/** + * Never serialize the provider exceptions: they may contain credentials. + * The host persists only this explicit ownership envelope, then retries teardown. + */ +export class PluginEnvironmentCreationCleanupError extends AggregateError { + readonly cleanup: PluginEnvironmentCreationCleanup; + + constructor(errors: unknown[], message: string, cleanup: PluginEnvironmentCreationCleanup) { + super(errors, message); + this.name = "PluginEnvironmentCreationCleanupError"; + this.cleanup = cleanup; + } +} + +export function environmentCreationCleanupErrorData(error: unknown): unknown { + if (!(error instanceof PluginEnvironmentCreationCleanupError)) return undefined; + const cleanup = readEnvironmentCreationCleanupError(error); + return cleanup ? { schema: SCHEMA, cleanup } : undefined; +} + +export function readEnvironmentCreationCleanupError(error: unknown): PluginEnvironmentCreationCleanup | null { + const data = error instanceof PluginEnvironmentCreationCleanupError + ? { schema: SCHEMA, cleanup: error.cleanup } + : error && typeof error === "object" && "data" in error ? error.data : null; + if (!data || typeof data !== "object" || !("schema" in data) || data.schema !== SCHEMA || !("cleanup" in data)) return null; + const value = data.cleanup; + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const row = value as Record; + const identifier = (value: unknown): value is string => typeof value === "string" && /^[a-zA-Z0-9_-]{1,200}$/.test(value); + if (![row.providerLeaseId, row.companyId, row.environmentId, row.attemptId].every(identifier)) return null; + if (row.runId !== undefined && !identifier(row.runId)) return null; + if (typeof row.accountFingerprint !== "string" || !/^[a-f0-9]{64}$/.test(row.accountFingerprint)) return null; + if (!row.labels || typeof row.labels !== "object" || Array.isArray(row.labels)) return null; + const labels = row.labels as Record; + if (Object.keys(labels).length > 16 || Object.entries(labels).some(([key, value]) => !/^paperclip-[a-z-]+$/.test(key) || !identifier(value))) return null; + return { + providerLeaseId: row.providerLeaseId as string, + companyId: row.companyId as string, + environmentId: row.environmentId as string, + ...(typeof row.runId === "string" ? { runId: row.runId } : {}), + attemptId: row.attemptId as string, + accountFingerprint: row.accountFingerprint, + labels: { ...labels } as Record, + }; +} diff --git a/packages/plugins/sdk/src/index.ts b/packages/plugins/sdk/src/index.ts index 1cf5adf0dda..788247f39ad 100644 --- a/packages/plugins/sdk/src/index.ts +++ b/packages/plugins/sdk/src/index.ts @@ -448,3 +448,6 @@ export { MEMBERSHIP_STATUSES, PRINCIPAL_TYPES, } from "@paperclipai/shared"; + +export { PluginEnvironmentCreationCleanupError, environmentCreationCleanupErrorData, readEnvironmentCreationCleanupError } from "./environment-creation-cleanup.js"; +export type { PluginEnvironmentCreationCleanup } from "./environment-creation-cleanup.js"; diff --git a/packages/plugins/sdk/src/worker-rpc-host.ts b/packages/plugins/sdk/src/worker-rpc-host.ts index 5e56ef200ca..1e98a7a8267 100644 --- a/packages/plugins/sdk/src/worker-rpc-host.ts +++ b/packages/plugins/sdk/src/worker-rpc-host.ts @@ -1,3 +1,4 @@ +import { environmentCreationCleanupErrorData } from "./environment-creation-cleanup.js"; /** * Worker-side RPC host — runs inside the child process spawned by the host. * @@ -1571,7 +1572,8 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost ? (err as any).code : PLUGIN_RPC_ERROR_CODES.WORKER_ERROR; - sendMessage(createErrorResponse(id, errorCode, errorMessage)); + sendMessage(createErrorResponse(id, errorCode, errorMessage, + method === "environmentAcquireLease" ? environmentCreationCleanupErrorData(err) : undefined)); } } diff --git a/packages/plugins/sdk/tests/environment-creation-cleanup.test.ts b/packages/plugins/sdk/tests/environment-creation-cleanup.test.ts new file mode 100644 index 00000000000..b91486dfc15 --- /dev/null +++ b/packages/plugins/sdk/tests/environment-creation-cleanup.test.ts @@ -0,0 +1,74 @@ +import { createInterface } from "node:readline"; +import { PassThrough } from "node:stream"; +import { describe, expect, it } from "vitest"; +import { definePlugin } from "../src/define-plugin.js"; +import { PluginEnvironmentCreationCleanupError, environmentCreationCleanupErrorData, readEnvironmentCreationCleanupError } from "../src/environment-creation-cleanup.js"; +import { createRequest, isJsonRpcResponse, JsonRpcCallError, parseMessage, serializeMessage, type JsonRpcResponse } from "../src/protocol.js"; +import { startWorkerRpcHost } from "../src/worker-rpc-host.js"; + +const ownership = { + providerLeaseId: "paperclip-create-attempt-1", companyId: "company-1", environmentId: "environment-1", + runId: "run-1", attemptId: "attempt-1", accountFingerprint: "a".repeat(64), + labels: { "paperclip-provider": "daytona" }, +}; +const schema = "paperclip/environment-creation-cleanup/v1"; + +async function invoke(method: string, error: Error) { + const stdin = new PassThrough(); + const stdout = new PassThrough(); + const lines = createInterface({ input: stdout }); + const worker = startWorkerRpcHost({ stdin, stdout, plugin: definePlugin({ + async setup() {}, + async onEnvironmentAcquireLease() { throw error; }, + async onEnvironmentProbe() { throw error; }, + }) }); + try { + const response = new Promise((resolve) => { + lines.on("line", (line) => { + const message = parseMessage(line); + if (isJsonRpcResponse(message)) resolve(message); + }); + }); + stdin.write(serializeMessage(createRequest(method, {}, "cleanup-test"))); + return await response; + } finally { + worker.stop(); lines.close(); stdin.destroy(); stdout.destroy(); + } +} + +describe("failed environment creation ownership", () => { + it("crosses the actual worker RPC as allowlisted ownership only", async () => { + const cleanup = { ...ownership, accessToken: "secret-token", config: { apiKey: "secret-key" } }; + const error = new PluginEnvironmentCreationCleanupError([new Error("secret-provider-response")], "Cleanup required", cleanup); + const response = await invoke("environmentAcquireLease", error); + expect("error" in response && response.error).toMatchObject({ data: { schema, cleanup: ownership } }); + if (!("error" in response) || !response.error) throw new Error("Expected RPC error"); + expect(readEnvironmentCreationCleanupError(new JsonRpcCallError(response.error))).toEqual(ownership); + expect(JSON.stringify(response)).not.toContain("secret-"); + }); + + it.each(["environmentAcquireLease", "environmentProbe"])("does not forward arbitrary error data for %s", async (method) => { + const response = await invoke(method, Object.assign(new Error("provider failure"), { data: { apiKey: "secret-key", schema, cleanup: ownership } })); + expect("error" in response && response.error).not.toHaveProperty("data"); + }); + + it("only forwards the typed envelope on supported acquisition calls", async () => { + const response = await invoke("environmentProbe", new PluginEnvironmentCreationCleanupError([], "Cleanup required", ownership)); + expect("error" in response && response.error).not.toHaveProperty("data"); + }); + + it.each([ + { providerLeaseId: "../other" }, { companyId: "" }, { environmentId: null }, { attemptId: "a".repeat(201) }, + { runId: "other/run" }, { accountFingerprint: "secret-key" }, { labels: { apiKey: "secret-key" } }, + { labels: { "paperclip-provider": "Bearer secret-key" } }, { labels: [] }, + { labels: Object.fromEntries(Array.from({ length: 17 }, (_, i) => [`paperclip-${String.fromCharCode(97 + i)}`, "x"])) }, + ])("rejects malformed evidence %j", (invalid) => { + const cleanup = { ...ownership, ...invalid } as typeof ownership; + expect(readEnvironmentCreationCleanupError({ data: { schema, cleanup } })).toBeNull(); + expect(environmentCreationCleanupErrorData(new PluginEnvironmentCreationCleanupError([], "invalid", cleanup))).toBeUndefined(); + }); + + it.each([null, {}, { data: { schema: "unknown", cleanup: ownership } }, { data: { schema, cleanup: [] } }])("rejects malformed envelopes %j", (value) => { + expect(readEnvironmentCreationCleanupError(value)).toBeNull(); + }); +}); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 37e8d315254..26cb64f2d5e 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -653,6 +653,58 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect(call.mock.calls.filter((entry) => entry[1] === "environmentResumeLease")).toHaveLength(shouldResume ? 1 : 0); }); + it.each(["pending", "inline", "foreign scope", "foreign environment", "foreign run", "malformed", "deleted environment"])("retains uncertain plugin creation for durable cleanup: %s", async (mode) => { + const seeded = await seedReusablePluginSandboxLease(); + await environmentService(db).releaseLease(seeded.reusableLease.id, "expired"); + const attemptId = randomUUID(); + const providerLeaseId = `paperclip-create-${attemptId}`; + const cleanup = { + providerLeaseId, attemptId, companyId: mode === "foreign scope" ? randomUUID() : seeded.companyId, + environmentId: mode === "foreign environment" ? randomUUID() : seeded.environment.id, + runId: mode === "foreign run" ? randomUUID() : seeded.runId, + accountFingerprint: mode === "malformed" ? "invalid" : "a".repeat(64), labels: { "paperclip-provider": "fake-plugin" }, + }; + const failure = Object.assign(new Error("Provider creation cleanup required"), { + data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup }, + }); + let available = mode === "inline"; + const call = vi.fn(async (_id: string, method: string, params: Record) => { + if (method === "environmentAcquireLease") throw failure; + if (method === "environmentDestroyLease") { + const pending = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, providerLeaseId)); + expect(pending).toHaveLength(1); + expect(pending[0].status).toBe("pending_cleanup"); + expect(params).toMatchObject({ providerLeaseId, companyId: seeded.companyId, environmentId: seeded.environment.id, leaseMetadata: { failedCreateCleanup: cleanup } }); + if (!available) throw new Error("provider not visible yet"); + return { providerLeaseId, state: "destroyed" }; + } + throw new Error(`Unexpected method ${method}`); + }); + const worker = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }), call } as unknown as PluginWorkerManager; + const candidate = environmentRuntimeService(db, { pluginWorkerManager: worker }); + await expect(candidate.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, + issueId: null, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: null })).rejects.toBe(failure); + const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, providerLeaseId)); + if (["foreign scope", "foreign environment", "foreign run", "malformed"].includes(mode)) { + expect(rows).toHaveLength(0); + expect(call.mock.calls.filter((c) => c[1] === "environmentDestroyLease")).toHaveLength(0); + return; + } + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ status: mode === "inline" ? "expired" : "pending_cleanup", + cleanupStatus: mode === "inline" ? "success" : "failed", companyId: seeded.companyId, + metadata: { pluginId: seeded.pluginId, sandboxProviderPlugin: true, failedCreateCleanup: cleanup } }); + if (mode === "pending" || mode === "deleted environment") { + available = true; + if (mode === "deleted environment") { + await db.update(environmentLeases).set({ environmentId: null }).where(eq(environmentLeases.id, rows[0].id)); + } + const restarted = environmentRuntimeService(db, { pluginWorkerManager: worker }); + const persisted = await environmentService(db).getLeaseById(rows[0].id); + await expect(restarted.retryPendingSandboxTeardown({ environment: null, lease: persisted! })).resolves.toEqual({ providerLeaseId, state: "destroyed" }); + } + }); + it("resumes the same provider lease on the next per-turn run", async () => { const seeded = await seedReusablePluginSandboxLease(); const workerManager = { diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 0a8f5ffc301..d6eb936ead0 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1,3 +1,4 @@ +import { readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk"; import { remoteTerminationReceipt } from "./remote-execution-termination.js"; import { createHash, randomUUID } from "node:crypto"; import { and, eq, inArray, sql } from "drizzle-orm"; @@ -2019,38 +2020,80 @@ function createSandboxEnvironmentDriver( }); } } - const acquiredLease = providerLease ?? await pluginWorkerManager.call( - pluginProvider.resolved.plugin.id, - "environmentAcquireLease", - { - driverKey: parsed.config.provider, - companyId: input.companyId, - environmentId: input.environment.id, - issueId: input.issueId, - config: workerConfig, - // Plugin SDK requires a string; ad-hoc test leases use a fresh - // UUID so providers that validate or persist the runId still see - // a well-formed identifier. - runId: input.heartbeatRunId ?? randomUUID(), - workspaceMode: input.executionWorkspaceMode ?? undefined, - agentId: input.agentId ?? undefined, - executionWorkspaceId: input.executionWorkspaceId ?? undefined, - // The agent's harness for THIS run, so the plugin picks the matching - // runtime image (per-run adapter, mixed-harness environments). - // NOTE: environment-runtime.ts has TWO drivers calling - // environmentAcquireLease; this plugin-sandbox one is the HEARTBEAT - // path. Omitting adapterType here silently falls back to the - // environment's default adapter image (a pi agent then runs in the - // opencode image and the harness binary is missing at exec time). - adapterType: input.adapterType ?? undefined, - // Forward a caller deadline so the provider configures a provider-side - // expiry at or before it and returns the real provider expiry. - ...(requestedExpiresAtParam(input.requestedExpiresAt) !== undefined - ? { requestedExpiresAt: requestedExpiresAtParam(input.requestedExpiresAt) } - : {}), - }, - resolvePluginSandboxRpcTimeoutMs(workerConfig), - ); + const acquisitionRunId = input.heartbeatRunId ?? randomUUID(); + const acquiredLease = providerLease ?? await (async () => { + try { + return await pluginWorkerManager.call( + pluginProvider.resolved.plugin.id, + "environmentAcquireLease", + { + driverKey: parsed.config.provider, + companyId: input.companyId, + environmentId: input.environment.id, + issueId: input.issueId, + config: workerConfig, + // Plugin SDK requires a string; ad-hoc test leases use a fresh + // UUID so providers that validate or persist the runId still see + // a well-formed identifier. + runId: acquisitionRunId, + workspaceMode: input.executionWorkspaceMode ?? undefined, + agentId: input.agentId ?? undefined, + executionWorkspaceId: input.executionWorkspaceId ?? undefined, + // The agent's harness for THIS run, so the plugin picks the matching + // runtime image (per-run adapter, mixed-harness environments). + // NOTE: environment-runtime.ts has TWO drivers calling + // environmentAcquireLease; this plugin-sandbox one is the HEARTBEAT + // path. Omitting adapterType here silently falls back to the + // environment's default adapter image (a pi agent then runs in the + // opencode image and the harness binary is missing at exec time). + adapterType: input.adapterType ?? undefined, + // Forward a caller deadline so the provider configures a provider-side + // expiry at or before it and returns the real provider expiry. + ...(requestedExpiresAtParam(input.requestedExpiresAt) !== undefined + ? { requestedExpiresAt: requestedExpiresAtParam(input.requestedExpiresAt) } + : {}), + }, + resolvePluginSandboxRpcTimeoutMs(workerConfig), + ); + } catch (error) { + const cleanup = readEnvironmentCreationCleanupError(error); + // The authenticated worker may identify its uncertain allocation, + // but cannot redirect cleanup to another company, environment, or run. + if (cleanup && cleanup.companyId === input.companyId && + cleanup.environmentId === input.environment.id && cleanup.runId === acquisitionRunId) { + const cleanupMetadata = { + ...sandboxConfigForLeaseMetadata(storedConfig), + failedCreateCleanup: cleanup, + driver: input.environment.driver, + pluginId: pluginProvider.resolved.plugin.id, + pluginKey: pluginProvider.resolved.plugin.pluginKey, + sandboxProviderPlugin: true, + }; + // Record before retrying the provider. Existing pending-cleanup + // recovery (including its durable spool) survives controller loss. + await cleanUpRejectedOrphanSandbox({ + record: { + companyId: input.companyId, environmentId: input.environment.id, + executionWorkspaceId: input.executionWorkspaceId ?? null, + issueId: input.issueId ?? null, heartbeatRunId: input.heartbeatRunId ?? null, + provider: parsed.config.provider, providerLeaseId: cleanup.providerLeaseId, + metadata: cleanupMetadata, + }, + cause: error, + canTeardown: pluginWorkerManager.isRunning(pluginProvider.resolved.plugin.id), + teardown: () => pluginWorkerManager.call(pluginProvider.resolved.plugin.id, "environmentDestroyLease", { + driverKey: parsed.config.provider, companyId: input.companyId, + environmentId: input.environment.id, issueId: input.issueId, + config: workerConfig, providerLeaseId: cleanup.providerLeaseId, + leaseMetadata: { failedCreateCleanup: cleanup }, + }, resolvePluginSandboxRpcTimeoutMs(workerConfig)), + }); + } + // A cleanup record is not a usable lease. Keep acquisition failed, + // even when its compensating deletion succeeds on this attempt. + throw error; + } + })(); // Ad-hoc test leases are never publishable for reuse: storing them // as `reuse_by_environment` would let a concurrent heartbeat resume @@ -2524,16 +2567,25 @@ function createSandboxEnvironmentDriver( { issueId: input.lease.issueId, heartbeatRunId: input.lease.heartbeatRunId }, ); const workerConfig = stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig); + const failedCreation = readEnvironmentCreationCleanupError({ data: { + schema: "paperclip/environment-creation-cleanup/v1", + cleanup: input.lease.metadata?.failedCreateCleanup, + } }); + // An environment deletion clears the lease FK. Retain the original + // ownership scope for uncertain allocations without accepting a foreign + // company, provider name, or changed environment from metadata. + if (failedCreation && (failedCreation.companyId !== input.lease.companyId || + failedCreation.providerLeaseId !== input.lease.providerLeaseId || + (input.lease.environmentId !== null && failedCreation.environmentId !== input.lease.environmentId))) { + throw new Error("Pending creation cleanup ownership does not match the lease"); + } return await pluginWorkerManager.call( pluginProvider.resolved.plugin.id, "environmentDestroyLease", { driverKey: recordedProvider, companyId: input.lease.companyId, - // The provider teardown keys on the provider lease id. The - // environment id is only context, and it is empty when a delete - // removed the environment before this orphan was recorded. - environmentId: input.lease.environmentId ?? input.environment?.id ?? "", + environmentId: failedCreation?.environmentId ?? input.lease.environmentId ?? input.environment?.id ?? "", issueId: input.lease.issueId, config: workerConfig, providerLeaseId: input.lease.providerLeaseId,