From 16dfa200ac8c94772109ba9b9cb2eea7f4d9b63d Mon Sep 17 00:00:00 2001 From: paperclipbugbot Date: Fri, 20 Nov 2026 04:57:36 -0800 Subject: [PATCH] fix(environments): clean up an unconfirmed setup sandbox and answer a provider outage with 503 (#22377) A provider outage during an interactive custom-image setup start left a provisional sandbox with no cleanup owner, and the raw RPC error reached the HTTP error handler as a 500 crash report. - Keep the creation-cleanup envelope and create status for environmentStartInteractiveSetup errors at the worker RPC boundary. - Record an in-scope unconfirmed setup sandbox as a pending-cleanup lease that the existing sweep destroys. - Pin failed-creation cleanup (probe and teardown) to the plugin that created the sandbox. - Answer a transient provider refusal with 503 environment_provider_unavailable instead of a crash report. Co-Authored-By: Paperclip Co-Authored-By: Claude Opus 5.5 --- packages/plugins/sdk/src/worker-rpc-host.ts | 4 +- .../tests/environment-acquire-error.test.ts | 15 +++- .../environment-custom-images-service.test.ts | 60 +++++++++++++++ .../src/__tests__/environment-runtime.test.ts | 52 +++++++++++++ ...dbox-provider-outage-error-handler.test.ts | 75 +++++++++++++++++++ server/src/middleware/error-handler.ts | 15 ++++ .../src/services/environment-custom-images.ts | 55 ++++++++++++-- .../environment-provider-unavailable-error.ts | 22 ++++++ server/src/services/environment-runtime.ts | 19 ++++- .../runner-e2e/source-sdk-public-inputs.json | 2 +- 10 files changed, 309 insertions(+), 10 deletions(-) create mode 100644 server/src/__tests__/sandbox-provider-outage-error-handler.test.ts diff --git a/packages/plugins/sdk/src/worker-rpc-host.ts b/packages/plugins/sdk/src/worker-rpc-host.ts index afb577313ed..820c73c7d4c 100644 --- a/packages/plugins/sdk/src/worker-rpc-host.ts +++ b/packages/plugins/sdk/src/worker-rpc-host.ts @@ -1675,7 +1675,9 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost : PLUGIN_RPC_ERROR_CODES.WORKER_ERROR; sendMessage(createErrorResponse(id, errorCode, errorMessage, - method === "environmentAcquireLease" ? environmentAcquireErrorData(err) + // An interactive setup creates a sandbox the same way an acquisition does, + // so a failed create keeps the same ownership envelope and create status. + method === "environmentAcquireLease" || method === "environmentStartInteractiveSetup" ? environmentAcquireErrorData(err) : method === "environmentDestroyLease" ? environmentCreationCleanupErrorData(err) : method === "environmentSyncOut" ? environmentSyncErrorData(err) : method === "environmentRunnerIngressEndpoint" ? environmentRunnerIngressErrorData(err) diff --git a/packages/plugins/sdk/tests/environment-acquire-error.test.ts b/packages/plugins/sdk/tests/environment-acquire-error.test.ts index a9d2f45f770..69fc10beece 100644 --- a/packages/plugins/sdk/tests/environment-acquire-error.test.ts +++ b/packages/plugins/sdk/tests/environment-acquire-error.test.ts @@ -12,13 +12,14 @@ import { import { createRequest, isJsonRpcResponse, JsonRpcCallError, parseMessage, serializeMessage, type JsonRpcResponse } from "../src/protocol.js"; import { startWorkerRpcHost } from "../src/worker-rpc-host.js"; -async function acquire(error: unknown) { +async function acquire(error: unknown, method: "environmentAcquireLease" | "environmentStartInteractiveSetup" = "environmentAcquireLease") { const stdin = new PassThrough(); const stdout = new PassThrough(); const lines = createInterface({ input: stdout }); const worker = startWorkerRpcHost({ stdin, stdout, plugin: definePlugin({ async setup() {}, async onEnvironmentAcquireLease() { throw error; }, + async onEnvironmentStartInteractiveSetup() { throw error; }, }) }); try { const response = new Promise((resolve) => { @@ -27,7 +28,7 @@ async function acquire(error: unknown) { if (isJsonRpcResponse(message)) resolve(message); }); }); - stdin.write(serializeMessage(createRequest("environmentAcquireLease", {}, "acquire-test"))); + stdin.write(serializeMessage(createRequest(method, {}, "acquire-test"))); const result = await response; if (!("error" in result) || !result.error) throw new Error("Expected RPC error"); return new JsonRpcCallError(result.error); @@ -59,6 +60,16 @@ describe("environment acquisition refusal diagnostics", () => { expect(readEnvironmentCreationCleanupError(callError)).toMatchObject({ providerLeaseId: cleanup.providerLeaseId, attemptId: cleanup.attemptId }); }); + it("keeps the cleanup envelope and create status when an interactive setup create fails", async () => { + const createError = Object.assign(new Error("Bad gateway at https://private.example/api"), { statusCode: 502 }); + const callError = await acquire(new PluginEnvironmentCreationCleanupError([createError, new Error("lookup failed")], + "Daytona sandbox creation failed (HTTP 502); cleanup could not be confirmed for paperclip-create-05f81d77", cleanup), + "environmentStartInteractiveSetup"); + expect(readEnvironmentAcquireErrorDiagnostic(callError)).toEqual({ httpStatus: 502 }); + expect(readEnvironmentCreationCleanupError(callError)).toMatchObject({ providerLeaseId: cleanup.providerLeaseId, attemptId: cleanup.attemptId }); + expect(JSON.stringify(callError.data)).not.toContain("private.example"); + }); + it("carries a closed connection errno and omits unstructured failures", async () => { expect(readEnvironmentAcquireErrorDiagnostic(await acquire(Object.assign(new Error("reset"), { code: "ECONNRESET" })))) .toEqual({ errorCode: "ECONNRESET" }); diff --git a/server/src/__tests__/environment-custom-images-service.test.ts b/server/src/__tests__/environment-custom-images-service.test.ts index 80b6c072450..23448af57ae 100644 --- a/server/src/__tests__/environment-custom-images-service.test.ts +++ b/server/src/__tests__/environment-custom-images-service.test.ts @@ -11,9 +11,11 @@ import { createDb, environmentCustomImageSetupSessions, environmentCustomImageTemplates, + environmentLeases, environments, plugins, } from "@paperclipai/db"; +import { JsonRpcCallError } from "@paperclipai/plugin-sdk"; import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { @@ -27,6 +29,8 @@ import { import { resolveEnvironmentDriverConfigForRuntime, } from "../services/environment-config.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; +import { environmentService } from "../services/environments.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, @@ -168,6 +172,7 @@ describeEmbeddedPostgres("environmentCustomImageService", () => { }); afterEach(async () => { + await db.delete(environmentLeases); await db.delete(environmentCustomImageSetupSessions); await db.delete(environmentCustomImageTemplates); await db.delete(plugins); @@ -608,6 +613,61 @@ describeEmbeddedPostgres("environmentCustomImageService", () => { sourceEnvironmentConfigFingerprint: expired.session.metadata!.sourceEnvironmentConfigFingerprint }); }); + it.each(["setup scope", "foreign company", "instance scope"])( + "hands an unconfirmed setup sandbox to the pending-cleanup sweep: %s", async (scope) => { + // PAPERCLIP-BACKEND-75: a Daytona 502 during setup creation left a + // provisional sandbox whose cleanup was never recorded anywhere. + const { companyId, environmentId } = await seed(); + const workerManager = createWorkerManager(); + const startCall = workerManager.call.getMockImplementation()!; + const providerLeaseId = `paperclip-create-${randomUUID()}`; + let cleanup: Record | undefined; + let destroyed: Record | undefined; + workerManager.call.mockImplementation(async (pluginId, method, params) => { + if (method === "environmentStartInteractiveSetup") { + cleanup = { + providerLeaseId, attemptId: providerLeaseId.slice("paperclip-create-".length), + companyId: scope === "foreign company" ? randomUUID() : params.companyId, environmentId, + accountFingerprint: "a".repeat(64), labels: { "paperclip-provider": "fake-plugin" }, + }; + throw new JsonRpcCallError({ code: -32000, + message: `Sandbox creation failed (HTTP 502); cleanup could not be confirmed for ${providerLeaseId}`, + data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup, providerDiagnostic: { httpStatus: 502 } } }); + } + if (method === "environmentDestroyLease") { + destroyed = params; + return { providerLeaseId: params.providerLeaseId, state: "destroyed" }; + } + return startCall(pluginId, method, params); + }); + const service = environmentCustomImageService(db, { pluginWorkerManager: workerManager }); + + const attempt = service.startSetupSession({ environmentId, actor: { userId: "user-1" }, + ...(scope === "instance scope" ? {} : { secretContextCompanyId: companyId }) }); + if (scope === "setup scope") await expect(attempt).rejects.toBeInstanceOf(JsonRpcCallError); + else await expect(attempt).rejects.toThrow("Interactive setup sandbox cleanup could not be recorded"); + + const [session] = await db.select().from(environmentCustomImageSetupSessions) + .where(eq(environmentCustomImageSetupSessions.environmentId, environmentId)); + expect(session.status).toBe("failed"); + const leases = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, providerLeaseId)); + if (scope !== "setup scope") { + expect(leases).toHaveLength(0); + return; + } + expect(leases).toEqual([expect.objectContaining({ companyId, environmentId, status: "pending_cleanup", + provider: "fake-plugin", failureReason: "interrupted_setup_creation" })]); + expect(leases[0].metadata).toMatchObject({ provider: "fake-plugin", image: "fake:base", purpose: "interactive_setup", + setupSessionId: session.id, failedCreateCleanup: cleanup }); + + // The existing sweep teardown accepts the recorded lease and destroys the allocation. + const runtime = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); + await expect(runtime.retryPendingSandboxTeardown({ environment: null, lease: (await environmentService(db).getLeaseById(leases[0].id))! })) + .resolves.toEqual({ providerLeaseId, state: "destroyed" }); + expect(destroyed).toMatchObject({ providerLeaseId, companyId, environmentId, leaseMetadata: { failedCreateCleanup: cleanup } }); + }, + ); + it("rejects interactive setup when the provider declares it but the worker omits a setup method", async () => { const { environmentId } = await seed(); // The manifest declares supportsInteractiveSetup, but the worker reports diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 0738854998a..97379964644 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -2411,6 +2411,58 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { ).resolves.toBe(true); }); + it("probes the plugin that recorded a failed creation when two plugins share its provider key", async () => { + // An interactive setup can create its sandbox through the second of two + // plugins with the same provider key, while the first is disabled. The + // cleanup must follow the recorded plugin, not the first one with the key. + const disabledPluginId = randomUUID(); + const creatorPluginId = randomUUID(); + const { companyId, environment, runId } = await seedEnvironment({ + driver: "sandbox", + name: "Shared Provider Key Cleanup", + config: { provider: "fake-plugin-shared", image: "fake:test", reuseLease: false }, + }); + const sharedDriver = [{ driverKey: "fake-plugin-shared", kind: "sandbox_provider", displayName: "Fake Shared", configSchema: { type: "object" } }]; + await insertSandboxProviderPlugin(disabledPluginId, { + pluginKey: "paperclip.fake-shared-disabled", packageName: "@paperclipai/plugin-fake-shared-disabled", + environmentDrivers: sharedDriver, status: "disabled", installOrder: 1, + }); + await insertSandboxProviderPlugin(creatorPluginId, { + pluginKey: "paperclip.fake-shared-creator", packageName: "@paperclipai/plugin-fake-shared-creator", + environmentDrivers: sharedDriver, status: "ready", installOrder: 2, + }); + const workerManager = { + isRunning: vi.fn((id: string) => id === creatorPluginId), + call: vi.fn(async () => { + throw new Error("call must not run during a readiness probe"); + }), + } as unknown as PluginWorkerManager; + const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); + const insertOrphan = async (providerLeaseId: string, metadata: Record) => { + const orphan = await environmentService(db).insertPendingCleanupLease({ + companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, + provider: "fake-plugin-shared", providerLeaseId, + metadata: { provider: "fake-plugin-shared", driver: "sandbox", reuseLease: false, ...metadata }, + failureReason: "interrupted_setup_creation", + }); + return (await environmentService(db).getLeaseById(orphan.id))!; + }; + + const setupLease = await insertOrphan("paperclip-create-shared", { + pluginId: creatorPluginId, + failedCreateCleanup: { + providerLeaseId: "paperclip-create-shared", companyId, environmentId: environment.id, + attemptId: "attempt-shared", accountFingerprint: "account-shared", labels: {}, + }, + }); + await expect(runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease: setupLease })).resolves.toBe(true); + + // The pinned plugin's worker is down: a transient window, not ready. + (workerManager.isRunning as ReturnType).mockReturnValue(false); + await expect(runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease: setupLease })).resolves.toBe(false); + expect((workerManager.call as ReturnType)).not.toHaveBeenCalled(); + }); + it("reports the plugin cleanup provider not ready while the plugin is not ready, then ready after it recovers", async () => { // A pending_cleanup orphan targets a plugin-backed provider. A plugin reload // or a plugin reinstall moves the plugin through a "not ready" status. The diff --git a/server/src/__tests__/sandbox-provider-outage-error-handler.test.ts b/server/src/__tests__/sandbox-provider-outage-error-handler.test.ts new file mode 100644 index 00000000000..8994228d8e8 --- /dev/null +++ b/server/src/__tests__/sandbox-provider-outage-error-handler.test.ts @@ -0,0 +1,75 @@ +import express from "express"; +import request from "supertest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { JsonRpcCallError } from "@paperclipai/plugin-sdk"; +import { errorHandler } from "../middleware/error-handler.js"; + +const captureExceptionMock = vi.hoisted(() => vi.fn()); + +vi.mock("../sentry.js", () => ({ captureException: captureExceptionMock })); +vi.mock("../telemetry.js", () => ({ getTelemetryClient: () => null })); + +const cleanup = { + providerLeaseId: "paperclip-create-7c3c865d", companyId: "company-1", environmentId: "environment-1", + attemptId: "7c3c865d", accountFingerprint: "a".repeat(64), labels: { "paperclip-provider": "daytona" }, +}; + +function refusal(data: unknown) { + return new JsonRpcCallError({ code: -32000, + message: "Daytona sandbox creation failed (HTTP 502); cleanup could not be confirmed for paperclip-create-7c3c865d", data }); +} + +function createApp(error: unknown) { + const app = express(); + app.post("/start", async () => { throw error; }); + app.use(errorHandler); + return app; +} + +describe("sandbox provider outages during a requested creation", () => { + beforeEach(() => captureExceptionMock.mockReset()); + + it("answers a transient create refusal with 503 and no crash report", async () => { + // PAPERCLIP-BACKEND-75: a Daytona 502 reached this handler as a raw RPC + // error, answering 500 with a crash report. + const res = await request(createApp(refusal({ schema: "paperclip/environment-creation-cleanup/v1", cleanup, + providerDiagnostic: { httpStatus: 502 } }))) + .post("/start"); + + expect(res.status).toBe(503); + expect(res.headers["retry-after"]).toBe("30"); + expect(res.body).toEqual({ + error: "The sandbox provider was temporarily unavailable (HTTP 502). Try again in a few minutes.", + code: "environment_provider_unavailable", + }); + expect(JSON.stringify(res.body)).not.toContain("paperclip-create-"); + expect(captureExceptionMock).not.toHaveBeenCalled(); + }); + + it("answers a connection failure and an expired creation deadline the same way", async () => { + const reset = await request(createApp(refusal({ schema: "paperclip/environment-acquire-error/v1", + providerDiagnostic: { errorCode: "ECONNRESET" } }))).post("/start"); + expect(reset.status).toBe(503); + expect(reset.body.error).toBe("The sandbox provider was temporarily unavailable (ECONNRESET). Try again in a few minutes."); + + const deadline = await request(createApp(refusal({ schema: "paperclip/environment-creation-cleanup/v1", cleanup, + acquisitionDiagnostic: { phase: "create", elapsedMs: 300_000, budgetMs: 300_000 } }))).post("/start"); + expect(deadline.status).toBe(503); + expect(deadline.body.error).toBe("The sandbox provider did not finish starting the sandbox in time. Try again in a few minutes."); + expect(captureExceptionMock).not.toHaveBeenCalled(); + }); + + it.each([ + ["a definite provider status", refusal({ schema: "paperclip/environment-acquire-error/v1", providerDiagnostic: { httpStatus: 401 } })], + ["no structured evidence", refusal(undefined)], + ["a forged envelope on another error", Object.assign(new Error("x"), { data: { schema: "paperclip/environment-acquire-error/v1", + providerDiagnostic: { httpStatus: 502 } } })], + ["a later Paperclip failure", new Error("Interactive setup sandbox cleanup could not be recorded", { + cause: refusal({ schema: "paperclip/environment-acquire-error/v1", providerDiagnostic: { httpStatus: 502 } }) })], + ])("still reports %s as a server fault", async (_label, error) => { + const res = await request(createApp(error)).post("/start"); + + expect(res.status).toBe(500); + expect(captureExceptionMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/server/src/middleware/error-handler.ts b/server/src/middleware/error-handler.ts index e311dd4e377..758c3b26c67 100644 --- a/server/src/middleware/error-handler.ts +++ b/server/src/middleware/error-handler.ts @@ -21,6 +21,7 @@ import { recordResponsibleUserDenialOnActiveRun } from "../services/responsible- import { isObjectJournalBusyError } from "../services/object-byte-journal.js"; import { isExpectedMcpConnectionFailure } from "../services/mcp-connection-failure.js"; import { isOpenRouterCatalogUnavailable } from "../services/openrouter-models.js"; +import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE, readTransientSandboxCreationRefusal } from "../services/environment-provider-unavailable-error.js"; import { isEndedDbPoolError } from "../database-retry.js"; import { ToolGatewayHttpError } from "../services/tool-gateway.js"; @@ -340,6 +341,20 @@ export function errorHandler( return; } + // A sandbox provider outage during a sandbox creation that this request + // started is not a server fault. The answer is fixed text: provider text and + // provisional sandbox names stay out, and cleanup stays with its own record. + const providerRefusal = readTransientSandboxCreationRefusal(rootError); + if (providerRefusal) { + if (!res.headersSent) { + res.set("Retry-After", "30"); + res.status(503).json({ error: providerRefusal.message, code: ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE }); + } else { + res.end(); + } + return; + } + const reportableError = sanitizeSecretSensitiveError(req, rootError); attachErrorContext( req, diff --git a/server/src/services/environment-custom-images.ts b/server/src/services/environment-custom-images.ts index 9ff4fd1c1df..9e98ecee01d 100644 --- a/server/src/services/environment-custom-images.ts +++ b/server/src/services/environment-custom-images.ts @@ -20,6 +20,7 @@ import { type SandboxEnvironmentConfig, redactEnvironmentCustomImageValue, } from "@paperclipai/shared"; +import { readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk"; import type { PluginEnvironmentCancelInteractiveSetupResult, PluginEnvironmentCaptureTemplateResult, @@ -442,6 +443,7 @@ export function environmentCustomImageService( provider, rpcCompanyId: storedRpcCompanyId ?? secretContextCompanyId ?? "instance", pluginId: resolved.plugin.id, + pluginKey: resolved.plugin.pluginKey, driver: resolved.driver, runtimeConfig: parsed.config, driverConfig: stripSandboxProviderEnvelope(parsed.config), @@ -645,6 +647,45 @@ export function environmentCustomImageService( return toSession(row); } + // A setup sandbox whose failed creation could not be cleaned up has no + // session lease to cancel. Hand it to the pending-cleanup sweep, the same + // owner as a failed run acquisition, so the provider allocation is not lost. + async function recordUnconfirmedSetupCreation(input: { + error: unknown; + environment: Environment; + provider: Awaited>; + sessionId: string; + }): Promise { + const cleanup = readEnvironmentCreationCleanupError(input.error); + if (!cleanup) return; + const stored = parseEnvironmentDriverConfig(input.environment); + // Accept only an allocation in this setup's own scope. An instance-scoped + // setup has no company to own the cleanup record, so it is reported instead. + if (input.provider.rpcCompanyId === "instance" || cleanup.companyId !== input.provider.rpcCompanyId || + cleanup.environmentId !== input.environment.id || cleanup.runId !== undefined || + stored.driver !== "sandbox" || stored.config.provider !== input.provider.provider) { + throw new Error("Interactive setup sandbox cleanup could not be recorded", { cause: input.error }); + } + // The stored config keeps secret references, never resolved values. + await environments.insertPendingCleanupLease({ + companyId: cleanup.companyId, + environmentId: input.environment.id, + provider: input.provider.provider, + providerLeaseId: cleanup.providerLeaseId, + metadata: { + ...stored.config, + driver: "sandbox", + pluginId: input.provider.pluginId, + pluginKey: input.provider.pluginKey, + sandboxProviderPlugin: true, + purpose: "interactive_setup", + setupSessionId: input.sessionId, + failedCreateCleanup: cleanup, + }, + failureReason: "interrupted_setup_creation", + }); + } + async function cancelSession( session: EnvironmentCustomImageSetupSession, reason: string | null, @@ -873,11 +914,15 @@ export function environmentCustomImageService( connectionPayload: providerSession.connectionPayload ?? null, }; } catch (error) { - await markSessionStatus({ - sessionId, - status: "failed", - failureReason: error instanceof Error ? error.message : String(error), - }); + try { + await recordUnconfirmedSetupCreation({ error, environment, provider, sessionId }); + } finally { + await markSessionStatus({ + sessionId, + status: "failed", + failureReason: error instanceof Error ? error.message : String(error), + }); + } throw error; } }, diff --git a/server/src/services/environment-provider-unavailable-error.ts b/server/src/services/environment-provider-unavailable-error.ts index c5724a8f749..c45c097864f 100644 --- a/server/src/services/environment-provider-unavailable-error.ts +++ b/server/src/services/environment-provider-unavailable-error.ts @@ -53,6 +53,28 @@ export function readTransientEnvironmentProviderFailure(error: unknown): Transie return null; } +/** + * A sandbox creation that a request started directly (an interactive setup or + * a login sandbox), refused by the provider with a transient status. Only the + * worker's validated RPC envelope on the error itself counts: a wrapper means + * that Paperclip failed afterwards, for example while recording cleanup, and + * must still be reported. Ownership of a provisional allocation stays with its + * creation-cleanup record; this only decides the HTTP answer. + */ +export function readTransientSandboxCreationRefusal(error: unknown): { message: string } | null { + if (!(error instanceof JsonRpcCallError)) return null; + const acquisition = readEnvironmentAcquisitionDiagnostic(error); + const envelope = readEnvironmentAcquireErrorDiagnostic(error); + const timedOut = acquisition !== null && acquisition.elapsedMs >= acquisition.budgetMs; + const httpStatus = envelope?.httpStatus !== undefined && isRetryableEnvironmentHttpStatus(envelope.httpStatus) ? envelope.httpStatus : undefined; + const errorCode = envelope?.errorCode !== undefined && isRetryableEnvironmentErrorCode(envelope.errorCode) ? envelope.errorCode : undefined; + if (!timedOut && httpStatus === undefined && errorCode === undefined) return null; + const evidence = httpStatus !== undefined ? ` (HTTP ${httpStatus})` : errorCode !== undefined ? ` (${errorCode})` : ""; + return { message: evidence === "" + ? "The sandbox provider did not finish starting the sandbox in time. Try again in a few minutes." + : `The sandbox provider was temporarily unavailable${evidence}. Try again in a few minutes.` }; +} + /** A definite non-transient HTTP status anywhere on the chain is authoritative: retrying cannot change it. */ function definiteNonTransientStatus(error: unknown): boolean { for (let cause = error, depth = 0; cause && depth < 5; cause = own(cause, "cause"), depth++) { diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 3b254f1fa25..bbf5cca1120 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -2508,7 +2508,7 @@ function createSandboxEnvironmentDriver( `Sandbox provider "${recordedProvider}" needs a plugin worker manager for cleanup, but none is available.`, ); } - const pinnedPluginId = exportResume || input.lease.metadata?.creationOperationId ? readString(input.lease.metadata?.pluginId) : null; + const pinnedPluginId = exportResume ? readString(input.lease.metadata?.pluginId) : pinnedCreationCleanupPluginId(input.lease); if (exportResume && !pinnedPluginId) throw new Error("Workspace export cleanup has no recorded provider plugin."); const pluginProvider = await resolveSandboxProviderPlugin({ provider: recordedProvider, ...(pinnedPluginId ? { pluginId: pinnedPluginId } : {}) }); if (pluginProvider.state !== "running") { @@ -2627,6 +2627,13 @@ function createSandboxEnvironmentDriver( return Boolean(pinned?.plugin.status === "ready" && pluginWorkerManager.isRunning(pinned.plugin.id) && pluginWorkerVerifiesLifecycleMethod(pinned.plugin.id, "environmentStopLease")); } + // A failed creation is torn down by the plugin that created it, so probe + // that plugin. A missing or not-ready pinned plugin is a transient window. + const pinnedCreationPluginId = pinnedCreationCleanupPluginId(input.lease); + if (pinnedCreationPluginId) { + const pinned = await resolvePluginSandboxProviderDriverById({ db, pluginId: pinnedCreationPluginId, driverKey: recordedProvider }); + return Boolean(pinned?.plugin.status === "ready" && pluginWorkerManager.isRunning(pinned.plugin.id)); + } // Resolve the installed plugin without a wait. A plugin reload or a plugin // reinstall can remove the plugin row for a short window, so a missing // plugin is a transient condition, not a permanent one. Report not ready, @@ -3360,6 +3367,16 @@ function readString(value: unknown): string | null { return typeof value === "string" && value.length > 0 ? value : null; } +/** + * A failed sandbox creation belongs to the plugin that started it. Two plugins + * can declare the same provider key, so the cleanup readiness probe and the + * teardown both use this pin instead of the first installed plugin with the key. + */ +function pinnedCreationCleanupPluginId(lease: { metadata: Record | null }): string | null { + const metadata = lease.metadata ?? {}; + return metadata.creationOperationId || metadata.failedCreateCleanup ? readString(metadata.pluginId) : null; +} + // Keys the runtime stores in the lease metadata that are not part of the // provider driver config. Some are host-internal control fields. `remoteCwd` is // a per-lease runtime value. The host reads `remoteCwd` from the lease metadata diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index 0fe138d417f..3273cb5e903 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -29,7 +29,7 @@ "packages/plugins/sdk/src/ui/index.ts": "f7d1cee64a8f0c924bd10a1bd20745a52fe2060e", "packages/plugins/sdk/src/ui/runtime.ts": "998428e9701eba3b762d3417236d32c2d0d5763d", "packages/plugins/sdk/src/ui/types.ts": "5a5644acf6554d2e5e1d000bfd03b97d57001eb2", - "packages/plugins/sdk/src/worker-rpc-host.ts": "afb577313ed099e20446fae6640e4f921089bafd", + "packages/plugins/sdk/src/worker-rpc-host.ts": "820c73c7d4c681296ef8c813a68ab2bd9901d339", "packages/plugins/sdk/src/workspace-operation-error.ts": "2cd247d002d83b163239229e631bca4e519e5075", "packages/plugins/sdk/src/workspace-provider.ts": "38e945b807f22150710c11b136af0a28aab2cb7c", "packages/plugins/sdk/tsconfig.json": "d62ebdff94bc4b75dabf1fcad187e1e7a90fc794",