diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index 342c202753a..ee5e0e48984 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -39,7 +39,7 @@ import plugin, { STUCK_STOPPING_FORCE_AFTER_MS, } from "./plugin.js"; import type { PluginContext } from "@paperclipai/plugin-sdk"; -import { environmentAcquireErrorData, environmentResumeErrorData, readEnvironmentAcquireErrorDiagnostic, environmentSyncErrorData, readEnvironmentSyncErrorDiagnostic, withEnvironmentSyncErrorCapture, environmentCreationCleanupErrorData, readEnvironmentCreationCleanupError, readEnvironmentAcquisitionDiagnostic, PluginEnvironmentStopError, environmentStopErrorData, readEnvironmentStopFailure } from "@paperclipai/plugin-sdk"; +import { PluginEnvironmentAcquisitionError, environmentAcquireErrorData, environmentResumeErrorData, readEnvironmentAcquireErrorDiagnostic, environmentSyncErrorData, readEnvironmentSyncErrorDiagnostic, withEnvironmentSyncErrorCapture, environmentCreationCleanupErrorData, readEnvironmentCreationCleanupError, readEnvironmentAcquisitionDiagnostic, PluginEnvironmentStopError, environmentStopErrorData, readEnvironmentStopFailure } from "@paperclipai/plugin-sdk"; import manifest from "./manifest.js"; import { parseTarVerboseListingLine, splitLinkEntryOnce } from "./file-sync.js"; import * as duplexCommands from "./duplex-command-stream.js"; @@ -1729,6 +1729,33 @@ describe("Daytona sandbox provider plugin", () => { expect(orphan!.process.executeCommand).not.toHaveBeenCalled(); }); + it("reports a sandbox that failed to start as a removed allocation with its observed step", async () => { + const startError = new Error("Sandbox private-sandbox-id failed to start with status: error, error reason: null"); + mockCreate.mockRejectedValue(startError); + let orphan: ReturnType; + mockGet.mockImplementation(async () => { + orphan = { ...ownedSandbox(), state: "error" }; + orphan.delete.mockImplementation(async () => { orphan.state = "destroyed"; }); + return orphan; + }); + const failure = await plugin.definition.onEnvironmentAcquireLease!(params).catch(error => error); + expect(orphan!.delete).toHaveBeenCalledWith(10, true); + expect(failure).toBeInstanceOf(PluginEnvironmentAcquisitionError); + expect(failure.message).not.toContain("private-sandbox-id"); + expect(failure.cause).toBe(startError); + expect(readEnvironmentCreationCleanupError(failure)).toBeNull(); + expect(readEnvironmentAcquisitionDiagnostic(failure)).toMatchObject({ phase: "create", providerState: "error" }); + expect(environmentAcquireErrorData(failure)).toMatchObject({ + schema: "paperclip/environment-acquire-error/v1", acquisitionDiagnostic: { phase: "create", providerState: "error" } }); + }); + + it.each(["build_failed", "started"])("keeps the create error when the removed sandbox was %s", async (state) => { + const startError = new Error("Sandbox failed to start"); + mockCreate.mockRejectedValue(startError); + mockGet.mockImplementation(async () => ({ ...ownedSandbox(), state })); + await expect(plugin.definition.onEnvironmentAcquireLease!(params)).rejects.toBe(startError); + }); + it.each(["paperclip-company-id", "paperclip-environment-id", "paperclip-run-id", "paperclip-create-attempt", "paperclip-provider"])("never deletes a sandbox with a mismatched %s", async (label) => { let orphan: ReturnType; mockGet.mockImplementation(async () => { diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 3306c05185c..31ff467792f 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -9,7 +9,7 @@ import type { Resources, Sandbox, } from "@daytonaio/sdk"; -import { decodeChannelBytes, definePlugin, NOOP_PLUGIN_TRACER, PluginEnvironmentCreationCleanupError, PluginEnvironmentRunnerIngressError, PluginEnvironmentStopError, readEnvironmentCreationCleanupError, withEnvironmentSyncTransferStep, preserveEnvironmentSyncErrorDiagnostic } from "@paperclipai/plugin-sdk"; +import { decodeChannelBytes, definePlugin, NOOP_PLUGIN_TRACER, PluginEnvironmentAcquisitionError, PluginEnvironmentCreationCleanupError, PluginEnvironmentRunnerIngressError, PluginEnvironmentStopError, readEnvironmentCreationCleanupError, withEnvironmentSyncTransferStep, preserveEnvironmentSyncErrorDiagnostic } from "@paperclipai/plugin-sdk"; import type { PluginContext, PluginEnvironmentCreationCleanup, @@ -1301,16 +1301,21 @@ async function createSandbox( if (options.claimCleanup && !options.claimCleanup()) throw createError; // Observe the create failure before compensating deletion can stall or fail. const diagnostic = options.onCreateFailure?.(); + let removedState: PluginEnvironmentAcquisitionProviderState | undefined; try { // A not-found lookup after an uncertain create is not a deletion receipt: // the provider may still materialize the request. Keep the name in the // error so cleanup can be reconciled rather than declaring success. - await destroyFailedCreation(config, cleanup, params.driverKey); + removedState = await destroyFailedCreation(config, cleanup, params.driverKey); } catch (cleanupError) { const status = creationHttpStatus(createError); throw new PluginEnvironmentCreationCleanupError([createError, cleanupError], `Daytona sandbox creation failed${status === undefined ? "" : ` (HTTP ${status})`}; cleanup could not be confirmed for ${name}`, cleanup, diagnostic); } + if (removedState === "error" && diagnostic && creationHttpStatus(createError) === undefined) { + throw new PluginEnvironmentAcquisitionError("Daytona could not start the sandbox; it was removed.", + { ...diagnostic, providerState: "error" }, createError); + } throw createError; } } @@ -1333,7 +1338,7 @@ async function destroyFailedCreation( config: DaytonaDriverConfig, cleanup: PluginEnvironmentCreationCleanup, driverKey: string, requireDurableObservation = false, -): Promise { +): Promise { if (cleanup.providerLeaseId !== `paperclip-create-${cleanup.attemptId}` || cleanup.accountFingerprint !== sandboxAccountDiscriminator(config) || cleanup.labels["paperclip-provider"] !== "daytona" || @@ -1360,7 +1365,7 @@ async function destroyFailedCreation( // observed with exact ownership cannot materialize as a new allocation. if (cleanup.observedProviderLeaseId && error instanceof DaytonaNotFoundError) { idleActivity.confirm(observationProof, true); - return; + return undefined; } throw error; } @@ -1377,8 +1382,11 @@ async function destroyFailedCreation( } idleActivity.observe(allocation, providerActivityKey(identity, config, orphan.id)); const deletionProof = idleActivity.proof(providerActivityKey(identity, config, orphan.id)); + const observedState = typeof orphan.state === "string" && REPORTED_CREATE_STATES.has(orphan.state) + ? orphan.state as PluginEnvironmentAcquisitionProviderState : undefined; await withLivenessTimeout("sandbox.failedCreateDelete", 15_000, () => orphan.delete(10, true)); idleActivity.confirm(deletionProof); + return observedState; } // Daytona states reported as-is after a create deadline; any other is `other`. diff --git a/packages/plugins/sdk/src/environment-acquire-error.ts b/packages/plugins/sdk/src/environment-acquire-error.ts index 50a74c3d1f8..4686b19d791 100644 --- a/packages/plugins/sdk/src/environment-acquire-error.ts +++ b/packages/plugins/sdk/src/environment-acquire-error.ts @@ -1,4 +1,11 @@ -import { PluginEnvironmentCreationCleanupError, environmentCreationCleanupErrorData } from "./environment-creation-cleanup.js"; +import { + ENVIRONMENT_ACQUIRE_ERROR_SCHEMA, + ENVIRONMENT_CREATION_CLEANUP_SCHEMA, + PluginEnvironmentAcquisitionError, + PluginEnvironmentCreationCleanupError, + environmentCreationCleanupErrorData, + readEnvironmentAcquisitionDiagnostic, +} from "./environment-creation-cleanup.js"; import { providerErrorDiagnostic, sanitizeProviderErrorDiagnostic, @@ -15,8 +22,8 @@ import { */ export type PluginEnvironmentAcquireErrorDiagnostic = PluginEnvironmentResumeErrorDiagnostic; -const SCHEMA = "paperclip/environment-acquire-error/v1"; -const CLEANUP_SCHEMA = "paperclip/environment-creation-cleanup/v1"; +const SCHEMA = ENVIRONMENT_ACQUIRE_ERROR_SCHEMA; +const CLEANUP_SCHEMA = ENVIRONMENT_CREATION_CLEANUP_SCHEMA; function field(value: unknown, key: string): unknown { try { return value && typeof value === "object" ? (value as Record)[key] : undefined; } @@ -35,7 +42,14 @@ export function environmentAcquireErrorData(error: unknown): unknown { if (cleanup && typeof cleanup === "object") { return providerDiagnostic ? { ...cleanup, providerDiagnostic } : cleanup; } - return providerDiagnostic ? { schema: SCHEMA, providerDiagnostic } : undefined; + const acquisitionDiagnostic = error instanceof PluginEnvironmentAcquisitionError + ? readEnvironmentAcquisitionDiagnostic(error) : null; + if (!providerDiagnostic && !acquisitionDiagnostic) return undefined; + return { + schema: SCHEMA, + ...(providerDiagnostic ? { providerDiagnostic } : {}), + ...(acquisitionDiagnostic ? { acquisitionDiagnostic } : {}), + }; } /** Host side: read and re-validate the provider diagnostic from an RPC error. */ diff --git a/packages/plugins/sdk/src/environment-creation-cleanup.ts b/packages/plugins/sdk/src/environment-creation-cleanup.ts index 35f23cd19ab..419ead5f876 100644 --- a/packages/plugins/sdk/src/environment-creation-cleanup.ts +++ b/packages/plugins/sdk/src/environment-creation-cleanup.ts @@ -11,7 +11,8 @@ export interface PluginEnvironmentCreationCleanup { labels: Record; } -const SCHEMA = "paperclip/environment-creation-cleanup/v1"; +export const ENVIRONMENT_CREATION_CLEANUP_SCHEMA = "paperclip/environment-creation-cleanup/v1"; +export const ENVIRONMENT_ACQUIRE_ERROR_SCHEMA = "paperclip/environment-acquire-error/v1"; /** The step observed at the first acquisition failure, not its upstream cause. */ export interface PluginEnvironmentAcquisitionDiagnostic { @@ -32,12 +33,19 @@ export const ENVIRONMENT_ACQUISITION_PROVIDER_STATES = [ ] as const; export type PluginEnvironmentAcquisitionProviderState = typeof ENVIRONMENT_ACQUISITION_PROVIDER_STATES[number]; +function acquisitionDiagnosticContainer(error: unknown): object | null { + if (error instanceof PluginEnvironmentAcquisitionError) return error; + if (readEnvironmentCreationCleanupError(error)) { + return error instanceof PluginEnvironmentCreationCleanupError ? error : (error as { data?: object }).data ?? null; + } + const data = error && typeof error === "object" ? Object.getOwnPropertyDescriptor(error, "data")?.value : undefined; + return data && typeof data === "object" && Object.getOwnPropertyDescriptor(data, "schema")?.value === ENVIRONMENT_ACQUIRE_ERROR_SCHEMA + ? data : null; +} + export function readEnvironmentAcquisitionDiagnostic(error: unknown): PluginEnvironmentAcquisitionDiagnostic | null { try { - // Diagnostics never make an invalid ownership envelope usable for cleanup. - if (!readEnvironmentCreationCleanupError(error)) return null; - const container = error instanceof PluginEnvironmentCreationCleanupError - ? error : (error as { data?: object }).data; + const container = acquisitionDiagnosticContainer(error); const value = container && Object.getOwnPropertyDescriptor(container, "acquisitionDiagnostic")?.value; if (!value || typeof value !== "object" || Array.isArray(value)) return null; const fields = Object.getOwnPropertyDescriptors(value); @@ -72,18 +80,28 @@ export class PluginEnvironmentCreationCleanupError extends AggregateError { } } +export class PluginEnvironmentAcquisitionError extends Error { + readonly acquisitionDiagnostic: PluginEnvironmentAcquisitionDiagnostic; + + constructor(message: string, acquisitionDiagnostic: PluginEnvironmentAcquisitionDiagnostic, cause?: unknown) { + super(message, { cause }); + this.name = "PluginEnvironmentAcquisitionError"; + this.acquisitionDiagnostic = acquisitionDiagnostic; + } +} + export function environmentCreationCleanupErrorData(error: unknown, includeAcquisitionDiagnostic = false): unknown { if (!(error instanceof PluginEnvironmentCreationCleanupError)) return undefined; const cleanup = readEnvironmentCreationCleanupError(error); const acquisitionDiagnostic = includeAcquisitionDiagnostic ? readEnvironmentAcquisitionDiagnostic(error) : null; - return cleanup ? { schema: SCHEMA, cleanup, ...(acquisitionDiagnostic ? { acquisitionDiagnostic } : {}) } : undefined; + return cleanup ? { schema: ENVIRONMENT_CREATION_CLEANUP_SCHEMA, cleanup, ...(acquisitionDiagnostic ? { acquisitionDiagnostic } : {}) } : undefined; } export function readEnvironmentCreationCleanupError(error: unknown): PluginEnvironmentCreationCleanup | null { const data = error instanceof PluginEnvironmentCreationCleanupError - ? { schema: SCHEMA, cleanup: error.cleanup } + ? { schema: ENVIRONMENT_CREATION_CLEANUP_SCHEMA, cleanup: error.cleanup } : error && typeof error === "object" && "data" in error ? error.data : null; - if (!data || typeof data !== "object" || !("schema" in data) || data.schema !== SCHEMA || !("cleanup" in data)) return null; + if (!data || typeof data !== "object" || !("schema" in data) || data.schema !== ENVIRONMENT_CREATION_CLEANUP_SCHEMA || !("cleanup" in data)) return null; const value = data.cleanup; if (!value || typeof value !== "object" || Array.isArray(value)) return null; const row = value as Record; diff --git a/packages/plugins/sdk/src/index.ts b/packages/plugins/sdk/src/index.ts index 7a10d0e8339..de69769668c 100644 --- a/packages/plugins/sdk/src/index.ts +++ b/packages/plugins/sdk/src/index.ts @@ -476,7 +476,7 @@ export { PRINCIPAL_TYPES, } from "@paperclipai/shared"; -export { PluginEnvironmentCreationCleanupError, environmentCreationCleanupErrorData, readEnvironmentCreationCleanupError, readEnvironmentAcquisitionDiagnostic, ENVIRONMENT_ACQUISITION_PROVIDER_STATES } from "./environment-creation-cleanup.js"; +export { PluginEnvironmentAcquisitionError, PluginEnvironmentCreationCleanupError, environmentCreationCleanupErrorData, readEnvironmentCreationCleanupError, readEnvironmentAcquisitionDiagnostic, ENVIRONMENT_ACQUISITION_PROVIDER_STATES } from "./environment-creation-cleanup.js"; export type { PluginEnvironmentCreationCleanup, PluginEnvironmentAcquisitionDiagnostic, PluginEnvironmentAcquisitionProviderState } from "./environment-creation-cleanup.js"; export { preserveEnvironmentSyncErrorDiagnostic, environmentSyncErrorData, readEnvironmentSyncErrorDiagnostic, withEnvironmentSyncErrorCapture, withEnvironmentSyncTransferStep, recordEnvironmentSyncError } from "./environment-sync-error.js"; export type { PluginEnvironmentSyncErrorDiagnostic } from "./environment-sync-error.js"; diff --git a/packages/plugins/sdk/tests/environment-acquire-error.test.ts b/packages/plugins/sdk/tests/environment-acquire-error.test.ts index 5e1bae48bee..a9d2f45f770 100644 --- a/packages/plugins/sdk/tests/environment-acquire-error.test.ts +++ b/packages/plugins/sdk/tests/environment-acquire-error.test.ts @@ -3,7 +3,12 @@ import { PassThrough } from "node:stream"; import { describe, expect, it } from "vitest"; import { definePlugin } from "../src/define-plugin.js"; import { readEnvironmentAcquireErrorDiagnostic } from "../src/environment-acquire-error.js"; -import { PluginEnvironmentCreationCleanupError, readEnvironmentCreationCleanupError } from "../src/environment-creation-cleanup.js"; +import { + PluginEnvironmentAcquisitionError, + PluginEnvironmentCreationCleanupError, + readEnvironmentAcquisitionDiagnostic, + readEnvironmentCreationCleanupError, +} from "../src/environment-creation-cleanup.js"; import { createRequest, isJsonRpcResponse, JsonRpcCallError, parseMessage, serializeMessage, type JsonRpcResponse } from "../src/protocol.js"; import { startWorkerRpcHost } from "../src/worker-rpc-host.js"; @@ -62,6 +67,25 @@ describe("environment acquisition refusal diagnostics", () => { expect(readEnvironmentAcquireErrorDiagnostic(plain)).toBeNull(); }); + it("carries the observed step of a removed failed allocation without provider text", async () => { + const callError = await acquire(new PluginEnvironmentAcquisitionError("Daytona could not start the sandbox.", + { phase: "create", elapsedMs: 38_000, budgetMs: 90_000, providerState: "error" }, + new Error("Sandbox private-sandbox-id failed to start with status: error"))); + expect(readEnvironmentAcquisitionDiagnostic(callError)).toEqual({ phase: "create", elapsedMs: 38_000, budgetMs: 90_000, providerState: "error" }); + expect(readEnvironmentCreationCleanupError(callError)).toBeNull(); + expect(readEnvironmentAcquireErrorDiagnostic(callError)).toBeNull(); + expect(JSON.stringify(callError.data)).not.toContain("private-sandbox-id"); + }); + + it("reads no acquisition step from an unstructured or forged refusal", async () => { + expect(readEnvironmentAcquisitionDiagnostic(await acquire(new Error("plain")))).toBeNull(); + const forged = (data: unknown) => Object.assign(new Error("x"), { data }); + expect(readEnvironmentAcquisitionDiagnostic(forged({ schema: "other", + acquisitionDiagnostic: { phase: "create", elapsedMs: 1, budgetMs: 2 } }))).toBeNull(); + expect(readEnvironmentAcquisitionDiagnostic(forged({ schema: "paperclip/environment-acquire-error/v1", + acquisitionDiagnostic: { phase: "anything", elapsedMs: 1, budgetMs: 2 } }))).toBeNull(); + }); + it("rejects forged or out-of-range envelopes", () => { const forged = (data: unknown) => Object.assign(new Error("x"), { data }); expect(readEnvironmentAcquireErrorDiagnostic(forged({ schema: "other", providerDiagnostic: { httpStatus: 502 } }))).toBeNull(); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 366c3b24f64..0738854998a 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -1,5 +1,6 @@ import { JsonRpcCallError, environmentResumeErrorData } from "@paperclipai/plugin-sdk"; import { collectRunFailureDiagnostics } from "../services/run-failure-diagnostics.js"; +import { readTransientLeaseAcquisitionFailure } from "../services/environment-provider-unavailable-error.js"; import { PluginWorkerUnavailableError } from "../services/plugin-worker-unavailable-error.js"; import { createSandboxOrphanCleanupSpool } from "../services/sandbox-orphan-cleanup-spool.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; @@ -1496,6 +1497,37 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { .toEqual([expect.objectContaining({ state: "pending", originalEnvironmentId: seeded.environment.id, runId: seeded.runId })]); }); + it.each(["worker", "forged error", "invalid worker diagnostic"])("records the step of a sandbox the provider could not start and removed: %s", async (mode) => { + const seeded = await seedReusablePluginSandboxLease(); + await environmentService(db).releaseLease(seeded.reusableLease.id, "expired"); + const data = { schema: "paperclip/environment-acquire-error/v1", + acquisitionDiagnostic: { phase: mode === "invalid worker diagnostic" ? "private-command" : "create", + elapsedMs: 38_000, budgetMs: 90_000, providerState: "error" } }; + const failure = mode === "forged error" + ? Object.assign(new Error("Daytona could not start the sandbox; it was removed."), { data }) + : new JsonRpcCallError({ code: -32002, message: "Daytona could not start the sandbox; it was removed.", data }); + const call = vi.fn(async (_id: string, method: string) => { + if (method === "environmentAcquireLease") throw failure; + throw new Error(`Unexpected method ${method}`); + }); + const worker = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }), call } as unknown as PluginWorkerManager; + const caught = await environmentRuntimeService(db, { pluginWorkerManager: worker }).acquireRunLease({ + companyId: seeded.companyId, environment: seeded.environment, issueId: null, heartbeatRunId: seeded.runId, + persistedExecutionWorkspace: null }).catch(error => error); + expect(caught).toBe(failure); + const run = { id: seeded.runId, companyId: seeded.companyId }; + const execution = collectRunFailureDiagnostics({ ...run, errorCode: "setup_failed", resultJson: null, + } as typeof heartbeatRuns.$inferSelect, { error: caught, phase: "setup" }).execution; + if (mode === "worker") { + expect(execution).toMatchObject({ environmentAcquisitionPhase: "create", environmentAcquisitionProviderState: "error" }); + expect(readTransientLeaseAcquisitionFailure(caught, run)).toEqual({ reason: "provider_start_failed" }); + } else { + expect(execution).not.toHaveProperty("environmentAcquisitionPhase"); + expect(readTransientLeaseAcquisitionFailure(caught, run)).toBeNull(); + } + expect(await db.select().from(environmentLeases).where(eq(environmentLeases.status, "pending_cleanup"))).toHaveLength(0); + }); + it.each(["deleted receipt lost", "observation response lost", "journal write failed", "foreign observation"])( "journals observed creation identity before deletion and recovers: %s", async (fault) => { const seeded = await seedReusablePluginSandboxLease(); diff --git a/server/src/services/environment-provider-unavailable-error.test.ts b/server/src/services/environment-provider-unavailable-error.test.ts index 2871ff9f247..c8b7ac67d87 100644 --- a/server/src/services/environment-provider-unavailable-error.test.ts +++ b/server/src/services/environment-provider-unavailable-error.test.ts @@ -154,6 +154,36 @@ describe("readTransientLeaseAcquisitionFailure", () => { expect(readTransientLeaseAcquisitionFailure(creationFailure(12_000), run)).toEqual({ reason: "acquisition_incomplete" }); }); + it("retries a sandbox the provider could not start and already removed", () => { + const error = new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, + message: "Daytona could not start the sandbox; it was removed.", + data: { schema: "paperclip/environment-acquire-error/v1", + acquisitionDiagnostic: { phase: "create", elapsedMs: 38_000, budgetMs: 90_000, providerState: "error" } } }); + captureEnvironmentAcquisitionDiagnostic(error, { companyId: run.companyId, environmentId: "env-1", runId: run.id }, + readEnvironmentAcquisitionDiagnostic(error)); + expect(readTransientLeaseAcquisitionFailure(new Error("lease", { cause: error }), run)).toEqual({ reason: "provider_start_failed" }); + const unavailable = new EnvironmentProviderUnavailableError({ environmentName: "Daytona warm", driver: "sandbox", + evidence: { reason: "provider_start_failed" }, cause: error }); + expect(unavailable.providerFailure).toMatchObject({ retryability: "transient", providerCode: "provider_start_failed" }); + expect(unavailable.message).toContain("the provider could not start the sandbox"); + }); + + it("names a later-step failure an unfinished acquisition, not a start failure", () => { + const error = new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, message: "x", + data: { schema: "paperclip/environment-acquire-error/v1", + acquisitionDiagnostic: { phase: "shell", elapsedMs: 38_000, budgetMs: 90_000, providerState: "error" } } }); + captureEnvironmentAcquisitionDiagnostic(error, { companyId: run.companyId, environmentId: "env-1", runId: run.id }, + readEnvironmentAcquisitionDiagnostic(error)); + expect(readTransientLeaseAcquisitionFailure(error, run)).toEqual({ reason: "acquisition_incomplete" }); + }); + + it("does not retry a removed sandbox whose step was never recorded for this run", () => { + const error = new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, message: "x", + data: { schema: "paperclip/environment-acquire-error/v1", + acquisitionDiagnostic: { phase: "create", elapsedMs: 38_000, budgetMs: 90_000, providerState: "error" } } }); + expect(readTransientLeaseAcquisitionFailure(error, run)).toBeNull(); + }); + it("ignores an unfinished-creation diagnostic recorded for another run", () => { expect(readTransientLeaseAcquisitionFailure(creationFailure(12_000, "run-2"), run)).toBeNull(); }); diff --git a/server/src/services/environment-provider-unavailable-error.ts b/server/src/services/environment-provider-unavailable-error.ts index 3f7ecba56c3..c5724a8f749 100644 --- a/server/src/services/environment-provider-unavailable-error.ts +++ b/server/src/services/environment-provider-unavailable-error.ts @@ -19,7 +19,7 @@ export const ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE = "environment_provider_unava * the provider reported a provisional creation that did not finish, or a * control-plane database query failed while the lease was taken. */ -type TransientLeaseFailureReason = "acquisition_incomplete" | "database_error"; +type TransientLeaseFailureReason = "acquisition_incomplete" | "provider_start_failed" | "database_error"; type TransientProviderEvidence = { httpStatus?: number; errorCode?: string; acquisitionTimedOut?: true; reason?: TransientLeaseFailureReason }; function own(value: unknown, key: string): unknown { @@ -104,7 +104,8 @@ export function readTransientLeaseAcquisitionFailure( for (let cause = error, depth = 0; cause && depth < 5; cause = own(cause, "cause"), depth++) { const acquisition = getEnvironmentAcquisitionDiagnostic(cause, run); if (acquisition) { - return acquisition.elapsedMs >= acquisition.budgetMs ? { acquisitionTimedOut: true } : { reason: "acquisition_incomplete" }; + if (acquisition.elapsedMs >= acquisition.budgetMs) return { acquisitionTimedOut: true }; + return { reason: acquisition.phase === "create" && acquisition.providerState === "error" ? "provider_start_failed" : "acquisition_incomplete" }; } if (cause instanceof JsonRpcCallError && cause.code === PLUGIN_RPC_ERROR_CODES.TIMEOUT) return { acquisitionTimedOut: true }; } @@ -113,6 +114,7 @@ export function readTransientLeaseAcquisitionFailure( const REASON_LABELS: Record = { acquisition_incomplete: "sandbox creation did not finish", + provider_start_failed: "the provider could not start the sandbox", database_error: "a database query failed while recording the lease", }; diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index c8c7d501201..3b254f1fa25 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -2047,8 +2047,9 @@ function createSandboxEnvironmentDriver( resolvePluginSandboxRpcTimeoutMs(workerConfig, pluginProvider.resolved.driver.defaultAcquireTimeoutMs)); } catch (error) { const cleanup = readEnvironmentCreationCleanupError(error); - const diagnostic = error instanceof JsonRpcCallError && cleanup?.companyId === input.companyId && - cleanup.environmentId === input.environment.id && cleanup.runId === identity.runId + const ownsReceipt = cleanup === null || (cleanup.companyId === input.companyId && + cleanup.environmentId === input.environment.id && cleanup.runId === identity.runId); + const diagnostic = error instanceof JsonRpcCallError && ownsReceipt ? readEnvironmentAcquisitionDiagnostic(error) : null; captureEnvironmentAcquisitionDiagnostic(error, { companyId: input.companyId, environmentId: input.environment.id, runId: identity.runId, diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index 55d9bfc53ac..99103671e41 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -6,8 +6,8 @@ "packages/plugins/sdk/src/define-plugin.ts": "a99e07ce77ef72fcf1df1347346aacb8ba12e284", "packages/plugins/sdk/src/dev-cli.ts": "7c1b0e2b6e4b7fd82027cca5321066e473a03d20", "packages/plugins/sdk/src/dev-server.ts": "2eadff81b74ca0706aec53aaf8637797fe4b4093", - "packages/plugins/sdk/src/environment-acquire-error.ts": "50a74c3d1f83150f4215d44a763aa8ce4d9469c2", - "packages/plugins/sdk/src/environment-creation-cleanup.ts": "35f23cd19ab0dd4148731846419a1c9b9942786b", + "packages/plugins/sdk/src/environment-acquire-error.ts": "4686b19d7918ae40e909028154a3614da4d17f35", + "packages/plugins/sdk/src/environment-creation-cleanup.ts": "419ead5f876d752f156e3852653b6da548a559a8", "packages/plugins/sdk/src/environment-resume-error.ts": "51ddfcd00e4e8c8a8b6bb39152655926b10aacad", "packages/plugins/sdk/src/environment-runner-ingress-error.ts": "fdb47797c2a1d2df82390e42b9593d4fa4cf49e7", "packages/plugins/sdk/src/environment-stop-error.ts": "b0e03f8aa2b2170ba095da5c7138050fced3f3cb", @@ -17,7 +17,7 @@ "packages/plugins/sdk/src/host-client-factory.ts": "e0301c390ea13f2340b669383f07af2455e48c8a", "packages/plugins/sdk/src/idle-drain.test.ts": "cc59f7881e7d46c89395a1d93d74f6c6b9b0a64a", "packages/plugins/sdk/src/idle-drain.ts": "ae6e4c6be4657a3d541396922b3bc551b8db557f", - "packages/plugins/sdk/src/index.ts": "7a10d0e833956d6265250eedbb768c7971c36d09", + "packages/plugins/sdk/src/index.ts": "de69769668c7d555d52576548d7fc1a35617627e", "packages/plugins/sdk/src/invocation-tracer.ts": "b28ac4aa4bc3e82658abac4a8081ed4508008696", "packages/plugins/sdk/src/protocol.duplex-channel.test.ts": "18255eb3d63038dc459240c111cbcb29fcba25cc", "packages/plugins/sdk/src/protocol.ts": "4861d78f1c57f1485a79e037932f39e8c91446b8",