diff --git a/server/src/services/__tests__/run-failure-diagnostics-provider-unavailable.test.ts b/server/src/services/__tests__/run-failure-diagnostics-provider-unavailable.test.ts new file mode 100644 index 00000000000..75a122c46ef --- /dev/null +++ b/server/src/services/__tests__/run-failure-diagnostics-provider-unavailable.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import type { heartbeatRuns } from "@paperclipai/db"; +import { captureEnvironmentAcquisitionDiagnostic } from "../environment-acquisition-diagnostics.js"; +import { collectRunFailureDiagnostics } from "../run-failure-diagnostics.js"; + +type Run = typeof heartbeatRuns.$inferSelect; +const run = (overrides: Partial = {}) => ({ resultJson: null, ...overrides }) as Run; + +describe("run failure diagnostics for a lease acquisition retried as a provider outage", () => { + it("reports the acquisition phase and provider state for the exact failed run", () => { + const scope = { companyId: "company-fixture", environmentId: "environment-fixture", runId: "run-fixture" }; + const rpcError = new Error("Daytona lease acquisition timed out; allocation cleanup is pending"); + captureEnvironmentAcquisitionDiagnostic(rpcError, scope, + { phase: "create", elapsedMs: 90_000, budgetMs: 90_000, providerState: "creating" }); + // The host wraps the RPC error as the transient provider failure, then as the lease failure. + const unavailable = Object.assign(new Error("Environment provider did not finish starting the sandbox in time.", + { cause: rpcError }), { code: "environment_provider_unavailable" }); + const failure = Object.assign(new Error("Failed to acquire lease", { cause: unavailable }), { code: "lease_acquire_failed" }); + const fixture = run({ id: scope.runId, companyId: scope.companyId, errorCode: "environment_provider_unavailable" }); + expect(collectRunFailureDiagnostics(fixture, { error: failure, phase: "setup" }).execution).toMatchObject({ + environmentAcquisitionPhase: "create", environmentAcquisitionElapsedMs: 90_000, + environmentAcquisitionBudgetMs: 90_000, environmentAcquisitionProviderState: "creating", + }); + expect(collectRunFailureDiagnostics(run({ ...fixture, id: "another-run" }), { error: failure, phase: "setup" }).execution) + .not.toHaveProperty("environmentAcquisitionPhase"); + expect(collectRunFailureDiagnostics(fixture, { error: failure, phase: "execute" }).execution) + .not.toHaveProperty("environmentAcquisitionPhase"); + }); +}); diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts index 5cbe30cc0f6..4059248a106 100644 --- a/server/src/services/run-failure-diagnostics.ts +++ b/server/src/services/run-failure-diagnostics.ts @@ -380,7 +380,10 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport break; } if (typeof error !== "object") break; - const acquisition = options.phase === "setup" && run.errorCode === "setup_failed" + // A lease acquisition timeout is retried as a transient provider failure; + // it needs the same phase and provider state as a plain setup failure. + const acquisition = options.phase === "setup" && (run.errorCode === "setup_failed" + || run.errorCode === "environment_provider_unavailable") ? getEnvironmentAcquisitionDiagnostic(error, run) : null; if (acquisition && execution.environmentAcquisitionPhase === undefined) { execution.environmentAcquisitionPhase = acquisition.phase;