From b4861d5e376fa60de2c471acf87da4acc2356be3 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 13 Nov 2026 12:04:25 -0800 Subject: [PATCH] fix(diagnostics): report Daytona acquisition state for runs retried as provider outages (#21476) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Before an agent starts on a sandbox environment, the host gets an environment lease. On Daytona, this creates a sandbox. > - When the create does not finish in time, the Daytona plugin records the failed phase, the elapsed time, the budget, and the Daytona sandbox state. #15736 and #15946 added these fields to the run failure report, so that operators can see where the acquisition stopped. > - #20857 changed the error code of these failures from `setup_failed` to `environment_provider_unavailable`, so that the host retries them. > - The run failure diagnostics read the acquisition fields only when the error code is `setup_failed`. Thus every Daytona create timeout now reaches the error report without its phase or provider state. > - On staging, more than 270 Daytona lease timeouts occurred in three days, and none of them shows the provider state. > - This pull request also reads the acquisition fields when the error code is `environment_provider_unavailable`. > - The benefit is that the error report shows again whether the sandbox was stuck in `creating`, in a snapshot state, or absent. This evidence is necessary to find the cause of the timeouts. ## Linked Issues or Issue Description Refs #20857, Refs #20828, Refs #15946, Refs #15736 **What happened?** A Daytona lease acquisition timeout is now classified as `environment_provider_unavailable` (#20857). The run failure report for this failure does not contain `environmentAcquisitionPhase`, `environmentAcquisitionElapsedMs`, `environmentAcquisitionBudgetMs` or `environmentAcquisitionProviderState`. **What did you expect?** The report contains the acquisition phase and the provider state, as it did for `setup_failed` runs before #20857. **Steps to reproduce** 1. Let a Daytona sandbox create pass its acquisition deadline. 2. The plugin returns "Daytona lease acquisition timed out; allocation cleanup is pending" with an acquisition diagnostic. 3. The host fails the run with `environment_provider_unavailable`. 4. Look at the `run_execution` context of the error report. The acquisition fields are not there. ## What Changed - `collectRunFailureDiagnostics` reads the private acquisition diagnostic for a setup failure with the error code `setup_failed` or `environment_provider_unavailable`. The other conditions do not change: the phase must be `setup`, and the diagnostic must belong to the same run and company. - A new test builds the real wrapper chain (lease failure, then provider failure, then the RPC error that holds the diagnostic). It checks that the fields are reported for the failed run, and that they are not reported for another run or for the execute phase. ## Verification - `cd server && npx vitest run src/services/__tests__/run-failure-diagnostics-provider-unavailable.test.ts` fails on master (the report has only `failurePhase`). It passes with this change. - `cd server && npx vitest run src/services/__tests__/run-failure-diagnostics-provider-unavailable.test.ts src/services/__tests__/run-failure-diagnostics.test.ts src/__tests__/environment-acquisition-diagnostics.test.ts`: 3 files, 139 tests pass. ## Risks Low risk. The change adds four bounded scalar fields to an error report. These fields already exist for `setup_failed` runs. The change does not alter retries, the error grouping, or the run result. ## Model Used Claude Opus 5.5 (`claude-opus-5-5`), with tool use and code execution in an agent harness. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- ...e-diagnostics-provider-unavailable.test.ts | 29 +++++++++++++++++++ .../src/services/run-failure-diagnostics.ts | 5 +++- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 server/src/services/__tests__/run-failure-diagnostics-provider-unavailable.test.ts diff --git a/server/src/services/__tests__/run-failure-diagnostics-provider-unavailable.test.ts b/server/src/services/__tests__/run-failure-diagnostics-provider-unavailable.test.ts new file mode 100644 index 00000000000..75a122c46ef --- /dev/null +++ b/server/src/services/__tests__/run-failure-diagnostics-provider-unavailable.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import type { heartbeatRuns } from "@paperclipai/db"; +import { captureEnvironmentAcquisitionDiagnostic } from "../environment-acquisition-diagnostics.js"; +import { collectRunFailureDiagnostics } from "../run-failure-diagnostics.js"; + +type Run = typeof heartbeatRuns.$inferSelect; +const run = (overrides: Partial = {}) => ({ resultJson: null, ...overrides }) as Run; + +describe("run failure diagnostics for a lease acquisition retried as a provider outage", () => { + it("reports the acquisition phase and provider state for the exact failed run", () => { + const scope = { companyId: "company-fixture", environmentId: "environment-fixture", runId: "run-fixture" }; + const rpcError = new Error("Daytona lease acquisition timed out; allocation cleanup is pending"); + captureEnvironmentAcquisitionDiagnostic(rpcError, scope, + { phase: "create", elapsedMs: 90_000, budgetMs: 90_000, providerState: "creating" }); + // The host wraps the RPC error as the transient provider failure, then as the lease failure. + const unavailable = Object.assign(new Error("Environment provider did not finish starting the sandbox in time.", + { cause: rpcError }), { code: "environment_provider_unavailable" }); + const failure = Object.assign(new Error("Failed to acquire lease", { cause: unavailable }), { code: "lease_acquire_failed" }); + const fixture = run({ id: scope.runId, companyId: scope.companyId, errorCode: "environment_provider_unavailable" }); + expect(collectRunFailureDiagnostics(fixture, { error: failure, phase: "setup" }).execution).toMatchObject({ + environmentAcquisitionPhase: "create", environmentAcquisitionElapsedMs: 90_000, + environmentAcquisitionBudgetMs: 90_000, environmentAcquisitionProviderState: "creating", + }); + expect(collectRunFailureDiagnostics(run({ ...fixture, id: "another-run" }), { error: failure, phase: "setup" }).execution) + .not.toHaveProperty("environmentAcquisitionPhase"); + expect(collectRunFailureDiagnostics(fixture, { error: failure, phase: "execute" }).execution) + .not.toHaveProperty("environmentAcquisitionPhase"); + }); +}); diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts index 5cbe30cc0f6..4059248a106 100644 --- a/server/src/services/run-failure-diagnostics.ts +++ b/server/src/services/run-failure-diagnostics.ts @@ -380,7 +380,10 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport break; } if (typeof error !== "object") break; - const acquisition = options.phase === "setup" && run.errorCode === "setup_failed" + // A lease acquisition timeout is retried as a transient provider failure; + // it needs the same phase and provider state as a plain setup failure. + const acquisition = options.phase === "setup" && (run.errorCode === "setup_failed" + || run.errorCode === "environment_provider_unavailable") ? getEnvironmentAcquisitionDiagnostic(error, run) : null; if (acquisition && execution.environmentAcquisitionPhase === undefined) { execution.environmentAcquisitionPhase = acquisition.phase;