From 4b958ecbd8f33a7782fe6f29ec825f80f636b833 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Mon, 12 Oct 2026 15:59:27 -0700 Subject: [PATCH] fix(server): keep classified provider quota runs out of Sentry (#16029) A run that stops because the user's provider account hit its usage limit is an expected wait, not a Paperclip defect. The adapter already classifies it (errorCode and resultJson.errorFamily are both provider_quota) and recovery schedules a retry at the reset time. Skip the Sentry report only when both adapter-owned fields agree, the run status is failed, no signal stopped it, and the failure did not happen during setup. Timeouts, cancellations, signals, other codes and missing or mismatched families still report. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Dotta --- .../__tests__/run-failure-report.test.ts | 33 +++++++++++++++++++ server/src/services/run-failure-report.ts | 11 +++++++ 2 files changed, 44 insertions(+) diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts index 0c1d83de7b7..111be4faf60 100644 --- a/server/src/services/__tests__/run-failure-report.test.ts +++ b/server/src/services/__tests__/run-failure-report.test.ts @@ -301,6 +301,39 @@ describeEmbeddedPostgres("reportRunFailure", () => { expect(mockCaptureRunFailure).toHaveBeenCalledWith(expect.objectContaining({ agentAdapter: "unknown" })); }); + function providerQuotaRun(overrides: Partial = {}) { + return buildRun({ + error: "ACP agent reported a terminal limit failure.\nYou've hit your session limit · resets 2am (UTC)", + errorCode: "provider_quota", executionStage: "dispatching", exitCode: 1, signal: null, + resultJson: { errorFamily: "provider_quota", stopReason: "adapter_failed" }, ...overrides, + }); + } + + it.each(["claude_local", "codex_local"])("keeps a classified %s provider quota wait local", async (adapterType) => { + await seedCompanyAndAgent({ adapterType }); + const [run] = await db.insert(heartbeatRuns).values(providerQuotaRun()).returning(); + const before = structuredClone(run); + for (const phase of [undefined, "execute"] as const) await reportRunFailure(db, run!, { phase }); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + expect(await db.select().from(heartbeatRuns)).toEqual([before]); + }); + + it.each([ + { status: "timed_out" }, { status: "cancelled", startedAt: new Date(0) }, { signal: "SIGTERM" }, + { errorCode: "adapter_failed" }, { errorCode: "claude_transient_upstream" }, + { resultJson: {} }, { resultJson: null }, { resultJson: { errorFamily: "transient_upstream" } }, + ])("reports a provider quota outcome without matching adapter evidence: %j", async (overrides) => { + await seedCompanyAndAgent({ adapterType: "claude_local" }); + await reportRunFailure(db, providerQuotaRun(overrides), { phase: "execute" }); + expect(mockCaptureRunFailure).toHaveBeenCalledOnce(); + }); + + it("keeps a provider quota code reportable during setup", async () => { + await seedCompanyAndAgent({ adapterType: "claude_local" }); + await reportRunFailure(db, providerQuotaRun(), { phase: "setup" }); + expect(mockCaptureRunFailure).toHaveBeenCalledOnce(); + }); + function aiSelectionRun(overrides: Partial = {}) { return buildRun({ errorCode: "configuration_incomplete", diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index a059bf523d0..60e6309e541 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -164,6 +164,16 @@ function isHermesConnectionFailure(run: HeartbeatRun, options: RunFailureReportO (connection.reason === "remote_unavailable" && code === "hermes_gateway_upstream_error"); } +/** + * The adapter proved the user's provider account is out of quota, and recovery + * waits for its reset. Both fields come from the adapter's own classifier. + */ +function isProviderQuotaFailure(run: HeartbeatRun, options: RunFailureReportOptions): boolean { + return run.status === "failed" && run.errorCode === "provider_quota" && + asRecord(run.resultJson)?.errorFamily === "provider_quota" && + options.phase !== "setup" && run.signal == null; +} + /** * Report a terminal run failure to Sentry. Returns at once for any status * other than failures and unexpected started cancellations. Never throws — a Sentry failure or a @@ -182,6 +192,7 @@ export function reportRunFailure(db: Db, run: HeartbeatRun, options: RunFailureR if (isAiConnectionSelectionBlocker(run, options)) return Promise.resolve(); if (isLocalPathWorkspaceConfigurationBlocker(run, options)) return Promise.resolve(); if (isGitConnectionFailure(run, options)) return Promise.resolve(); + if (isProviderQuotaFailure(run, options)) return Promise.resolve(); const runStatus = run.status; const report = captureTerminalRunFailure(db, run, runStatus, options); pendingRunFailureReports.add(report);