diff --git a/doc/observability.md b/doc/observability.md index 78ae54df3a6..741dced29a7 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -637,6 +637,14 @@ whether remote work may have been accepted. identifiers, untyped terminal failures, and timeouts after accepted work remain reportable. Stream/poll errors retain their existing retry and logging behavior. +A Codex ACP model capacity failure also stays local. The Codex ACP classifier +must match a typed `service` terminal failure that reports the model at capacity +or the server overloaded. It records `codex_transient_upstream` with the +`transient_upstream` family, and the bounded transient retry owns recovery. The +report must come after setup and carry no signal. Other `service` failures, +untyped capacity text, and Codex CLI transient upstream failures remain reportable. +The run keeps its failed state and its terminal diagnostic. + Expected pre-provider waits for a shared project workspace, a credential rotation lock, or an exhausted AI connection pool carry a server-owned expected cancellation receipt. They retain existing retry and lock behavior without a diff --git a/packages/adapters/codex-local/src/server/acp.quota.test.ts b/packages/adapters/codex-local/src/server/acp.quota.test.ts index 67b228664a8..fe371d95f1f 100644 --- a/packages/adapters/codex-local/src/server/acp.quota.test.ts +++ b/packages/adapters/codex-local/src/server/acp.quota.test.ts @@ -130,3 +130,44 @@ it.each(["Usage limit reached", "Usage limit exceeded", "You’ve hit your usage .toEqual({ errorCode: "provider_quota", errorFamily: "provider_quota" }); }, ); + +it.each([ + "Selected model is at capacity. Please try a different model.", + "This model is at capacity right now", + "We're at capacity for this model. Try again soon.", + "codexErrorInfo: serverOverloaded", +])("retries a typed Codex model capacity failure as transient upstream: %s", (title) => { + expect(classifyCodexTerminalSessionFailure({ category: "service", title }, now)) + .toEqual({ errorCode: "codex_transient_upstream", errorFamily: "transient_upstream" }); + expect(classifyCodexTerminalSessionFailure({ category: "service", title: "Codex turn failed", details: title }, now)) + .toEqual({ errorCode: "codex_transient_upstream", errorFamily: "transient_upstream" }); +}); + +it.each([ + ["HTTP 500 internal server error", "service"], + ["Workspace storage capacity limit reached", "service"], + ["Selected model is at capacity. Please try a different model.", "limit"], + ["Selected model is at capacity. Please try a different model.", "request"], + ["Selected model is at capacity. Please try a different model.", "connection"], +])("keeps other typed failures out of capacity retry: %s (%s)", (title, category) => { + expect(classifyCodexTerminalSessionFailure({ category, title }, now)).toBeNull(); +}); + +it("keeps the historical generic terminal-service error unclassified", () => { + expect(classifyCodexTerminalSessionFailure({ + category: "service", + title: "ACP agent reported a terminal service failure.", + }, now)).toBeNull(); +}); + +it("arms the bounded transient retry for a typed Codex model capacity failure", async () => { + const title = "Selected model is at capacity. Please try a different model."; + const { result } = await executeFailure(title, "service"); + expect(result).toMatchObject({ + exitCode: 1, + errorCode: "codex_transient_upstream", + errorFamily: "transient_upstream", + resultJson: { errorFamily: "transient_upstream", terminalSessionFailure: { category: "service" } }, + }); + expect(result.retryNotBefore).toBeUndefined(); +}); diff --git a/packages/adapters/codex-local/src/server/acp.ts b/packages/adapters/codex-local/src/server/acp.ts index ed57d8aa983..6b3fb4f83d4 100644 --- a/packages/adapters/codex-local/src/server/acp.ts +++ b/packages/adapters/codex-local/src/server/acp.ts @@ -266,10 +266,19 @@ async function prepareCodexRemoteManagedHome( }; } +const CODEX_MODEL_CAPACITY_RE = + /\b(?:selected model is at capacity|model is at capacity|at capacity for this model|serverOverloaded|server overloaded)\b/i; + export function classifyCodexTerminalSessionFailure( failure: AcpxTerminalSessionFailure, now: Date, ): AcpxTerminalFailureClassification | null { + if (failure.category === "service") { + const text = [failure.title, failure.details].filter(Boolean).join("\n"); + return CODEX_MODEL_CAPACITY_RE.test(text) + ? { errorCode: "codex_transient_upstream", errorFamily: "transient_upstream" } + : null; + } // ACP's `limit` also covers context, turn, rate and configured budget limits. // Require explicit usage exhaustion; the CLI's broader capacity matcher would // also match a context/storage capacity limit and defer the wrong failure. diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts index 6feb238bc71..d6269880635 100644 --- a/server/src/services/__tests__/run-failure-report.test.ts +++ b/server/src/services/__tests__/run-failure-report.test.ts @@ -393,6 +393,43 @@ describeEmbeddedPostgres("reportRunFailure", () => { expect(mockCaptureRunFailure).toHaveBeenCalledOnce(); }); + function codexModelCapacityRun(overrides: Partial = {}) { + return buildRun({ + error: "ACP agent reported a terminal service failure.\nSelected model is at capacity. Please try a different model.", + errorCode: "codex_transient_upstream", executionStage: "dispatching", exitCode: 1, signal: null, + resultJson: { + errorFamily: "transient_upstream", stopReason: "adapter_failed", + terminalSessionFailure: { category: "service", title: "Selected model is at capacity. Please try a different model." }, + }, + ...overrides, + }); + } + + it("keeps a classified Codex ACP model capacity retry local", async () => { + await seedCompanyAndAgent({ adapterType: "codex_local" }); + for (const phase of [undefined, "execute"] as const) await reportRunFailure(db, codexModelCapacityRun(), { phase }); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + }); + + it.each([ + { status: "timed_out" }, { signal: "SIGTERM" }, { errorCode: "acpx_turn_failed" }, + { resultJson: { errorFamily: "transient_upstream" } }, + { resultJson: { errorFamily: "provider_quota", terminalSessionFailure: { category: "service" } } }, + { resultJson: { errorFamily: "transient_upstream", terminalSessionFailure: { category: "connection" } } }, + ])("reports a Codex transient outcome without typed capacity evidence: %j", async (overrides) => { + await seedCompanyAndAgent({ adapterType: "codex_local" }); + await reportRunFailure(db, codexModelCapacityRun(overrides), { phase: "execute" }); + expect(mockCaptureRunFailure).toHaveBeenCalledOnce(); + }); + + it("reports a Codex ACP model capacity failure during setup or in all mode", async () => { + await seedCompanyAndAgent({ adapterType: "codex_local" }); + await reportRunFailure(db, codexModelCapacityRun(), { phase: "setup" }); + vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all"); + await reportRunFailure(db, codexModelCapacityRun(), { phase: "execute" }); + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(2); + }); + function restartInterruptedSetupRun(overrides: Partial = {}) { return buildRun({ error: "Process lost -- server may have restarted; retrying once", diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index 0fe6b277688..aedc4046a23 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -193,6 +193,14 @@ function isProviderQuotaFailure(run: HeartbeatRun, options: RunFailureReportOpti options.phase !== "setup" && run.signal == null; } +function isCodexAcpModelCapacityFailure(run: HeartbeatRun, options: RunFailureReportOptions): boolean { + const result = asRecord(run.resultJson); + return run.status === "failed" && run.errorCode === "codex_transient_upstream" && + result?.errorFamily === "transient_upstream" && + asRecord(result.terminalSessionFailure)?.category === "service" && + options.phase !== "setup" && run.signal == null; +} + /** * The Grok endpoint refused every work wake with an HTTP 4xx, so the offer * expired unaccepted. Durable delivery receipts prove the external cause. A @@ -236,6 +244,7 @@ export function reportRunFailure(db: Db, run: HeartbeatRun, options: RunFailureR isLocalPathWorkspaceConfigurationBlocker(run, options) || isGitConnectionFailure(run, options) || isProviderQuotaFailure(run, options) || + isCodexAcpModelCapacityFailure(run, options) || isRestartInterruptedSetup(run) )) return Promise.resolve(); const runStatus = run.status;