fix(codex-local): retry ACP model capacity failures as transient upstream (#16407)

Classify typed Codex ACP `service` failures about model capacity or server overload as `codex_transient_upstream` / `transient_upstream`, so the existing bounded transient retry and continuation recovery apply. In `unexpected` Sentry mode, skip only runs carrying that full classified evidence; `all` mode still reports them.

doc/observability.md

@@ -637,6 +637,14 @@ whether remote work may have been accepted.
   identifiers, untyped terminal failures, and timeouts after accepted work remain
   reportable. Stream/poll errors retain their existing retry and logging behavior.
 
+A Codex ACP model capacity failure also stays local. The Codex ACP classifier
+must match a typed `service` terminal failure that reports the model at capacity
+or the server overloaded. It records `codex_transient_upstream` with the
+`transient_upstream` family, and the bounded transient retry owns recovery. The
+report must come after setup and carry no signal. Other `service` failures,
+untyped capacity text, and Codex CLI transient upstream failures remain reportable.
+The run keeps its failed state and its terminal diagnostic.
+
 Expected pre-provider waits for a shared project workspace, a credential rotation
 lock, or an exhausted AI connection pool carry a server-owned expected
 cancellation receipt. They retain existing retry and lock behavior without a

packages/adapters/codex-local/src/server/acp.quota.test.ts

@@ -130,3 +130,44 @@ it.each(["Usage limit reached", "Usage limit exceeded", "You’ve hit your usage
       .toEqual({ errorCode: "provider_quota", errorFamily: "provider_quota" });
   },
 );
+
+it.each([
+  "Selected model is at capacity. Please try a different model.",
+  "This model is at capacity right now",
+  "We're at capacity for this model. Try again soon.",
+  "codexErrorInfo: serverOverloaded",
+])("retries a typed Codex model capacity failure as transient upstream: %s", (title) => {
+  expect(classifyCodexTerminalSessionFailure({ category: "service", title }, now))
+    .toEqual({ errorCode: "codex_transient_upstream", errorFamily: "transient_upstream" });
+  expect(classifyCodexTerminalSessionFailure({ category: "service", title: "Codex turn failed", details: title }, now))
+    .toEqual({ errorCode: "codex_transient_upstream", errorFamily: "transient_upstream" });
+});
+
+it.each([
+  ["HTTP 500 internal server error", "service"],
+  ["Workspace storage capacity limit reached", "service"],
+  ["Selected model is at capacity. Please try a different model.", "limit"],
+  ["Selected model is at capacity. Please try a different model.", "request"],
+  ["Selected model is at capacity. Please try a different model.", "connection"],
+])("keeps other typed failures out of capacity retry: %s (%s)", (title, category) => {
+  expect(classifyCodexTerminalSessionFailure({ category, title }, now)).toBeNull();
+});
+
+it("keeps the historical generic terminal-service error unclassified", () => {
+  expect(classifyCodexTerminalSessionFailure({
+    category: "service",
+    title: "ACP agent reported a terminal service failure.",
+  }, now)).toBeNull();
+});
+
+it("arms the bounded transient retry for a typed Codex model capacity failure", async () => {
+  const title = "Selected model is at capacity. Please try a different model.";
+  const { result } = await executeFailure(title, "service");
+  expect(result).toMatchObject({
+    exitCode: 1,
+    errorCode: "codex_transient_upstream",
+    errorFamily: "transient_upstream",
+    resultJson: { errorFamily: "transient_upstream", terminalSessionFailure: { category: "service" } },
+  });
+  expect(result.retryNotBefore).toBeUndefined();
+});

packages/adapters/codex-local/src/server/acp.ts

@@ -266,10 +266,19 @@ async function prepareCodexRemoteManagedHome(
   };
 }
 
+const CODEX_MODEL_CAPACITY_RE =
+  /\b(?:selected model is at capacity|model is at capacity|at capacity for this model|serverOverloaded|server overloaded)\b/i;
+
 export function classifyCodexTerminalSessionFailure(
   failure: AcpxTerminalSessionFailure,
   now: Date,
 ): AcpxTerminalFailureClassification | null {
+  if (failure.category === "service") {
+    const text = [failure.title, failure.details].filter(Boolean).join("\n");
+    return CODEX_MODEL_CAPACITY_RE.test(text)
+      ? { errorCode: "codex_transient_upstream", errorFamily: "transient_upstream" }
+      : null;
+  }
   // ACP's `limit` also covers context, turn, rate and configured budget limits.
   // Require explicit usage exhaustion; the CLI's broader capacity matcher would
   // also match a context/storage capacity limit and defer the wrong failure.

server/src/services/__tests__/run-failure-report.test.ts

@@ -393,6 +393,43 @@ describeEmbeddedPostgres("reportRunFailure", () => {
     expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
   });
 
+  function codexModelCapacityRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
+    return buildRun({
+      error: "ACP agent reported a terminal service failure.\nSelected model is at capacity. Please try a different model.",
+      errorCode: "codex_transient_upstream", executionStage: "dispatching", exitCode: 1, signal: null,
+      resultJson: {
+        errorFamily: "transient_upstream", stopReason: "adapter_failed",
+        terminalSessionFailure: { category: "service", title: "Selected model is at capacity. Please try a different model." },
+      },
+      ...overrides,
+    });
+  }
+
+  it("keeps a classified Codex ACP model capacity retry local", async () => {
+    await seedCompanyAndAgent({ adapterType: "codex_local" });
+    for (const phase of [undefined, "execute"] as const) await reportRunFailure(db, codexModelCapacityRun(), { phase });
+    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
+  });
+
+  it.each([
+    { status: "timed_out" }, { signal: "SIGTERM" }, { errorCode: "acpx_turn_failed" },
+    { resultJson: { errorFamily: "transient_upstream" } },
+    { resultJson: { errorFamily: "provider_quota", terminalSessionFailure: { category: "service" } } },
+    { resultJson: { errorFamily: "transient_upstream", terminalSessionFailure: { category: "connection" } } },
+  ])("reports a Codex transient outcome without typed capacity evidence: %j", async (overrides) => {
+    await seedCompanyAndAgent({ adapterType: "codex_local" });
+    await reportRunFailure(db, codexModelCapacityRun(overrides), { phase: "execute" });
+    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
+  });
+
+  it("reports a Codex ACP model capacity failure during setup or in all mode", async () => {
+    await seedCompanyAndAgent({ adapterType: "codex_local" });
+    await reportRunFailure(db, codexModelCapacityRun(), { phase: "setup" });
+    vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all");
+    await reportRunFailure(db, codexModelCapacityRun(), { phase: "execute" });
+    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(2);
+  });
+
   function restartInterruptedSetupRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
     return buildRun({
       error: "Process lost -- server may have restarted; retrying once",

server/src/services/run-failure-report.ts

@@ -193,6 +193,14 @@ function isProviderQuotaFailure(run: HeartbeatRun, options: RunFailureReportOpti
     options.phase !== "setup" && run.signal == null;
 }
 
+function isCodexAcpModelCapacityFailure(run: HeartbeatRun, options: RunFailureReportOptions): boolean {
+  const result = asRecord(run.resultJson);
+  return run.status === "failed" && run.errorCode === "codex_transient_upstream" &&
+    result?.errorFamily === "transient_upstream" &&
+    asRecord(result.terminalSessionFailure)?.category === "service" &&
+    options.phase !== "setup" && run.signal == null;
+}
+
 /**
  * The Grok endpoint refused every work wake with an HTTP 4xx, so the offer
  * expired unaccepted. Durable delivery receipts prove the external cause. A
@@ -236,6 +244,7 @@ export function reportRunFailure(db: Db, run: HeartbeatRun, options: RunFailureR
     isLocalPathWorkspaceConfigurationBlocker(run, options) ||
     isGitConnectionFailure(run, options) ||
     isProviderQuotaFailure(run, options) ||
+    isCodexAcpModelCapacityFailure(run, options) ||
     isRestartInterruptedSetup(run)
   )) return Promise.resolve();
   const runStatus = run.status;