diff --git a/doc/observability.md b/doc/observability.md index 8da760ebf8d..94103a3a5a2 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -1070,6 +1070,15 @@ report must come after setup and carry no signal. Other `service` failures, untyped capacity text, and Codex CLI transient upstream failures remain reportable. The run keeps its failed state and its terminal diagnostic. +A native Codex model capacity failure stays local only while its automatic +retry owns recovery. The run must carry `native_provider_overloaded`, the closed +`codex`/`server_overloaded` diagnostic, and a committed finalization whose +reason is `native_provider_overloaded`. The report also requires the +`native_provider_overloaded` retry run that the same status transaction created. +Exhausted or held capacity failures, other retry lanes, setup failures, signals, +and unreadable retry evidence remain reportable. A failed retry reports its own +outcome. + Expected pre-provider waits for a shared project workspace, a credential rotation lock, or an exhausted AI connection pool carry a server-owned expected cancellation receipt. They retain existing retry and lock behavior without a diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts index 2ce0d6891fe..e471046ae03 100644 --- a/server/src/services/__tests__/run-failure-report.test.ts +++ b/server/src/services/__tests__/run-failure-report.test.ts @@ -430,6 +430,79 @@ describeEmbeddedPostgres("reportRunFailure", () => { expect(mockCaptureRunFailure).toHaveBeenCalledTimes(2); }); + /** The persisted shape after the native finalizer commits a capacity decision. */ + async function nativeCapacityFailure(input: { + run?: Partial; + resultJson?: Record; + retry?: Partial | null; + } = {}) { + await seedCompanyAndAgent({ adapterType: "paperclip_runner" }); + const run = buildRun({ + error: "Selected model is at capacity. Please try a different model.", + errorCode: "native_provider_overloaded", runtimeMode: "native", nativePhase: "committed", + executionStage: "preparing", exitCode: null, signal: null, + resultJson: { + nativeProviderFailure: { provider: "codex", category: "server_overloaded" }, + finalizationPhase: "committed", finalizationReasonCode: "native_provider_overloaded", + ...input.resultJson, + }, + ...input.run, + }); + const [stored] = await db.insert(heartbeatRuns).values(run).returning(); + if (input.retry !== null) { + await db.insert(heartbeatRuns).values({ + companyId, agentId, invocationSource: "automation", triggerDetail: "system", status: "scheduled_retry", + retryOfRunId: run.id, scheduledRetryReason: "native_provider_overloaded", scheduledRetryAttempt: 1, + scheduledRetryAt: new Date(Date.now() + 60_000), errorCode: "native_provider_overloaded", + contextSnapshot: { issueId: run.nativeIssueId, retryOfRunId: run.id }, ...input.retry, + }); + } + return stored!; + } + + it("keeps a native model capacity failure local once its automatic retry is scheduled", async () => { + const run = await nativeCapacityFailure(); + await reportRunFailure(db, run); + await reportRunFailure(db, run, { phase: "execute" }); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + }); + + it.each([ + ["the retries are exhausted", { resultJson: { finalizationReasonCode: "native_provider_overloaded_exhausted" }, retry: null }], + ["the task holds the retry", { resultJson: { finalizationReasonCode: "native_provider_overloaded_held" }, retry: null }], + ["no retry run exists", { retry: null }], + ["the successor belongs to another retry lane", { retry: { scheduledRetryReason: "native_safe_replacement" } }], + ["the successor belongs to another task", { retry: { contextSnapshot: { issueId: randomUUID() } } }], + ["the run has no native task", { run: { nativeIssueId: null } }], + ["finalization is not committed", { resultJson: { finalizationPhase: "retryable_failure" } }], + ["the diagnostic is not typed capacity evidence", { resultJson: { nativeProviderFailure: { provider: "codex", category: "turn_failed" } } }], + ["the diagnostic is missing", { resultJson: { nativeProviderFailure: undefined } }], + ["the run is legacy", { run: { runtimeMode: "legacy" } }], + ["the run has a signal", { run: { signal: "SIGTERM" } }], + ["the run timed out", { run: { status: "timed_out" } }], + ] as const)("reports a native capacity failure when %s", async (_label, input) => { + const run = await nativeCapacityFailure(input as Parameters[0]); + await reportRunFailure(db, run); + expect(mockCaptureRunFailure).toHaveBeenCalledOnce(); + }); + + it("reports a native capacity retry during setup, in all mode, or when its retry cannot be read", async () => { + const run = await nativeCapacityFailure(); + await reportRunFailure(db, run, { phase: "setup" }); + const select = db.select.bind(db); + // The agent lookup succeeds; the retry read fails. + const spy = vi.spyOn(db, "select").mockImplementationOnce(select as typeof db.select) + .mockImplementation(() => { throw new Error("fixture retry read failed"); }); + try { + await reportRunFailure(db, run); + } finally { + spy.mockRestore(); + } + vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all"); + await reportRunFailure(db, run); + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(3); + }); + function providerAccessRun(overrides: Partial = {}) { return buildRun({ error: "ACP agent reported a terminal access failure.\nInvalid API key ยท Fix external API key", diff --git a/server/src/services/native-runtime/native-provider-overload.integration.test.ts b/server/src/services/native-runtime/native-provider-overload.integration.test.ts index 862a7389bdd..82516e895d2 100644 --- a/server/src/services/native-runtime/native-provider-overload.integration.test.ts +++ b/server/src/services/native-runtime/native-provider-overload.integration.test.ts @@ -7,6 +7,10 @@ import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embed import { CONTROL_PLANE_CONFORMANCE_RESULT } from "../../vendor/paperclip-runner/testing.js"; const provider = vi.hoisted(() => ({ execute: vi.fn() })); +const sentry = vi.hoisted(() => ({ captureRunFailure: vi.fn() })); +vi.mock("../../sentry.js", async (importOriginal) => ({ + ...(await importOriginal()), captureRunFailure: sentry.captureRunFailure, +})); vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({ ...(await importOriginal()), executeNativeSession: provider.execute, })); @@ -20,12 +24,14 @@ import { heartbeatService } from "../heartbeat.js"; import { createPostgresWakeQueueAdapter } from "../../modules/wake-queue/adapters/postgres.js"; import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; import { buildExecutionContinuation } from "../execution-continuation.js"; +import { waitForPendingRunFailureReports } from "../run-failure-report.js"; describe("durable native provider capacity retry", () => { let temporary: Awaited>; let db: ReturnType; const companyId = randomUUID(), agentId = randomUUID(), otherAgentId = randomUUID(); beforeAll(async () => { + vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "unexpected"); temporary = await startEmbeddedPostgresTestDatabase("native-capacity-retry-"); db = createDb(temporary.connectionString); await db.insert(companies).values({ id: companyId, name: "Capacity regression", issuePrefix: "CAP" }); @@ -34,7 +40,7 @@ describe("durable native provider capacity retry", () => { { id: otherAgentId, companyId, name: "New owner", status: "active", adapterType: "paperclip_runner" }, ]); }); - afterAll(async () => { await temporary?.cleanup(); }); + afterAll(async () => { vi.unstubAllEnvs(); await temporary?.cleanup(); }); async function acceptCapacityFailure(runId: string, issueId: string, mutateOwner = false) { const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); @@ -95,10 +101,15 @@ describe("durable native provider capacity retry", () => { }); for (let failure = 0; failure <= 2; failure++) { const before = Date.now(); + sentry.captureRunFailure.mockClear(); await acceptCapacityFailure(runId, issueId); + await waitForPendingRunFailureReports(); const successors = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.retryOfRunId, runId))); if (failure === 2) { expect(successors).toHaveLength(0); + // Exhaustion ends automatic recovery, so Sentry receives the failure. + expect(sentry.captureRunFailure).toHaveBeenCalledOnce(); + expect(sentry.captureRunFailure.mock.calls[0]![0]).toMatchObject({ runId, runStatus: "failed" }); expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ status: "blocked", unblockDescriptor: { action: expect.stringContaining("Automatic retries exhausted") } }); break; } @@ -107,6 +118,8 @@ describe("durable native provider capacity retry", () => { expect(retry).toMatchObject({ status: "scheduled_retry", scheduledRetryAttempt: failure + 1, scheduledRetryReason: "native_provider_overloaded", error: NATIVE_PROVIDER_OVERLOADED_MESSAGE, contextSnapshot: { issueId, forceFreshSession: true, executionRetryAccounting: { version: 1, failureRetries: failure + 1, maxTurnContinuations: 0 } } }); expect(retry.contextSnapshot).not.toHaveProperty("wakeCommentId"); + // The scheduled retry owns recovery; this failure stays out of Sentry. + expect(sentry.captureRunFailure).not.toHaveBeenCalled(); expect(retry.scheduledRetryAt!.getTime() - before).toBeGreaterThanOrEqual(60_000 * 2 ** failure); expect(retry.scheduledRetryAt!.getTime() - Date.now()).toBeLessThanOrEqual(60_000 * 2 ** failure); const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.runId, retry.id)); @@ -170,8 +183,12 @@ describe("durable native provider capacity retry", () => { const issueId = randomUUID(), runId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId, title: "Reassigned capacity task", status: "in_progress", assigneeAgentId: agentId, responsibleUserId: "capacity-owner" }); await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, responsibleUserId: "capacity-owner", contextSnapshot: { issueId } }); + sentry.captureRunFailure.mockClear(); await acceptCapacityFailure(runId, issueId, true); + await waitForPendingRunFailureReports(); expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0); + // No retry owns this failure, so it remains reportable. + expect(sentry.captureRunFailure).toHaveBeenCalledOnce(); expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ status: "in_progress", assigneeAgentId: otherAgentId }); expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId))).filter(wake => wake.payload?.issueId === issueId)).toHaveLength(0); }); diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index c912b19a0d3..71b5494ca3a 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -1,4 +1,4 @@ -import { and, eq } from "drizzle-orm"; +import { and, eq, gte, sql } from "drizzle-orm"; import { agents, dotRunnerAssignments, grokBotDispatches, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db"; import { captureRunFailure, type RunFailureStatus } from "../sentry.js"; import { @@ -15,6 +15,7 @@ import { readConnectionFailure, type ConnectionFailure } from "@paperclipai/adap import { isAiConnectionConfigurationReason } from "./ai-connection-configuration-failure.js"; import { readRunFailureReportingMode, type RunFailureReportingMode } from "../sentry-run-failure-policy.js"; import { RESTRICTED_WORKSPACE_SOURCE_REASON } from "./workspace-source-restriction.js"; +import { NATIVE_PROVIDER_OVERLOADED_CODE, NATIVE_PROVIDER_OVERLOADED_DIAGNOSTIC } from "./native-runtime/native-provider-failure.js"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; @@ -236,6 +237,32 @@ function isCodexAcpModelCapacityFailure(run: HeartbeatRun, options: RunFailureRe options.phase !== "setup" && run.signal == null; } +/** + * A committed Codex `serverOverloaded` terminal chose the bounded capacity + * retry, and the same status transaction created that retry run. The retry + * owns recovery; a later failure reports itself. Held and exhausted capacity + * failures have no automatic retry and stay reportable. The retry carries the + * task in its context, so the company/context-issue index bounds the read to + * that task's runs created since this one. + */ +async function isScheduledNativeCapacityRetry(db: Db, run: HeartbeatRun, options: RunFailureReportOptions): Promise { + const result = asRecord(run.resultJson); + const diagnostic = asRecord(result?.nativeProviderFailure); + if (run.status !== "failed" || run.runtimeMode !== "native" || run.errorCode !== NATIVE_PROVIDER_OVERLOADED_CODE || + run.nativeIssueId == null || + options.phase === "setup" || run.signal != null || result?.finalizationPhase !== "committed" || + result.finalizationReasonCode !== NATIVE_PROVIDER_OVERLOADED_CODE || + diagnostic?.provider !== NATIVE_PROVIDER_OVERLOADED_DIAGNOSTIC.provider || + diagnostic.category !== NATIVE_PROVIDER_OVERLOADED_DIAGNOSTIC.category) return false; + const [retry] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, run.companyId), + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${run.nativeIssueId}`, gte(heartbeatRuns.createdAt, run.createdAt), + eq(heartbeatRuns.agentId, run.agentId), + eq(heartbeatRuns.retryOfRunId, run.id), eq(heartbeatRuns.scheduledRetryReason, NATIVE_PROVIDER_OVERLOADED_CODE), + )).limit(1); + return retry !== undefined; +} + const providerAuthErrorCodes = new Set(["claude_auth_required", "acpx_auth_required"]); /** @@ -336,6 +363,11 @@ async function captureTerminalRunFailure( return false; }); if (rejectedExternally) return; + const capacityRetry = await isScheduledNativeCapacityRetry(db, run, options).catch((err) => { + logger.warn({ err, runId: run.id }, "failed to read native capacity retry for run failure report"); + return false; + }); + if (capacityRetry) return; } const taskId = readTaskId(run);