diff --git a/doc/observability.md b/doc/observability.md index 3b1df075f26..00d8ea08c0e 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -1134,10 +1134,11 @@ untyped capacity text, and Codex CLI transient upstream failures remain reportab The run keeps its failed state and its terminal diagnostic. A native Codex model capacity failure stays local only while its automatic -retry owns recovery. The run must carry `native_provider_overloaded`, the closed -`codex`/`server_overloaded` diagnostic, and a committed finalization whose -reason is `native_provider_overloaded`. The report also requires the -`native_provider_overloaded` retry run that the same status transaction created. +retry owns recovery. The run must carry `native_provider_overloaded` and the +closed `codex`/`server_overloaded` diagnostic. The proof of the retry is the +`native_provider_overloaded` successor run for the same task, which only the +committed capacity status decision creates. The run's own finalization fields +are not required: a late terminal write can replace them with an earlier result. Exhausted or held capacity failures, other retry lanes, setup failures, signals, and unreadable retry evidence remain reportable. A failed retry reports its own outcome. diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts index e471046ae03..04a41cb627d 100644 --- a/server/src/services/__tests__/run-failure-report.test.ts +++ b/server/src/services/__tests__/run-failure-report.test.ts @@ -467,6 +467,17 @@ describeEmbeddedPostgres("reportRunFailure", () => { expect(mockCaptureRunFailure).not.toHaveBeenCalled(); }); + it.each([ + ["no finalization fields", { finalizationPhase: undefined, finalizationReasonCode: undefined }], + ["an earlier finalization phase", { finalizationPhase: "retryable_failure", finalizationReasonCode: undefined }], + ] as const)("keeps a scheduled native capacity retry local when the run row has %s", async (_label, resultJson) => { + // A late terminal write can replace the finalizer's fields with an earlier + // resultJson. The successor created by the committed decision still proves it. + const run = await nativeCapacityFailure({ resultJson }); + await reportRunFailure(db, run); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + }); + it.each([ ["the retries are exhausted", { resultJson: { finalizationReasonCode: "native_provider_overloaded_exhausted" }, retry: null }], ["the task holds the retry", { resultJson: { finalizationReasonCode: "native_provider_overloaded_held" }, retry: null }], @@ -474,7 +485,7 @@ describeEmbeddedPostgres("reportRunFailure", () => { ["the successor belongs to another retry lane", { retry: { scheduledRetryReason: "native_safe_replacement" } }], ["the successor belongs to another task", { retry: { contextSnapshot: { issueId: randomUUID() } } }], ["the run has no native task", { run: { nativeIssueId: null } }], - ["finalization is not committed", { resultJson: { finalizationPhase: "retryable_failure" } }], + ["finalization has no fields and no retry run exists", { resultJson: { finalizationPhase: undefined, finalizationReasonCode: undefined }, retry: null }], ["the diagnostic is not typed capacity evidence", { resultJson: { nativeProviderFailure: { provider: "codex", category: "turn_failed" } } }], ["the diagnostic is missing", { resultJson: { nativeProviderFailure: undefined } }], ["the run is legacy", { run: { runtimeMode: "legacy" } }], diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index 71b5494ca3a..c9162ba08bb 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -238,20 +238,19 @@ function isCodexAcpModelCapacityFailure(run: HeartbeatRun, options: RunFailureRe } /** - * A committed Codex `serverOverloaded` terminal chose the bounded capacity - * retry, and the same status transaction created that retry run. The retry - * owns recovery; a later failure reports itself. Held and exhausted capacity - * failures have no automatic retry and stay reportable. The retry carries the - * task in its context, so the company/context-issue index bounds the read to - * that task's runs created since this one. + * A Codex `serverOverloaded` terminal chose the bounded capacity retry. Only + * that committed status decision creates a successor with this retry reason, + * so the successor row is the proof. The run's own finalization fields are + * not: a late terminal write can replace them with an earlier `resultJson`. + * The retry owns recovery; a later failure reports itself. Held and exhausted + * capacity failures have no automatic retry and stay reportable. The retry + * carries the task in its context, so the company/context-issue index bounds + * the read to that task's runs created since this one. */ async function isScheduledNativeCapacityRetry(db: Db, run: HeartbeatRun, options: RunFailureReportOptions): Promise { - const result = asRecord(run.resultJson); - const diagnostic = asRecord(result?.nativeProviderFailure); + const diagnostic = asRecord(asRecord(run.resultJson)?.nativeProviderFailure); if (run.status !== "failed" || run.runtimeMode !== "native" || run.errorCode !== NATIVE_PROVIDER_OVERLOADED_CODE || - run.nativeIssueId == null || - options.phase === "setup" || run.signal != null || result?.finalizationPhase !== "committed" || - result.finalizationReasonCode !== NATIVE_PROVIDER_OVERLOADED_CODE || + run.nativeIssueId == null || options.phase === "setup" || run.signal != null || diagnostic?.provider !== NATIVE_PROVIDER_OVERLOADED_DIAGNOSTIC.provider || diagnostic.category !== NATIVE_PROVIDER_OVERLOADED_DIAGNOSTIC.category) return false; const [retry] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(