diff --git a/doc/observability.md b/doc/observability.md index 00d8ea08c0e..e9f5e842ce9 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -1134,11 +1134,12 @@ untyped capacity text, and Codex CLI transient upstream failures remain reportab The run keeps its failed state and its terminal diagnostic. A native Codex model capacity failure stays local only while its automatic -retry owns recovery. The run must carry `native_provider_overloaded` and the -closed `codex`/`server_overloaded` diagnostic. The proof of the retry is the -`native_provider_overloaded` successor run for the same task, which only the -committed capacity status decision creates. The run's own finalization fields -are not required: a late terminal write can replace them with an earlier result. +retry owns recovery. The run must carry `native_provider_overloaded`. The proof +of the retry is the `native_provider_overloaded` successor run for the same +task, which only the committed capacity status decision creates, and only for +the typed `codex`/`server_overloaded` diagnostic. The run's own `resultJson` is +not required: the live heartbeat's failed-status write replaces it with the +adapter's copy, which lacks the finalizer's diagnostic and finalization fields. Exhausted or held capacity failures, other retry lanes, setup failures, signals, and unreadable retry evidence remain reportable. A failed retry reports its own outcome. diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts index 04a41cb627d..c533e59fee9 100644 --- a/server/src/services/__tests__/run-failure-report.test.ts +++ b/server/src/services/__tests__/run-failure-report.test.ts @@ -470,9 +470,11 @@ describeEmbeddedPostgres("reportRunFailure", () => { it.each([ ["no finalization fields", { finalizationPhase: undefined, finalizationReasonCode: undefined }], ["an earlier finalization phase", { finalizationPhase: "retryable_failure", finalizationReasonCode: undefined }], + ["only the adapter's resultJson", { nativeProviderFailure: undefined, finalizationPhase: undefined, finalizationReasonCode: undefined }], ] as const)("keeps a scheduled native capacity retry local when the run row has %s", async (_label, resultJson) => { - // A late terminal write can replace the finalizer's fields with an earlier - // resultJson. The successor created by the committed decision still proves it. + // The live heartbeat's failed-status write replaces the finalizer's + // resultJson with the adapter's copy. The successor created by the + // committed decision still proves the scheduled retry. const run = await nativeCapacityFailure({ resultJson }); await reportRunFailure(db, run); expect(mockCaptureRunFailure).not.toHaveBeenCalled(); @@ -486,8 +488,7 @@ describeEmbeddedPostgres("reportRunFailure", () => { ["the successor belongs to another task", { retry: { contextSnapshot: { issueId: randomUUID() } } }], ["the run has no native task", { run: { nativeIssueId: null } }], ["finalization has no fields and no retry run exists", { resultJson: { finalizationPhase: undefined, finalizationReasonCode: undefined }, retry: null }], - ["the diagnostic is not typed capacity evidence", { resultJson: { nativeProviderFailure: { provider: "codex", category: "turn_failed" } } }], - ["the diagnostic is missing", { resultJson: { nativeProviderFailure: undefined } }], + ["the diagnostic is missing and no retry run exists", { resultJson: { nativeProviderFailure: undefined }, retry: null }], ["the run is legacy", { run: { runtimeMode: "legacy" } }], ["the run has a signal", { run: { signal: "SIGTERM" } }], ["the run timed out", { run: { status: "timed_out" } }], diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index c9162ba08bb..fcf4baee0a4 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -15,7 +15,7 @@ import { readConnectionFailure, type ConnectionFailure } from "@paperclipai/adap import { isAiConnectionConfigurationReason } from "./ai-connection-configuration-failure.js"; import { readRunFailureReportingMode, type RunFailureReportingMode } from "../sentry-run-failure-policy.js"; import { RESTRICTED_WORKSPACE_SOURCE_REASON } from "./workspace-source-restriction.js"; -import { NATIVE_PROVIDER_OVERLOADED_CODE, NATIVE_PROVIDER_OVERLOADED_DIAGNOSTIC } from "./native-runtime/native-provider-failure.js"; +import { NATIVE_PROVIDER_OVERLOADED_CODE } from "./native-runtime/native-provider-failure.js"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; @@ -240,19 +240,17 @@ function isCodexAcpModelCapacityFailure(run: HeartbeatRun, options: RunFailureRe /** * A Codex `serverOverloaded` terminal chose the bounded capacity retry. Only * that committed status decision creates a successor with this retry reason, - * so the successor row is the proof. The run's own finalization fields are - * not: a late terminal write can replace them with an earlier `resultJson`. + * so the successor row is the proof. The run's own `resultJson` is not: the + * live heartbeat's failed-status write replaces it with the adapter's copy, + * which lacks the finalizer's diagnostic and finalization fields. * The retry owns recovery; a later failure reports itself. Held and exhausted * capacity failures have no automatic retry and stay reportable. The retry * carries the task in its context, so the company/context-issue index bounds * the read to that task's runs created since this one. */ async function isScheduledNativeCapacityRetry(db: Db, run: HeartbeatRun, options: RunFailureReportOptions): Promise { - const diagnostic = asRecord(asRecord(run.resultJson)?.nativeProviderFailure); if (run.status !== "failed" || run.runtimeMode !== "native" || run.errorCode !== NATIVE_PROVIDER_OVERLOADED_CODE || - run.nativeIssueId == null || options.phase === "setup" || run.signal != null || - diagnostic?.provider !== NATIVE_PROVIDER_OVERLOADED_DIAGNOSTIC.provider || - diagnostic.category !== NATIVE_PROVIDER_OVERLOADED_DIAGNOSTIC.category) return false; + run.nativeIssueId == null || options.phase === "setup" || run.signal != null) return false; const [retry] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, run.companyId), sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${run.nativeIssueId}`, gte(heartbeatRuns.createdAt, run.createdAt),