From aa4a06653c6f6c0a96664efc2bc89df1079aaf8a Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 13 Nov 2026 11:22:55 -0800 Subject: [PATCH] fix(native): name the recorded turn failure when a native run fails (#21235) Failed native runs now report the runner's recorded turn-failure code instead of the generic unclassified fallback, and the Sentry run-failure fingerprint includes runner:. Co-Authored-By: Paperclip Co-Authored-By: Claude Opus 5.5 --- .../native-run-projection-work.test.ts | 15 ++++++++++ .../src/__tests__/run-failure-sentry.test.ts | 23 ++++++++++++++ server/src/sentry.ts | 8 ++++- ...ative-provider-failure.integration.test.ts | 17 +++++++++-- .../native-run-failure-explanation.test.ts | 30 +++++++++++++++++++ .../native-run-failure-explanation.ts | 21 +++++++++++++ .../native-runtime/native-run-finalizer.ts | 7 +++-- .../native-runtime/native-run-projection.ts | 12 ++++---- .../native-runtime/native-session-executor.ts | 4 ++- 9 files changed, 125 insertions(+), 12 deletions(-) create mode 100644 server/src/services/native-runtime/native-run-failure-explanation.test.ts diff --git a/server/src/__tests__/native-run-projection-work.test.ts b/server/src/__tests__/native-run-projection-work.test.ts index 84f80b02e34..ab217da5d85 100644 --- a/server/src/__tests__/native-run-projection-work.test.ts +++ b/server/src/__tests__/native-run-projection-work.test.ts @@ -485,6 +485,21 @@ describe("committed native run projection lifecycle", () => { await projectCommittedRun({ db, run, coordinator: await state(binding.runId) }); expect(await read(binding.runId)).toMatchObject({ status: "failed", errorCode: expectedCode, error: expectedError }); }); + it("names a recorded runner turn failure in committed recovery projection", async () => { + const binding = await accepted("failed"); + await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); + const [unfinished] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, binding.runId)); + await db.update(heartbeatRuns).set({ errorCode: null, error: null, resultJson: { ...unfinished.resultJson, + nativeTurnFailure: { provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_empty_response", message: "private provider text" } } }) + .where(eq(heartbeatRuns.id, binding.runId)); + const run = await read(binding.runId); + expect(run.status).toBe("running"); + await projectCommittedRun({ db, run, coordinator: await state(binding.runId) }); + const projected = await read(binding.runId); + expect(projected).toMatchObject({ status: "failed", errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode, + error: "The OpenCode turn failed (opencode_empty_response). Inspect the run events for the last recorded step." }); + expect(projected.error).not.toContain("private"); + }); it("ignores routine company-only finalization hints and still repairs a committed run", async () => { await start(); await flush(); await quiet(); const previous = db.$client.options.debug; diff --git a/server/src/__tests__/run-failure-sentry.test.ts b/server/src/__tests__/run-failure-sentry.test.ts index 6f42a6867d4..b4be99438ee 100644 --- a/server/src/__tests__/run-failure-sentry.test.ts +++ b/server/src/__tests__/run-failure-sentry.test.ts @@ -233,6 +233,29 @@ describe("captureRunFailure", () => { expect(JSON.stringify(fingerprints)).not.toMatch(/workspace|timed out|plain failure/i); }); + it("separates native turn failures by their runner failure code", async () => { + const { sentryModule, captureException } = await importFreshSentryWithGateOpen(); + const turnFailure = (provider: NonNullable["provider"]) => baseEvent({ + errorCode: "native_run_failed", agentAdapter: "paperclip_runner", + diagnostics: { execution: {}, adapter: {}, provider, exceptions: [], truncatedFields: [] }, + }); + + sentryModule.captureRunFailure(turnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_empty_response" })); + sentryModule.captureRunFailure(turnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited" })); + sentryModule.captureRunFailure(turnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "unrecognized" })); + sentryModule.captureRunFailure(turnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "not a code" })); + sentryModule.captureRunFailure(turnFailure({})); + + const prefix = ["native_run_failed", "paperclip_runner"]; + expect(captureException.mock.calls.map(([, context]) => context?.fingerprint)).toEqual([ + [...prefix, "runner:opencode_empty_response"], + [...prefix, "runner:opencode_server_exited"], + prefix, + prefix, + prefix, + ]); + }); + it("keeps an identifier root code as the only root grouping element", async () => { const { sentryModule, captureException } = await importFreshSentryWithGateOpen(); diff --git a/server/src/sentry.ts b/server/src/sentry.ts index e3d0787416d..24b2637db7c 100644 --- a/server/src/sentry.ts +++ b/server/src/sentry.ts @@ -288,6 +288,11 @@ function captureRunFailureEvent(event: RunFailureEvent): string | null { rootFallback.push(`code:${rootException.code}`); } } + // A native turn failure's runner-issued code is closed vocabulary. Without + // it, every unclassified native turn failure shares one group. + const runnerCodeCandidate = diagnostics?.provider.runnerFailureCode; + const runnerFailureCode = typeof runnerCodeCandidate === "string" && runnerCodeCandidate !== "unrecognized" && + /^[a-z][a-z0-9_]{1,99}$/.test(runnerCodeCandidate) ? runnerCodeCandidate : undefined; const phase = diagnostics?.execution.failurePhase; const failurePhase = phase === "setup" || phase === "execute" ? phase : undefined; const contexts: Record> = {}; @@ -363,7 +368,8 @@ function captureRunFailureEvent(event: RunFailureEvent): string | null { signal: normalizeRunSignal(event.signal), }, }, - fingerprint: [errorCode, event.agentAdapter, ...(failurePhase ? [failurePhase] : []), ...(rootErrorCode ? [rootErrorCode] : rootFallback)], + fingerprint: [errorCode, event.agentAdapter, ...(failurePhase ? [failurePhase] : []), ...(rootErrorCode ? [rootErrorCode] : rootFallback), + ...(runnerFailureCode ? [`runner:${runnerFailureCode}`] : [])], }); } catch (err) { // eslint-disable-next-line no-console diff --git a/server/src/services/native-runtime/native-provider-failure.integration.test.ts b/server/src/services/native-runtime/native-provider-failure.integration.test.ts index efe8b057285..0c5f1eeb252 100644 --- a/server/src/services/native-runtime/native-provider-failure.integration.test.ts +++ b/server/src/services/native-runtime/native-provider-failure.integration.test.ts @@ -17,6 +17,7 @@ import { buildNativeCompletionContract } from "./completion-contracts.js"; import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; import { NATIVE_MODEL_NOT_FOUND_DIAGNOSTIC, NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_REJECTION_MESSAGE } from "./native-provider-failure.js"; +import { NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js"; import { collectRunFailureDiagnostics } from "../run-failure-diagnostics.js"; import { finalizeNativeRun } from "./native-run-finalizer.js"; import { readPersistedNativeFailureEvidence, readPersistedNativeModelRejection } from "./native-provider-failure-evidence.js"; @@ -148,7 +149,10 @@ describe("native completed failure diagnostics", () => { const reviewAuthorityEnded = scenario === "superseded-review" || scenario === "resolved-review"; const ownerChanged = scenario === "reassigned" || scenario === "successor-race"; expect(adapter.errorCode).toBe(classified ? "native_provider_model_rejected" : undefined); - expect(adapter.errorMessage).toBe(classified ? NATIVE_MODEL_REJECTION_MESSAGE : scenario === "succeeded" ? null : `Native session ${terminal.runTerminalState}`); + // A runner-issued turn failure code names the cause instead of the generic text. + const openCodeExplanation = "The OpenCode turn failed (opencode_server_exited). Inspect the run events for the last recorded step."; + expect(adapter.errorMessage).toBe(classified ? NATIVE_MODEL_REJECTION_MESSAGE : scenario === "succeeded" ? null + : opencode ? openCodeExplanation : `Native session ${terminal.runTerminalState}`); expect(adapter.resultJson?.nativeProviderFailure).toEqual(classified ? expectedDiagnostic : undefined); // Only a committed, unclassified failure of the terminal turn explains the generic message. const turnFailure = scenario === "unknown-error" ? { provider: "codex", category: "turn_failed", codexErrorInfo: "other" } @@ -292,12 +296,21 @@ describe("native completed failure diagnostics", () => { expect(finalRun.resultJson?.nativeTurnFailure).toEqual(turnFailure); const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); expect(wakes.filter(wake => wake.payload?.issueId === issueId)).toHaveLength(1); + if (opencode) { + // The projected failure names the recorded runner cause, not an unexplained failure. + await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + const [settled] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(settled).toMatchObject({ status: "failed", errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode, error: openCodeExplanation }); + expect(settled.error).not.toContain("Private"); + } if (scenario === "unknown-error") { await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const [settled] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); expect(coordinator.phase).toBe("committed"); - expect(settled).toMatchObject({ status: "failed", processPid: EXITED_PID, processGroupId: EXITED_PID }); + // A Codex `other` failure carries no specific cause, so it stays unexplained. + expect(settled).toMatchObject({ status: "failed", processPid: EXITED_PID, processGroupId: EXITED_PID, + errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode, error: NATIVE_RUN_FAILED_UNEXPLAINED.error }); await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 1_000, eventType: "harness.ready" }); expect(await releaseSettledNativeRunProcess(db, runId)).toBe("released"); const [released] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); diff --git a/server/src/services/native-runtime/native-run-failure-explanation.test.ts b/server/src/services/native-runtime/native-run-failure-explanation.test.ts new file mode 100644 index 00000000000..4454d3076f3 --- /dev/null +++ b/server/src/services/native-runtime/native-run-failure-explanation.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { explainNativeTurnFailure, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js"; + +describe("explainNativeTurnFailure", () => { + const explained = (provider: string, cause: string) => ({ + errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode, + error: `The ${provider} turn failed (${cause}). Inspect the run events for the last recorded step.`, + }); + + it("names a runner-issued failure code", () => { + expect(explainNativeTurnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_empty_response" })) + .toEqual(explained("OpenCode", "opencode_empty_response")); + expect(explainNativeTurnFailure({ provider: "codex", category: "turn_failed", codexErrorInfo: "other", runnerFailureCode: "turn_binding_mismatch" })) + .toEqual(explained("Codex", "turn_binding_mismatch")); + }); + + it("names a specific Codex error kind when no runner code was issued", () => { + expect(explainNativeTurnFailure({ provider: "codex", category: "turn_failed", codexErrorInfo: "usageLimitExceeded", httpStatus: 429 })) + .toEqual(explained("Codex", "usageLimitExceeded")); + }); + + it("leaves a failure without a specific cause unexplained", () => { + expect(explainNativeTurnFailure(null)).toBeNull(); + expect(explainNativeTurnFailure({ provider: "opencode", category: "turn_failed" })).toBeNull(); + expect(explainNativeTurnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "unrecognized" })).toBeNull(); + for (const codexErrorInfo of ["absent", "unrecognized", "other"]) { + expect(explainNativeTurnFailure({ provider: "codex", category: "turn_failed", codexErrorInfo, runnerFailureCode: "unrecognized" })).toBeNull(); + } + }); +}); diff --git a/server/src/services/native-runtime/native-run-failure-explanation.ts b/server/src/services/native-runtime/native-run-failure-explanation.ts index 73e7c0df69b..5750f3cc821 100644 --- a/server/src/services/native-runtime/native-run-failure-explanation.ts +++ b/server/src/services/native-runtime/native-run-failure-explanation.ts @@ -1,5 +1,6 @@ import { sql, type SQL } from "drizzle-orm"; import { heartbeatRuns } from "@paperclipai/db"; +import type { NativeTurnFailureDiagnostic } from "./native-provider-failure.js"; /** Used when a native run reaches `failed` and no writer recorded a more specific cause. */ export const NATIVE_RUN_FAILED_UNEXPLAINED = { @@ -9,6 +10,26 @@ export const NATIVE_RUN_FAILED_UNEXPLAINED = { export type NativeRunFailureExplanation = { errorCode: string; error: string }; +const TURN_FAILURE_PROVIDER_LABELS = { codex: "Codex", opencode: "OpenCode" } as const; + +/** + * Name the cause of a failed turn that no provider classifier claimed, using + * only the diagnostic's closed vocabulary: a runner-issued failure code, else a + * specific Codex error kind. Returns null when the diagnostic has no such cause. + */ +export function explainNativeTurnFailure(diagnostic: NativeTurnFailureDiagnostic | null | undefined): NativeRunFailureExplanation | null { + if (!diagnostic) return null; + const runnerCode = diagnostic.runnerFailureCode; + const codexKind = diagnostic.provider === "codex" ? diagnostic.codexErrorInfo : undefined; + const cause = runnerCode && runnerCode !== "unrecognized" ? runnerCode + : codexKind && !["absent", "unrecognized", "other"].includes(codexKind) ? codexKind : null; + if (!cause) return null; + return { + errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode, + error: `The ${TURN_FAILURE_PROVIDER_LABELS[diagnostic.provider]} turn failed (${cause}). Inspect the run events for the last recorded step.`, + }; +} + /** * A failed run must always say why. These SET expressions keep an explanation * already on the row, and otherwise record `fallback`. A run that is already diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index ea8a3884daf..70882ed0a8a 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -32,7 +32,7 @@ import { settleExternalConversation } from "../slack-conversation-lifecycle.js"; import { executionFailureRetryCount } from "../execution-recovery-attempt.js"; import { isFailedTurnContinuationRun } from "./native-failed-turn-continuation.js"; import { readPersistedNativeFailureEvidence } from "./native-provider-failure-evidence.js"; -import { explainFailedRun, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js"; +import { explainFailedRun, explainNativeTurnFailure, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js"; import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js"; import { findSupersededExternalReviewRequests } from "./superseded-external-review-requests.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js"; @@ -2015,10 +2015,11 @@ export async function finalizeNativeRun(input: { const nextErrorCode = providerFailure ? sql`coalesce(${heartbeatRuns.errorCode}, ${providerFailure.errorCode})` : clearExecutionFailure ? sql`null` : sql`${heartbeatRuns.errorCode}`; // A run projected to failed always records why, even when the provider - // reported a failed terminal without a classified failure. + // reported a failed terminal without a classified failure. A recorded + // runner turn failure names that cause. const projectsFailedStatus = input.projectRunStatus && projectedStatus === "failed"; const explained = projectsFailedStatus - ? explainFailedRun(NATIVE_RUN_FAILED_UNEXPLAINED, { error: nextError, errorCode: nextErrorCode }) + ? explainFailedRun(explainNativeTurnFailure(turnFailure) ?? NATIVE_RUN_FAILED_UNEXPLAINED, { error: nextError, errorCode: nextErrorCode }) : null; const projectedError = explained?.error ?? sql`case when ${heartbeatRuns.status} = 'cancelled' then ${heartbeatRuns.error} else ${nextError} end`; diff --git a/server/src/services/native-runtime/native-run-projection.ts b/server/src/services/native-runtime/native-run-projection.ts index 92efb555d18..c36e9b251dd 100644 --- a/server/src/services/native-runtime/native-run-projection.ts +++ b/server/src/services/native-runtime/native-run-projection.ts @@ -19,7 +19,8 @@ import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js" import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; import { isNewTerminalRunFailure, reportRunFailure } from "../run-failure-report.js"; import { nativeRunnerOwnershipNotHeldCondition } from "./native-runner-ownership.js"; -import { explainFailedRun, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js"; +import { explainFailedRun, explainNativeTurnFailure, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js"; +import { readNativeTurnFailureDiagnostic } from "./native-provider-failure.js"; export function projectNativeTerminalRunStatus( @@ -161,7 +162,7 @@ export async function projectCommittedRun(input: { const projected = await input.db.transaction(async transaction => { const tx = transaction as unknown as Db; const [before] = await tx.select({ status: heartbeatRuns.status, runtimeMode: heartbeatRuns.runtimeMode, - nativePhase: heartbeatRuns.nativePhase, finishedAt: heartbeatRuns.finishedAt }) + nativePhase: heartbeatRuns.nativePhase, finishedAt: heartbeatRuns.finishedAt, resultJson: heartbeatRuns.resultJson }) .from(heartbeatRuns).where(needsRepair).for("update").limit(1); if (!before) return null; await notifyExternalAgentSettlement(tx, { companyId: input.run.companyId, runId: input.run.id }); @@ -195,9 +196,10 @@ export async function projectCommittedRun(input: { || jsonb_build_object('finalizationPhase', 'committed', 'failureCode', null, 'originalFailureCode', null, 'nextAttemptAt', null)`, } : { - // A failed accepted result keeps any recorded cause, and otherwise - // names the unexplained failure. - ...(terminalState === "failed" ? explainFailedRun(NATIVE_RUN_FAILED_UNEXPLAINED, { preserveCancelled: false }) : {}), + // A failed accepted result keeps any recorded cause, then names a + // recorded runner turn failure, and otherwise the unexplained failure. + ...(terminalState === "failed" ? explainFailedRun(explainNativeTurnFailure(readNativeTurnFailureDiagnostic( + before.resultJson?.nativeTurnFailure)) ?? NATIVE_RUN_FAILED_UNEXPLAINED, { preserveCancelled: false }) : {}), resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || jsonb_build_object('finalizationPhase', 'committed', 'failureCode', null, 'originalFailureCode', null, 'nextAttemptAt', null)`, }), updatedAt: now, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 64ff926b87f..54913e0bb22 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -106,6 +106,7 @@ import { redactNativeFailureError } from "./native-failure-redaction.js"; import { parseCodexCliVersion } from "./codex-runtime-compatibility.js"; import { createNativeToolTrace, type NativeToolTrace } from "./native-tool-trace.js"; import { createNativeProviderFailureObservation } from "./native-provider-failure.js"; +import { explainNativeTurnFailure } from "./native-run-failure-explanation.js"; import { nativeStartupRetryExhaustedAction, nativeStartupRetryWait, planNativeStartupRetry } from "./native-startup-retry.js"; import { admitUnclaimedNativeBootstrapRetry } from "./native-bootstrap-retry.js"; import { RUNNERD_CONTROL_PLANE_STATE_SCHEMAS, runnerdStateIsIncompleteBootstrap, @@ -13826,7 +13827,8 @@ async function executePaperclipNativeSessionAttempt( errorMessage: native.terminal.runTerminalState === "succeeded" ? null - : providerFailure?.errorMessage ?? `Native session ${native.terminal.runTerminalState}`, + : providerFailure?.errorMessage ?? explainNativeTurnFailure(unclassifiedTurnFailure)?.error + ?? `Native session ${native.terminal.runTerminalState}`, ...(providerFailure ? { errorCode: providerFailure.errorCode } : {}), resultJson: { nativeResult: native.result as unknown as Record,