fix(native): name the recorded turn failure when a native run fails (#21235)

Failed native runs now report the runner's recorded turn-failure code instead of the generic unclassified fallback, and the Sentry run-failure fingerprint includes runner:<code>.

Co-Authored-By: Paperclip <Paperclip-Paperclip@users.noreply.github.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-11-13 11:22:55 -08:00
1 parent 38821e061f
commit aa4a06653c
9 files changed
+125 -12

No files matched your search

@@ -485,6 +485,21 @@ describe("committed native run projection lifecycle", () => {
await projectCommittedRun({ db, run, coordinator: await state(binding.runId) });
expect(await read(binding.runId)).toMatchObject({ status: "failed", errorCode: expectedCode, error: expectedError });
});
it("names a recorded runner turn failure in committed recovery projection", async () => {
const binding = await accepted("failed");
await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" });
const [unfinished] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, binding.runId));
await db.update(heartbeatRuns).set({ errorCode: null, error: null, resultJson: { ...unfinished.resultJson,
nativeTurnFailure: { provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_empty_response", message: "private provider text" } } })
.where(eq(heartbeatRuns.id, binding.runId));
const run = await read(binding.runId);
expect(run.status).toBe("running");
await projectCommittedRun({ db, run, coordinator: await state(binding.runId) });
const projected = await read(binding.runId);
expect(projected).toMatchObject({ status: "failed", errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode,
error: "The OpenCode turn failed (opencode_empty_response). Inspect the run events for the last recorded step." });
expect(projected.error).not.toContain("private");
});
it("ignores routine company-only finalization hints and still repairs a committed run", async () => {
await start(); await flush(); await quiet();
const previous = db.$client.options.debug;
@@ -233,6 +233,29 @@ describe("captureRunFailure", () => {
expect(JSON.stringify(fingerprints)).not.toMatch(/workspace|timed out|plain failure/i);
});
it("separates native turn failures by their runner failure code", async () => {
const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
const turnFailure = (provider: NonNullable<RunFailureEvent["diagnostics"]>["provider"]) => baseEvent({
errorCode: "native_run_failed", agentAdapter: "paperclip_runner",
diagnostics: { execution: {}, adapter: {}, provider, exceptions: [], truncatedFields: [] },
});
sentryModule.captureRunFailure(turnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_empty_response" }));
sentryModule.captureRunFailure(turnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited" }));
sentryModule.captureRunFailure(turnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "unrecognized" }));
sentryModule.captureRunFailure(turnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "not a code" }));
sentryModule.captureRunFailure(turnFailure({}));
const prefix = ["native_run_failed", "paperclip_runner"];
expect(captureException.mock.calls.map(([, context]) => context?.fingerprint)).toEqual([
[...prefix, "runner:opencode_empty_response"],
[...prefix, "runner:opencode_server_exited"],
prefix,
prefix,
prefix,
]);
});
it("keeps an identifier root code as the only root grouping element", async () => {
const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
+7 -1
View File
@@ -288,6 +288,11 @@ function captureRunFailureEvent(event: RunFailureEvent): string | null {
rootFallback.push(`code:${rootException.code}`);
}
}
// A native turn failure's runner-issued code is closed vocabulary. Without
// it, every unclassified native turn failure shares one group.
const runnerCodeCandidate = diagnostics?.provider.runnerFailureCode;
const runnerFailureCode = typeof runnerCodeCandidate === "string" && runnerCodeCandidate !== "unrecognized" &&
/^[a-z][a-z0-9_]{1,99}$/.test(runnerCodeCandidate) ? runnerCodeCandidate : undefined;
const phase = diagnostics?.execution.failurePhase;
const failurePhase = phase === "setup" || phase === "execute" ? phase : undefined;
const contexts: Record<string, Record<string, unknown>> = {};
@@ -363,7 +368,8 @@ function captureRunFailureEvent(event: RunFailureEvent): string | null {
signal: normalizeRunSignal(event.signal),
},
},
fingerprint: [errorCode, event.agentAdapter, ...(failurePhase ? [failurePhase] : []), ...(rootErrorCode ? [rootErrorCode] : rootFallback)],
fingerprint: [errorCode, event.agentAdapter, ...(failurePhase ? [failurePhase] : []), ...(rootErrorCode ? [rootErrorCode] : rootFallback),
...(runnerFailureCode ? [`runner:${runnerFailureCode}`] : [])],
});
} catch (err) {
// eslint-disable-next-line no-console
@@ -17,6 +17,7 @@ import { buildNativeCompletionContract } from "./completion-contracts.js";
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
import { prepareNativeHeartbeatRun } from "./prepare-native-run.js";
import { NATIVE_MODEL_NOT_FOUND_DIAGNOSTIC, NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_REJECTION_MESSAGE } from "./native-provider-failure.js";
import { NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js";
import { collectRunFailureDiagnostics } from "../run-failure-diagnostics.js";
import { finalizeNativeRun } from "./native-run-finalizer.js";
import { readPersistedNativeFailureEvidence, readPersistedNativeModelRejection } from "./native-provider-failure-evidence.js";
@@ -148,7 +149,10 @@ describe("native completed failure diagnostics", () => {
const reviewAuthorityEnded = scenario === "superseded-review" || scenario === "resolved-review";
const ownerChanged = scenario === "reassigned" || scenario === "successor-race";
expect(adapter.errorCode).toBe(classified ? "native_provider_model_rejected" : undefined);
expect(adapter.errorMessage).toBe(classified ? NATIVE_MODEL_REJECTION_MESSAGE : scenario === "succeeded" ? null : `Native session ${terminal.runTerminalState}`);
// A runner-issued turn failure code names the cause instead of the generic text.
const openCodeExplanation = "The OpenCode turn failed (opencode_server_exited). Inspect the run events for the last recorded step.";
expect(adapter.errorMessage).toBe(classified ? NATIVE_MODEL_REJECTION_MESSAGE : scenario === "succeeded" ? null
: opencode ? openCodeExplanation : `Native session ${terminal.runTerminalState}`);
expect(adapter.resultJson?.nativeProviderFailure).toEqual(classified ? expectedDiagnostic : undefined);
// Only a committed, unclassified failure of the terminal turn explains the generic message.
const turnFailure = scenario === "unknown-error" ? { provider: "codex", category: "turn_failed", codexErrorInfo: "other" }
@@ -292,12 +296,21 @@ describe("native completed failure diagnostics", () => {
expect(finalRun.resultJson?.nativeTurnFailure).toEqual(turnFailure);
const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId));
expect(wakes.filter(wake => wake.payload?.issueId === issueId)).toHaveLength(1);
if (opencode) {
// The projected failure names the recorded runner cause, not an unexplained failure.
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
const [settled] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
expect(settled).toMatchObject({ status: "failed", errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode, error: openCodeExplanation });
expect(settled.error).not.toContain("Private");
}
if (scenario === "unknown-error") {
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
const [settled] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId));
expect(coordinator.phase).toBe("committed");
expect(settled).toMatchObject({ status: "failed", processPid: EXITED_PID, processGroupId: EXITED_PID });
// A Codex `other` failure carries no specific cause, so it stays unexplained.
expect(settled).toMatchObject({ status: "failed", processPid: EXITED_PID, processGroupId: EXITED_PID,
errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode, error: NATIVE_RUN_FAILED_UNEXPLAINED.error });
await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 1_000, eventType: "harness.ready" });
expect(await releaseSettledNativeRunProcess(db, runId)).toBe("released");
const [released] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
@@ -0,0 +1,30 @@
import { describe, expect, it } from "vitest";
import { explainNativeTurnFailure, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js";
describe("explainNativeTurnFailure", () => {
const explained = (provider: string, cause: string) => ({
errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode,
error: `The ${provider} turn failed (${cause}). Inspect the run events for the last recorded step.`,
});
it("names a runner-issued failure code", () => {
expect(explainNativeTurnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_empty_response" }))
.toEqual(explained("OpenCode", "opencode_empty_response"));
expect(explainNativeTurnFailure({ provider: "codex", category: "turn_failed", codexErrorInfo: "other", runnerFailureCode: "turn_binding_mismatch" }))
.toEqual(explained("Codex", "turn_binding_mismatch"));
});
it("names a specific Codex error kind when no runner code was issued", () => {
expect(explainNativeTurnFailure({ provider: "codex", category: "turn_failed", codexErrorInfo: "usageLimitExceeded", httpStatus: 429 }))
.toEqual(explained("Codex", "usageLimitExceeded"));
});
it("leaves a failure without a specific cause unexplained", () => {
expect(explainNativeTurnFailure(null)).toBeNull();
expect(explainNativeTurnFailure({ provider: "opencode", category: "turn_failed" })).toBeNull();
expect(explainNativeTurnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "unrecognized" })).toBeNull();
for (const codexErrorInfo of ["absent", "unrecognized", "other"]) {
expect(explainNativeTurnFailure({ provider: "codex", category: "turn_failed", codexErrorInfo, runnerFailureCode: "unrecognized" })).toBeNull();
}
});
});
@@ -1,5 +1,6 @@
import { sql, type SQL } from "drizzle-orm";
import { heartbeatRuns } from "@paperclipai/db";
import type { NativeTurnFailureDiagnostic } from "./native-provider-failure.js";
/** Used when a native run reaches `failed` and no writer recorded a more specific cause. */
export const NATIVE_RUN_FAILED_UNEXPLAINED = {
@@ -9,6 +10,26 @@ export const NATIVE_RUN_FAILED_UNEXPLAINED = {
export type NativeRunFailureExplanation = { errorCode: string; error: string };
const TURN_FAILURE_PROVIDER_LABELS = { codex: "Codex", opencode: "OpenCode" } as const;
/**
* Name the cause of a failed turn that no provider classifier claimed, using
* only the diagnostic's closed vocabulary: a runner-issued failure code, else a
* specific Codex error kind. Returns null when the diagnostic has no such cause.
*/
export function explainNativeTurnFailure(diagnostic: NativeTurnFailureDiagnostic | null | undefined): NativeRunFailureExplanation | null {
if (!diagnostic) return null;
const runnerCode = diagnostic.runnerFailureCode;
const codexKind = diagnostic.provider === "codex" ? diagnostic.codexErrorInfo : undefined;
const cause = runnerCode && runnerCode !== "unrecognized" ? runnerCode
: codexKind && !["absent", "unrecognized", "other"].includes(codexKind) ? codexKind : null;
if (!cause) return null;
return {
errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode,
error: `The ${TURN_FAILURE_PROVIDER_LABELS[diagnostic.provider]} turn failed (${cause}). Inspect the run events for the last recorded step.`,
};
}
/**
* A failed run must always say why. These SET expressions keep an explanation
* already on the row, and otherwise record `fallback`. A run that is already
@@ -32,7 +32,7 @@ import { settleExternalConversation } from "../slack-conversation-lifecycle.js";
import { executionFailureRetryCount } from "../execution-recovery-attempt.js";
import { isFailedTurnContinuationRun } from "./native-failed-turn-continuation.js";
import { readPersistedNativeFailureEvidence } from "./native-provider-failure-evidence.js";
import { explainFailedRun, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js";
import { explainFailedRun, explainNativeTurnFailure, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js";
import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js";
import { findSupersededExternalReviewRequests } from "./superseded-external-review-requests.js";
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js";
@@ -2015,10 +2015,11 @@ export async function finalizeNativeRun(input: {
const nextErrorCode = providerFailure ? sql`coalesce(${heartbeatRuns.errorCode}, ${providerFailure.errorCode})`
: clearExecutionFailure ? sql`null` : sql`${heartbeatRuns.errorCode}`;
// A run projected to failed always records why, even when the provider
// reported a failed terminal without a classified failure.
// reported a failed terminal without a classified failure. A recorded
// runner turn failure names that cause.
const projectsFailedStatus = input.projectRunStatus && projectedStatus === "failed";
const explained = projectsFailedStatus
? explainFailedRun(NATIVE_RUN_FAILED_UNEXPLAINED, { error: nextError, errorCode: nextErrorCode })
? explainFailedRun(explainNativeTurnFailure(turnFailure) ?? NATIVE_RUN_FAILED_UNEXPLAINED, { error: nextError, errorCode: nextErrorCode })
: null;
const projectedError = explained?.error ?? sql`case when ${heartbeatRuns.status} = 'cancelled' then ${heartbeatRuns.error}
else ${nextError} end`;
@@ -19,7 +19,8 @@ import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"
import { emitAgentTaskRun } from "../agent-task-run-telemetry.js";
import { isNewTerminalRunFailure, reportRunFailure } from "../run-failure-report.js";
import { nativeRunnerOwnershipNotHeldCondition } from "./native-runner-ownership.js";
import { explainFailedRun, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js";
import { explainFailedRun, explainNativeTurnFailure, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js";
import { readNativeTurnFailureDiagnostic } from "./native-provider-failure.js";
export function projectNativeTerminalRunStatus(
@@ -161,7 +162,7 @@ export async function projectCommittedRun(input: {
const projected = await input.db.transaction(async transaction => {
const tx = transaction as unknown as Db;
const [before] = await tx.select({ status: heartbeatRuns.status, runtimeMode: heartbeatRuns.runtimeMode,
nativePhase: heartbeatRuns.nativePhase, finishedAt: heartbeatRuns.finishedAt })
nativePhase: heartbeatRuns.nativePhase, finishedAt: heartbeatRuns.finishedAt, resultJson: heartbeatRuns.resultJson })
.from(heartbeatRuns).where(needsRepair).for("update").limit(1);
if (!before) return null;
await notifyExternalAgentSettlement(tx, { companyId: input.run.companyId, runId: input.run.id });
@@ -195,9 +196,10 @@ export async function projectCommittedRun(input: {
|| jsonb_build_object('finalizationPhase', 'committed', 'failureCode', null, 'originalFailureCode', null, 'nextAttemptAt', null)`,
}
: {
// A failed accepted result keeps any recorded cause, and otherwise
// names the unexplained failure.
...(terminalState === "failed" ? explainFailedRun(NATIVE_RUN_FAILED_UNEXPLAINED, { preserveCancelled: false }) : {}),
// A failed accepted result keeps any recorded cause, then names a
// recorded runner turn failure, and otherwise the unexplained failure.
...(terminalState === "failed" ? explainFailedRun(explainNativeTurnFailure(readNativeTurnFailureDiagnostic(
before.resultJson?.nativeTurnFailure)) ?? NATIVE_RUN_FAILED_UNEXPLAINED, { preserveCancelled: false }) : {}),
resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || jsonb_build_object('finalizationPhase', 'committed', 'failureCode', null, 'originalFailureCode', null, 'nextAttemptAt', null)`,
}),
updatedAt: now,
@@ -106,6 +106,7 @@ import { redactNativeFailureError } from "./native-failure-redaction.js";
import { parseCodexCliVersion } from "./codex-runtime-compatibility.js";
import { createNativeToolTrace, type NativeToolTrace } from "./native-tool-trace.js";
import { createNativeProviderFailureObservation } from "./native-provider-failure.js";
import { explainNativeTurnFailure } from "./native-run-failure-explanation.js";
import { nativeStartupRetryExhaustedAction, nativeStartupRetryWait, planNativeStartupRetry } from "./native-startup-retry.js";
import { admitUnclaimedNativeBootstrapRetry } from "./native-bootstrap-retry.js";
import { RUNNERD_CONTROL_PLANE_STATE_SCHEMAS, runnerdStateIsIncompleteBootstrap,
@@ -13826,7 +13827,8 @@ async function executePaperclipNativeSessionAttempt(
errorMessage:
native.terminal.runTerminalState === "succeeded"
? null
: providerFailure?.errorMessage ?? `Native session ${native.terminal.runTerminalState}`,
: providerFailure?.errorMessage ?? explainNativeTurnFailure(unclassifiedTurnFailure)?.error
?? `Native session ${native.terminal.runTerminalState}`,
...(providerFailure ? { errorCode: providerFailure.errorCode } : {}),
resultJson: {
nativeResult: native.result as unknown as Record<string, unknown>,