mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-17 04:21:24 +01:00
fix(native): name the recorded turn failure when a native run fails (#21235)
Failed native runs now report the runner's recorded turn-failure code instead of the generic unclassified fallback, and the Sentry run-failure fingerprint includes runner:<code>. Co-Authored-By: Paperclip <Paperclip-Paperclip@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
9 files changed
+125
-12
No files matched your search
@@ -485,6 +485,21 @@ describe("committed native run projection lifecycle", () => {
|
||||
await projectCommittedRun({ db, run, coordinator: await state(binding.runId) });
|
||||
expect(await read(binding.runId)).toMatchObject({ status: "failed", errorCode: expectedCode, error: expectedError });
|
||||
});
|
||||
it("names a recorded runner turn failure in committed recovery projection", async () => {
|
||||
const binding = await accepted("failed");
|
||||
await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" });
|
||||
const [unfinished] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, binding.runId));
|
||||
await db.update(heartbeatRuns).set({ errorCode: null, error: null, resultJson: { ...unfinished.resultJson,
|
||||
nativeTurnFailure: { provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_empty_response", message: "private provider text" } } })
|
||||
.where(eq(heartbeatRuns.id, binding.runId));
|
||||
const run = await read(binding.runId);
|
||||
expect(run.status).toBe("running");
|
||||
await projectCommittedRun({ db, run, coordinator: await state(binding.runId) });
|
||||
const projected = await read(binding.runId);
|
||||
expect(projected).toMatchObject({ status: "failed", errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode,
|
||||
error: "The OpenCode turn failed (opencode_empty_response). Inspect the run events for the last recorded step." });
|
||||
expect(projected.error).not.toContain("private");
|
||||
});
|
||||
it("ignores routine company-only finalization hints and still repairs a committed run", async () => {
|
||||
await start(); await flush(); await quiet();
|
||||
const previous = db.$client.options.debug;
|
||||
|
||||
@@ -233,6 +233,29 @@ describe("captureRunFailure", () => {
|
||||
expect(JSON.stringify(fingerprints)).not.toMatch(/workspace|timed out|plain failure/i);
|
||||
});
|
||||
|
||||
it("separates native turn failures by their runner failure code", async () => {
|
||||
const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
|
||||
const turnFailure = (provider: NonNullable<RunFailureEvent["diagnostics"]>["provider"]) => baseEvent({
|
||||
errorCode: "native_run_failed", agentAdapter: "paperclip_runner",
|
||||
diagnostics: { execution: {}, adapter: {}, provider, exceptions: [], truncatedFields: [] },
|
||||
});
|
||||
|
||||
sentryModule.captureRunFailure(turnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_empty_response" }));
|
||||
sentryModule.captureRunFailure(turnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited" }));
|
||||
sentryModule.captureRunFailure(turnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "unrecognized" }));
|
||||
sentryModule.captureRunFailure(turnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "not a code" }));
|
||||
sentryModule.captureRunFailure(turnFailure({}));
|
||||
|
||||
const prefix = ["native_run_failed", "paperclip_runner"];
|
||||
expect(captureException.mock.calls.map(([, context]) => context?.fingerprint)).toEqual([
|
||||
[...prefix, "runner:opencode_empty_response"],
|
||||
[...prefix, "runner:opencode_server_exited"],
|
||||
prefix,
|
||||
prefix,
|
||||
prefix,
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps an identifier root code as the only root grouping element", async () => {
|
||||
const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
|
||||
|
||||
|
||||
@@ -288,6 +288,11 @@ function captureRunFailureEvent(event: RunFailureEvent): string | null {
|
||||
rootFallback.push(`code:${rootException.code}`);
|
||||
}
|
||||
}
|
||||
// A native turn failure's runner-issued code is closed vocabulary. Without
|
||||
// it, every unclassified native turn failure shares one group.
|
||||
const runnerCodeCandidate = diagnostics?.provider.runnerFailureCode;
|
||||
const runnerFailureCode = typeof runnerCodeCandidate === "string" && runnerCodeCandidate !== "unrecognized" &&
|
||||
/^[a-z][a-z0-9_]{1,99}$/.test(runnerCodeCandidate) ? runnerCodeCandidate : undefined;
|
||||
const phase = diagnostics?.execution.failurePhase;
|
||||
const failurePhase = phase === "setup" || phase === "execute" ? phase : undefined;
|
||||
const contexts: Record<string, Record<string, unknown>> = {};
|
||||
@@ -363,7 +368,8 @@ function captureRunFailureEvent(event: RunFailureEvent): string | null {
|
||||
signal: normalizeRunSignal(event.signal),
|
||||
},
|
||||
},
|
||||
fingerprint: [errorCode, event.agentAdapter, ...(failurePhase ? [failurePhase] : []), ...(rootErrorCode ? [rootErrorCode] : rootFallback)],
|
||||
fingerprint: [errorCode, event.agentAdapter, ...(failurePhase ? [failurePhase] : []), ...(rootErrorCode ? [rootErrorCode] : rootFallback),
|
||||
...(runnerFailureCode ? [`runner:${runnerFailureCode}`] : [])],
|
||||
});
|
||||
} catch (err) {
|
||||
// eslint-disable-next-line no-console
|
||||
|
||||
@@ -17,6 +17,7 @@ import { buildNativeCompletionContract } from "./completion-contracts.js";
|
||||
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
|
||||
import { prepareNativeHeartbeatRun } from "./prepare-native-run.js";
|
||||
import { NATIVE_MODEL_NOT_FOUND_DIAGNOSTIC, NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_REJECTION_MESSAGE } from "./native-provider-failure.js";
|
||||
import { NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js";
|
||||
import { collectRunFailureDiagnostics } from "../run-failure-diagnostics.js";
|
||||
import { finalizeNativeRun } from "./native-run-finalizer.js";
|
||||
import { readPersistedNativeFailureEvidence, readPersistedNativeModelRejection } from "./native-provider-failure-evidence.js";
|
||||
@@ -148,7 +149,10 @@ describe("native completed failure diagnostics", () => {
|
||||
const reviewAuthorityEnded = scenario === "superseded-review" || scenario === "resolved-review";
|
||||
const ownerChanged = scenario === "reassigned" || scenario === "successor-race";
|
||||
expect(adapter.errorCode).toBe(classified ? "native_provider_model_rejected" : undefined);
|
||||
expect(adapter.errorMessage).toBe(classified ? NATIVE_MODEL_REJECTION_MESSAGE : scenario === "succeeded" ? null : `Native session ${terminal.runTerminalState}`);
|
||||
// A runner-issued turn failure code names the cause instead of the generic text.
|
||||
const openCodeExplanation = "The OpenCode turn failed (opencode_server_exited). Inspect the run events for the last recorded step.";
|
||||
expect(adapter.errorMessage).toBe(classified ? NATIVE_MODEL_REJECTION_MESSAGE : scenario === "succeeded" ? null
|
||||
: opencode ? openCodeExplanation : `Native session ${terminal.runTerminalState}`);
|
||||
expect(adapter.resultJson?.nativeProviderFailure).toEqual(classified ? expectedDiagnostic : undefined);
|
||||
// Only a committed, unclassified failure of the terminal turn explains the generic message.
|
||||
const turnFailure = scenario === "unknown-error" ? { provider: "codex", category: "turn_failed", codexErrorInfo: "other" }
|
||||
@@ -292,12 +296,21 @@ describe("native completed failure diagnostics", () => {
|
||||
expect(finalRun.resultJson?.nativeTurnFailure).toEqual(turnFailure);
|
||||
const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId));
|
||||
expect(wakes.filter(wake => wake.payload?.issueId === issueId)).toHaveLength(1);
|
||||
if (opencode) {
|
||||
// The projected failure names the recorded runner cause, not an unexplained failure.
|
||||
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
const [settled] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
expect(settled).toMatchObject({ status: "failed", errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode, error: openCodeExplanation });
|
||||
expect(settled.error).not.toContain("Private");
|
||||
}
|
||||
if (scenario === "unknown-error") {
|
||||
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
const [settled] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId));
|
||||
expect(coordinator.phase).toBe("committed");
|
||||
expect(settled).toMatchObject({ status: "failed", processPid: EXITED_PID, processGroupId: EXITED_PID });
|
||||
// A Codex `other` failure carries no specific cause, so it stays unexplained.
|
||||
expect(settled).toMatchObject({ status: "failed", processPid: EXITED_PID, processGroupId: EXITED_PID,
|
||||
errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode, error: NATIVE_RUN_FAILED_UNEXPLAINED.error });
|
||||
await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 1_000, eventType: "harness.ready" });
|
||||
expect(await releaseSettledNativeRunProcess(db, runId)).toBe("released");
|
||||
const [released] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { explainNativeTurnFailure, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js";
|
||||
|
||||
describe("explainNativeTurnFailure", () => {
|
||||
const explained = (provider: string, cause: string) => ({
|
||||
errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode,
|
||||
error: `The ${provider} turn failed (${cause}). Inspect the run events for the last recorded step.`,
|
||||
});
|
||||
|
||||
it("names a runner-issued failure code", () => {
|
||||
expect(explainNativeTurnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_empty_response" }))
|
||||
.toEqual(explained("OpenCode", "opencode_empty_response"));
|
||||
expect(explainNativeTurnFailure({ provider: "codex", category: "turn_failed", codexErrorInfo: "other", runnerFailureCode: "turn_binding_mismatch" }))
|
||||
.toEqual(explained("Codex", "turn_binding_mismatch"));
|
||||
});
|
||||
|
||||
it("names a specific Codex error kind when no runner code was issued", () => {
|
||||
expect(explainNativeTurnFailure({ provider: "codex", category: "turn_failed", codexErrorInfo: "usageLimitExceeded", httpStatus: 429 }))
|
||||
.toEqual(explained("Codex", "usageLimitExceeded"));
|
||||
});
|
||||
|
||||
it("leaves a failure without a specific cause unexplained", () => {
|
||||
expect(explainNativeTurnFailure(null)).toBeNull();
|
||||
expect(explainNativeTurnFailure({ provider: "opencode", category: "turn_failed" })).toBeNull();
|
||||
expect(explainNativeTurnFailure({ provider: "opencode", category: "turn_failed", runnerFailureCode: "unrecognized" })).toBeNull();
|
||||
for (const codexErrorInfo of ["absent", "unrecognized", "other"]) {
|
||||
expect(explainNativeTurnFailure({ provider: "codex", category: "turn_failed", codexErrorInfo, runnerFailureCode: "unrecognized" })).toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { sql, type SQL } from "drizzle-orm";
|
||||
import { heartbeatRuns } from "@paperclipai/db";
|
||||
import type { NativeTurnFailureDiagnostic } from "./native-provider-failure.js";
|
||||
|
||||
/** Used when a native run reaches `failed` and no writer recorded a more specific cause. */
|
||||
export const NATIVE_RUN_FAILED_UNEXPLAINED = {
|
||||
@@ -9,6 +10,26 @@ export const NATIVE_RUN_FAILED_UNEXPLAINED = {
|
||||
|
||||
export type NativeRunFailureExplanation = { errorCode: string; error: string };
|
||||
|
||||
const TURN_FAILURE_PROVIDER_LABELS = { codex: "Codex", opencode: "OpenCode" } as const;
|
||||
|
||||
/**
|
||||
* Name the cause of a failed turn that no provider classifier claimed, using
|
||||
* only the diagnostic's closed vocabulary: a runner-issued failure code, else a
|
||||
* specific Codex error kind. Returns null when the diagnostic has no such cause.
|
||||
*/
|
||||
export function explainNativeTurnFailure(diagnostic: NativeTurnFailureDiagnostic | null | undefined): NativeRunFailureExplanation | null {
|
||||
if (!diagnostic) return null;
|
||||
const runnerCode = diagnostic.runnerFailureCode;
|
||||
const codexKind = diagnostic.provider === "codex" ? diagnostic.codexErrorInfo : undefined;
|
||||
const cause = runnerCode && runnerCode !== "unrecognized" ? runnerCode
|
||||
: codexKind && !["absent", "unrecognized", "other"].includes(codexKind) ? codexKind : null;
|
||||
if (!cause) return null;
|
||||
return {
|
||||
errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode,
|
||||
error: `The ${TURN_FAILURE_PROVIDER_LABELS[diagnostic.provider]} turn failed (${cause}). Inspect the run events for the last recorded step.`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A failed run must always say why. These SET expressions keep an explanation
|
||||
* already on the row, and otherwise record `fallback`. A run that is already
|
||||
|
||||
@@ -32,7 +32,7 @@ import { settleExternalConversation } from "../slack-conversation-lifecycle.js";
|
||||
import { executionFailureRetryCount } from "../execution-recovery-attempt.js";
|
||||
import { isFailedTurnContinuationRun } from "./native-failed-turn-continuation.js";
|
||||
import { readPersistedNativeFailureEvidence } from "./native-provider-failure-evidence.js";
|
||||
import { explainFailedRun, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js";
|
||||
import { explainFailedRun, explainNativeTurnFailure, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js";
|
||||
import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js";
|
||||
import { findSupersededExternalReviewRequests } from "./superseded-external-review-requests.js";
|
||||
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js";
|
||||
@@ -2015,10 +2015,11 @@ export async function finalizeNativeRun(input: {
|
||||
const nextErrorCode = providerFailure ? sql`coalesce(${heartbeatRuns.errorCode}, ${providerFailure.errorCode})`
|
||||
: clearExecutionFailure ? sql`null` : sql`${heartbeatRuns.errorCode}`;
|
||||
// A run projected to failed always records why, even when the provider
|
||||
// reported a failed terminal without a classified failure.
|
||||
// reported a failed terminal without a classified failure. A recorded
|
||||
// runner turn failure names that cause.
|
||||
const projectsFailedStatus = input.projectRunStatus && projectedStatus === "failed";
|
||||
const explained = projectsFailedStatus
|
||||
? explainFailedRun(NATIVE_RUN_FAILED_UNEXPLAINED, { error: nextError, errorCode: nextErrorCode })
|
||||
? explainFailedRun(explainNativeTurnFailure(turnFailure) ?? NATIVE_RUN_FAILED_UNEXPLAINED, { error: nextError, errorCode: nextErrorCode })
|
||||
: null;
|
||||
const projectedError = explained?.error ?? sql`case when ${heartbeatRuns.status} = 'cancelled' then ${heartbeatRuns.error}
|
||||
else ${nextError} end`;
|
||||
|
||||
@@ -19,7 +19,8 @@ import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"
|
||||
import { emitAgentTaskRun } from "../agent-task-run-telemetry.js";
|
||||
import { isNewTerminalRunFailure, reportRunFailure } from "../run-failure-report.js";
|
||||
import { nativeRunnerOwnershipNotHeldCondition } from "./native-runner-ownership.js";
|
||||
import { explainFailedRun, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js";
|
||||
import { explainFailedRun, explainNativeTurnFailure, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js";
|
||||
import { readNativeTurnFailureDiagnostic } from "./native-provider-failure.js";
|
||||
|
||||
|
||||
export function projectNativeTerminalRunStatus(
|
||||
@@ -161,7 +162,7 @@ export async function projectCommittedRun(input: {
|
||||
const projected = await input.db.transaction(async transaction => {
|
||||
const tx = transaction as unknown as Db;
|
||||
const [before] = await tx.select({ status: heartbeatRuns.status, runtimeMode: heartbeatRuns.runtimeMode,
|
||||
nativePhase: heartbeatRuns.nativePhase, finishedAt: heartbeatRuns.finishedAt })
|
||||
nativePhase: heartbeatRuns.nativePhase, finishedAt: heartbeatRuns.finishedAt, resultJson: heartbeatRuns.resultJson })
|
||||
.from(heartbeatRuns).where(needsRepair).for("update").limit(1);
|
||||
if (!before) return null;
|
||||
await notifyExternalAgentSettlement(tx, { companyId: input.run.companyId, runId: input.run.id });
|
||||
@@ -195,9 +196,10 @@ export async function projectCommittedRun(input: {
|
||||
|| jsonb_build_object('finalizationPhase', 'committed', 'failureCode', null, 'originalFailureCode', null, 'nextAttemptAt', null)`,
|
||||
}
|
||||
: {
|
||||
// A failed accepted result keeps any recorded cause, and otherwise
|
||||
// names the unexplained failure.
|
||||
...(terminalState === "failed" ? explainFailedRun(NATIVE_RUN_FAILED_UNEXPLAINED, { preserveCancelled: false }) : {}),
|
||||
// A failed accepted result keeps any recorded cause, then names a
|
||||
// recorded runner turn failure, and otherwise the unexplained failure.
|
||||
...(terminalState === "failed" ? explainFailedRun(explainNativeTurnFailure(readNativeTurnFailureDiagnostic(
|
||||
before.resultJson?.nativeTurnFailure)) ?? NATIVE_RUN_FAILED_UNEXPLAINED, { preserveCancelled: false }) : {}),
|
||||
resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || jsonb_build_object('finalizationPhase', 'committed', 'failureCode', null, 'originalFailureCode', null, 'nextAttemptAt', null)`,
|
||||
}),
|
||||
updatedAt: now,
|
||||
|
||||
@@ -106,6 +106,7 @@ import { redactNativeFailureError } from "./native-failure-redaction.js";
|
||||
import { parseCodexCliVersion } from "./codex-runtime-compatibility.js";
|
||||
import { createNativeToolTrace, type NativeToolTrace } from "./native-tool-trace.js";
|
||||
import { createNativeProviderFailureObservation } from "./native-provider-failure.js";
|
||||
import { explainNativeTurnFailure } from "./native-run-failure-explanation.js";
|
||||
import { nativeStartupRetryExhaustedAction, nativeStartupRetryWait, planNativeStartupRetry } from "./native-startup-retry.js";
|
||||
import { admitUnclaimedNativeBootstrapRetry } from "./native-bootstrap-retry.js";
|
||||
import { RUNNERD_CONTROL_PLANE_STATE_SCHEMAS, runnerdStateIsIncompleteBootstrap,
|
||||
@@ -13826,7 +13827,8 @@ async function executePaperclipNativeSessionAttempt(
|
||||
errorMessage:
|
||||
native.terminal.runTerminalState === "succeeded"
|
||||
? null
|
||||
: providerFailure?.errorMessage ?? `Native session ${native.terminal.runTerminalState}`,
|
||||
: providerFailure?.errorMessage ?? explainNativeTurnFailure(unclassifiedTurnFailure)?.error
|
||||
?? `Native session ${native.terminal.runTerminalState}`,
|
||||
...(providerFailure ? { errorCode: providerFailure.errorCode } : {}),
|
||||
resultJson: {
|
||||
nativeResult: native.result as unknown as Record<string, unknown>,
|
||||
|
||||
Reference in new issue
Block a user