diff --git a/server/src/__tests__/native-run-projection-work.test.ts b/server/src/__tests__/native-run-projection-work.test.ts index 462f2d57ae5..678e66b0633 100644 --- a/server/src/__tests__/native-run-projection-work.test.ts +++ b/server/src/__tests__/native-run-projection-work.test.ts @@ -15,6 +15,7 @@ import { agentWakeupRequests, agents, companies, completionContracts, createDb, import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { nativeRunProjectionWork } from "../services/native-runtime/native-run-projection-work.js"; import { projectCommittedRun } from "../services/native-runtime/native-run-projection.js"; +import { NATIVE_RUN_FAILED_UNEXPLAINED } from "../services/native-runtime/native-run-failure-explanation.js"; import { notifyNativeFinalizationWork } from "../services/native-finalization-notifications.js"; import { finalizeNativeRun, recordNativeFinalizationFailure } from "../services/native-runtime/native-run-finalizer.js"; import * as statusCommitter from "../services/native-runtime/status-decision-committer.js"; @@ -430,6 +431,25 @@ describe("committed native run projection lifecycle", () => { expect((await read(binding.runId)).executionStatusDeliveryId).toBe(settled.executionStatusDeliveryId); expect(emitAgentTaskRun).toHaveBeenCalledTimes(1); }); + it("explains a failed accepted result when the finalizer projects the run", async () => { + const binding = await accepted("failed"); + await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + const run = await read(binding.runId); + expect(run).toMatchObject({ status: "failed", errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode, error: NATIVE_RUN_FAILED_UNEXPLAINED.error }); + }); + it.each([ + ["no recorded cause", null, null, NATIVE_RUN_FAILED_UNEXPLAINED.errorCode, NATIVE_RUN_FAILED_UNEXPLAINED.error], + ["an empty error", "adapter_failed", " ", "adapter_failed", NATIVE_RUN_FAILED_UNEXPLAINED.error], + ["a recorded cause", "adapter_failed", "Adapter exited 1", "adapter_failed", "Adapter exited 1"], + ])("explains a failed accepted result in committed recovery projection with %s", async (_label, errorCode, error, expectedCode, expectedError) => { + const binding = await accepted("failed"); + await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); + await db.update(heartbeatRuns).set({ errorCode, error }).where(eq(heartbeatRuns.id, binding.runId)); + const run = await read(binding.runId); + expect(run.status).toBe("running"); + await projectCommittedRun({ db, run, coordinator: await state(binding.runId) }); + expect(await read(binding.runId)).toMatchObject({ status: "failed", errorCode: expectedCode, error: expectedError }); + }); it("ignores routine company-only finalization hints and still repairs a committed run", async () => { await start(); await flush(); await quiet(); const previous = db.$client.options.debug; diff --git a/server/src/services/native-runtime/native-run-failure-explanation.ts b/server/src/services/native-runtime/native-run-failure-explanation.ts new file mode 100644 index 00000000000..73e7c0df69b --- /dev/null +++ b/server/src/services/native-runtime/native-run-failure-explanation.ts @@ -0,0 +1,36 @@ +import { sql, type SQL } from "drizzle-orm"; +import { heartbeatRuns } from "@paperclipai/db"; + +/** Used when a native run reaches `failed` and no writer recorded a more specific cause. */ +export const NATIVE_RUN_FAILED_UNEXPLAINED = { + errorCode: "native_run_failed", + error: "The native run ended in a failed state without a classified provider error. Inspect the run events for the last recorded step.", +} as const; + +export type NativeRunFailureExplanation = { errorCode: string; error: string }; + +/** + * A failed run must always say why. These SET expressions keep an explanation + * already on the row, and otherwise record `fallback`. A run that is already + * cancelled keeps its own fields, because the status writers preserve a + * concurrent cancellation. In an UPDATE, column references read the row before + * the update, which matches the status `case` expressions of the callers. + * + * `current` overrides the expression used for the existing value, for writers + * that already compute a projected value for that field. A writer that sets + * `failed` unconditionally passes `preserveCancelled: false`. + */ +export function explainFailedRun( + fallback: NativeRunFailureExplanation, + current: { errorCode?: SQL; error?: SQL; preserveCancelled?: boolean } = {}, +) { + const errorCode = current.errorCode ?? sql`${heartbeatRuns.errorCode}`; + const error = current.error ?? sql`${heartbeatRuns.error}`; + const explainedCode = sql`coalesce(nullif(btrim(${errorCode}), ''), ${fallback.errorCode})`; + const explainedError = sql`coalesce(nullif(btrim(${error}), ''), ${fallback.error})`; + if (current.preserveCancelled === false) return { errorCode: explainedCode, error: explainedError }; + return { + errorCode: sql`case when ${heartbeatRuns.status} = 'cancelled' then ${heartbeatRuns.errorCode} else ${explainedCode} end`, + error: sql`case when ${heartbeatRuns.status} = 'cancelled' then ${heartbeatRuns.error} else ${explainedError} end`, + }; +} diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index d9079d58b70..08baf07c3b9 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -26,6 +26,7 @@ import { hasPendingNativeChildCompletion } from "./native-child-completion-deliv import { settleExternalConversation } from "../slack-conversation-lifecycle.js"; import { executionFailureRetryCount } from "../execution-recovery-attempt.js"; import { readPersistedNativeFailureEvidence } from "./native-provider-failure-evidence.js"; +import { explainFailedRun, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js"; import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js"; import { findSupersededExternalReviewRequests } from "./superseded-external-review-requests.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js"; @@ -1629,12 +1630,20 @@ export async function finalizeNativeRun(input: { || ${JSON.stringify(finalizationMetadata)}::jsonb || ${clearExecutionFailure ? sql`case when ${heartbeatRuns.status} = 'cancelled' then '{}'::jsonb else ${recoveredExecutionFailureMetadata()} end` : sql`'{}'::jsonb`}`; - const projectedError = sql`case when ${heartbeatRuns.status} = 'cancelled' then ${heartbeatRuns.error} - else ${providerFailure ? sql`coalesce(${heartbeatRuns.error}, ${providerFailure.errorMessage})` - : clearExecutionFailure ? sql`null` : heartbeatRuns.error} end`; - const projectedErrorCode = sql`case when ${heartbeatRuns.status} = 'cancelled' then ${heartbeatRuns.errorCode} - else ${providerFailure ? sql`coalesce(${heartbeatRuns.errorCode}, ${providerFailure.errorCode})` - : clearExecutionFailure ? sql`null` : heartbeatRuns.errorCode} end`; + const nextError = providerFailure ? sql`coalesce(${heartbeatRuns.error}, ${providerFailure.errorMessage})` + : clearExecutionFailure ? sql`null` : sql`${heartbeatRuns.error}`; + const nextErrorCode = providerFailure ? sql`coalesce(${heartbeatRuns.errorCode}, ${providerFailure.errorCode})` + : clearExecutionFailure ? sql`null` : sql`${heartbeatRuns.errorCode}`; + // A run projected to failed always records why, even when the provider + // reported a failed terminal without a classified failure. + const projectsFailedStatus = input.projectRunStatus && projectedStatus === "failed"; + const explained = projectsFailedStatus + ? explainFailedRun(NATIVE_RUN_FAILED_UNEXPLAINED, { error: nextError, errorCode: nextErrorCode }) + : null; + const projectedError = explained?.error ?? sql`case when ${heartbeatRuns.status} = 'cancelled' then ${heartbeatRuns.error} + else ${nextError} end`; + const projectedErrorCode = explained?.errorCode ?? sql`case when ${heartbeatRuns.status} = 'cancelled' then ${heartbeatRuns.errorCode} + else ${nextErrorCode} end`; const projectionChanged = or( sql`${heartbeatRuns.nativePhase} is distinct from ${finalizationPhase}`, sql`${heartbeatRuns.resultJson} is distinct from ${projectedResult}`, @@ -1659,7 +1668,7 @@ export async function finalizeNativeRun(input: { : {}), nativePhase: finalizationPhase, nativePhaseUpdatedAt: now, - ...(clearExecutionFailure || providerFailure ? { error: projectedError, errorCode: projectedErrorCode } : {}), + ...(clearExecutionFailure || providerFailure || projectsFailedStatus ? { error: projectedError, errorCode: projectedErrorCode } : {}), resultJson: projectedResult, updatedAt: now, }) diff --git a/server/src/services/native-runtime/native-run-projection.ts b/server/src/services/native-runtime/native-run-projection.ts index 706f80e7366..77dd186a30e 100644 --- a/server/src/services/native-runtime/native-run-projection.ts +++ b/server/src/services/native-runtime/native-run-projection.ts @@ -18,6 +18,7 @@ import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js" import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; import { isNewTerminalRunFailure, reportRunFailure } from "../run-failure-report.js"; import { nativeRunnerOwnershipNotHeldCondition } from "./native-runner-ownership.js"; +import { explainFailedRun, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js"; export function projectNativeTerminalRunStatus( @@ -192,7 +193,12 @@ export async function projectCommittedRun(input: { || ${recoveredExecutionFailureMetadata()} || jsonb_build_object('finalizationPhase', 'committed', 'failureCode', null, 'originalFailureCode', null, 'nextAttemptAt', null)`, } - : { resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || jsonb_build_object('finalizationPhase', 'committed', 'failureCode', null, 'originalFailureCode', null, 'nextAttemptAt', null)` }), + : { + // A failed accepted result keeps any recorded cause, and otherwise + // names the unexplained failure. + ...(terminalState === "failed" ? explainFailedRun(NATIVE_RUN_FAILED_UNEXPLAINED, { preserveCancelled: false }) : {}), + resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || jsonb_build_object('finalizationPhase', 'committed', 'failureCode', null, 'originalFailureCode', null, 'nextAttemptAt', null)`, + }), updatedAt: now, }) .where(needsRepair)