Files
PaperClipAI/server/src/services/native-runtime/native-provider-failure-evidence.ts
T
d747752496 fix(runner): record why an OpenCode turn fails when opencode serve stops (#16863)
When `opencode serve` stops during a turn, the OpenCode driver now fails the
active turn itself with a closed-vocabulary cause (server exit code or known
signal, allowlisted socket error code, HTTP status, or closed stream) instead
of leaving runnerd to close it with `error: null`. The proxy flushes stdout
before a fatal exit so the failed turn reaches runnerd, and the server records
OpenCode failed turns as `nativeTurnFailure` (live and persisted-recovery
paths) so the run-failure diagnostic names the cause. Provider text is never
copied. Run status, retries and recovery behaviour do not change.

Co-Authored-By: Paperclip Sentry triage <Paperclip-Paperclip@users.noreply.github.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 12:20:24 -07:00

75 lines
3.8 KiB
TypeScript

import { and, desc, eq, sql } from "drizzle-orm";
import { heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db";
import { parseNativeExecutionInput, validatePrpEvent, type PrpTerminalState } from "../../vendor/paperclip-runner/index.js";
import { nativeSha256 } from "./canonical.js";
import { createNativeProviderFailureObservation } from "./native-provider-failure.js";
function record(value: unknown): Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value)
? value as Record<string, unknown> : {};
}
async function observePersistedNativeTurnFailure(
db: Db,
run: typeof heartbeatRuns.$inferSelect,
turnId: string | null,
terminal: PrpTerminalState,
) {
if (run.runtimeMode !== "native" || !turnId || !run.nativeSessionId || !run.runnerInstanceId ||
terminal.turnTerminalState !== "failed" || terminal.runTerminalState !== "failed") return null;
let execution;
try { execution = parseNativeExecutionInput(record(run.runnerProfileJson).nativeExecutionInput); } catch { return null; }
if ((execution.provider.kind !== "codex" && execution.provider.kind !== "opencode") ||
execution.binding.runId !== run.id || execution.binding.companyId !== run.companyId ||
execution.binding.agentId !== run.agentId || execution.binding.issueId !== run.nativeIssueId ||
execution.session.normalizedSessionId !== run.nativeSessionId ||
execution.completionContract.id !== run.completionContractId ||
execution.completionContract.sha256 !== run.completionContractSha256) return null;
const [row] = await db.select().from(heartbeatRunEvents).where(and(
eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id),
eq(heartbeatRunEvents.sourceInstanceId, run.runnerInstanceId), eq(heartbeatRunEvents.eventType, "turn.failed"),
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'turnId' = ${turnId}`,
)).orderBy(desc(heartbeatRunEvents.seq)).limit(1);
if (!row) return null;
const parsed = validatePrpEvent(record(row.payload).prpEvent);
if (!parsed.ok || parsed.event.runId !== run.id || parsed.event.normalizedSessionId !== run.nativeSessionId ||
parsed.event.sourceKind !== "runner" || parsed.event.sourceInstanceId !== run.runnerInstanceId ||
parsed.event.sourceEventId !== row.sourceEventId || parsed.event.sourceSeq !== row.sourceSeq ||
parsed.event.eventType !== row.eventType || parsed.event.turnId !== turnId ||
row.sourcePayloadSha256 !== nativeSha256(parsed.event)) return null;
const observed = createNativeProviderFailureObservation(execution.provider);
observed.observe(parsed.event);
return observed;
}
/** Re-derive authority from immutable run-bound evidence, including after restart. */
export async function readPersistedNativeProviderFailure(
...args: Parameters<typeof observePersistedNativeTurnFailure>
) {
const [, , turnId, terminal] = args;
return (await observePersistedNativeTurnFailure(...args))?.forTerminal(turnId, terminal) ?? null;
}
/**
* Recovery can finalize before heartbeat saves the adapter result. Rebuild the
* classified failure and the diagnostics-only turn failure from one event.
*/
export async function readPersistedNativeFailureEvidence(
...args: Parameters<typeof observePersistedNativeTurnFailure>
) {
const [, , turnId, terminal] = args;
const observed = await observePersistedNativeTurnFailure(...args);
return {
providerFailure: observed?.forTerminal(turnId, terminal) ?? null,
turnFailure: observed?.unclassifiedTurnFailure(turnId, terminal) ?? null,
};
}
export async function readPersistedNativeModelRejection(
...args: Parameters<typeof readPersistedNativeProviderFailure>
) {
const failure = await readPersistedNativeProviderFailure(...args);
return failure?.errorCode === "native_provider_model_rejected" ? failure : null;
}