fix(runner): record why an OpenCode turn fails when opencode serve stops (#16863)

When `opencode serve` stops during a turn, the OpenCode driver now fails the
active turn itself with a closed-vocabulary cause (server exit code or known
signal, allowlisted socket error code, HTTP status, or closed stream) instead
of leaving runnerd to close it with `error: null`. The proxy flushes stdout
before a fatal exit so the failed turn reaches runnerd, and the server records
OpenCode failed turns as `nativeTurnFailure` (live and persisted-recovery
paths) so the run-failure diagnostic names the cause. Provider text is never
copied. Run status, retries and recovery behaviour do not change.

Co-Authored-By: Paperclip Sentry triage <Paperclip-Paperclip@users.noreply.github.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-10-19 12:20:24 -07:00
1 parent 8193abb936
commit d747752496
7 files changed
+314 -20

No files matched your search

@@ -471,6 +471,9 @@ function shutdown(exitCode = 0): Promise<void> {
?.close({ reason: "proxy_shutdown", force: true })
.catch(() => {});
await eventPump?.catch(() => {});
// A failed turn can be the last frame before a fatal exit. Pipe writes
// are asynchronous on some platforms, so let it reach runnerd first.
await new Promise<void>((resolve) => process.stdout.write("", () => resolve()));
})().finally(() => process.exit(exitCode));
return shutdownPromise;
}
@@ -14,7 +14,8 @@ import {
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { createServer as createHttpServer } from "node:http";
import { execFile } from "node:child_process";
import { execFile, type ChildProcess } from "node:child_process";
import { EventEmitter } from "node:events";
import { promisify } from "node:util";
import { getCACertificates } from "node:tls";
import { join, resolve } from "node:path";
@@ -2554,6 +2555,79 @@ describe("OpenCodeServerDriver", () => {
}
});
it("fails the running turn with the exit cause when OpenCode stops mid-turn", async () => {
await chmod(fixture, 0o755);
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-server-lost-"));
const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-server-lost-workspace-"));
roots.push(root, workspace);
let stream!: ReadableStreamDefaultController<Uint8Array>;
let streamOpened!: () => void;
const opened = new Promise<void>(resolve => { streamOpened = resolve; });
let serverPid: number | undefined;
let eventRequests = 0;
const refused = () => new TypeError("fetch failed", {
cause: Object.assign(new Error("connect ECONNREFUSED 127.0.0.1"), { code: "ECONNREFUSED" }),
});
const driver = new OpenCodeServerDriver({
model: "openrouter/deepseek/deepseek-v4-flash-0731", runtimeDirectory: root, command: fixture,
environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" },
onSpawn: async ({ pid }) => { serverPid = pid; },
// Own the event stream so the fixture cannot finish the turn on its own,
// then report the refused reconnects a dead server produces.
fetch: async (input, init) => {
if (!String(input).endsWith("/event")) return fetch(input, init);
eventRequests += 1;
if (eventRequests > 1) throw refused();
return new Response(new ReadableStream<Uint8Array>({ start(controller) { stream = controller; streamOpened(); } }),
{ headers: { "Content-Type": "text/event-stream" } });
},
});
const session = await driver.openSession({ runId: "server-lost", normalizedSessionId: "server-lost", workingDirectory: workspace });
try {
const events = collectTurnEvents(session.events());
const turn = await session.startTurn({ message: { role: "user", text: "install dependencies" } });
await opened;
process.kill(serverPid!, "SIGKILL");
stream.error(refused());
const observed = await events;
const failed = observed.at(-1);
expect(failed).toMatchObject({
eventType: "turn.failed",
turnId: turn.turnId,
payload: {
status: "failed",
providerTerminalObserved: false,
error: {
code: "opencode_server_exited",
opencodeFailure: { kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" },
},
},
});
const diagnostic = observed.find(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_sse_failed");
expect(diagnostic?.payload.failure).toEqual({ kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" });
expect(eventRequests).toBe(4);
} finally {
await session.close({ reason: "test complete" });
}
});
it("classifies a lost event stream from closed error fields only", async () => {
const { openCodeEventStreamFailure } = openCodeServerDriverInternals;
const running = Object.assign(new EventEmitter(), { exitCode: null, signalCode: null }) as unknown as ChildProcess;
const exited = Object.assign(new EventEmitter(), { exitCode: 137, signalCode: null }) as unknown as ChildProcess;
const reset = new TypeError("fetch failed", { cause: Object.assign(new Error("private socket detail"), { code: "ECONNRESET" }) });
expect(await openCodeEventStreamFailure(reset, exited)).toEqual({ kind: "server_exited", exitCode: 137, connectionError: "ECONNRESET" });
expect(await openCodeEventStreamFailure(Object.assign(new Error("HTTP 502"), { httpStatus: 502 }), running))
.toEqual({ kind: "http_error", httpStatus: 502 });
expect(await openCodeEventStreamFailure(Object.assign(new Error("closed"), { streamClosed: true }), running))
.toEqual({ kind: "stream_closed" });
expect(await openCodeEventStreamFailure(new Error("private provider text"), running)).toEqual({ kind: "stream_error" });
const unknownCode = new TypeError("fetch failed", { cause: Object.assign(new Error("x"), { code: "EPRIVATE" }) });
const started = Date.now();
expect(await openCodeEventStreamFailure(unknownCode, running)).toEqual({ kind: "connection_failed", connectionError: "unrecognized" });
expect(Date.now() - started).toBeGreaterThanOrEqual(900);
});
it("restarts OpenCode when session startup fails transiently", async () => {
await chmod(fixture, 0o755);
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-"));
@@ -15,6 +15,7 @@ import {
writeFile,
} from "node:fs/promises";
import { createServer } from "node:net";
import { constants as osConstants } from "node:os";
import { Agent as HttpAgent, createServer as createHttpServer, request as requestHttp, type IncomingMessage } from "node:http";
import { Agent as HttpsAgent, request as requestHttps } from "node:https";
import { getCACertificates } from "node:tls";
@@ -1353,8 +1354,9 @@ class OpenCodeHarnessSession implements HarnessSession {
signal: this.#abort.signal,
});
if (!response.ok || !response.body)
throw new Error(
`OpenCode event stream returned HTTP ${response.status}`,
throw Object.assign(
new Error(`OpenCode event stream returned HTTP ${response.status}`),
{ httpStatus: response.status },
);
const outboundFrameId = this.#runtime.trace?.frame({
direction: "client_to_provider",
@@ -1452,8 +1454,9 @@ class OpenCodeHarnessSession implements HarnessSession {
}
this.#mapProviderEvent(event, frameId);
}
throw new Error(
"OpenCode event stream closed before the session became terminal",
throw Object.assign(
new Error("OpenCode event stream closed before the session became terminal"),
{ streamClosed: true },
);
} catch (error) {
if (this.#closed || this.#abort.signal.aborted) return;
@@ -1461,10 +1464,14 @@ class OpenCodeHarnessSession implements HarnessSession {
if (attempts > 3) {
await this.#completionSettlement;
if (this.#closed) return;
const failure = await openCodeEventStreamFailure(error, this.#runtime.process);
if (this.#closed) return;
this.#emit("harness.diagnostic", {
code: "opencode_sse_failed",
message: redact(String(error), this.#runtime.sensitiveValues),
failure,
});
this.#failActiveTurnOnStreamLoss(failure);
this.#events.fail(error);
return;
}
@@ -1473,6 +1480,34 @@ class OpenCodeHarnessSession implements HarnessSession {
}
}
/**
* The event stream is the only path for this turn's terminal event. Once it
* is gone the turn cannot finish, so record why before the stream ends
* instead of leaving runnerd to close it later with no cause.
*/
#failActiveTurnOnStreamLoss(failure: OpenCodeEventStreamFailure): void {
const turnId = this.#activeTurnId;
if (!turnId || this.#terminalTurns.has(turnId)) return;
this.#settlePendingRuntimeRequestsForTurn(turnId);
this.#emit(
"turn.failed",
{
status: "failed",
providerTerminalObserved: false,
error: {
code: failure.kind === "server_exited" ? "opencode_server_exited" : "opencode_event_stream_lost",
message: failure.kind === "server_exited"
? "OpenCode stopped while the turn was running."
: "The OpenCode event stream failed while the turn was running.",
opencodeFailure: failure,
},
},
{ turnId },
);
this.#terminalTurns.set(turnId, canonicalJson({ status: "failed" }));
this.#activeTurnId = null;
}
#mapProviderEvent(value: unknown, traceFrameId: number | null = null): void {
this.#activeTraceFrameId = traceFrameId;
this.#activeTraceEmittedEventIds = [];
@@ -3031,7 +3066,55 @@ async function waitForExit(
});
}
export const openCodeServerDriverInternals = { parseSse };
/** Closed vocabulary for a lost OpenCode event stream. Never provider text. */
export interface OpenCodeEventStreamFailure {
kind: "server_exited" | "connection_failed" | "http_error" | "stream_closed" | "stream_error";
exitCode?: number;
signal?: string;
connectionError?: string;
httpStatus?: number;
}
const OPENCODE_CONNECTION_ERROR_CODES: ReadonlySet<string> = new Set([
"ECONNREFUSED", "ECONNRESET", "EPIPE", "ETIMEDOUT", "EHOSTUNREACH", "ENETUNREACH",
"UND_ERR_SOCKET", "UND_ERR_CLOSED", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT",
"UND_ERR_BODY_TIMEOUT",
]);
function connectionErrorCode(error: unknown): string | undefined {
// undici reports `TypeError: fetch failed` and keeps the socket error in `cause`.
for (let current = error, depth = 0; current && depth < 4; current = record(current).cause, depth += 1) {
const code = record(current).code;
if (typeof code === "string") return OPENCODE_CONNECTION_ERROR_CODES.has(code) ? code : "unrecognized";
}
return undefined;
}
async function openCodeEventStreamFailure(
error: unknown,
process: ChildProcess,
): Promise<OpenCodeEventStreamFailure> {
const connectionError = connectionErrorCode(error);
// A refused or reset connection usually means `opencode serve` has died.
// Give its exit a moment to be observed so the cause is the exit itself.
if (connectionError) await waitForExit(process, 1_000);
if (process.exitCode !== null || process.signalCode !== null) {
return {
kind: "server_exited",
...(process.exitCode !== null ? { exitCode: process.exitCode } : {}),
...(process.signalCode !== null && process.signalCode in osConstants.signals ? { signal: process.signalCode } : {}),
...(connectionError ? { connectionError } : {}),
};
}
const httpStatus = record(error).httpStatus;
if (typeof httpStatus === "number" && Number.isInteger(httpStatus) && httpStatus >= 100 && httpStatus <= 599)
return { kind: "http_error", httpStatus };
if (connectionError) return { kind: "connection_failed", connectionError };
if (record(error).streamClosed === true) return { kind: "stream_closed" };
return { kind: "stream_error" };
}
export const openCodeServerDriverInternals = { parseSse, openCodeEventStreamFailure };
class AsyncQueue<T> implements AsyncIterable<T> {
#items: T[] = [];
@@ -19,7 +19,7 @@ async function observePersistedNativeTurnFailure(
terminal.turnTerminalState !== "failed" || terminal.runTerminalState !== "failed") return null;
let execution;
try { execution = parseNativeExecutionInput(record(run.runnerProfileJson).nativeExecutionInput); } catch { return null; }
if (execution.provider.kind !== "codex" ||
if ((execution.provider.kind !== "codex" && execution.provider.kind !== "opencode") ||
execution.binding.runId !== run.id || execution.binding.companyId !== run.companyId ||
execution.binding.agentId !== run.agentId || execution.binding.issueId !== run.nativeIssueId ||
execution.session.normalizedSessionId !== run.nativeSessionId ||
@@ -46,10 +46,12 @@ describe("native completed failure diagnostics", () => {
});
afterAll(async () => { await temporary?.cleanup(); });
it.each(["committed", "duplicate", "pending-review", "superseded-review", "resolved-review", "reassigned", "successor-race", "different-turn", "unknown-error", "untrusted-source", "untrusted-run", "untrusted-session", "succeeded", "cancelled"] as const)(
it.each(["committed", "duplicate", "pending-review", "superseded-review", "resolved-review", "reassigned", "successor-race", "different-turn", "unknown-error", "opencode-lost-server", "untrusted-source", "untrusted-run", "untrusted-session", "succeeded", "cancelled"] as const)(
"preserves accepted results and only labels the matching provider rejection (%s)", async (scenario) => {
const issueId = randomUUID();
const runId = randomUUID();
const opencode = scenario === "opencode-lost-server";
const executionProvider = opencode ? { kind: "opencode" as const, model: `openrouter/${model}` } : { kind: "codex" as const, model };
const [issue] = await db.insert(issues).values({ id: issueId, companyId, title: "Explain a provider rejection", status: "in_progress", workMode: "standard", assigneeAgentId: agentId }).returning();
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId, invocationSource: "assignment", triggerDetail: "system", contextSnapshot: { issueId } }).returning();
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
@@ -60,10 +62,10 @@ describe("native completed failure diagnostics", () => {
const execution: NativeExecutionInputV1 = {
schema: "paperclip.native-execution-input.v1",
binding: { companyId, runId, issueId, agentId, executionWorkspaceId: runId },
provider: { kind: "codex", model },
provider: executionProvider,
task: { identifier: issueId, title: issue.title, description: null, prompt: "Reply to the current message.", workMode: "standard" },
workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null },
session: { normalizedSessionId: native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } },
session: { normalizedSessionId: native.normalizedSessionId, driverKind: opencode ? "opencode_server" : "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } },
completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", contract: completionInput },
interactionResponses: [], credentialBindings: [],
};
@@ -94,7 +96,10 @@ describe("native completed failure diagnostics", () => {
schema: "paperclip.prp.event.v1", runId, normalizedSessionId: native.normalizedSessionId,
turnId: native.turnId, sourceInstanceId: native.runnerInstanceId, sourceEventId: "model-terminal", sourceSeq: 1,
sourceKind: "runner", eventType: "turn.failed", schemaVersion: 1, priority: 0, emittedAt: new Date().toISOString(),
payload: { status: "failed", error: { codexErrorInfo: "other", message: JSON.stringify({
payload: opencode ? { status: "failed", providerTerminalObserved: false, error: {
code: "opencode_server_exited", message: "Private stream failure",
opencodeFailure: { kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" },
} } : { status: "failed", error: { codexErrorInfo: "other", message: JSON.stringify({
type: "error", status: 400, error: { type: "invalid_request_error", message: scenario === "unknown-error" ? "Private unknown failure" : `The '${model}' model is not supported when using Codex with a ChatGPT account.` },
}) } },
};
@@ -124,7 +129,7 @@ describe("native completed failure diagnostics", () => {
expect((await options.controlPlane.appendEvent(event)).disposition).toBe(scenario === "duplicate" ? "duplicate" : "committed");
}
await options.controlPlane.completeRun({ result, terminal, turnId });
return { result, terminal, turnId, normalizedSessionId: native.normalizedSessionId, providerSessionId: "fixture-provider-session", driverKind: "codex_app_server", driverVersion: "fixture", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
return { result, terminal, turnId, normalizedSessionId: native.normalizedSessionId, providerSessionId: "fixture-provider-session", driverKind: execution.session.driverKind, driverVersion: "fixture", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
});
const adapter = await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, backend: {
descriptor: async () => { throw new Error("Unexpected provider call"); },
@@ -137,11 +142,13 @@ describe("native completed failure diagnostics", () => {
expect(adapter.errorMessage).toBe(classified ? NATIVE_MODEL_REJECTION_MESSAGE : scenario === "succeeded" ? null : `Native session ${terminal.runTerminalState}`);
expect(adapter.resultJson?.nativeProviderFailure).toEqual(classified ? NATIVE_MODEL_REJECTION_DIAGNOSTIC : undefined);
// Only a committed, unclassified failure of the terminal turn explains the generic message.
const turnFailure = scenario === "unknown-error" ? { provider: "codex", category: "turn_failed", codexErrorInfo: "other" } : undefined;
const turnFailure = scenario === "unknown-error" ? { provider: "codex", category: "turn_failed", codexErrorInfo: "other" }
: opencode ? { provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited", streamFailure: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" }
: undefined;
expect(adapter.resultJson?.nativeTurnFailure).toEqual(turnFailure);
expect(adapter.nativeFinalization).toMatchObject({ result, terminal, turnId, workspaceFinalizeStatus: "pending" });
expect(provider.execute).toHaveBeenCalledTimes(1);
expect(execution.provider).toEqual({ kind: "codex", model });
expect(execution.provider).toEqual(executionProvider);
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId));
expect(coordinator).toMatchObject({ phase: "workspace_finalizing", leaseOwner: null, attempt: 1, nextAttemptAt: null });
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toHaveLength(1);
@@ -263,7 +270,7 @@ describe("native completed failure diagnostics", () => {
expect(provider.execute).toHaveBeenCalledTimes(1);
await db.update(agents).set({ adapterConfig: { provider: "codex", model } }).where(eq(agents.id, agentId));
}
} else if (scenario === "unknown-error") {
} else if (scenario === "unknown-error" || opencode) {
// A lookalike diagnostic flag does not alter unknown failed-result policy.
await db.update(heartbeatRuns).set({ resultJson: { nativeProviderFailure: NATIVE_MODEL_REJECTION_DIAGNOSTIC } }).where(eq(heartbeatRuns.id, runId));
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded" });
@@ -133,6 +133,57 @@ describe("native provider failure observation", () => {
})).toEqual({ provider: "codex", category: "turn_failed", codexErrorInfo: "badRequest", runnerFailureCode: "unrecognized" });
});
it("names a lost OpenCode server without copying provider text", () => {
const observed = createNativeProviderFailureObservation({ kind: "opencode", model });
observed.observe({ ...event, payload: { status: "failed", providerTerminalObserved: false, error: {
code: "opencode_server_exited", message: "Private provider message",
opencodeFailure: { kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED", detail: "private" },
} } });
expect(observed.forTerminal("turn", terminal)).toBeNull();
const diagnostic = observed.unclassifiedTurnFailure("turn", terminal);
expect(diagnostic).toEqual({
provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited",
streamFailure: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED",
});
expect(JSON.stringify(diagnostic)).not.toMatch(/private/i);
expect(readNativeTurnFailureDiagnostic(diagnostic)).toEqual(diagnostic);
});
it("still identifies an OpenCode turn that failed with no structured cause", () => {
const observed = createNativeProviderFailureObservation({ kind: "opencode", model });
observed.observe({ ...event, payload: { status: "failed", error: null } });
expect(observed.unclassifiedTurnFailure("turn", terminal)).toEqual({ provider: "opencode", category: "turn_failed" });
const providerError = createNativeProviderFailureObservation({ kind: "opencode", model });
providerError.observe({ ...event, payload: { status: "failed", error: { code: "private_provider_code", name: "APIError" } } });
expect(providerError.unclassifiedTurnFailure("turn", terminal)).toEqual({
provider: "opencode", category: "turn_failed", runnerFailureCode: "unrecognized",
});
});
it("drops values outside the OpenCode stream-failure vocabulary", () => {
expect(readNativeTurnFailureDiagnostic({
provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_event_stream_lost",
streamFailure: "connection_failed", connectionError: "EPRIVATE", exitCode: 300, signal: "private signal", httpStatus: 42,
})).toEqual({
provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_event_stream_lost", streamFailure: "connection_failed",
});
expect(readNativeTurnFailureDiagnostic({
provider: "opencode", category: "turn_failed", streamFailure: "private", exitCode: 137,
})).toEqual({ provider: "opencode", category: "turn_failed" });
expect(readNativeTurnFailureDiagnostic({ provider: "opencode", category: "server_overloaded" })).toBeNull();
});
it("records only known signal names for a lost OpenCode server", () => {
const observed = createNativeProviderFailureObservation({ kind: "opencode", model });
observed.observe({ ...event, payload: { status: "failed", error: {
code: "opencode_server_exited", opencodeFailure: { kind: "server_exited", signal: "SIGPRIVATE" },
} } });
const expected = { provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited", streamFailure: "server_exited" };
expect(observed.unclassifiedTurnFailure("turn", terminal)).toEqual(expected);
expect(readNativeTurnFailureDiagnostic({ ...expected, signal: "SIGPRIVATE" })).toEqual(expected);
expect(readNativeTurnFailureDiagnostic({ ...expected, signal: "SIGTERM" })).toEqual({ ...expected, signal: "SIGTERM" });
});
it.each([{ kind: "claude", model }, { kind: "codex" }, { kind: "codex", model: "other" }])(
"does not apply a rejection to another selected provider/model (%j)", (provider) => {
const observed = createNativeProviderFailureObservation(provider);
@@ -62,7 +62,7 @@ const NATIVE_TURN_FAILURE_RUNNER_CODES: ReadonlySet<string> = new Set([
* Closed vocabulary for a failed Codex turn that no classifier above claims.
* It explains generic `Native session failed` reports without provider text.
*/
export interface NativeTurnFailureDiagnostic {
export interface NativeCodexTurnFailureDiagnostic {
provider: "codex";
category: "turn_failed";
/** A known `CodexErrorInfo` kind, `unrecognized`, or `absent`. */
@@ -72,17 +72,79 @@ export interface NativeTurnFailureDiagnostic {
runnerFailureCode?: string;
}
/**
* `error.code` values the OpenCode driver writes when it fails a turn itself.
* Other OpenCode failures carry provider error objects and stay `unrecognized`.
*/
const OPENCODE_TURN_FAILURE_RUNNER_CODES: ReadonlySet<string> = new Set([
"opencode_server_exited", "opencode_event_stream_lost", "provider_terminal_error_missing",
]);
const OPENCODE_STREAM_FAILURE_KINDS: ReadonlySet<string> = new Set([
"server_exited", "connection_failed", "http_error", "stream_closed", "stream_error",
]);
const OPENCODE_CONNECTION_ERRORS: ReadonlySet<string> = new Set([
"ECONNREFUSED", "ECONNRESET", "EPIPE", "ETIMEDOUT", "EHOSTUNREACH", "ENETUNREACH",
"UND_ERR_SOCKET", "UND_ERR_CLOSED", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT",
"UND_ERR_BODY_TIMEOUT", "unrecognized",
]);
/** POSIX, Linux and macOS signal names; any other `signal` value is dropped. */
const OPENCODE_EXIT_SIGNALS: ReadonlySet<string> = new Set([
"SIGHUP", "SIGINT", "SIGQUIT", "SIGILL", "SIGTRAP", "SIGABRT", "SIGIOT", "SIGBUS", "SIGEMT", "SIGFPE",
"SIGKILL", "SIGUSR1", "SIGSEGV", "SIGUSR2", "SIGPIPE", "SIGALRM", "SIGTERM", "SIGSTKFLT", "SIGCHLD",
"SIGCONT", "SIGSTOP", "SIGTSTP", "SIGTTIN", "SIGTTOU", "SIGURG", "SIGXCPU", "SIGXFSZ", "SIGVTALRM",
"SIGPROF", "SIGWINCH", "SIGIO", "SIGPOLL", "SIGPWR", "SIGINFO", "SIGLOST", "SIGSYS", "SIGUNUSED",
]);
/** Closed vocabulary for a failed OpenCode turn, including a lost `opencode serve`. */
export interface NativeOpenCodeTurnFailureDiagnostic {
provider: "opencode";
category: "turn_failed";
/** A known runner-issued failure code, or `unrecognized` for any other code. */
runnerFailureCode?: string;
/** How the driver lost its event stream, when it failed the turn itself. */
streamFailure?: string;
exitCode?: number;
signal?: string;
connectionError?: string;
httpStatus?: number;
}
export type NativeTurnFailureDiagnostic = NativeCodexTurnFailureDiagnostic | NativeOpenCodeTurnFailureDiagnostic;
function httpStatusField(value: unknown) {
return typeof value === "number" && Number.isInteger(value) && value >= 100 && value <= 599
? { httpStatus: value } : {};
}
function runnerFailureCodeField(value: unknown) {
function runnerFailureCodeField(value: unknown, codes: ReadonlySet<string> = NATIVE_TURN_FAILURE_RUNNER_CODES) {
if (typeof value !== "string") return {};
return { runnerFailureCode: NATIVE_TURN_FAILURE_RUNNER_CODES.has(value) ? value : "unrecognized" };
return { runnerFailureCode: codes.has(value) ? value : "unrecognized" };
}
function codexErrorInfoFields(value: unknown): Pick<NativeTurnFailureDiagnostic, "codexErrorInfo" | "httpStatus"> {
function openCodeStreamFailureFields(value: unknown): Omit<NativeOpenCodeTurnFailureDiagnostic, "provider" | "category" | "runnerFailureCode"> {
const entry = record(value);
const kind = entry.kind ?? entry.streamFailure;
if (typeof kind !== "string" || !OPENCODE_STREAM_FAILURE_KINDS.has(kind)) return {};
const { exitCode, signal, connectionError } = entry;
return {
streamFailure: kind,
...(typeof exitCode === "number" && Number.isInteger(exitCode) && exitCode >= 0 && exitCode <= 255 ? { exitCode } : {}),
...(typeof signal === "string" && OPENCODE_EXIT_SIGNALS.has(signal) ? { signal } : {}),
...(typeof connectionError === "string" && OPENCODE_CONNECTION_ERRORS.has(connectionError) ? { connectionError } : {}),
...httpStatusField(entry.httpStatus),
};
}
function openCodeTurnFailure(failure: Record<string, unknown>): NativeOpenCodeTurnFailureDiagnostic {
return {
provider: "opencode",
category: "turn_failed",
...runnerFailureCodeField(failure.code, OPENCODE_TURN_FAILURE_RUNNER_CODES),
...openCodeStreamFailureFields(failure.opencodeFailure),
};
}
function codexErrorInfoFields(value: unknown): Pick<NativeCodexTurnFailureDiagnostic, "codexErrorInfo" | "httpStatus"> {
if (value === undefined || value === null) return { codexErrorInfo: "absent" };
if (typeof value === "string") return { codexErrorInfo: CODEX_ERROR_INFO_KINDS.has(value) ? value : "unrecognized" };
const entry = record(value);
@@ -95,6 +157,15 @@ function codexErrorInfoFields(value: unknown): Pick<NativeTurnFailureDiagnostic,
export function readNativeTurnFailureDiagnostic(value: unknown): NativeTurnFailureDiagnostic | null {
try {
const entry = record(value);
if (entry.provider === "opencode") {
if (entry.category !== "turn_failed") return null;
return {
provider: "opencode",
category: "turn_failed",
...runnerFailureCodeField(entry.runnerFailureCode, OPENCODE_TURN_FAILURE_RUNNER_CODES),
...openCodeStreamFailureFields(entry),
};
}
const kind = entry.codexErrorInfo;
if (entry.provider !== "codex" || entry.category !== "turn_failed" || typeof kind !== "string" ||
(kind !== "absent" && kind !== "unrecognized" && !CODEX_ERROR_INFO_KINDS.has(kind))) return null;
@@ -115,10 +186,15 @@ export function createNativeProviderFailureObservation(provider: { kind: string;
const failedTurns = new Map<string, NativeTurnFailureDiagnostic>();
return {
observe(event: PrpEvent) {
if (provider.kind !== "codex" ||
if ((provider.kind !== "codex" && provider.kind !== "opencode") ||
event.sourceKind !== "runner" || event.eventType !== "turn.failed" || !event.turnId) return;
const payload = record(event.payload);
const failure = record(payload.error);
if (provider.kind === "opencode") {
// OpenCode has no classifier above, so every failed turn is diagnosed.
if (payload.status === "failed") failedTurns.set(event.turnId, openCodeTurnFailure(failure));
return;
}
if (payload.status === "failed") {
failedTurns.set(event.turnId, {
provider: "codex",