From d747752496f3e6f589dc50aa74d33b24afeab797 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Mon, 19 Oct 2026 12:20:24 -0700 Subject: [PATCH] fix(runner): record why an OpenCode turn fails when opencode serve stops (#16863) When `opencode serve` stops during a turn, the OpenCode driver now fails the active turn itself with a closed-vocabulary cause (server exit code or known signal, allowlisted socket error code, HTTP status, or closed stream) instead of leaving runnerd to close it with `error: null`. The proxy flushes stdout before a fatal exit so the failed turn reaches runnerd, and the server records OpenCode failed turns as `nativeTurnFailure` (live and persisted-recovery paths) so the run-failure diagnostic names the cause. Provider text is never copied. Run status, retries and recovery behaviour do not change. Co-Authored-By: Paperclip Sentry triage Co-Authored-By: Claude Opus 5.5 --- .../src/cli/opencode-app-server-proxy.ts | 3 + .../opencode/opencode-server-driver.test.ts | 76 ++++++++++++++- .../opencode/opencode-server-driver.ts | 93 ++++++++++++++++++- .../native-provider-failure-evidence.ts | 2 +- ...ative-provider-failure.integration.test.ts | 23 +++-- .../native-provider-failure.test.ts | 51 ++++++++++ .../native-runtime/native-provider-failure.ts | 86 ++++++++++++++++- 7 files changed, 314 insertions(+), 20 deletions(-) diff --git a/packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts b/packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts index 0e60ac1f245..971c15f0e59 100644 --- a/packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts +++ b/packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts @@ -471,6 +471,9 @@ function shutdown(exitCode = 0): Promise { ?.close({ reason: "proxy_shutdown", force: true }) .catch(() => {}); await eventPump?.catch(() => {}); + // A failed turn can be the last frame before a fatal exit. Pipe writes + // are asynchronous on some platforms, so let it reach runnerd first. + await new Promise((resolve) => process.stdout.write("", () => resolve())); })().finally(() => process.exit(exitCode)); return shutdownPromise; } diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index 06381b79b9b..30ed4b970a6 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -14,7 +14,8 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import { createServer as createHttpServer } from "node:http"; -import { execFile } from "node:child_process"; +import { execFile, type ChildProcess } from "node:child_process"; +import { EventEmitter } from "node:events"; import { promisify } from "node:util"; import { getCACertificates } from "node:tls"; import { join, resolve } from "node:path"; @@ -2554,6 +2555,79 @@ describe("OpenCodeServerDriver", () => { } }); + it("fails the running turn with the exit cause when OpenCode stops mid-turn", async () => { + await chmod(fixture, 0o755); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-server-lost-")); + const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-server-lost-workspace-")); + roots.push(root, workspace); + let stream!: ReadableStreamDefaultController; + let streamOpened!: () => void; + const opened = new Promise(resolve => { streamOpened = resolve; }); + let serverPid: number | undefined; + let eventRequests = 0; + const refused = () => new TypeError("fetch failed", { + cause: Object.assign(new Error("connect ECONNREFUSED 127.0.0.1"), { code: "ECONNREFUSED" }), + }); + const driver = new OpenCodeServerDriver({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", runtimeDirectory: root, command: fixture, + environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" }, + onSpawn: async ({ pid }) => { serverPid = pid; }, + // Own the event stream so the fixture cannot finish the turn on its own, + // then report the refused reconnects a dead server produces. + fetch: async (input, init) => { + if (!String(input).endsWith("/event")) return fetch(input, init); + eventRequests += 1; + if (eventRequests > 1) throw refused(); + return new Response(new ReadableStream({ start(controller) { stream = controller; streamOpened(); } }), + { headers: { "Content-Type": "text/event-stream" } }); + }, + }); + const session = await driver.openSession({ runId: "server-lost", normalizedSessionId: "server-lost", workingDirectory: workspace }); + try { + const events = collectTurnEvents(session.events()); + const turn = await session.startTurn({ message: { role: "user", text: "install dependencies" } }); + await opened; + process.kill(serverPid!, "SIGKILL"); + stream.error(refused()); + const observed = await events; + const failed = observed.at(-1); + expect(failed).toMatchObject({ + eventType: "turn.failed", + turnId: turn.turnId, + payload: { + status: "failed", + providerTerminalObserved: false, + error: { + code: "opencode_server_exited", + opencodeFailure: { kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" }, + }, + }, + }); + const diagnostic = observed.find(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_sse_failed"); + expect(diagnostic?.payload.failure).toEqual({ kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" }); + expect(eventRequests).toBe(4); + } finally { + await session.close({ reason: "test complete" }); + } + }); + + it("classifies a lost event stream from closed error fields only", async () => { + const { openCodeEventStreamFailure } = openCodeServerDriverInternals; + const running = Object.assign(new EventEmitter(), { exitCode: null, signalCode: null }) as unknown as ChildProcess; + const exited = Object.assign(new EventEmitter(), { exitCode: 137, signalCode: null }) as unknown as ChildProcess; + const reset = new TypeError("fetch failed", { cause: Object.assign(new Error("private socket detail"), { code: "ECONNRESET" }) }); + expect(await openCodeEventStreamFailure(reset, exited)).toEqual({ kind: "server_exited", exitCode: 137, connectionError: "ECONNRESET" }); + expect(await openCodeEventStreamFailure(Object.assign(new Error("HTTP 502"), { httpStatus: 502 }), running)) + .toEqual({ kind: "http_error", httpStatus: 502 }); + expect(await openCodeEventStreamFailure(Object.assign(new Error("closed"), { streamClosed: true }), running)) + .toEqual({ kind: "stream_closed" }); + expect(await openCodeEventStreamFailure(new Error("private provider text"), running)).toEqual({ kind: "stream_error" }); + const unknownCode = new TypeError("fetch failed", { cause: Object.assign(new Error("x"), { code: "EPRIVATE" }) }); + const started = Date.now(); + expect(await openCodeEventStreamFailure(unknownCode, running)).toEqual({ kind: "connection_failed", connectionError: "unrecognized" }); + expect(Date.now() - started).toBeGreaterThanOrEqual(900); + }); + it("restarts OpenCode when session startup fails transiently", async () => { await chmod(fixture, 0o755); const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-")); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index 8e58c6543e4..a7f80aac85e 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -15,6 +15,7 @@ import { writeFile, } from "node:fs/promises"; import { createServer } from "node:net"; +import { constants as osConstants } from "node:os"; import { Agent as HttpAgent, createServer as createHttpServer, request as requestHttp, type IncomingMessage } from "node:http"; import { Agent as HttpsAgent, request as requestHttps } from "node:https"; import { getCACertificates } from "node:tls"; @@ -1353,8 +1354,9 @@ class OpenCodeHarnessSession implements HarnessSession { signal: this.#abort.signal, }); if (!response.ok || !response.body) - throw new Error( - `OpenCode event stream returned HTTP ${response.status}`, + throw Object.assign( + new Error(`OpenCode event stream returned HTTP ${response.status}`), + { httpStatus: response.status }, ); const outboundFrameId = this.#runtime.trace?.frame({ direction: "client_to_provider", @@ -1452,8 +1454,9 @@ class OpenCodeHarnessSession implements HarnessSession { } this.#mapProviderEvent(event, frameId); } - throw new Error( - "OpenCode event stream closed before the session became terminal", + throw Object.assign( + new Error("OpenCode event stream closed before the session became terminal"), + { streamClosed: true }, ); } catch (error) { if (this.#closed || this.#abort.signal.aborted) return; @@ -1461,10 +1464,14 @@ class OpenCodeHarnessSession implements HarnessSession { if (attempts > 3) { await this.#completionSettlement; if (this.#closed) return; + const failure = await openCodeEventStreamFailure(error, this.#runtime.process); + if (this.#closed) return; this.#emit("harness.diagnostic", { code: "opencode_sse_failed", message: redact(String(error), this.#runtime.sensitiveValues), + failure, }); + this.#failActiveTurnOnStreamLoss(failure); this.#events.fail(error); return; } @@ -1473,6 +1480,34 @@ class OpenCodeHarnessSession implements HarnessSession { } } + /** + * The event stream is the only path for this turn's terminal event. Once it + * is gone the turn cannot finish, so record why before the stream ends + * instead of leaving runnerd to close it later with no cause. + */ + #failActiveTurnOnStreamLoss(failure: OpenCodeEventStreamFailure): void { + const turnId = this.#activeTurnId; + if (!turnId || this.#terminalTurns.has(turnId)) return; + this.#settlePendingRuntimeRequestsForTurn(turnId); + this.#emit( + "turn.failed", + { + status: "failed", + providerTerminalObserved: false, + error: { + code: failure.kind === "server_exited" ? "opencode_server_exited" : "opencode_event_stream_lost", + message: failure.kind === "server_exited" + ? "OpenCode stopped while the turn was running." + : "The OpenCode event stream failed while the turn was running.", + opencodeFailure: failure, + }, + }, + { turnId }, + ); + this.#terminalTurns.set(turnId, canonicalJson({ status: "failed" })); + this.#activeTurnId = null; + } + #mapProviderEvent(value: unknown, traceFrameId: number | null = null): void { this.#activeTraceFrameId = traceFrameId; this.#activeTraceEmittedEventIds = []; @@ -3031,7 +3066,55 @@ async function waitForExit( }); } -export const openCodeServerDriverInternals = { parseSse }; +/** Closed vocabulary for a lost OpenCode event stream. Never provider text. */ +export interface OpenCodeEventStreamFailure { + kind: "server_exited" | "connection_failed" | "http_error" | "stream_closed" | "stream_error"; + exitCode?: number; + signal?: string; + connectionError?: string; + httpStatus?: number; +} + +const OPENCODE_CONNECTION_ERROR_CODES: ReadonlySet = new Set([ + "ECONNREFUSED", "ECONNRESET", "EPIPE", "ETIMEDOUT", "EHOSTUNREACH", "ENETUNREACH", + "UND_ERR_SOCKET", "UND_ERR_CLOSED", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT", + "UND_ERR_BODY_TIMEOUT", +]); + +function connectionErrorCode(error: unknown): string | undefined { + // undici reports `TypeError: fetch failed` and keeps the socket error in `cause`. + for (let current = error, depth = 0; current && depth < 4; current = record(current).cause, depth += 1) { + const code = record(current).code; + if (typeof code === "string") return OPENCODE_CONNECTION_ERROR_CODES.has(code) ? code : "unrecognized"; + } + return undefined; +} + +async function openCodeEventStreamFailure( + error: unknown, + process: ChildProcess, +): Promise { + const connectionError = connectionErrorCode(error); + // A refused or reset connection usually means `opencode serve` has died. + // Give its exit a moment to be observed so the cause is the exit itself. + if (connectionError) await waitForExit(process, 1_000); + if (process.exitCode !== null || process.signalCode !== null) { + return { + kind: "server_exited", + ...(process.exitCode !== null ? { exitCode: process.exitCode } : {}), + ...(process.signalCode !== null && process.signalCode in osConstants.signals ? { signal: process.signalCode } : {}), + ...(connectionError ? { connectionError } : {}), + }; + } + const httpStatus = record(error).httpStatus; + if (typeof httpStatus === "number" && Number.isInteger(httpStatus) && httpStatus >= 100 && httpStatus <= 599) + return { kind: "http_error", httpStatus }; + if (connectionError) return { kind: "connection_failed", connectionError }; + if (record(error).streamClosed === true) return { kind: "stream_closed" }; + return { kind: "stream_error" }; +} + +export const openCodeServerDriverInternals = { parseSse, openCodeEventStreamFailure }; class AsyncQueue implements AsyncIterable { #items: T[] = []; diff --git a/server/src/services/native-runtime/native-provider-failure-evidence.ts b/server/src/services/native-runtime/native-provider-failure-evidence.ts index 1aa539fccdc..cc89f4130b4 100644 --- a/server/src/services/native-runtime/native-provider-failure-evidence.ts +++ b/server/src/services/native-runtime/native-provider-failure-evidence.ts @@ -19,7 +19,7 @@ async function observePersistedNativeTurnFailure( terminal.turnTerminalState !== "failed" || terminal.runTerminalState !== "failed") return null; let execution; try { execution = parseNativeExecutionInput(record(run.runnerProfileJson).nativeExecutionInput); } catch { return null; } - if (execution.provider.kind !== "codex" || + if ((execution.provider.kind !== "codex" && execution.provider.kind !== "opencode") || execution.binding.runId !== run.id || execution.binding.companyId !== run.companyId || execution.binding.agentId !== run.agentId || execution.binding.issueId !== run.nativeIssueId || execution.session.normalizedSessionId !== run.nativeSessionId || diff --git a/server/src/services/native-runtime/native-provider-failure.integration.test.ts b/server/src/services/native-runtime/native-provider-failure.integration.test.ts index 8592144f2ce..60bf7a24ed7 100644 --- a/server/src/services/native-runtime/native-provider-failure.integration.test.ts +++ b/server/src/services/native-runtime/native-provider-failure.integration.test.ts @@ -46,10 +46,12 @@ describe("native completed failure diagnostics", () => { }); afterAll(async () => { await temporary?.cleanup(); }); - it.each(["committed", "duplicate", "pending-review", "superseded-review", "resolved-review", "reassigned", "successor-race", "different-turn", "unknown-error", "untrusted-source", "untrusted-run", "untrusted-session", "succeeded", "cancelled"] as const)( + it.each(["committed", "duplicate", "pending-review", "superseded-review", "resolved-review", "reassigned", "successor-race", "different-turn", "unknown-error", "opencode-lost-server", "untrusted-source", "untrusted-run", "untrusted-session", "succeeded", "cancelled"] as const)( "preserves accepted results and only labels the matching provider rejection (%s)", async (scenario) => { const issueId = randomUUID(); const runId = randomUUID(); + const opencode = scenario === "opencode-lost-server"; + const executionProvider = opencode ? { kind: "opencode" as const, model: `openrouter/${model}` } : { kind: "codex" as const, model }; const [issue] = await db.insert(issues).values({ id: issueId, companyId, title: "Explain a provider rejection", status: "in_progress", workMode: "standard", assigneeAgentId: agentId }).returning(); const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId, invocationSource: "assignment", triggerDetail: "system", contextSnapshot: { issueId } }).returning(); await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); @@ -60,10 +62,10 @@ describe("native completed failure diagnostics", () => { const execution: NativeExecutionInputV1 = { schema: "paperclip.native-execution-input.v1", binding: { companyId, runId, issueId, agentId, executionWorkspaceId: runId }, - provider: { kind: "codex", model }, + provider: executionProvider, task: { identifier: issueId, title: issue.title, description: null, prompt: "Reply to the current message.", workMode: "standard" }, workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null }, - session: { normalizedSessionId: native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + session: { normalizedSessionId: native.normalizedSessionId, driverKind: opencode ? "opencode_server" : "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", contract: completionInput }, interactionResponses: [], credentialBindings: [], }; @@ -94,7 +96,10 @@ describe("native completed failure diagnostics", () => { schema: "paperclip.prp.event.v1", runId, normalizedSessionId: native.normalizedSessionId, turnId: native.turnId, sourceInstanceId: native.runnerInstanceId, sourceEventId: "model-terminal", sourceSeq: 1, sourceKind: "runner", eventType: "turn.failed", schemaVersion: 1, priority: 0, emittedAt: new Date().toISOString(), - payload: { status: "failed", error: { codexErrorInfo: "other", message: JSON.stringify({ + payload: opencode ? { status: "failed", providerTerminalObserved: false, error: { + code: "opencode_server_exited", message: "Private stream failure", + opencodeFailure: { kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" }, + } } : { status: "failed", error: { codexErrorInfo: "other", message: JSON.stringify({ type: "error", status: 400, error: { type: "invalid_request_error", message: scenario === "unknown-error" ? "Private unknown failure" : `The '${model}' model is not supported when using Codex with a ChatGPT account.` }, }) } }, }; @@ -124,7 +129,7 @@ describe("native completed failure diagnostics", () => { expect((await options.controlPlane.appendEvent(event)).disposition).toBe(scenario === "duplicate" ? "duplicate" : "committed"); } await options.controlPlane.completeRun({ result, terminal, turnId }); - return { result, terminal, turnId, normalizedSessionId: native.normalizedSessionId, providerSessionId: "fixture-provider-session", driverKind: "codex_app_server", driverVersion: "fixture", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + return { result, terminal, turnId, normalizedSessionId: native.normalizedSessionId, providerSessionId: "fixture-provider-session", driverKind: execution.session.driverKind, driverVersion: "fixture", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; }); const adapter = await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, backend: { descriptor: async () => { throw new Error("Unexpected provider call"); }, @@ -137,11 +142,13 @@ describe("native completed failure diagnostics", () => { expect(adapter.errorMessage).toBe(classified ? NATIVE_MODEL_REJECTION_MESSAGE : scenario === "succeeded" ? null : `Native session ${terminal.runTerminalState}`); expect(adapter.resultJson?.nativeProviderFailure).toEqual(classified ? NATIVE_MODEL_REJECTION_DIAGNOSTIC : undefined); // Only a committed, unclassified failure of the terminal turn explains the generic message. - const turnFailure = scenario === "unknown-error" ? { provider: "codex", category: "turn_failed", codexErrorInfo: "other" } : undefined; + const turnFailure = scenario === "unknown-error" ? { provider: "codex", category: "turn_failed", codexErrorInfo: "other" } + : opencode ? { provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited", streamFailure: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" } + : undefined; expect(adapter.resultJson?.nativeTurnFailure).toEqual(turnFailure); expect(adapter.nativeFinalization).toMatchObject({ result, terminal, turnId, workspaceFinalizeStatus: "pending" }); expect(provider.execute).toHaveBeenCalledTimes(1); - expect(execution.provider).toEqual({ kind: "codex", model }); + expect(execution.provider).toEqual(executionProvider); const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); expect(coordinator).toMatchObject({ phase: "workspace_finalizing", leaseOwner: null, attempt: 1, nextAttemptAt: null }); expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toHaveLength(1); @@ -263,7 +270,7 @@ describe("native completed failure diagnostics", () => { expect(provider.execute).toHaveBeenCalledTimes(1); await db.update(agents).set({ adapterConfig: { provider: "codex", model } }).where(eq(agents.id, agentId)); } - } else if (scenario === "unknown-error") { + } else if (scenario === "unknown-error" || opencode) { // A lookalike diagnostic flag does not alter unknown failed-result policy. await db.update(heartbeatRuns).set({ resultJson: { nativeProviderFailure: NATIVE_MODEL_REJECTION_DIAGNOSTIC } }).where(eq(heartbeatRuns.id, runId)); await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded" }); diff --git a/server/src/services/native-runtime/native-provider-failure.test.ts b/server/src/services/native-runtime/native-provider-failure.test.ts index fc2e349a57c..f8ce42f76f5 100644 --- a/server/src/services/native-runtime/native-provider-failure.test.ts +++ b/server/src/services/native-runtime/native-provider-failure.test.ts @@ -133,6 +133,57 @@ describe("native provider failure observation", () => { })).toEqual({ provider: "codex", category: "turn_failed", codexErrorInfo: "badRequest", runnerFailureCode: "unrecognized" }); }); + it("names a lost OpenCode server without copying provider text", () => { + const observed = createNativeProviderFailureObservation({ kind: "opencode", model }); + observed.observe({ ...event, payload: { status: "failed", providerTerminalObserved: false, error: { + code: "opencode_server_exited", message: "Private provider message", + opencodeFailure: { kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED", detail: "private" }, + } } }); + expect(observed.forTerminal("turn", terminal)).toBeNull(); + const diagnostic = observed.unclassifiedTurnFailure("turn", terminal); + expect(diagnostic).toEqual({ + provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited", + streamFailure: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED", + }); + expect(JSON.stringify(diagnostic)).not.toMatch(/private/i); + expect(readNativeTurnFailureDiagnostic(diagnostic)).toEqual(diagnostic); + }); + + it("still identifies an OpenCode turn that failed with no structured cause", () => { + const observed = createNativeProviderFailureObservation({ kind: "opencode", model }); + observed.observe({ ...event, payload: { status: "failed", error: null } }); + expect(observed.unclassifiedTurnFailure("turn", terminal)).toEqual({ provider: "opencode", category: "turn_failed" }); + const providerError = createNativeProviderFailureObservation({ kind: "opencode", model }); + providerError.observe({ ...event, payload: { status: "failed", error: { code: "private_provider_code", name: "APIError" } } }); + expect(providerError.unclassifiedTurnFailure("turn", terminal)).toEqual({ + provider: "opencode", category: "turn_failed", runnerFailureCode: "unrecognized", + }); + }); + + it("drops values outside the OpenCode stream-failure vocabulary", () => { + expect(readNativeTurnFailureDiagnostic({ + provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_event_stream_lost", + streamFailure: "connection_failed", connectionError: "EPRIVATE", exitCode: 300, signal: "private signal", httpStatus: 42, + })).toEqual({ + provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_event_stream_lost", streamFailure: "connection_failed", + }); + expect(readNativeTurnFailureDiagnostic({ + provider: "opencode", category: "turn_failed", streamFailure: "private", exitCode: 137, + })).toEqual({ provider: "opencode", category: "turn_failed" }); + expect(readNativeTurnFailureDiagnostic({ provider: "opencode", category: "server_overloaded" })).toBeNull(); + }); + + it("records only known signal names for a lost OpenCode server", () => { + const observed = createNativeProviderFailureObservation({ kind: "opencode", model }); + observed.observe({ ...event, payload: { status: "failed", error: { + code: "opencode_server_exited", opencodeFailure: { kind: "server_exited", signal: "SIGPRIVATE" }, + } } }); + const expected = { provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited", streamFailure: "server_exited" }; + expect(observed.unclassifiedTurnFailure("turn", terminal)).toEqual(expected); + expect(readNativeTurnFailureDiagnostic({ ...expected, signal: "SIGPRIVATE" })).toEqual(expected); + expect(readNativeTurnFailureDiagnostic({ ...expected, signal: "SIGTERM" })).toEqual({ ...expected, signal: "SIGTERM" }); + }); + it.each([{ kind: "claude", model }, { kind: "codex" }, { kind: "codex", model: "other" }])( "does not apply a rejection to another selected provider/model (%j)", (provider) => { const observed = createNativeProviderFailureObservation(provider); diff --git a/server/src/services/native-runtime/native-provider-failure.ts b/server/src/services/native-runtime/native-provider-failure.ts index 4f9b29fa4c0..db5d0e28814 100644 --- a/server/src/services/native-runtime/native-provider-failure.ts +++ b/server/src/services/native-runtime/native-provider-failure.ts @@ -62,7 +62,7 @@ const NATIVE_TURN_FAILURE_RUNNER_CODES: ReadonlySet = new Set([ * Closed vocabulary for a failed Codex turn that no classifier above claims. * It explains generic `Native session failed` reports without provider text. */ -export interface NativeTurnFailureDiagnostic { +export interface NativeCodexTurnFailureDiagnostic { provider: "codex"; category: "turn_failed"; /** A known `CodexErrorInfo` kind, `unrecognized`, or `absent`. */ @@ -72,17 +72,79 @@ export interface NativeTurnFailureDiagnostic { runnerFailureCode?: string; } +/** + * `error.code` values the OpenCode driver writes when it fails a turn itself. + * Other OpenCode failures carry provider error objects and stay `unrecognized`. + */ +const OPENCODE_TURN_FAILURE_RUNNER_CODES: ReadonlySet = new Set([ + "opencode_server_exited", "opencode_event_stream_lost", "provider_terminal_error_missing", +]); +const OPENCODE_STREAM_FAILURE_KINDS: ReadonlySet = new Set([ + "server_exited", "connection_failed", "http_error", "stream_closed", "stream_error", +]); +const OPENCODE_CONNECTION_ERRORS: ReadonlySet = new Set([ + "ECONNREFUSED", "ECONNRESET", "EPIPE", "ETIMEDOUT", "EHOSTUNREACH", "ENETUNREACH", + "UND_ERR_SOCKET", "UND_ERR_CLOSED", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT", + "UND_ERR_BODY_TIMEOUT", "unrecognized", +]); +/** POSIX, Linux and macOS signal names; any other `signal` value is dropped. */ +const OPENCODE_EXIT_SIGNALS: ReadonlySet = new Set([ + "SIGHUP", "SIGINT", "SIGQUIT", "SIGILL", "SIGTRAP", "SIGABRT", "SIGIOT", "SIGBUS", "SIGEMT", "SIGFPE", + "SIGKILL", "SIGUSR1", "SIGSEGV", "SIGUSR2", "SIGPIPE", "SIGALRM", "SIGTERM", "SIGSTKFLT", "SIGCHLD", + "SIGCONT", "SIGSTOP", "SIGTSTP", "SIGTTIN", "SIGTTOU", "SIGURG", "SIGXCPU", "SIGXFSZ", "SIGVTALRM", + "SIGPROF", "SIGWINCH", "SIGIO", "SIGPOLL", "SIGPWR", "SIGINFO", "SIGLOST", "SIGSYS", "SIGUNUSED", +]); + +/** Closed vocabulary for a failed OpenCode turn, including a lost `opencode serve`. */ +export interface NativeOpenCodeTurnFailureDiagnostic { + provider: "opencode"; + category: "turn_failed"; + /** A known runner-issued failure code, or `unrecognized` for any other code. */ + runnerFailureCode?: string; + /** How the driver lost its event stream, when it failed the turn itself. */ + streamFailure?: string; + exitCode?: number; + signal?: string; + connectionError?: string; + httpStatus?: number; +} + +export type NativeTurnFailureDiagnostic = NativeCodexTurnFailureDiagnostic | NativeOpenCodeTurnFailureDiagnostic; + function httpStatusField(value: unknown) { return typeof value === "number" && Number.isInteger(value) && value >= 100 && value <= 599 ? { httpStatus: value } : {}; } -function runnerFailureCodeField(value: unknown) { +function runnerFailureCodeField(value: unknown, codes: ReadonlySet = NATIVE_TURN_FAILURE_RUNNER_CODES) { if (typeof value !== "string") return {}; - return { runnerFailureCode: NATIVE_TURN_FAILURE_RUNNER_CODES.has(value) ? value : "unrecognized" }; + return { runnerFailureCode: codes.has(value) ? value : "unrecognized" }; } -function codexErrorInfoFields(value: unknown): Pick { +function openCodeStreamFailureFields(value: unknown): Omit { + const entry = record(value); + const kind = entry.kind ?? entry.streamFailure; + if (typeof kind !== "string" || !OPENCODE_STREAM_FAILURE_KINDS.has(kind)) return {}; + const { exitCode, signal, connectionError } = entry; + return { + streamFailure: kind, + ...(typeof exitCode === "number" && Number.isInteger(exitCode) && exitCode >= 0 && exitCode <= 255 ? { exitCode } : {}), + ...(typeof signal === "string" && OPENCODE_EXIT_SIGNALS.has(signal) ? { signal } : {}), + ...(typeof connectionError === "string" && OPENCODE_CONNECTION_ERRORS.has(connectionError) ? { connectionError } : {}), + ...httpStatusField(entry.httpStatus), + }; +} + +function openCodeTurnFailure(failure: Record): NativeOpenCodeTurnFailureDiagnostic { + return { + provider: "opencode", + category: "turn_failed", + ...runnerFailureCodeField(failure.code, OPENCODE_TURN_FAILURE_RUNNER_CODES), + ...openCodeStreamFailureFields(failure.opencodeFailure), + }; +} + +function codexErrorInfoFields(value: unknown): Pick { if (value === undefined || value === null) return { codexErrorInfo: "absent" }; if (typeof value === "string") return { codexErrorInfo: CODEX_ERROR_INFO_KINDS.has(value) ? value : "unrecognized" }; const entry = record(value); @@ -95,6 +157,15 @@ function codexErrorInfoFields(value: unknown): Pick(); return { observe(event: PrpEvent) { - if (provider.kind !== "codex" || + if ((provider.kind !== "codex" && provider.kind !== "opencode") || event.sourceKind !== "runner" || event.eventType !== "turn.failed" || !event.turnId) return; const payload = record(event.payload); const failure = record(payload.error); + if (provider.kind === "opencode") { + // OpenCode has no classifier above, so every failed turn is diagnosed. + if (payload.status === "failed") failedTurns.set(event.turnId, openCodeTurnFailure(failure)); + return; + } if (payload.status === "failed") { failedTurns.set(event.turnId, { provider: "codex",