mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-29 21:07:40 +01:00
fix(runner): record why an OpenCode turn fails when opencode serve stops (#16863)
When `opencode serve` stops during a turn, the OpenCode driver now fails the active turn itself with a closed-vocabulary cause (server exit code or known signal, allowlisted socket error code, HTTP status, or closed stream) instead of leaving runnerd to close it with `error: null`. The proxy flushes stdout before a fatal exit so the failed turn reaches runnerd, and the server records OpenCode failed turns as `nativeTurnFailure` (live and persisted-recovery paths) so the run-failure diagnostic names the cause. Provider text is never copied. Run status, retries and recovery behaviour do not change. Co-Authored-By: Paperclip Sentry triage <Paperclip-Paperclip@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
7 files changed
+314
-20
No files matched your search
@@ -471,6 +471,9 @@ function shutdown(exitCode = 0): Promise<void> {
|
||||
?.close({ reason: "proxy_shutdown", force: true })
|
||||
.catch(() => {});
|
||||
await eventPump?.catch(() => {});
|
||||
// A failed turn can be the last frame before a fatal exit. Pipe writes
|
||||
// are asynchronous on some platforms, so let it reach runnerd first.
|
||||
await new Promise<void>((resolve) => process.stdout.write("", () => resolve()));
|
||||
})().finally(() => process.exit(exitCode));
|
||||
return shutdownPromise;
|
||||
}
|
||||
|
||||
@@ -14,7 +14,8 @@ import {
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { createServer as createHttpServer } from "node:http";
|
||||
import { execFile } from "node:child_process";
|
||||
import { execFile, type ChildProcess } from "node:child_process";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { promisify } from "node:util";
|
||||
import { getCACertificates } from "node:tls";
|
||||
import { join, resolve } from "node:path";
|
||||
@@ -2554,6 +2555,79 @@ describe("OpenCodeServerDriver", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("fails the running turn with the exit cause when OpenCode stops mid-turn", async () => {
|
||||
await chmod(fixture, 0o755);
|
||||
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-server-lost-"));
|
||||
const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-server-lost-workspace-"));
|
||||
roots.push(root, workspace);
|
||||
let stream!: ReadableStreamDefaultController<Uint8Array>;
|
||||
let streamOpened!: () => void;
|
||||
const opened = new Promise<void>(resolve => { streamOpened = resolve; });
|
||||
let serverPid: number | undefined;
|
||||
let eventRequests = 0;
|
||||
const refused = () => new TypeError("fetch failed", {
|
||||
cause: Object.assign(new Error("connect ECONNREFUSED 127.0.0.1"), { code: "ECONNREFUSED" }),
|
||||
});
|
||||
const driver = new OpenCodeServerDriver({
|
||||
model: "openrouter/deepseek/deepseek-v4-flash-0731", runtimeDirectory: root, command: fixture,
|
||||
environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" },
|
||||
onSpawn: async ({ pid }) => { serverPid = pid; },
|
||||
// Own the event stream so the fixture cannot finish the turn on its own,
|
||||
// then report the refused reconnects a dead server produces.
|
||||
fetch: async (input, init) => {
|
||||
if (!String(input).endsWith("/event")) return fetch(input, init);
|
||||
eventRequests += 1;
|
||||
if (eventRequests > 1) throw refused();
|
||||
return new Response(new ReadableStream<Uint8Array>({ start(controller) { stream = controller; streamOpened(); } }),
|
||||
{ headers: { "Content-Type": "text/event-stream" } });
|
||||
},
|
||||
});
|
||||
const session = await driver.openSession({ runId: "server-lost", normalizedSessionId: "server-lost", workingDirectory: workspace });
|
||||
try {
|
||||
const events = collectTurnEvents(session.events());
|
||||
const turn = await session.startTurn({ message: { role: "user", text: "install dependencies" } });
|
||||
await opened;
|
||||
process.kill(serverPid!, "SIGKILL");
|
||||
stream.error(refused());
|
||||
const observed = await events;
|
||||
const failed = observed.at(-1);
|
||||
expect(failed).toMatchObject({
|
||||
eventType: "turn.failed",
|
||||
turnId: turn.turnId,
|
||||
payload: {
|
||||
status: "failed",
|
||||
providerTerminalObserved: false,
|
||||
error: {
|
||||
code: "opencode_server_exited",
|
||||
opencodeFailure: { kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" },
|
||||
},
|
||||
},
|
||||
});
|
||||
const diagnostic = observed.find(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_sse_failed");
|
||||
expect(diagnostic?.payload.failure).toEqual({ kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" });
|
||||
expect(eventRequests).toBe(4);
|
||||
} finally {
|
||||
await session.close({ reason: "test complete" });
|
||||
}
|
||||
});
|
||||
|
||||
it("classifies a lost event stream from closed error fields only", async () => {
|
||||
const { openCodeEventStreamFailure } = openCodeServerDriverInternals;
|
||||
const running = Object.assign(new EventEmitter(), { exitCode: null, signalCode: null }) as unknown as ChildProcess;
|
||||
const exited = Object.assign(new EventEmitter(), { exitCode: 137, signalCode: null }) as unknown as ChildProcess;
|
||||
const reset = new TypeError("fetch failed", { cause: Object.assign(new Error("private socket detail"), { code: "ECONNRESET" }) });
|
||||
expect(await openCodeEventStreamFailure(reset, exited)).toEqual({ kind: "server_exited", exitCode: 137, connectionError: "ECONNRESET" });
|
||||
expect(await openCodeEventStreamFailure(Object.assign(new Error("HTTP 502"), { httpStatus: 502 }), running))
|
||||
.toEqual({ kind: "http_error", httpStatus: 502 });
|
||||
expect(await openCodeEventStreamFailure(Object.assign(new Error("closed"), { streamClosed: true }), running))
|
||||
.toEqual({ kind: "stream_closed" });
|
||||
expect(await openCodeEventStreamFailure(new Error("private provider text"), running)).toEqual({ kind: "stream_error" });
|
||||
const unknownCode = new TypeError("fetch failed", { cause: Object.assign(new Error("x"), { code: "EPRIVATE" }) });
|
||||
const started = Date.now();
|
||||
expect(await openCodeEventStreamFailure(unknownCode, running)).toEqual({ kind: "connection_failed", connectionError: "unrecognized" });
|
||||
expect(Date.now() - started).toBeGreaterThanOrEqual(900);
|
||||
});
|
||||
|
||||
it("restarts OpenCode when session startup fails transiently", async () => {
|
||||
await chmod(fixture, 0o755);
|
||||
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-"));
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { createServer } from "node:net";
|
||||
import { constants as osConstants } from "node:os";
|
||||
import { Agent as HttpAgent, createServer as createHttpServer, request as requestHttp, type IncomingMessage } from "node:http";
|
||||
import { Agent as HttpsAgent, request as requestHttps } from "node:https";
|
||||
import { getCACertificates } from "node:tls";
|
||||
@@ -1353,8 +1354,9 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
signal: this.#abort.signal,
|
||||
});
|
||||
if (!response.ok || !response.body)
|
||||
throw new Error(
|
||||
`OpenCode event stream returned HTTP ${response.status}`,
|
||||
throw Object.assign(
|
||||
new Error(`OpenCode event stream returned HTTP ${response.status}`),
|
||||
{ httpStatus: response.status },
|
||||
);
|
||||
const outboundFrameId = this.#runtime.trace?.frame({
|
||||
direction: "client_to_provider",
|
||||
@@ -1452,8 +1454,9 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
}
|
||||
this.#mapProviderEvent(event, frameId);
|
||||
}
|
||||
throw new Error(
|
||||
"OpenCode event stream closed before the session became terminal",
|
||||
throw Object.assign(
|
||||
new Error("OpenCode event stream closed before the session became terminal"),
|
||||
{ streamClosed: true },
|
||||
);
|
||||
} catch (error) {
|
||||
if (this.#closed || this.#abort.signal.aborted) return;
|
||||
@@ -1461,10 +1464,14 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
if (attempts > 3) {
|
||||
await this.#completionSettlement;
|
||||
if (this.#closed) return;
|
||||
const failure = await openCodeEventStreamFailure(error, this.#runtime.process);
|
||||
if (this.#closed) return;
|
||||
this.#emit("harness.diagnostic", {
|
||||
code: "opencode_sse_failed",
|
||||
message: redact(String(error), this.#runtime.sensitiveValues),
|
||||
failure,
|
||||
});
|
||||
this.#failActiveTurnOnStreamLoss(failure);
|
||||
this.#events.fail(error);
|
||||
return;
|
||||
}
|
||||
@@ -1473,6 +1480,34 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The event stream is the only path for this turn's terminal event. Once it
|
||||
* is gone the turn cannot finish, so record why before the stream ends
|
||||
* instead of leaving runnerd to close it later with no cause.
|
||||
*/
|
||||
#failActiveTurnOnStreamLoss(failure: OpenCodeEventStreamFailure): void {
|
||||
const turnId = this.#activeTurnId;
|
||||
if (!turnId || this.#terminalTurns.has(turnId)) return;
|
||||
this.#settlePendingRuntimeRequestsForTurn(turnId);
|
||||
this.#emit(
|
||||
"turn.failed",
|
||||
{
|
||||
status: "failed",
|
||||
providerTerminalObserved: false,
|
||||
error: {
|
||||
code: failure.kind === "server_exited" ? "opencode_server_exited" : "opencode_event_stream_lost",
|
||||
message: failure.kind === "server_exited"
|
||||
? "OpenCode stopped while the turn was running."
|
||||
: "The OpenCode event stream failed while the turn was running.",
|
||||
opencodeFailure: failure,
|
||||
},
|
||||
},
|
||||
{ turnId },
|
||||
);
|
||||
this.#terminalTurns.set(turnId, canonicalJson({ status: "failed" }));
|
||||
this.#activeTurnId = null;
|
||||
}
|
||||
|
||||
#mapProviderEvent(value: unknown, traceFrameId: number | null = null): void {
|
||||
this.#activeTraceFrameId = traceFrameId;
|
||||
this.#activeTraceEmittedEventIds = [];
|
||||
@@ -3031,7 +3066,55 @@ async function waitForExit(
|
||||
});
|
||||
}
|
||||
|
||||
export const openCodeServerDriverInternals = { parseSse };
|
||||
/** Closed vocabulary for a lost OpenCode event stream. Never provider text. */
|
||||
export interface OpenCodeEventStreamFailure {
|
||||
kind: "server_exited" | "connection_failed" | "http_error" | "stream_closed" | "stream_error";
|
||||
exitCode?: number;
|
||||
signal?: string;
|
||||
connectionError?: string;
|
||||
httpStatus?: number;
|
||||
}
|
||||
|
||||
const OPENCODE_CONNECTION_ERROR_CODES: ReadonlySet<string> = new Set([
|
||||
"ECONNREFUSED", "ECONNRESET", "EPIPE", "ETIMEDOUT", "EHOSTUNREACH", "ENETUNREACH",
|
||||
"UND_ERR_SOCKET", "UND_ERR_CLOSED", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT",
|
||||
"UND_ERR_BODY_TIMEOUT",
|
||||
]);
|
||||
|
||||
function connectionErrorCode(error: unknown): string | undefined {
|
||||
// undici reports `TypeError: fetch failed` and keeps the socket error in `cause`.
|
||||
for (let current = error, depth = 0; current && depth < 4; current = record(current).cause, depth += 1) {
|
||||
const code = record(current).code;
|
||||
if (typeof code === "string") return OPENCODE_CONNECTION_ERROR_CODES.has(code) ? code : "unrecognized";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function openCodeEventStreamFailure(
|
||||
error: unknown,
|
||||
process: ChildProcess,
|
||||
): Promise<OpenCodeEventStreamFailure> {
|
||||
const connectionError = connectionErrorCode(error);
|
||||
// A refused or reset connection usually means `opencode serve` has died.
|
||||
// Give its exit a moment to be observed so the cause is the exit itself.
|
||||
if (connectionError) await waitForExit(process, 1_000);
|
||||
if (process.exitCode !== null || process.signalCode !== null) {
|
||||
return {
|
||||
kind: "server_exited",
|
||||
...(process.exitCode !== null ? { exitCode: process.exitCode } : {}),
|
||||
...(process.signalCode !== null && process.signalCode in osConstants.signals ? { signal: process.signalCode } : {}),
|
||||
...(connectionError ? { connectionError } : {}),
|
||||
};
|
||||
}
|
||||
const httpStatus = record(error).httpStatus;
|
||||
if (typeof httpStatus === "number" && Number.isInteger(httpStatus) && httpStatus >= 100 && httpStatus <= 599)
|
||||
return { kind: "http_error", httpStatus };
|
||||
if (connectionError) return { kind: "connection_failed", connectionError };
|
||||
if (record(error).streamClosed === true) return { kind: "stream_closed" };
|
||||
return { kind: "stream_error" };
|
||||
}
|
||||
|
||||
export const openCodeServerDriverInternals = { parseSse, openCodeEventStreamFailure };
|
||||
|
||||
class AsyncQueue<T> implements AsyncIterable<T> {
|
||||
#items: T[] = [];
|
||||
|
||||
@@ -19,7 +19,7 @@ async function observePersistedNativeTurnFailure(
|
||||
terminal.turnTerminalState !== "failed" || terminal.runTerminalState !== "failed") return null;
|
||||
let execution;
|
||||
try { execution = parseNativeExecutionInput(record(run.runnerProfileJson).nativeExecutionInput); } catch { return null; }
|
||||
if (execution.provider.kind !== "codex" ||
|
||||
if ((execution.provider.kind !== "codex" && execution.provider.kind !== "opencode") ||
|
||||
execution.binding.runId !== run.id || execution.binding.companyId !== run.companyId ||
|
||||
execution.binding.agentId !== run.agentId || execution.binding.issueId !== run.nativeIssueId ||
|
||||
execution.session.normalizedSessionId !== run.nativeSessionId ||
|
||||
|
||||
@@ -46,10 +46,12 @@ describe("native completed failure diagnostics", () => {
|
||||
});
|
||||
afterAll(async () => { await temporary?.cleanup(); });
|
||||
|
||||
it.each(["committed", "duplicate", "pending-review", "superseded-review", "resolved-review", "reassigned", "successor-race", "different-turn", "unknown-error", "untrusted-source", "untrusted-run", "untrusted-session", "succeeded", "cancelled"] as const)(
|
||||
it.each(["committed", "duplicate", "pending-review", "superseded-review", "resolved-review", "reassigned", "successor-race", "different-turn", "unknown-error", "opencode-lost-server", "untrusted-source", "untrusted-run", "untrusted-session", "succeeded", "cancelled"] as const)(
|
||||
"preserves accepted results and only labels the matching provider rejection (%s)", async (scenario) => {
|
||||
const issueId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
const opencode = scenario === "opencode-lost-server";
|
||||
const executionProvider = opencode ? { kind: "opencode" as const, model: `openrouter/${model}` } : { kind: "codex" as const, model };
|
||||
const [issue] = await db.insert(issues).values({ id: issueId, companyId, title: "Explain a provider rejection", status: "in_progress", workMode: "standard", assigneeAgentId: agentId }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId, invocationSource: "assignment", triggerDetail: "system", contextSnapshot: { issueId } }).returning();
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
@@ -60,10 +62,10 @@ describe("native completed failure diagnostics", () => {
|
||||
const execution: NativeExecutionInputV1 = {
|
||||
schema: "paperclip.native-execution-input.v1",
|
||||
binding: { companyId, runId, issueId, agentId, executionWorkspaceId: runId },
|
||||
provider: { kind: "codex", model },
|
||||
provider: executionProvider,
|
||||
task: { identifier: issueId, title: issue.title, description: null, prompt: "Reply to the current message.", workMode: "standard" },
|
||||
workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null },
|
||||
session: { normalizedSessionId: native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } },
|
||||
session: { normalizedSessionId: native.normalizedSessionId, driverKind: opencode ? "opencode_server" : "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } },
|
||||
completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", contract: completionInput },
|
||||
interactionResponses: [], credentialBindings: [],
|
||||
};
|
||||
@@ -94,7 +96,10 @@ describe("native completed failure diagnostics", () => {
|
||||
schema: "paperclip.prp.event.v1", runId, normalizedSessionId: native.normalizedSessionId,
|
||||
turnId: native.turnId, sourceInstanceId: native.runnerInstanceId, sourceEventId: "model-terminal", sourceSeq: 1,
|
||||
sourceKind: "runner", eventType: "turn.failed", schemaVersion: 1, priority: 0, emittedAt: new Date().toISOString(),
|
||||
payload: { status: "failed", error: { codexErrorInfo: "other", message: JSON.stringify({
|
||||
payload: opencode ? { status: "failed", providerTerminalObserved: false, error: {
|
||||
code: "opencode_server_exited", message: "Private stream failure",
|
||||
opencodeFailure: { kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" },
|
||||
} } : { status: "failed", error: { codexErrorInfo: "other", message: JSON.stringify({
|
||||
type: "error", status: 400, error: { type: "invalid_request_error", message: scenario === "unknown-error" ? "Private unknown failure" : `The '${model}' model is not supported when using Codex with a ChatGPT account.` },
|
||||
}) } },
|
||||
};
|
||||
@@ -124,7 +129,7 @@ describe("native completed failure diagnostics", () => {
|
||||
expect((await options.controlPlane.appendEvent(event)).disposition).toBe(scenario === "duplicate" ? "duplicate" : "committed");
|
||||
}
|
||||
await options.controlPlane.completeRun({ result, terminal, turnId });
|
||||
return { result, terminal, turnId, normalizedSessionId: native.normalizedSessionId, providerSessionId: "fixture-provider-session", driverKind: "codex_app_server", driverVersion: "fixture", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
|
||||
return { result, terminal, turnId, normalizedSessionId: native.normalizedSessionId, providerSessionId: "fixture-provider-session", driverKind: execution.session.driverKind, driverVersion: "fixture", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
|
||||
});
|
||||
const adapter = await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, backend: {
|
||||
descriptor: async () => { throw new Error("Unexpected provider call"); },
|
||||
@@ -137,11 +142,13 @@ describe("native completed failure diagnostics", () => {
|
||||
expect(adapter.errorMessage).toBe(classified ? NATIVE_MODEL_REJECTION_MESSAGE : scenario === "succeeded" ? null : `Native session ${terminal.runTerminalState}`);
|
||||
expect(adapter.resultJson?.nativeProviderFailure).toEqual(classified ? NATIVE_MODEL_REJECTION_DIAGNOSTIC : undefined);
|
||||
// Only a committed, unclassified failure of the terminal turn explains the generic message.
|
||||
const turnFailure = scenario === "unknown-error" ? { provider: "codex", category: "turn_failed", codexErrorInfo: "other" } : undefined;
|
||||
const turnFailure = scenario === "unknown-error" ? { provider: "codex", category: "turn_failed", codexErrorInfo: "other" }
|
||||
: opencode ? { provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited", streamFailure: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" }
|
||||
: undefined;
|
||||
expect(adapter.resultJson?.nativeTurnFailure).toEqual(turnFailure);
|
||||
expect(adapter.nativeFinalization).toMatchObject({ result, terminal, turnId, workspaceFinalizeStatus: "pending" });
|
||||
expect(provider.execute).toHaveBeenCalledTimes(1);
|
||||
expect(execution.provider).toEqual({ kind: "codex", model });
|
||||
expect(execution.provider).toEqual(executionProvider);
|
||||
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId));
|
||||
expect(coordinator).toMatchObject({ phase: "workspace_finalizing", leaseOwner: null, attempt: 1, nextAttemptAt: null });
|
||||
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toHaveLength(1);
|
||||
@@ -263,7 +270,7 @@ describe("native completed failure diagnostics", () => {
|
||||
expect(provider.execute).toHaveBeenCalledTimes(1);
|
||||
await db.update(agents).set({ adapterConfig: { provider: "codex", model } }).where(eq(agents.id, agentId));
|
||||
}
|
||||
} else if (scenario === "unknown-error") {
|
||||
} else if (scenario === "unknown-error" || opencode) {
|
||||
// A lookalike diagnostic flag does not alter unknown failed-result policy.
|
||||
await db.update(heartbeatRuns).set({ resultJson: { nativeProviderFailure: NATIVE_MODEL_REJECTION_DIAGNOSTIC } }).where(eq(heartbeatRuns.id, runId));
|
||||
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded" });
|
||||
|
||||
@@ -133,6 +133,57 @@ describe("native provider failure observation", () => {
|
||||
})).toEqual({ provider: "codex", category: "turn_failed", codexErrorInfo: "badRequest", runnerFailureCode: "unrecognized" });
|
||||
});
|
||||
|
||||
it("names a lost OpenCode server without copying provider text", () => {
|
||||
const observed = createNativeProviderFailureObservation({ kind: "opencode", model });
|
||||
observed.observe({ ...event, payload: { status: "failed", providerTerminalObserved: false, error: {
|
||||
code: "opencode_server_exited", message: "Private provider message",
|
||||
opencodeFailure: { kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED", detail: "private" },
|
||||
} } });
|
||||
expect(observed.forTerminal("turn", terminal)).toBeNull();
|
||||
const diagnostic = observed.unclassifiedTurnFailure("turn", terminal);
|
||||
expect(diagnostic).toEqual({
|
||||
provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited",
|
||||
streamFailure: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED",
|
||||
});
|
||||
expect(JSON.stringify(diagnostic)).not.toMatch(/private/i);
|
||||
expect(readNativeTurnFailureDiagnostic(diagnostic)).toEqual(diagnostic);
|
||||
});
|
||||
|
||||
it("still identifies an OpenCode turn that failed with no structured cause", () => {
|
||||
const observed = createNativeProviderFailureObservation({ kind: "opencode", model });
|
||||
observed.observe({ ...event, payload: { status: "failed", error: null } });
|
||||
expect(observed.unclassifiedTurnFailure("turn", terminal)).toEqual({ provider: "opencode", category: "turn_failed" });
|
||||
const providerError = createNativeProviderFailureObservation({ kind: "opencode", model });
|
||||
providerError.observe({ ...event, payload: { status: "failed", error: { code: "private_provider_code", name: "APIError" } } });
|
||||
expect(providerError.unclassifiedTurnFailure("turn", terminal)).toEqual({
|
||||
provider: "opencode", category: "turn_failed", runnerFailureCode: "unrecognized",
|
||||
});
|
||||
});
|
||||
|
||||
it("drops values outside the OpenCode stream-failure vocabulary", () => {
|
||||
expect(readNativeTurnFailureDiagnostic({
|
||||
provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_event_stream_lost",
|
||||
streamFailure: "connection_failed", connectionError: "EPRIVATE", exitCode: 300, signal: "private signal", httpStatus: 42,
|
||||
})).toEqual({
|
||||
provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_event_stream_lost", streamFailure: "connection_failed",
|
||||
});
|
||||
expect(readNativeTurnFailureDiagnostic({
|
||||
provider: "opencode", category: "turn_failed", streamFailure: "private", exitCode: 137,
|
||||
})).toEqual({ provider: "opencode", category: "turn_failed" });
|
||||
expect(readNativeTurnFailureDiagnostic({ provider: "opencode", category: "server_overloaded" })).toBeNull();
|
||||
});
|
||||
|
||||
it("records only known signal names for a lost OpenCode server", () => {
|
||||
const observed = createNativeProviderFailureObservation({ kind: "opencode", model });
|
||||
observed.observe({ ...event, payload: { status: "failed", error: {
|
||||
code: "opencode_server_exited", opencodeFailure: { kind: "server_exited", signal: "SIGPRIVATE" },
|
||||
} } });
|
||||
const expected = { provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited", streamFailure: "server_exited" };
|
||||
expect(observed.unclassifiedTurnFailure("turn", terminal)).toEqual(expected);
|
||||
expect(readNativeTurnFailureDiagnostic({ ...expected, signal: "SIGPRIVATE" })).toEqual(expected);
|
||||
expect(readNativeTurnFailureDiagnostic({ ...expected, signal: "SIGTERM" })).toEqual({ ...expected, signal: "SIGTERM" });
|
||||
});
|
||||
|
||||
it.each([{ kind: "claude", model }, { kind: "codex" }, { kind: "codex", model: "other" }])(
|
||||
"does not apply a rejection to another selected provider/model (%j)", (provider) => {
|
||||
const observed = createNativeProviderFailureObservation(provider);
|
||||
|
||||
@@ -62,7 +62,7 @@ const NATIVE_TURN_FAILURE_RUNNER_CODES: ReadonlySet<string> = new Set([
|
||||
* Closed vocabulary for a failed Codex turn that no classifier above claims.
|
||||
* It explains generic `Native session failed` reports without provider text.
|
||||
*/
|
||||
export interface NativeTurnFailureDiagnostic {
|
||||
export interface NativeCodexTurnFailureDiagnostic {
|
||||
provider: "codex";
|
||||
category: "turn_failed";
|
||||
/** A known `CodexErrorInfo` kind, `unrecognized`, or `absent`. */
|
||||
@@ -72,17 +72,79 @@ export interface NativeTurnFailureDiagnostic {
|
||||
runnerFailureCode?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* `error.code` values the OpenCode driver writes when it fails a turn itself.
|
||||
* Other OpenCode failures carry provider error objects and stay `unrecognized`.
|
||||
*/
|
||||
const OPENCODE_TURN_FAILURE_RUNNER_CODES: ReadonlySet<string> = new Set([
|
||||
"opencode_server_exited", "opencode_event_stream_lost", "provider_terminal_error_missing",
|
||||
]);
|
||||
const OPENCODE_STREAM_FAILURE_KINDS: ReadonlySet<string> = new Set([
|
||||
"server_exited", "connection_failed", "http_error", "stream_closed", "stream_error",
|
||||
]);
|
||||
const OPENCODE_CONNECTION_ERRORS: ReadonlySet<string> = new Set([
|
||||
"ECONNREFUSED", "ECONNRESET", "EPIPE", "ETIMEDOUT", "EHOSTUNREACH", "ENETUNREACH",
|
||||
"UND_ERR_SOCKET", "UND_ERR_CLOSED", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT",
|
||||
"UND_ERR_BODY_TIMEOUT", "unrecognized",
|
||||
]);
|
||||
/** POSIX, Linux and macOS signal names; any other `signal` value is dropped. */
|
||||
const OPENCODE_EXIT_SIGNALS: ReadonlySet<string> = new Set([
|
||||
"SIGHUP", "SIGINT", "SIGQUIT", "SIGILL", "SIGTRAP", "SIGABRT", "SIGIOT", "SIGBUS", "SIGEMT", "SIGFPE",
|
||||
"SIGKILL", "SIGUSR1", "SIGSEGV", "SIGUSR2", "SIGPIPE", "SIGALRM", "SIGTERM", "SIGSTKFLT", "SIGCHLD",
|
||||
"SIGCONT", "SIGSTOP", "SIGTSTP", "SIGTTIN", "SIGTTOU", "SIGURG", "SIGXCPU", "SIGXFSZ", "SIGVTALRM",
|
||||
"SIGPROF", "SIGWINCH", "SIGIO", "SIGPOLL", "SIGPWR", "SIGINFO", "SIGLOST", "SIGSYS", "SIGUNUSED",
|
||||
]);
|
||||
|
||||
/** Closed vocabulary for a failed OpenCode turn, including a lost `opencode serve`. */
|
||||
export interface NativeOpenCodeTurnFailureDiagnostic {
|
||||
provider: "opencode";
|
||||
category: "turn_failed";
|
||||
/** A known runner-issued failure code, or `unrecognized` for any other code. */
|
||||
runnerFailureCode?: string;
|
||||
/** How the driver lost its event stream, when it failed the turn itself. */
|
||||
streamFailure?: string;
|
||||
exitCode?: number;
|
||||
signal?: string;
|
||||
connectionError?: string;
|
||||
httpStatus?: number;
|
||||
}
|
||||
|
||||
export type NativeTurnFailureDiagnostic = NativeCodexTurnFailureDiagnostic | NativeOpenCodeTurnFailureDiagnostic;
|
||||
|
||||
function httpStatusField(value: unknown) {
|
||||
return typeof value === "number" && Number.isInteger(value) && value >= 100 && value <= 599
|
||||
? { httpStatus: value } : {};
|
||||
}
|
||||
|
||||
function runnerFailureCodeField(value: unknown) {
|
||||
function runnerFailureCodeField(value: unknown, codes: ReadonlySet<string> = NATIVE_TURN_FAILURE_RUNNER_CODES) {
|
||||
if (typeof value !== "string") return {};
|
||||
return { runnerFailureCode: NATIVE_TURN_FAILURE_RUNNER_CODES.has(value) ? value : "unrecognized" };
|
||||
return { runnerFailureCode: codes.has(value) ? value : "unrecognized" };
|
||||
}
|
||||
|
||||
function codexErrorInfoFields(value: unknown): Pick<NativeTurnFailureDiagnostic, "codexErrorInfo" | "httpStatus"> {
|
||||
function openCodeStreamFailureFields(value: unknown): Omit<NativeOpenCodeTurnFailureDiagnostic, "provider" | "category" | "runnerFailureCode"> {
|
||||
const entry = record(value);
|
||||
const kind = entry.kind ?? entry.streamFailure;
|
||||
if (typeof kind !== "string" || !OPENCODE_STREAM_FAILURE_KINDS.has(kind)) return {};
|
||||
const { exitCode, signal, connectionError } = entry;
|
||||
return {
|
||||
streamFailure: kind,
|
||||
...(typeof exitCode === "number" && Number.isInteger(exitCode) && exitCode >= 0 && exitCode <= 255 ? { exitCode } : {}),
|
||||
...(typeof signal === "string" && OPENCODE_EXIT_SIGNALS.has(signal) ? { signal } : {}),
|
||||
...(typeof connectionError === "string" && OPENCODE_CONNECTION_ERRORS.has(connectionError) ? { connectionError } : {}),
|
||||
...httpStatusField(entry.httpStatus),
|
||||
};
|
||||
}
|
||||
|
||||
function openCodeTurnFailure(failure: Record<string, unknown>): NativeOpenCodeTurnFailureDiagnostic {
|
||||
return {
|
||||
provider: "opencode",
|
||||
category: "turn_failed",
|
||||
...runnerFailureCodeField(failure.code, OPENCODE_TURN_FAILURE_RUNNER_CODES),
|
||||
...openCodeStreamFailureFields(failure.opencodeFailure),
|
||||
};
|
||||
}
|
||||
|
||||
function codexErrorInfoFields(value: unknown): Pick<NativeCodexTurnFailureDiagnostic, "codexErrorInfo" | "httpStatus"> {
|
||||
if (value === undefined || value === null) return { codexErrorInfo: "absent" };
|
||||
if (typeof value === "string") return { codexErrorInfo: CODEX_ERROR_INFO_KINDS.has(value) ? value : "unrecognized" };
|
||||
const entry = record(value);
|
||||
@@ -95,6 +157,15 @@ function codexErrorInfoFields(value: unknown): Pick<NativeTurnFailureDiagnostic,
|
||||
export function readNativeTurnFailureDiagnostic(value: unknown): NativeTurnFailureDiagnostic | null {
|
||||
try {
|
||||
const entry = record(value);
|
||||
if (entry.provider === "opencode") {
|
||||
if (entry.category !== "turn_failed") return null;
|
||||
return {
|
||||
provider: "opencode",
|
||||
category: "turn_failed",
|
||||
...runnerFailureCodeField(entry.runnerFailureCode, OPENCODE_TURN_FAILURE_RUNNER_CODES),
|
||||
...openCodeStreamFailureFields(entry),
|
||||
};
|
||||
}
|
||||
const kind = entry.codexErrorInfo;
|
||||
if (entry.provider !== "codex" || entry.category !== "turn_failed" || typeof kind !== "string" ||
|
||||
(kind !== "absent" && kind !== "unrecognized" && !CODEX_ERROR_INFO_KINDS.has(kind))) return null;
|
||||
@@ -115,10 +186,15 @@ export function createNativeProviderFailureObservation(provider: { kind: string;
|
||||
const failedTurns = new Map<string, NativeTurnFailureDiagnostic>();
|
||||
return {
|
||||
observe(event: PrpEvent) {
|
||||
if (provider.kind !== "codex" ||
|
||||
if ((provider.kind !== "codex" && provider.kind !== "opencode") ||
|
||||
event.sourceKind !== "runner" || event.eventType !== "turn.failed" || !event.turnId) return;
|
||||
const payload = record(event.payload);
|
||||
const failure = record(payload.error);
|
||||
if (provider.kind === "opencode") {
|
||||
// OpenCode has no classifier above, so every failed turn is diagnosed.
|
||||
if (payload.status === "failed") failedTurns.set(event.turnId, openCodeTurnFailure(failure));
|
||||
return;
|
||||
}
|
||||
if (payload.status === "failed") {
|
||||
failedTurns.set(event.turnId, {
|
||||
provider: "codex",
|
||||
|
||||
Reference in new issue
Block a user