mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-08 22:13:59 +01:00
Devin FoleyandClaude Opus 5.5
7f9ef38273
OpenCode reports some OpenRouter failures as an UnknownError whose message is the provider's JSON error body. The provider failure now reads its HTTP status, error type and message from that body, so the run shows the upstream message instead of raw JSON. The OpenCode nativeTurnFailure diagnostic now carries the provider failure's category, retryability and HTTP status (closed values only; no provider text or code), and recognizes the driver's opencode_empty_response code. Run status, error codes and retries do not change. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
247 lines
17 KiB
TypeScript
247 lines
17 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import type { PrpEvent, PrpTerminalState } from "../../vendor/paperclip-runner/index.js";
|
|
import { createNativeProviderFailureObservation, NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_REJECTION_MESSAGE, NATIVE_PROVIDER_OVERLOADED_MESSAGE, readNativeTurnFailureDiagnostic } from "./native-provider-failure.js";
|
|
|
|
const model = "test-model";
|
|
const response = {
|
|
type: "error", status: 400,
|
|
error: { type: "invalid_request_error", message: `The '${model}' model is not supported when using Codex with a ChatGPT account.` },
|
|
};
|
|
const event: PrpEvent = {
|
|
schema: "paperclip.prp.event.v1", runId: "run", normalizedSessionId: "session", turnId: "turn",
|
|
sourceInstanceId: "runner", sourceEventId: "failure", sourceSeq: 1, sourceKind: "runner",
|
|
eventType: "turn.failed", schemaVersion: 1, priority: 0, emittedAt: "2026-01-01T00:00:00.000Z",
|
|
payload: { status: "failed", error: { message: JSON.stringify(response), codexErrorInfo: "other" } },
|
|
};
|
|
const terminal: PrpTerminalState = {
|
|
schema: "paperclip.prp.terminal.v1", turnTerminalState: "failed", runTerminalState: "failed", reportedWorkDisposition: "blocked",
|
|
};
|
|
const observation = () => createNativeProviderFailureObservation({ kind: "codex", model });
|
|
|
|
describe("native provider failure observation", () => {
|
|
it.each(["acpx", "codex", "opencode"])("retains a typed explanation for %s without granting recovery authority", kind => {
|
|
const observed = createNativeProviderFailureObservation({ kind });
|
|
const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "new-provider", phase: "turn",
|
|
category: "unknown", message: "An unfamiliar provider rejection.", retryability: "unknown" };
|
|
const failed = { ...event, payload: { status: "failed", error: { providerFailure } } };
|
|
observed.observe({ ...failed, sourceKind: "control_plane" });
|
|
expect(observed.providerFailureForTerminal("turn", terminal)).toBeNull();
|
|
observed.observe(failed);
|
|
expect(observed.providerFailureForTerminal("turn", terminal)).toEqual(providerFailure);
|
|
expect(observed.forTerminal("turn", terminal)).toBeNull();
|
|
expect(observed.providerFailureForTerminal("other-turn", terminal)).toBeNull();
|
|
expect(observed.providerFailureForTerminal("turn", { ...terminal, runTerminalState: "succeeded" })).toBeNull();
|
|
});
|
|
it("surfaces a structured capacity failure without copying arbitrary provider text", () => {
|
|
const observed = createNativeProviderFailureObservation({ kind: "codex" });
|
|
observed.observe({ ...event, payload: { status: "failed", error: { codexErrorInfo: "serverOverloaded", message: "Private arbitrary response" } } });
|
|
expect(observed.forTerminal("turn", terminal)).toEqual({ errorCode: "native_provider_overloaded", errorMessage: NATIVE_PROVIDER_OVERLOADED_MESSAGE, diagnostic: { provider: "codex", category: "server_overloaded" } });
|
|
expect(observed.forTerminal("different-turn", terminal)).toBeNull();
|
|
expect(observed.forTerminal("turn", { ...terminal, runTerminalState: "succeeded" })).toBeNull();
|
|
});
|
|
|
|
it.each(["usageLimitExceeded", "other", "unauthorized", undefined])("does not retry unrelated provider errors (%s)", (codexErrorInfo) => {
|
|
const observed = observation();
|
|
observed.observe({ ...event, payload: { status: "failed", error: { codexErrorInfo, message: NATIVE_PROVIDER_OVERLOADED_MESSAGE } } });
|
|
expect(observed.forTerminal("turn", terminal)).toBeNull();
|
|
});
|
|
it("binds the structured rejection to the failed turn and emits no response or model text", () => {
|
|
const observed = observation();
|
|
observed.observe(event);
|
|
observed.observe(event); // A byte-identical committed replay remains idempotent.
|
|
const result = observed.forTerminal("turn", terminal);
|
|
expect(result).toEqual({ errorCode: "native_provider_model_rejected", errorMessage: NATIVE_MODEL_REJECTION_MESSAGE, diagnostic: NATIVE_MODEL_REJECTION_DIAGNOSTIC });
|
|
expect(JSON.stringify(result)).not.toContain(model);
|
|
expect(JSON.stringify(result)).not.toContain("invalid_request_error");
|
|
expect(observed.forTerminal("later-turn", terminal)).toBeNull();
|
|
expect(observed.forTerminal(null, terminal)).toBeNull();
|
|
expect(observed.forTerminal("turn", { ...terminal, runTerminalState: "succeeded" })).toBeNull();
|
|
expect(observed.forTerminal("turn", { ...terminal, turnTerminalState: "cancelled" })).toBeNull();
|
|
});
|
|
|
|
it.each([
|
|
{ ...event, sourceKind: "control_plane" as const },
|
|
{ ...event, eventType: "item.completed" as const },
|
|
{ ...event, eventType: "harness.diagnostic" as const },
|
|
{ ...event, turnId: undefined },
|
|
{ ...event, payload: { ...event.payload, status: "completed" } },
|
|
{ ...event, payload: { ...event.payload, recoverable: true } },
|
|
{ ...event, payload: { status: "failed", error: { message: JSON.stringify(response), recoverable: true } } },
|
|
])("ignores non-authoritative or retryable evidence %#", (unrelated) => {
|
|
const observed = observation();
|
|
observed.observe(unrelated);
|
|
expect(observed.forTerminal("turn", terminal)).toBeNull();
|
|
});
|
|
|
|
it.each([
|
|
response.error.message, "malformed JSON", " ".repeat(4_097),
|
|
JSON.stringify({ ...response, status: 401 }),
|
|
JSON.stringify({ ...response, status: "400" }),
|
|
JSON.stringify({ ...response, type: "notice" }),
|
|
JSON.stringify({ ...response, error: { ...response.error, type: "server_error" } }),
|
|
JSON.stringify({ ...response, error: { ...response.error, message: "Private arbitrary provider text" } }),
|
|
JSON.stringify({ ...response, error: { ...response.error, message: response.error.message + " Private suffix" } }),
|
|
JSON.stringify({ ...response, error: { ...response.error, message: response.error.message.replace(model, "another-model") } }),
|
|
JSON.stringify([response]), "null",
|
|
])("leaves unknown failures unchanged %#", (message) => {
|
|
const observed = observation();
|
|
observed.observe({ ...event, payload: { status: "failed", error: { message } } });
|
|
expect(observed.forTerminal("turn", terminal)).toBeNull();
|
|
});
|
|
|
|
it.each([
|
|
[{ codexErrorInfo: "contextWindowExceeded", message: "Private provider text" }, { codexErrorInfo: "contextWindowExceeded" }],
|
|
[{ codexErrorInfo: { responseStreamDisconnected: { httpStatusCode: 502 } }, message: "Private" }, { codexErrorInfo: "responseStreamDisconnected", httpStatus: 502 }],
|
|
[{ codexErrorInfo: { httpConnectionFailed: { httpStatusCode: "502" } } }, { codexErrorInfo: "httpConnectionFailed" }],
|
|
[{ codexErrorInfo: { httpConnectionFailed: { httpStatusCode: 9_999 } } }, { codexErrorInfo: "httpConnectionFailed" }],
|
|
[{ codexErrorInfo: "privateUnknownKind" }, { codexErrorInfo: "unrecognized" }],
|
|
[{ codexErrorInfo: { privateUnknownKind: {} } }, { codexErrorInfo: "unrecognized" }],
|
|
[{ codexErrorInfo: { other: {}, badRequest: {} } }, { codexErrorInfo: "unrecognized" }],
|
|
[{ code: "provider_event_type_invalid", message: "Unknown canonical provider event type.", recoverable: false }, { codexErrorInfo: "absent", runnerFailureCode: "provider_event_type_invalid" }],
|
|
[{ code: "Private Code With Spaces", message: "Private" }, { codexErrorInfo: "absent", runnerFailureCode: "unrecognized" }],
|
|
[{ code: "private_prompt_slug", message: "Private" }, { codexErrorInfo: "absent", runnerFailureCode: "unrecognized" }],
|
|
[{ code: 42, message: "Private" }, { codexErrorInfo: "absent" }],
|
|
[{ message: NATIVE_PROVIDER_OVERLOADED_MESSAGE }, { codexErrorInfo: "absent" }],
|
|
])("describes an unclassified failed turn with closed vocabulary only %#", (error, expected) => {
|
|
const observed = observation();
|
|
observed.observe({ ...event, payload: { status: "failed", error } });
|
|
expect(observed.forTerminal("turn", terminal)).toBeNull();
|
|
const diagnostic = observed.unclassifiedTurnFailure("turn", terminal);
|
|
expect(diagnostic).toEqual({ provider: "codex", category: "turn_failed", ...expected });
|
|
expect(JSON.stringify(diagnostic)).not.toMatch(/private/i);
|
|
expect(readNativeTurnFailureDiagnostic(diagnostic)).toEqual(diagnostic);
|
|
});
|
|
|
|
it("does not describe a classified, unfailed, or unrelated turn as unclassified", () => {
|
|
const rejected = observation();
|
|
rejected.observe(event);
|
|
expect(rejected.unclassifiedTurnFailure("turn", terminal)).toBeNull();
|
|
const overloaded = observation();
|
|
overloaded.observe({ ...event, payload: { status: "failed", error: { codexErrorInfo: "serverOverloaded" } } });
|
|
expect(overloaded.unclassifiedTurnFailure("turn", terminal)).toBeNull();
|
|
const other = observation();
|
|
other.observe({ ...event, payload: { status: "failed", error: { codexErrorInfo: "sandboxError" } } });
|
|
expect(other.unclassifiedTurnFailure("later-turn", terminal)).toBeNull();
|
|
expect(other.unclassifiedTurnFailure(null, terminal)).toBeNull();
|
|
expect(other.unclassifiedTurnFailure("turn", { ...terminal, runTerminalState: "succeeded" })).toBeNull();
|
|
expect(other.unclassifiedTurnFailure("turn", { ...terminal, turnTerminalState: "cancelled" })).toBeNull();
|
|
const claude = createNativeProviderFailureObservation({ kind: "claude", model });
|
|
claude.observe({ ...event, payload: { status: "failed", error: { codexErrorInfo: "sandboxError" } } });
|
|
expect(claude.unclassifiedTurnFailure("turn", terminal)).toBeNull();
|
|
});
|
|
|
|
it.each([
|
|
null, [], "turn_failed",
|
|
{ provider: "codex", category: "turn_failed", codexErrorInfo: "private-kind" },
|
|
{ provider: "claude", category: "turn_failed", codexErrorInfo: "other" },
|
|
{ provider: "codex", category: "server_overloaded", codexErrorInfo: "other" },
|
|
])("rejects a persisted turn-failure diagnostic outside the vocabulary %#", (value) => {
|
|
expect(readNativeTurnFailureDiagnostic(value)).toBeNull();
|
|
});
|
|
|
|
it("drops unbounded fields from a persisted turn-failure diagnostic", () => {
|
|
expect(readNativeTurnFailureDiagnostic({
|
|
provider: "codex", category: "turn_failed", codexErrorInfo: "badRequest", httpStatus: 1.5,
|
|
runnerFailureCode: "private_prompt_slug", message: "private provider response",
|
|
})).toEqual({ provider: "codex", category: "turn_failed", codexErrorInfo: "badRequest", runnerFailureCode: "unrecognized" });
|
|
});
|
|
|
|
it("names a lost OpenCode server without copying provider text", () => {
|
|
const observed = createNativeProviderFailureObservation({ kind: "opencode", model });
|
|
observed.observe({ ...event, payload: { status: "failed", providerTerminalObserved: false, error: {
|
|
code: "opencode_server_exited", message: "Private provider message",
|
|
opencodeFailure: { kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED", detail: "private" },
|
|
} } });
|
|
expect(observed.forTerminal("turn", terminal)).toBeNull();
|
|
const diagnostic = observed.unclassifiedTurnFailure("turn", terminal);
|
|
expect(diagnostic).toEqual({
|
|
provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited",
|
|
streamFailure: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED",
|
|
});
|
|
expect(JSON.stringify(diagnostic)).not.toMatch(/private/i);
|
|
expect(readNativeTurnFailureDiagnostic(diagnostic)).toEqual(diagnostic);
|
|
});
|
|
|
|
it("still identifies an OpenCode turn that failed with no structured cause", () => {
|
|
const observed = createNativeProviderFailureObservation({ kind: "opencode", model });
|
|
observed.observe({ ...event, payload: { status: "failed", error: null } });
|
|
expect(observed.unclassifiedTurnFailure("turn", terminal)).toEqual({ provider: "opencode", category: "turn_failed" });
|
|
const providerError = createNativeProviderFailureObservation({ kind: "opencode", model });
|
|
providerError.observe({ ...event, payload: { status: "failed", error: { code: "private_provider_code", name: "APIError" } } });
|
|
expect(providerError.unclassifiedTurnFailure("turn", terminal)).toEqual({
|
|
provider: "opencode", category: "turn_failed", runnerFailureCode: "unrecognized",
|
|
});
|
|
});
|
|
|
|
it("drops values outside the OpenCode stream-failure vocabulary", () => {
|
|
expect(readNativeTurnFailureDiagnostic({
|
|
provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_event_stream_lost",
|
|
streamFailure: "connection_failed", connectionError: "EPRIVATE", exitCode: 300, signal: "private signal", httpStatus: 42,
|
|
})).toEqual({
|
|
provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_event_stream_lost", streamFailure: "connection_failed",
|
|
});
|
|
expect(readNativeTurnFailureDiagnostic({
|
|
provider: "opencode", category: "turn_failed", streamFailure: "private", exitCode: 137,
|
|
})).toEqual({ provider: "opencode", category: "turn_failed" });
|
|
expect(readNativeTurnFailureDiagnostic({ provider: "opencode", category: "server_overloaded" })).toBeNull();
|
|
});
|
|
|
|
it("records only known signal names for a lost OpenCode server", () => {
|
|
const observed = createNativeProviderFailureObservation({ kind: "opencode", model });
|
|
observed.observe({ ...event, payload: { status: "failed", error: {
|
|
code: "opencode_server_exited", opencodeFailure: { kind: "server_exited", signal: "SIGPRIVATE" },
|
|
} } });
|
|
const expected = { provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited", streamFailure: "server_exited" };
|
|
expect(observed.unclassifiedTurnFailure("turn", terminal)).toEqual(expected);
|
|
expect(readNativeTurnFailureDiagnostic({ ...expected, signal: "SIGPRIVATE" })).toEqual(expected);
|
|
expect(readNativeTurnFailureDiagnostic({ ...expected, signal: "SIGTERM" })).toEqual({ ...expected, signal: "SIGTERM" });
|
|
});
|
|
|
|
it("records an OpenCode upstream provider failure's closed fields without its message or code", () => {
|
|
const observed = createNativeProviderFailureObservation({ kind: "opencode", model });
|
|
const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "openrouter", phase: "turn",
|
|
category: "unknown", message: "Private upstream explanation", providerCode: "private_provider_type",
|
|
httpStatus: 502, retryability: "unknown" };
|
|
observed.observe({ ...event, payload: { status: "failed", error: {
|
|
name: "UnknownError", data: { message: "Private upstream body" }, providerFailure,
|
|
} } });
|
|
expect(observed.forTerminal("turn", terminal)).toBeNull();
|
|
expect(observed.providerFailureForTerminal("turn", terminal)).toEqual(providerFailure);
|
|
const diagnostic = observed.unclassifiedTurnFailure("turn", terminal);
|
|
expect(diagnostic).toEqual({
|
|
provider: "opencode", category: "turn_failed",
|
|
providerFailureCategory: "unknown", providerFailureRetryability: "unknown", httpStatus: 502,
|
|
});
|
|
expect(JSON.stringify(diagnostic)).not.toMatch(/private/i);
|
|
expect(readNativeTurnFailureDiagnostic(diagnostic)).toEqual(diagnostic);
|
|
// A malformed provider failure adds nothing, and a persisted record keeps only known values.
|
|
const malformed = createNativeProviderFailureObservation({ kind: "opencode", model });
|
|
malformed.observe({ ...event, payload: { status: "failed", error: { providerFailure: { ...providerFailure, category: "private" } } } });
|
|
expect(malformed.unclassifiedTurnFailure("turn", terminal)).toEqual({ provider: "opencode", category: "turn_failed" });
|
|
expect(readNativeTurnFailureDiagnostic({ provider: "opencode", category: "turn_failed",
|
|
providerFailureCategory: "private", providerFailureRetryability: "unknown", httpStatus: 502 }))
|
|
.toEqual({ provider: "opencode", category: "turn_failed" });
|
|
expect(readNativeTurnFailureDiagnostic({ provider: "opencode", category: "turn_failed",
|
|
providerFailureCategory: "overloaded", providerFailureRetryability: "private", httpStatus: 42 }))
|
|
.toEqual({ provider: "opencode", category: "turn_failed", providerFailureCategory: "overloaded" });
|
|
});
|
|
|
|
it("names the OpenCode empty-response failure the driver writes", () => {
|
|
const observed = createNativeProviderFailureObservation({ kind: "opencode", model });
|
|
observed.observe({ ...event, payload: { status: "failed", error: {
|
|
code: "opencode_empty_response", message: "OpenCode ended the turn without a final response.", retryable: false,
|
|
} } });
|
|
const diagnostic = observed.unclassifiedTurnFailure("turn", terminal);
|
|
expect(diagnostic).toEqual({ provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_empty_response" });
|
|
expect(readNativeTurnFailureDiagnostic(diagnostic)).toEqual(diagnostic);
|
|
});
|
|
|
|
it.each([{ kind: "claude", model }, { kind: "codex" }, { kind: "codex", model: "other" }])(
|
|
"does not apply a rejection to another selected provider/model (%j)", (provider) => {
|
|
const observed = createNativeProviderFailureObservation(provider);
|
|
observed.observe(event);
|
|
expect(observed.forTerminal("turn", terminal)).toBeNull();
|
|
},
|
|
);
|
|
});
|