fix(native): classify a Codex API-key unknown model as a model rejection

With API-key auth, Codex (0.160.0) answers an unknown configured model
with a Responses API 404, retries it for about a minute, and fails the
turn with "unexpected status 404 Not Found: The model `<id>` does not
exist or you do not have access to it." and codexErrorInfo
httpConnectionFailed/404. The observation only recognized the ChatGPT
400 envelope, so the run failed as a generic adapter_failed ("Native
session failed") and recovery opened a reconciliation incident for a
turn that never did work.

Recognize that terminal shape, bound to the configured model, as
native_provider_model_rejected with a closed api_key/404 diagnostic. The
task then asks the user to choose a model, and Retry runs with the
repaired model. The task interruption also carries the rejected model id
so the task page names it.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
DottaandClaude committed 2026-11-10 02:46:07 -06:00
1 parent 182a7cd4fe
commit 21befa1c5c
10 files changed
+122 -15

No files matched your search

@@ -136,6 +136,8 @@ export interface TaskInterruptionPresentation {
output: string | null;
message: string;
};
/** With `userAction: "choose_model"`: the configured model id the provider rejected. */
rejectedModel?: string;
/** A persisted Retry admission wait, normalized to public text by the server. */
retryWait?: { reason: string; message: string };
intentId: string | null;
@@ -403,6 +403,21 @@ describe.skipIf(!support.supported)("task interruption latest run query", () =>
} finally { vi.unstubAllEnvs(); }
});
it("names the configured model a provider rejected on task detail and list reads (CF3 model_id)", async () => {
const f = await seedInterruptionFixture(db, { selected: 1, history: 1, unrelated: 0, payloadBytes: 0 });
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId));
await db.update(heartbeatRuns).set({ status: "failed", errorCode: "native_provider_model_rejected",
runnerProfileJson: { nativeExecutionInput: { provider: { kind: "codex", model: "chaos-missing-model-1234" } } } }).where(eq(heartbeatRuns.id, run!.id));
for (const presentation of [(await taskInterruptionPresentations(db, f.companyId, f.input)).get(f.input[0]!.id),
await taskInterruptionPresentation(db, f.input[0]!)]) {
expect(presentation).toMatchObject({ responsibility: "user", state: "waiting_for_user", userAction: "choose_model",
rejectedModel: "chaos-missing-model-1234" });
}
// Only a model rejection discloses the configured model.
await db.update(heartbeatRuns).set({ errorCode: "adapter_failed" }).where(eq(heartbeatRuns.id, run!.id));
expect(await taskInterruptionPresentation(db, f.input[0]!)).not.toHaveProperty("rejectedModel");
});
it.each(["decryption failure", "expanded message"])("keeps task reads available after provider display %s", async scenario => {
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64));
try {
@@ -16,7 +16,7 @@ import { executePaperclipNativeSession } from "./native-session-executor.js";
import { buildNativeCompletionContract } from "./completion-contracts.js";
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
import { prepareNativeHeartbeatRun } from "./prepare-native-run.js";
import { NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_REJECTION_MESSAGE } from "./native-provider-failure.js";
import { NATIVE_MODEL_NOT_FOUND_DIAGNOSTIC, NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_REJECTION_MESSAGE } from "./native-provider-failure.js";
import { collectRunFailureDiagnostics } from "../run-failure-diagnostics.js";
import { finalizeNativeRun } from "./native-run-finalizer.js";
import { readPersistedNativeFailureEvidence, readPersistedNativeModelRejection } from "./native-provider-failure-evidence.js";
@@ -50,7 +50,7 @@ describe("native completed failure diagnostics", () => {
});
afterAll(async () => { await temporary?.cleanup(); });
it.each(["committed", "duplicate", "pending-review", "superseded-review", "resolved-review", "reassigned", "successor-race", "different-turn", "unknown-error", "opencode-lost-server", "untrusted-source", "untrusted-run", "untrusted-session", "succeeded", "cancelled"] as const)(
it.each(["committed", "api-key", "duplicate", "pending-review", "superseded-review", "resolved-review", "reassigned", "successor-race", "different-turn", "unknown-error", "opencode-lost-server", "untrusted-source", "untrusted-run", "untrusted-session", "succeeded", "cancelled"] as const)(
"preserves accepted results and only labels the matching provider rejection (%s)", async (scenario) => {
const issueId = randomUUID();
const runId = randomUUID();
@@ -103,6 +103,10 @@ describe("native completed failure diagnostics", () => {
payload: opencode ? { status: "failed", providerTerminalObserved: false, error: {
code: "opencode_server_exited", message: "Private stream failure",
opencodeFailure: { kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" },
} } : scenario === "api-key" ? { status: "failed", error: {
// The terminal error codex 0.160.0 app-server sends with API-key auth, after its own retries.
message: `unexpected status 404 Not Found: The model \`${model}\` does not exist or you do not have access to it., url: https://api.openai.com/v1/responses, cf-ray: a5833f01da2716ce-IAH, request id: req_584e320775c34b2c9c6d98af40e0ea3d`,
codexErrorInfo: { httpConnectionFailed: { httpStatusCode: 404 } }, additionalDetails: null, misalignment: null,
} } : { status: "failed", error: { codexErrorInfo: "other", message: JSON.stringify({
type: "error", status: 400, error: { type: "invalid_request_error", message: scenario === "unknown-error" ? "Private unknown failure" : `The '${model}' model is not supported when using Codex with a ChatGPT account.` },
}) } },
@@ -139,12 +143,13 @@ describe("native completed failure diagnostics", () => {
descriptor: async () => { throw new Error("Unexpected provider call"); },
openSession: async () => { throw new Error("Unexpected provider call"); },
} });
const classified = scenario === "committed" || scenario === "duplicate" || scenario.endsWith("-review") || scenario === "reassigned" || scenario === "successor-race";
const expectedDiagnostic = scenario === "api-key" ? NATIVE_MODEL_NOT_FOUND_DIAGNOSTIC : NATIVE_MODEL_REJECTION_DIAGNOSTIC;
const classified = scenario === "committed" || scenario === "api-key" || scenario === "duplicate" || scenario.endsWith("-review") || scenario === "reassigned" || scenario === "successor-race";
const reviewAuthorityEnded = scenario === "superseded-review" || scenario === "resolved-review";
const ownerChanged = scenario === "reassigned" || scenario === "successor-race";
expect(adapter.errorCode).toBe(classified ? "native_provider_model_rejected" : undefined);
expect(adapter.errorMessage).toBe(classified ? NATIVE_MODEL_REJECTION_MESSAGE : scenario === "succeeded" ? null : `Native session ${terminal.runTerminalState}`);
expect(adapter.resultJson?.nativeProviderFailure).toEqual(classified ? NATIVE_MODEL_REJECTION_DIAGNOSTIC : undefined);
expect(adapter.resultJson?.nativeProviderFailure).toEqual(classified ? expectedDiagnostic : undefined);
// Only a committed, unclassified failure of the terminal turn explains the generic message.
const turnFailure = scenario === "unknown-error" ? { provider: "codex", category: "turn_failed", codexErrorInfo: "other" }
: opencode ? { provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited", streamFailure: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" }
@@ -159,13 +164,13 @@ describe("native completed failure diagnostics", () => {
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0);
expect(await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId))).toHaveLength(scenario.startsWith("untrusted-") ? 0 : 1);
const diagnostics = collectRunFailureDiagnostics({ ...run, resultJson: adapter.resultJson ?? null }, {});
expect(diagnostics.provider).toEqual(classified ? NATIVE_MODEL_REJECTION_DIAGNOSTIC : turnFailure ?? {});
expect(diagnostics.provider).toEqual(classified ? expectedDiagnostic : turnFailure ?? {});
expect(JSON.stringify(diagnostics)).not.toContain(model);
expect(JSON.stringify(diagnostics)).not.toContain("Private");
const [persistedRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
expect(persistedRun.errorCode).toBeNull(); // No adapter metadata has been saved yet.
expect(await readPersistedNativeModelRejection(db, persistedRun, turnId, terminal)).toEqual(classified ? {
errorCode: "native_provider_model_rejected", errorMessage: NATIVE_MODEL_REJECTION_MESSAGE, diagnostic: NATIVE_MODEL_REJECTION_DIAGNOSTIC,
errorCode: "native_provider_model_rejected", errorMessage: NATIVE_MODEL_REJECTION_MESSAGE, diagnostic: expectedDiagnostic,
} : null);
expect((await readPersistedNativeFailureEvidence(db, persistedRun, turnId, terminal)).turnFailure).toEqual(turnFailure ?? null);
if (classified) {
@@ -1,6 +1,6 @@
import { describe, expect, it } from "vitest";
import type { PrpEvent, PrpTerminalState } from "../../vendor/paperclip-runner/index.js";
import { createNativeProviderFailureObservation, NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_REJECTION_MESSAGE, NATIVE_PROVIDER_OVERLOADED_MESSAGE, readNativeTurnFailureDiagnostic } from "./native-provider-failure.js";
import { createNativeProviderFailureObservation, NATIVE_MODEL_NOT_FOUND_DIAGNOSTIC, NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_REJECTION_MESSAGE, NATIVE_PROVIDER_OVERLOADED_MESSAGE, readNativeTurnFailureDiagnostic } from "./native-provider-failure.js";
const model = "test-model";
const response = {
@@ -52,6 +52,30 @@ describe("native provider failure observation", () => {
observed.observe({ ...event, payload: { status: "failed", error: { providerFailure: { schema: "paperclip.provider-failure.v1", provider: "openrouter", phase: "turn", category: "model_unavailable", message: "busy", retryability } } } });
expect(observed.forTerminal("turn", terminal)).toBeNull();
});
it("classifies the API-key 404 for the configured model as a model rejection (codex 0.160.0 shape)", () => {
const apiKeyFailure = (message: string, info: unknown = { httpConnectionFailed: { httpStatusCode: 404 } }) => ({ ...event, payload: { status: "failed", error: {
message, codexErrorInfo: info, additionalDetails: null, misalignment: null } } });
const real = `unexpected status 404 Not Found: The model \`${model}\` does not exist or you do not have access to it., url: https://api.openai.com/v1/responses, cf-ray: a5833f01da2716ce-IAH, request id: req_584e320775c34b2c9c6d98af40e0ea3d`;
const observed = observation();
observed.observe(apiKeyFailure(real));
const result = observed.forTerminal("turn", terminal);
expect(result).toEqual({ errorCode: "native_provider_model_rejected", errorMessage: NATIVE_MODEL_REJECTION_MESSAGE, diagnostic: NATIVE_MODEL_NOT_FOUND_DIAGNOSTIC });
expect(JSON.stringify(result)).not.toContain(model);
expect(observed.unclassifiedTurnFailure("turn", terminal)).toBeNull();
expect(observed.forTerminal("other-turn", terminal)).toBeNull();
// Another model's 404, another status, or a retryable turn is not this rejection.
for (const [message, info, recoverable] of [
[real.replace(model, "other-model"), undefined, false],
[real, { httpConnectionFailed: { httpStatusCode: 500 } }, false],
[real, "other", false],
[real, undefined, true],
] as const) {
const other = observation();
const failed = apiKeyFailure(message, info);
other.observe(recoverable ? { ...failed, payload: { ...failed.payload, recoverable: true } } : failed);
expect(other.forTerminal("turn", terminal)).toBeNull();
}
});
it("surfaces a structured capacity failure without copying arbitrary provider text", () => {
const observed = createNativeProviderFailureObservation({ kind: "codex" });
observed.observe({ ...event, payload: { status: "failed", error: { codexErrorInfo: "serverOverloaded", message: "Private arbitrary response" } } });
@@ -27,13 +27,22 @@ function record(value: unknown): Record<string, unknown> {
: {};
}
/** An API-key Codex turn whose configured model the Responses API answers with 404. */
export const NATIVE_MODEL_NOT_FOUND_DIAGNOSTIC = {
provider: "codex",
category: "model_not_found",
status: 404,
authMode: "api_key",
} as const;
/** Reconstruct only the closed diagnostic vocabulary, never provider text. */
export function readNativeModelRejectionDiagnostic(value: unknown) {
try {
const entry = record(value);
return Object.entries(NATIVE_MODEL_REJECTION_DIAGNOSTIC).every(([key, expected]) => entry[key] === expected)
? { ...NATIVE_MODEL_REJECTION_DIAGNOSTIC }
: null;
for (const diagnostic of [NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_NOT_FOUND_DIAGNOSTIC]) {
if (Object.entries(diagnostic).every(([key, expected]) => entry[key] === expected)) return { ...diagnostic };
}
return null;
} catch { return null; }
}
@@ -213,6 +222,8 @@ export function readNativeTurnFailureDiagnostic(value: unknown): NativeTurnFailu
/** Observe only events accepted by the bound control-plane persistence port. */
export function createNativeProviderFailureObservation(provider: { kind: string; model?: string | null }) {
let rejectedTurnId: string | null = null;
/** A Codex API-key turn whose configured model the Responses API does not serve (404). */
let missingModelTurnId: string | null = null;
let overloadedTurnId: string | null = null;
const failedTurns = new Map<string, NativeTurnFailureDiagnostic>();
const providerFailures = new Map<string, ProviderFailure>();
@@ -251,6 +262,16 @@ export function createNativeProviderFailureObservation(provider: { kind: string;
}
if (!provider.model || payload.status !== "failed" || payload.recoverable === true || failure.recoverable === true ||
typeof failure.message !== "string" || failure.message.length > 4_096) return;
// With an API key, Codex retries the Responses API 404 and then fails the turn with this
// terminal text (codex 0.160.0). Bind it to the configured model; store neither name nor text.
const info = record(failure.codexErrorInfo);
const infoKinds = Object.keys(info);
if (infoKinds.length === 1 && ["httpConnectionFailed", "responseStreamDisconnected"].includes(infoKinds[0]!) &&
record(info[infoKinds[0]!]).httpStatusCode === 404 &&
failure.message.startsWith(`unexpected status 404 Not Found: The model \`${provider.model}\` does not exist or you do not have access to it.`)) {
missingModelTurnId = event.turnId;
return;
}
let response: Record<string, unknown>;
try { response = record(JSON.parse(failure.message)); } catch { return; }
const error = record(response.error);
@@ -272,6 +293,11 @@ export function createNativeProviderFailureObservation(provider: { kind: string;
errorMessage: NATIVE_PROVIDER_OVERLOADED_MESSAGE,
diagnostic: { ...NATIVE_PROVIDER_OVERLOADED_DIAGNOSTIC },
};
if (turnId === missingModelTurnId) return {
errorCode: "native_provider_model_rejected" as const,
errorMessage: NATIVE_MODEL_REJECTION_MESSAGE,
diagnostic: { ...NATIVE_MODEL_NOT_FOUND_DIAGNOSTIC },
};
if (turnId !== rejectedTurnId) {
// A driver's closed category marks a rejected model for any provider. Only an
// action-required rejection qualifies; a transient one may succeed on retry.
@@ -292,7 +318,8 @@ export function createNativeProviderFailureObservation(provider: { kind: string;
/** Diagnostics only: the failed turn that `forTerminal` did not classify. */
unclassifiedTurnFailure(turnId: string | null | undefined, terminal: PrpTerminalState) {
if (!turnId || terminal.turnTerminalState !== "failed" || terminal.runTerminalState !== "failed" ||
turnId === overloadedTurnId || turnId === rejectedTurnId || modelUnavailableTurn(turnId)) return null;
turnId === overloadedTurnId || turnId === rejectedTurnId || turnId === missingModelTurnId ||
modelUnavailableTurn(turnId)) return null;
const diagnostic = failedTurns.get(turnId);
return diagnostic ? { ...diagnostic } : null;
},
@@ -44,6 +44,8 @@ type ProjectionRun = Pick<typeof heartbeatRuns.$inferSelect,
nativeResumeAt?: Date | null;
/** When the native coordinator saved this run's terminal failure, if it did. */
nativeFailedAt?: Date | null;
/** The configured model id the provider rejected (native_provider_model_rejected only). */
rejectedModel?: string | null;
};
type ProjectionWait = { id: string; requestedAt: Date; sourceRunId?: string | null; executionWaitReason?: string | null;
preparationStatus?: string | null; preparationRetryAt?: number | string | null; preparationUpdatedAt?: Date | null;
@@ -207,7 +209,8 @@ export function projectTaskInterruption(issue: ProjectionIssue, input: Projectio
: run.errorCode === "native_provider_model_rejected" ? "choose_model"
: [agent?.pauseReason, project?.pauseReason, company?.pauseReason].some(reason => reason === "budget" || reason === "budget_exceeded")
? "budget" : input.paused || project?.pausedAt || agent?.status === "paused" || company?.status === "paused" ? "resume" : null;
if (userAction) return { ...base, responsibility: "user", state: "waiting_for_user", userAction };
if (userAction) return { ...base, responsibility: "user", state: "waiting_for_user", userAction,
...(userAction === "choose_model" && run.rejectedModel ? { rejectedModel: run.rejectedModel } : {}) };
// The previous controller cannot confirm its own stop. Name the one board
// action instead of showing the fenced run as working (RC-47).
if (input.ownerStopRequired) return { ...base, responsibility: "user", state: "waiting_for_user", userAction: "confirm_owner_stop" };
@@ -343,6 +346,9 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i
and ${heartbeatRuns.error} ~ '^(Execution workspace provision|Workspace seed|Runtime provision) command "'
then ${heartbeatRuns.error} else null end`,
errorCode: heartbeatRuns.errorCode, livenessState: heartbeatRuns.livenessState, scheduledRetryAt: heartbeatRuns.scheduledRetryAt,
// The model this run asked the provider for; configuration, not provider text.
rejectedModel: sql<string | null>`case when ${heartbeatRuns.errorCode} = 'native_provider_model_rejected'
then ${boundedJsonString(sql`${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'provider'->'model'`, 200)} end`,
scheduledRetryReason: heartbeatRuns.scheduledRetryReason,
// Candidate only: the coordinator also pushes nextAttemptAt forward while
// it defers an ineligible run. The scheduler's own admission check below
+4 -1
View File
@@ -170,7 +170,9 @@ export interface ConfigSnapshot {
runs: Array<{ id: string; status: string; createdAt: number | null; startedAt: number | null; errorCode: string | null; error: string | null }>;
interruption: { state: string | null; canRetry: boolean; userAction: string | null; workspaceFailure: string | null;
/** The named setup command failure (`setup_failed`), as the task page shows it. */
setupFailure?: string | null; observedRunId: string | null } | null;
setupFailure?: string | null;
/** The model id the provider rejected, as the task page names it. */
rejectedModel?: string | null; observedRunId: string | null } | null;
execution: { label: string | null; cause: string | null; nextAction: string | null } | null;
/** Pending thread interactions on the task (connection cards and similar). */
cards: Array<{ id: string; kind: string; status: string; serviceSlug: string | null; agentSetup: boolean }>;
@@ -186,6 +188,7 @@ export function taskLevelTexts(snapshot: ConfigSnapshot): string[] {
if (interruption?.workspaceFailure) texts.push(interruption.workspaceFailure);
if (interruption?.setupFailure) texts.push(interruption.setupFailure);
if (interruption?.userAction) texts.push(`userAction:${interruption.userAction}`);
if (interruption?.rejectedModel) texts.push(`rejected model ${interruption.rejectedModel}`);
if (snapshot.execution?.label) texts.push(snapshot.execution.label);
if (snapshot.execution?.nextAction) texts.push(snapshot.execution.nextAction);
for (const card of snapshot.cards) {
+2 -1
View File
@@ -233,7 +233,8 @@ async function snapshot(api: ChaosApi, issueId: string, agentId: string, errors:
return {
at, reachable: true, issueStatus: String(issue.status), runs: runSamples,
interruption: interruption ? { state: interruption.state ?? null, canRetry: interruption.canRetry === true, userAction: interruption.userAction ?? null,
workspaceFailure: interruption.workspaceFailure?.message ?? null, setupFailure: interruption.setupFailure?.message ?? null, observedRunId: interruption.observedRunId ?? null } : null,
workspaceFailure: interruption.workspaceFailure?.message ?? null, setupFailure: interruption.setupFailure?.message ?? null,
rejectedModel: interruption.rejectedModel ?? null, observedRunId: interruption.observedRunId ?? null } : null,
execution: projection ? { label: projection.label ?? null, cause: projection.cause ?? null, nextAction: projection.nextAction ?? null } : null,
cards: (Array.isArray(interactions) ? interactions : []).map(card => ({ id: String(card.id), kind: String(card.kind), status: String(card.status),
serviceSlug: card.payload?.serviceSlug ?? null, agentSetup: card.payload?.agentSetup === true })),
+22
View File
@@ -2665,6 +2665,28 @@ describe("TaskChatThread runtime transcript selection", () => {
expect(container.textContent).not.toContain("after returning a final response");
});
it("names the rejected model id that the task interruption carries", () => {
renderThread({
comments: [],
onAdd: async () => {},
interruption: { intentId: null, responsibility: "user", state: "waiting_for_user", nextRetryAt: null,
canRetry: false, userAction: "choose_model", rejectedModel: "chaos-missing-model-1234", sourceRunIds: ["model-missing"], observedRunId: "model-missing" },
linkedRuns: [{
runId: "model-missing", runtimeMode: "native", status: "failed",
errorCode: "native_provider_model_rejected",
error: "The selected model is not available through the current AI connection. Choose a supported model or a compatible AI connection.",
agentId: "agent-1", agentName: "Runner", adapterType: "paperclip_runner",
createdAt: "2026-08-25T18:00:00.000Z", startedAt: "2026-08-25T18:00:00.000Z",
finishedAt: "2026-08-25T18:00:02.000Z",
}],
});
const marker = container.querySelector('[data-testid="task-chat-collapsible-marker"]');
expect(marker?.textContent).toContain("Model unavailable");
flushSync(() => marker!.querySelector<HTMLButtonElement>('button[aria-expanded="false"]')!.click());
expect(container.textContent).toContain('The provider rejected the model "chaos-missing-model-1234".');
expect(container.textContent).toContain("Choose a supported model");
});
it("keeps workspace contention out of the conversation's cancellation markers", () => {
renderThread({
comments: [],
+3 -1
View File
@@ -1868,7 +1868,9 @@ export function TaskChatThread(props: TaskChatThreadProps) {
: code === "native_provider_approval_required"
? "This operation requires approval, but this runner has no interactive approval handler. Review the operation and update the agent's permission setting before retrying."
: code === "native_provider_model_rejected"
? `${meta?.error || "The provider rejected the selected model."} Choose a supported model or clear the task's model override, then retry.`
? `${props.interruption?.observedRunId === source.id && props.interruption.rejectedModel
? `The provider rejected the model "${props.interruption.rejectedModel}".`
: meta?.error || "The provider rejected the selected model."} Choose a supported model or clear the task's model override, then retry.`
: code === "native_provider_usage_limit" &&
source.status === "failed"
? "The model provider has reached its current usage limit. Try again after the limit resets."