diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index 48649b9f7ce..831d8228d3d 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -780,6 +780,18 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].statusVersion).toBe(blockedIssue.statusVersion); }); + it("does not open a reconciliation incident for a native run whose model the provider rejected", async () => { + const companyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" }); + const runId = await seedRun({ companyId, agentId, status: "failed", contextSnapshot: { issueId } }); + await db.update(heartbeatRuns).set({ runtimeMode: "native", errorCode: "native_provider_model_rejected" }).where(eq(heartbeatRuns.id, runId)); + const adapter = createPostgresWakeQueueAdapter(db, stubDeps); + await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async () => { throw new Error("must not restart a rejected model"); }); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].status).toBe("blocked"); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0); + }); + it.each(["active", "escalated"])("repairs a failed native task with an existing %s recovery action", async (status) => { const companyId = await seedCompany(); const agentId = await seedAgent({ companyId }); diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index f8c748820fd..100e56b3143 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -1369,6 +1369,13 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd }; } + // A native run blocked for a rejected model or incomplete configuration waits for that + // change and a retry. The provider refused the request, so there is no execution to + // reconcile; a reconciliation incident would also hide the run's Retry. + if (!handoffAssigneeAgentId && run.runtimeMode === "native" && isConfigurationIncompleteFailedRun(run) + && issueRow.status === "blocked" && issueRow.assigneeAgentId === run.agentId) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } if (!handoffAssigneeAgentId && await recordNativeTerminalRecoveryIfNeeded(tx, run, issueRow, input.now)) { return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; } diff --git a/server/src/services/native-runtime/native-provider-failure.test.ts b/server/src/services/native-runtime/native-provider-failure.test.ts index ef1f17e40d4..dbab03ed300 100644 --- a/server/src/services/native-runtime/native-provider-failure.test.ts +++ b/server/src/services/native-runtime/native-provider-failure.test.ts @@ -32,6 +32,26 @@ describe("native provider failure observation", () => { expect(observed.providerFailureForTerminal("other-turn", terminal)).toBeNull(); expect(observed.providerFailureForTerminal("turn", { ...terminal, runTerminalState: "succeeded" })).toBeNull(); }); + it.each(["opencode", "codex"])("classifies an action-required model_unavailable failure from %s as a model rejection", kind => { + const observed = createNativeProviderFailureObservation({ kind }); + const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "openrouter", phase: "turn", + category: "model_unavailable", message: "vendor/private-model is not a valid model ID", retryability: "requires_action", httpStatus: 400 }; + observed.observe({ ...event, payload: { status: "failed", error: { providerFailure } } }); + const result = observed.forTerminal("turn", terminal); + expect(result).toMatchObject({ errorCode: "native_provider_model_rejected", errorMessage: NATIVE_MODEL_REJECTION_MESSAGE }); + expect(JSON.stringify(result)).not.toContain("private-model"); + // A closed diagnostic only, and the classified turn is not also reported as unclassified. + expect(result?.diagnostic).toEqual(expect.objectContaining({ provider: kind })); + expect(JSON.stringify(result?.diagnostic)).not.toContain("not a valid model"); + expect(observed.unclassifiedTurnFailure("turn", terminal)).toBeNull(); + expect(observed.forTerminal("other-turn", terminal)).toBeNull(); + expect(observed.forTerminal("turn", { ...terminal, runTerminalState: "succeeded" })).toBeNull(); + }); + it.each(["transient", "unknown"])("does not treat a %s model_unavailable failure as a model rejection", retryability => { + const observed = createNativeProviderFailureObservation({ kind: "opencode" }); + observed.observe({ ...event, payload: { status: "failed", error: { providerFailure: { schema: "paperclip.provider-failure.v1", provider: "openrouter", phase: "turn", category: "model_unavailable", message: "busy", retryability } } } }); + expect(observed.forTerminal("turn", terminal)).toBeNull(); + }); it("surfaces a structured capacity failure without copying arbitrary provider text", () => { const observed = createNativeProviderFailureObservation({ kind: "codex" }); observed.observe({ ...event, payload: { status: "failed", error: { codexErrorInfo: "serverOverloaded", message: "Private arbitrary response" } } }); diff --git a/server/src/services/native-runtime/native-provider-failure.ts b/server/src/services/native-runtime/native-provider-failure.ts index 71d040566bb..ecbd4ed837a 100644 --- a/server/src/services/native-runtime/native-provider-failure.ts +++ b/server/src/services/native-runtime/native-provider-failure.ts @@ -1,8 +1,9 @@ import type { PrpEvent, PrpTerminalState } from "../../vendor/paperclip-runner/index.js"; import { readProviderFailure, type ProviderFailure } from "@paperclipai/shared"; +// Shared by every provider whose selected model is rejected, so it names no specific connection. export const NATIVE_MODEL_REJECTION_MESSAGE = - "The selected model is not supported by the current ChatGPT connection. Choose a supported model or a compatible AI connection."; + "The selected model is not available through the current AI connection. Choose a supported model or a compatible AI connection."; export const NATIVE_PROVIDER_OVERLOADED_MESSAGE = "Selected model is at capacity. Please try a different model."; @@ -215,6 +216,11 @@ export function createNativeProviderFailureObservation(provider: { kind: string; let overloadedTurnId: string | null = null; const failedTurns = new Map(); const providerFailures = new Map(); + /** A driver-classified, action-required model rejection; `forTerminal` classifies it, so it is not "unclassified". */ + const modelUnavailableTurn = (turnId: string) => { + const failure = providerFailures.get(turnId); + return failure?.category === "model_unavailable" && failure.retryability === "requires_action"; + }; return { observe(event: PrpEvent) { if (event.sourceKind === "runner" && event.eventType === "turn.failed" && event.turnId @@ -266,7 +272,17 @@ export function createNativeProviderFailureObservation(provider: { kind: string; errorMessage: NATIVE_PROVIDER_OVERLOADED_MESSAGE, diagnostic: { ...NATIVE_PROVIDER_OVERLOADED_DIAGNOSTIC }, }; - if (turnId !== rejectedTurnId) return null; + if (turnId !== rejectedTurnId) { + // A driver's closed category marks a rejected model for any provider. Only an + // action-required rejection qualifies; a transient one may succeed on retry. + if (!modelUnavailableTurn(turnId)) return null; + const failure = providerFailures.get(turnId)!; + return { + errorCode: "native_provider_model_rejected" as const, + errorMessage: NATIVE_MODEL_REJECTION_MESSAGE, + diagnostic: failedTurns.get(turnId) ?? { provider: failure.provider, category: failure.category }, + }; + } return { errorCode: "native_provider_model_rejected" as const, errorMessage: NATIVE_MODEL_REJECTION_MESSAGE, @@ -276,7 +292,7 @@ export function createNativeProviderFailureObservation(provider: { kind: string; /** Diagnostics only: the failed turn that `forTerminal` did not classify. */ unclassifiedTurnFailure(turnId: string | null | undefined, terminal: PrpTerminalState) { if (!turnId || terminal.turnTerminalState !== "failed" || terminal.runTerminalState !== "failed" || - turnId === overloadedTurnId || turnId === rejectedTurnId) return null; + turnId === overloadedTurnId || turnId === rejectedTurnId || modelUnavailableTurn(turnId)) return null; const diagnostic = failedTurns.get(turnId); return diagnostic ? { ...diagnostic } : null; },