diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.test.ts new file mode 100644 index 00000000000..97eb42a3f79 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.test.ts @@ -0,0 +1,14 @@ +import { it, expect } from "vitest"; +import { admittedModelFailure } from "./runtime-native-failure.js"; +function valid() { return { companyId: "company", agentId: "morgan", issueId: "website", model: "unavailable", + run: { id: "actual", companyId: "company", agentId: "morgan", runtimeMode: "native", status: "failed", startedAt: "2026-10-26T00:00:00Z", + errorCode: "native_provider_model_rejected", error: "The selected model was rejected", nativeIssueId: "website", + runnerProfileJson: { nativeExecutionInput: { provider: { kind: "codex", model: "unavailable" } } } }, + events: [{ seq: 1, eventType: "run.terminal" }] }; } +it("accepts only admitted native failed evidence bound to the actual unavailable model and task", () => expect(admittedModelFailure(valid())).toBe(true)); +it("rejects cancellation relabeled as a failure fixture", () => { const o = valid(); o.run.status = "cancelled"; expect(admittedModelFailure(o)).toBe(false); }); +it("rejects a failed tool command in a successful native turn", () => { const o = valid(); o.run.status = "succeeded"; expect(admittedModelFailure(o)).toBe(false); }); +it("rejects configuration rejection before native admission", () => { const o = valid(); o.run.startedAt = ""; expect(admittedModelFailure(o)).toBe(false); }); +it("rejects an unrelated prior failure", () => { const o = valid(); o.run.nativeIssueId = "other"; expect(admittedModelFailure(o)).toBe(false); }); +it("rejects a changed or substituted provider model", () => { const o = valid(); o.run.runnerProfileJson.nativeExecutionInput.provider.model = "supported"; expect(admittedModelFailure(o)).toBe(false); }); +it("rejects absent durable terminal events", () => { const o = valid(); o.events = []; expect(admittedModelFailure(o)).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.ts b/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.ts new file mode 100644 index 00000000000..f43c1851b79 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.ts @@ -0,0 +1,59 @@ +import { pollUntil } from "../api.js"; +import { isDeepStrictEqual } from "node:util"; +import { collectRunEvents } from "../run-observations.js"; +import type { ChatRun } from "../chat-flow.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { RuntimeAgent } from "./runtime-support.js"; +interface NativeFailureRun extends ChatRun { + nativeIssueId?: string | null; + runnerProfileJson?: { nativeExecutionInput?: { provider?: { kind?: string; model?: string | null } } }; +} +interface NativeEvent { seq?: number; eventType?: string; payload?: { prpEvent?: { eventType?: string } } } +export function admittedModelFailure(input: { run?: NativeFailureRun; events: NativeEvent[]; companyId: string; agentId: string; issueId: string; model: string }) { + const r = input.run, provider = r?.runnerProfileJson?.nativeExecutionInput?.provider; + return !!r && r.runtimeMode === "native" && r.status === "failed" && !!r.startedAt && !!r.errorCode && !!r.error + && r.companyId === input.companyId && r.agentId === input.agentId + && [r.nativeIssueId, r.contextSnapshot?.issueId, r.contextSnapshot?.taskId].includes(input.issueId) + && provider?.kind === "codex" && provider.model === input.model + && input.events.some(event => [event.eventType, event.payload?.prpEvent?.eventType].some(type => type === "run.terminal" || type === "turn.failed")); +} +/** A real public configuration/invocation fixture, never a synthetic failure. + * The caller supplies exactly one ordinary task/routine invocation and retains + * its own invocation receipt. HTTP/preflight rejection is a setup blocker. + * This source-qualified recipe is not live qualification until this predicate + * observes an actually admitted native failed row and durable terminal events. + */ +export async function unavailableNativeModelFailure(ctx: ChatCatalogContext, worker: RuntimeAgent, + invoke: () => Promise<{ issueId: string }>) { + const endpoint = `/api/agents/${worker.id}`; + const original = await ctx.input.api.get(endpoint); + if (original.lifecycleState !== "ready" || !["active", "idle"].includes(original.status ?? "") + || original.adapterType !== "paperclip_runner" || original.adapterConfig?.provider !== "codex") + throw new Error("Native failure prerequisite requires an actually ready native Codex worker"); + const before = await ctx.allRuns(); const model = `paperclip-eval-unavailable-${ctx.input.nonce}`; + let failed: NativeFailureRun | undefined; + try { + const configured = await ctx.input.api.patch(endpoint, { adapterConfig: { ...original.adapterConfig, model } }); + if (configured.adapterConfig?.model !== model || configured.lifecycleState !== "ready") + throw new Error("Unavailable-model configuration was not retained on the admitted worker; no failed-native coverage"); + const receipt = await invoke(); + if (!receipt.issueId) throw new Error("Native failure invocation did not return an actual source task"); + const settled = await pollUntil({ label: "Actual admitted unavailable-model native failure", deadlineAt: Date.now() + 150_000, + load: async () => (await ctx.allRuns()).filter(run => !before.some(prior => prior.id === run.id)), + accept: rows => rows.length === 1 && rows[0]?.status === "failed", + reject: rows => rows.length > 1 ? "Failure prerequisite admitted more than its one declared invocation" + : rows.some(run => ["succeeded", "cancelled", "timed_out"].includes(run.status)) ? "Invocation did not produce genuine native failed history" : undefined }); + failed = await ctx.input.api.get(`/api/heartbeat-runs/${settled[0]!.id}`); + const events = await collectRunEvents((afterSeq, limit) => ctx.input.api.get(`/api/heartbeat-runs/${failed!.id}/events?afterSeq=${afterSeq}&limit=${limit}`)); + if (!admittedModelFailure({ run: failed, events, companyId: ctx.input.fixtures.company.id, agentId: worker.id, issueId: receipt.issueId, model })) + throw new Error("Actual invocation lacks admitted model-bound native failure evidence; no failure coverage claimed"); + return { failedRun: failed, failedRunEvents: events, failedIssueId: receipt.issueId, failedModel: model, + failureKind: "native-failure-under-explicit-unavailable-model-configuration", originalConfiguration: original }; + } finally { + await ctx.input.api.patch(endpoint, { adapterConfig: original.adapterConfig }); + const current = await ctx.input.api.get(endpoint); + if (current.status === "error") await ctx.input.api.post(`${endpoint}/clear-error`, {}); + const restored = await ctx.input.api.get(endpoint); + if (!isDeepStrictEqual(restored.adapterConfig, original.adapterConfig)) throw new Error("Original admitted model configuration was not restored"); + } +}