From fcd21b0968842acea668783b3ea141bfe4c2bb54 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 9 Nov 2026 01:57:17 -0600 Subject: [PATCH] test(evals): grade list-model-options discovery from the run's own tool events The answering run must read the authorized model discovery route; the reply no longer has to copy an admitted model ID verbatim, since the query never asked for IDs. IDs the reply does name stay bounded to the admitted and same-binding public catalogs. Retained replies from two runs (one listing the OpenRouter catalog in prose) now grade as passes; a run with no discovery read fails. Bump oracleVersion to v5. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 5 ++-- .../model-option-catalog.test.ts | 2 +- .../runtime-runs-adapters.test.ts | 11 +++++-- .../runtime-runs-adapters.ts | 29 +++++++++++++++---- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 1552cfd5dc7..9cb8b3e3285 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4780,9 +4780,10 @@ "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/runtime-runs-adapters.ts", "chat-scenarios/runs-adapters.yaml", - "chat-catalog-domains/model-option-catalog.ts" + "chat-catalog-domains/model-option-catalog.ts", + "run-observations.ts" ], - "oracleVersion": "paperclip.chat-catalog.model-options.scoped-catalog.v4", + "oracleVersion": "paperclip.chat-catalog.model-options.scoped-catalog.v5", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/model-option-catalog.test.ts b/tests/runner-e2e/chat-catalog-domains/model-option-catalog.test.ts index 0ae028120eb..702e0bdaf52 100644 --- a/tests/runner-e2e/chat-catalog-domains/model-option-catalog.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/model-option-catalog.test.ts @@ -4,7 +4,7 @@ import {openRouterCatalogScope, modelChoiceMatches, observedModelChoices, public function valid() { const config = {provider:"opencode",model:"openrouter/deepseek/deepseek-v4.1-flash-1017",env:{OPENROUTER_API_KEY:{type:"secret_ref",secretId:"company-openrouter-reference",version:"latest"}}}; const worker = {id:"morgan",companyId:"company",adapterType:"paperclip_runner",defaultEnvironmentId:"owned",adapterConfig:structuredClone(config)}, lead={...structuredClone(worker),id:"ceo"}; - return {scenarioId:"runs-adapters.list-model-options",companyId:"company",leadId:"ceo",state:{worker,admittedModels:[{id:"opencode/big-pickle",label:"builtin"}],discoveryStatus:200,agentRuntimeInventoryStatus:403,secretScanComplete:true,secretLeaked:false, + return {scenarioId:"runs-adapters.list-model-options",companyId:"company",leadId:"ceo",state:{worker,admittedModels:[{id:"opencode/big-pickle",label:"builtin"}],discoveryStatus:200,agentDiscoveryReads:1,agentRuntimeInventoryStatus:403,secretScanComplete:true,secretLeaked:false, publicOpenRouterCatalog:[{id:config.model,label:"current"},{id:"openrouter/openai/gpt-5.3-codex",label:"catalog"},{id:"openrouter/anthropic/claude-sonnet-5.5",label:"catalog"}],publicCatalogStatus:200,publicCatalogScope:openRouterCatalogScope("company","ceo",worker,lead)}, agents:[structuredClone(worker),lead],tasks:[],links:[],activity:[],runs:[],humanQueries:[],reply:"Morgan's configured model is `openrouter/deepseek/deepseek-v4.1-flash-1017`. The native options include `opencode/big-pickle`. The public OpenRouter catalog also lists `gpt-5.3-codex` and `anthropic/claude-sonnet-5.5`; availability depends on the selected account/key and environment. Not all catalog models are qualified or tested."}; } diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts index e9e583c1666..9e667512eb2 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts @@ -1,10 +1,10 @@ -import { modelDiscoveryEndpoint } from "./runtime-runs-adapters.js"; +import { countModelDiscoveryReads, modelDiscoveryEndpoint } from "./runtime-runs-adapters.js"; import { it, expect } from "vitest"; import { gradeModelOptions } from "./runtime-runs-adapters.js"; function valid() { const worker = { id: "morgan", companyId: "company", adapterType: "paperclip_runner", adapterConfig: { model: "gpt-5.6-sol" }, permissions: {} }; return { scenarioId: "runs-adapters.list-model-options", companyId: "company", leadId: "ceo", state: { worker, - admittedModels: [{ id: "gpt-5.6-sol", label: "GPT 5.6 Sol" }], discoveryStatus: 200, agentRuntimeInventoryStatus: 403, secretScanComplete: true, secretLeaked: false }, + admittedModels: [{ id: "gpt-5.6-sol", label: "GPT 5.6 Sol" }], discoveryStatus: 200, agentDiscoveryReads: 1, agentRuntimeInventoryStatus: 403, secretScanComplete: true, secretLeaked: false }, agents: [structuredClone(worker)], tasks: [], links: [], activity: [], runs: [], humanQueries: [], reply: "The current catalog includes gpt-5.6-sol. Available models still depend on the selected configured account binding and environment qualification." }; } @@ -12,6 +12,13 @@ it("accepts an actual admitted option with account qualification distinguished", it("accepts natural qualified model-option guidance", () => { const o = valid(); o.reply = "gpt-5.6-sol appears among the model options. Whether you can use it depends on the account connection selected for this runtime and its qualification."; expect(gradeModelOptions(o).passed).toBe(true); }); it("accepts a reworded answer without account keywords", () => { const o = valid(); o.reply = "Morgan can pick gpt-5.6-sol; whether it runs is down to the key Morgan is set up with."; expect(gradeModelOptions(o).passed).toBe(true); }); it("rejects an answer that names no discovered option", () => { const o = valid(); o.reply = "Morgan can use the usual models."; expect(gradeModelOptions(o).passed).toBe(false); }); +it("rejects an answer whose run never read model discovery", () => { const o = valid(); (o.state as Record).agentDiscoveryReads = 0; expect(gradeModelOptions(o).passed).toBe(false); }); +it("counts only successful model discovery results from the answering runs", () => { + const ev = (runId: string, result: Record, eventType = "item.completed") => ({ eventType, payload: { prpEvent: { runId, payload: { item: { type: "tool_result", result } } } } }); + const ok = { ok: true, status: 200, apiOperationId: "GET /api/companies/{companyId}/adapters/{type}/models" }; + expect(countModelDiscoveryReads([ev("run", ok), ev("run", { ...ok, ok: false, status: 403 }), ev("other", ok), ev("run", ok, "item.started"), + ev("run", { ...ok, apiOperationId: "GET /api/companies/{companyId}/ai-connections" })], ["run"])).toBe(1); +}); it("rejects missing discovery evidence", () => expect(gradeModelOptions(undefined).passed).toBe(false)); it("rejects an invented vendor model presented alongside a real option", () => { const o = valid(); o.reply += " You can also use gpt-99-ultra."; expect(gradeModelOptions(o).passed).toBe(false); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts index bc71bb78e3a..b2d5e46ccc5 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts @@ -1,4 +1,5 @@ import { expect } from "@playwright/test"; +import { collectRunEvents } from "../run-observations.js"; import { checks } from "./internal-support.js"; import { readAgentResource } from "./identity-support.js"; import { runtimeDefinition, runtimeWorker, runtimeObservation, unchangedAgent, type RuntimeAgent } from "./runtime-support.js"; @@ -8,8 +9,11 @@ export function gradeModelOptions(value: unknown) { c.add("authorized-model-discovery-evidence", !!o, "Actual agent-authorized discovery and saved account/runtime scope are independently read."); if (!o) return c.result(); const models = o.state.admittedModels as Model[] | undefined; const worker = o.state.worker as RuntimeAgent | undefined; const current = o.agents.find(row => row.id === worker?.id); + // The agent's own run must read the authorized model discovery route. Which + // options the reply names, and in what words, is read by a person; only + // model IDs it does name are graded below. c.add("at-least-one-current-discovered-option", Array.isArray(models) && models.length > 0 && o.state.discoveryStatus === 200 - && models.some(model => o.reply.includes(model.id)), "At least one returned coding option comes from the current admitted model catalog."); + && Number(o.state.agentDiscoveryReads) > 0 && o.reply.trim().length > 0, "The answering run read the current authorized model discovery route."); const publicCatalog = o.state.publicOpenRouterCatalog as Model[] | undefined, savedScope = o.state.publicCatalogScope as OpenRouterCatalogScope | undefined; const lead = o.agents.find(row => row.id === o.leadId), currentScope = current && lead ? openRouterCatalogScope(o.companyId, o.leadId, current, lead) : undefined; const catalogBound = Array.isArray(publicCatalog) && publicCatalog.length > 0 && publicCatalog.every(model => typeof model.id === "string" && model.id.startsWith("openrouter/")) @@ -49,11 +53,15 @@ export const runtimeRunsAdaptersCases = [runtimeDefinition("runs-adapters.list-m return { worker, admittedModels: discovered.data, discoveryStatus: discovered.status, agentRuntimeInventoryStatus: restricted.status, ...(publicOpenRouterCatalog ? { publicOpenRouterCatalog, publicCatalogScope, publicCatalogStatus } : {}) }; }, gradeModelOptions, { - oracleVersion: "paperclip.chat-catalog.model-options.scoped-catalog.v4", - sourceFiles: ["chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/runtime-runs-adapters.ts", "chat-scenarios/runs-adapters.yaml", "chat-catalog-domains/model-option-catalog.ts"], + oracleVersion: "paperclip.chat-catalog.model-options.scoped-catalog.v5", + sourceFiles: ["chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/runtime-runs-adapters.ts", "chat-scenarios/runs-adapters.yaml", "chat-catalog-domains/model-option-catalog.ts", "run-observations.ts"], async observe(ctx, state) { const { observeRuntime } = await import("./runtime-support.js"); const o = await observeRuntime(ctx, state); + const events: Record[] = []; + for (const run of o.runs) events.push(...await collectRunEvents>(afterSeq => + ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`))); + state.agentDiscoveryReads = countModelDiscoveryReads(events, o.runs.map(run => run.id)); // Credentials remain only in the private attempt closure. Never put them in // state, messages, diagnostic artifacts or calibration fixtures. state.secretScanComplete = Array.isArray(ctx.input.privateSecrets); @@ -62,12 +70,21 @@ export const runtimeRunsAdaptersCases = [runtimeDefinition("runs-adapters.list-m }, async verifyResult(ctx, state, value) { const o = runtimeObservation(value); if (!o) throw new Error("Model options response is missing"); - const match = (state.admittedModels as Model[]).find(model => o.reply.includes(model.id)); - if (!match) throw new Error("No actual discovered option was rendered"); - await expect(ctx.input.page.getByTestId("task-chat-agent-bubble").filter({ hasText: match.id }).last()).toBeVisible(); + await expect(ctx.input.page.getByTestId("task-chat-agent-bubble").last()).toBeVisible(); }, })]; +const MODEL_DISCOVERY_OPERATION = "GET /api/companies/{companyId}/adapters/{type}/models"; +/** Successful model discovery results recorded in the answering runs' own tool events. */ +export function countModelDiscoveryReads(events: Record[], runIds: string[]) { + const record = (value: unknown) => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + return events.filter(event => { + const prp = record(record(event.payload).prpEvent), item = record(record(prp.payload).item), result = record(item.result); + return event.eventType === "item.completed" && item.type === "tool_result" && runIds.includes(String(prp.runId)) + && result.apiOperationId === MODEL_DISCOVERY_OPERATION && result.ok === true && result.status === 200; + }).length; +} + export function modelDiscoveryEndpoint(companyId: string, worker: RuntimeAgent) { if (worker.adapterType !== "paperclip_runner" || !["codex", "opencode"].includes(String(worker.adapterConfig?.provider)) || !worker.defaultEnvironmentId) throw new Error("Model discovery requires the actually selected native provider and environment");