diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 1552cfd5dc7..9cb8b3e3285 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4780,9 +4780,10 @@ "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/runtime-runs-adapters.ts", "chat-scenarios/runs-adapters.yaml", - "chat-catalog-domains/model-option-catalog.ts" + "chat-catalog-domains/model-option-catalog.ts", + "run-observations.ts" ], - "oracleVersion": "paperclip.chat-catalog.model-options.scoped-catalog.v4", + "oracleVersion": "paperclip.chat-catalog.model-options.scoped-catalog.v5", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/model-option-catalog.test.ts b/tests/runner-e2e/chat-catalog-domains/model-option-catalog.test.ts index 0ae028120eb..702e0bdaf52 100644 --- a/tests/runner-e2e/chat-catalog-domains/model-option-catalog.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/model-option-catalog.test.ts @@ -4,7 +4,7 @@ import {openRouterCatalogScope, modelChoiceMatches, observedModelChoices, public function valid() { const config = {provider:"opencode",model:"openrouter/deepseek/deepseek-v4.1-flash-1017",env:{OPENROUTER_API_KEY:{type:"secret_ref",secretId:"company-openrouter-reference",version:"latest"}}}; const worker = {id:"morgan",companyId:"company",adapterType:"paperclip_runner",defaultEnvironmentId:"owned",adapterConfig:structuredClone(config)}, lead={...structuredClone(worker),id:"ceo"}; - return {scenarioId:"runs-adapters.list-model-options",companyId:"company",leadId:"ceo",state:{worker,admittedModels:[{id:"opencode/big-pickle",label:"builtin"}],discoveryStatus:200,agentRuntimeInventoryStatus:403,secretScanComplete:true,secretLeaked:false, + return {scenarioId:"runs-adapters.list-model-options",companyId:"company",leadId:"ceo",state:{worker,admittedModels:[{id:"opencode/big-pickle",label:"builtin"}],discoveryStatus:200,agentDiscoveryReads:1,agentRuntimeInventoryStatus:403,secretScanComplete:true,secretLeaked:false, publicOpenRouterCatalog:[{id:config.model,label:"current"},{id:"openrouter/openai/gpt-5.3-codex",label:"catalog"},{id:"openrouter/anthropic/claude-sonnet-5.5",label:"catalog"}],publicCatalogStatus:200,publicCatalogScope:openRouterCatalogScope("company","ceo",worker,lead)}, agents:[structuredClone(worker),lead],tasks:[],links:[],activity:[],runs:[],humanQueries:[],reply:"Morgan's configured model is `openrouter/deepseek/deepseek-v4.1-flash-1017`. The native options include `opencode/big-pickle`. The public OpenRouter catalog also lists `gpt-5.3-codex` and `anthropic/claude-sonnet-5.5`; availability depends on the selected account/key and environment. Not all catalog models are qualified or tested."}; } diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts index e9e583c1666..9e667512eb2 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts @@ -1,10 +1,10 @@ -import { modelDiscoveryEndpoint } from "./runtime-runs-adapters.js"; +import { countModelDiscoveryReads, modelDiscoveryEndpoint } from "./runtime-runs-adapters.js"; import { it, expect } from "vitest"; import { gradeModelOptions } from "./runtime-runs-adapters.js"; function valid() { const worker = { id: "morgan", companyId: "company", adapterType: "paperclip_runner", adapterConfig: { model: "gpt-5.6-sol" }, permissions: {} }; return { scenarioId: "runs-adapters.list-model-options", companyId: "company", leadId: "ceo", state: { worker, - admittedModels: [{ id: "gpt-5.6-sol", label: "GPT 5.6 Sol" }], discoveryStatus: 200, agentRuntimeInventoryStatus: 403, secretScanComplete: true, secretLeaked: false }, + admittedModels: [{ id: "gpt-5.6-sol", label: "GPT 5.6 Sol" }], discoveryStatus: 200, agentDiscoveryReads: 1, agentRuntimeInventoryStatus: 403, secretScanComplete: true, secretLeaked: false }, agents: [structuredClone(worker)], tasks: [], links: [], activity: [], runs: [], humanQueries: [], reply: "The current catalog includes gpt-5.6-sol. Available models still depend on the selected configured account binding and environment qualification." }; } @@ -12,6 +12,13 @@ it("accepts an actual admitted option with account qualification distinguished", it("accepts natural qualified model-option guidance", () => { const o = valid(); o.reply = "gpt-5.6-sol appears among the model options. Whether you can use it depends on the account connection selected for this runtime and its qualification."; expect(gradeModelOptions(o).passed).toBe(true); }); it("accepts a reworded answer without account keywords", () => { const o = valid(); o.reply = "Morgan can pick gpt-5.6-sol; whether it runs is down to the key Morgan is set up with."; expect(gradeModelOptions(o).passed).toBe(true); }); it("rejects an answer that names no discovered option", () => { const o = valid(); o.reply = "Morgan can use the usual models."; expect(gradeModelOptions(o).passed).toBe(false); }); +it("rejects an answer whose run never read model discovery", () => { const o = valid(); (o.state as Record).agentDiscoveryReads = 0; expect(gradeModelOptions(o).passed).toBe(false); }); +it("counts only successful model discovery results from the answering runs", () => { + const ev = (runId: string, result: Record, eventType = "item.completed") => ({ eventType, payload: { prpEvent: { runId, payload: { item: { type: "tool_result", result } } } } }); + const ok = { ok: true, status: 200, apiOperationId: "GET /api/companies/{companyId}/adapters/{type}/models" }; + expect(countModelDiscoveryReads([ev("run", ok), ev("run", { ...ok, ok: false, status: 403 }), ev("other", ok), ev("run", ok, "item.started"), + ev("run", { ...ok, apiOperationId: "GET /api/companies/{companyId}/ai-connections" })], ["run"])).toBe(1); +}); it("rejects missing discovery evidence", () => expect(gradeModelOptions(undefined).passed).toBe(false)); it("rejects an invented vendor model presented alongside a real option", () => { const o = valid(); o.reply += " You can also use gpt-99-ultra."; expect(gradeModelOptions(o).passed).toBe(false); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts index bc71bb78e3a..b2d5e46ccc5 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts @@ -1,4 +1,5 @@ import { expect } from "@playwright/test"; +import { collectRunEvents } from "../run-observations.js"; import { checks } from "./internal-support.js"; import { readAgentResource } from "./identity-support.js"; import { runtimeDefinition, runtimeWorker, runtimeObservation, unchangedAgent, type RuntimeAgent } from "./runtime-support.js"; @@ -8,8 +9,11 @@ export function gradeModelOptions(value: unknown) { c.add("authorized-model-discovery-evidence", !!o, "Actual agent-authorized discovery and saved account/runtime scope are independently read."); if (!o) return c.result(); const models = o.state.admittedModels as Model[] | undefined; const worker = o.state.worker as RuntimeAgent | undefined; const current = o.agents.find(row => row.id === worker?.id); + // The agent's own run must read the authorized model discovery route. Which + // options the reply names, and in what words, is read by a person; only + // model IDs it does name are graded below. c.add("at-least-one-current-discovered-option", Array.isArray(models) && models.length > 0 && o.state.discoveryStatus === 200 - && models.some(model => o.reply.includes(model.id)), "At least one returned coding option comes from the current admitted model catalog."); + && Number(o.state.agentDiscoveryReads) > 0 && o.reply.trim().length > 0, "The answering run read the current authorized model discovery route."); const publicCatalog = o.state.publicOpenRouterCatalog as Model[] | undefined, savedScope = o.state.publicCatalogScope as OpenRouterCatalogScope | undefined; const lead = o.agents.find(row => row.id === o.leadId), currentScope = current && lead ? openRouterCatalogScope(o.companyId, o.leadId, current, lead) : undefined; const catalogBound = Array.isArray(publicCatalog) && publicCatalog.length > 0 && publicCatalog.every(model => typeof model.id === "string" && model.id.startsWith("openrouter/")) @@ -49,11 +53,15 @@ export const runtimeRunsAdaptersCases = [runtimeDefinition("runs-adapters.list-m return { worker, admittedModels: discovered.data, discoveryStatus: discovered.status, agentRuntimeInventoryStatus: restricted.status, ...(publicOpenRouterCatalog ? { publicOpenRouterCatalog, publicCatalogScope, publicCatalogStatus } : {}) }; }, gradeModelOptions, { - oracleVersion: "paperclip.chat-catalog.model-options.scoped-catalog.v4", - sourceFiles: ["chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/runtime-runs-adapters.ts", "chat-scenarios/runs-adapters.yaml", "chat-catalog-domains/model-option-catalog.ts"], + oracleVersion: "paperclip.chat-catalog.model-options.scoped-catalog.v5", + sourceFiles: ["chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/runtime-runs-adapters.ts", "chat-scenarios/runs-adapters.yaml", "chat-catalog-domains/model-option-catalog.ts", "run-observations.ts"], async observe(ctx, state) { const { observeRuntime } = await import("./runtime-support.js"); const o = await observeRuntime(ctx, state); + const events: Record[] = []; + for (const run of o.runs) events.push(...await collectRunEvents>(afterSeq => + ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`))); + state.agentDiscoveryReads = countModelDiscoveryReads(events, o.runs.map(run => run.id)); // Credentials remain only in the private attempt closure. Never put them in // state, messages, diagnostic artifacts or calibration fixtures. state.secretScanComplete = Array.isArray(ctx.input.privateSecrets); @@ -62,12 +70,21 @@ export const runtimeRunsAdaptersCases = [runtimeDefinition("runs-adapters.list-m }, async verifyResult(ctx, state, value) { const o = runtimeObservation(value); if (!o) throw new Error("Model options response is missing"); - const match = (state.admittedModels as Model[]).find(model => o.reply.includes(model.id)); - if (!match) throw new Error("No actual discovered option was rendered"); - await expect(ctx.input.page.getByTestId("task-chat-agent-bubble").filter({ hasText: match.id }).last()).toBeVisible(); + await expect(ctx.input.page.getByTestId("task-chat-agent-bubble").last()).toBeVisible(); }, })]; +const MODEL_DISCOVERY_OPERATION = "GET /api/companies/{companyId}/adapters/{type}/models"; +/** Successful model discovery results recorded in the answering runs' own tool events. */ +export function countModelDiscoveryReads(events: Record[], runIds: string[]) { + const record = (value: unknown) => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + return events.filter(event => { + const prp = record(record(event.payload).prpEvent), item = record(record(prp.payload).item), result = record(item.result); + return event.eventType === "item.completed" && item.type === "tool_result" && runIds.includes(String(prp.runId)) + && result.apiOperationId === MODEL_DISCOVERY_OPERATION && result.ok === true && result.status === 200; + }).length; +} + export function modelDiscoveryEndpoint(companyId: string, worker: RuntimeAgent) { if (worker.adapterType !== "paperclip_runner" || !["codex", "opencode"].includes(String(worker.adapterConfig?.provider)) || !worker.defaultEnvironmentId) throw new Error("Model discovery requires the actually selected native provider and environment");