From 94829e2c1f07a70cadd01b2c4145314a39d4e6d7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 9 Nov 2026 01:38:37 -0600 Subject: [PATCH] test(evals): reach Connectors through the Coordinator More flyout; treat GPT-5.x family names as prose - unreachable-mcp: the Coordinator sidebar keeps Connectors under More. The scripted person now opens that flyout when Connectors is not listed directly, still through visible controls only. Add CoordinatorSidebar.tsx to the case's source files. - list-model-options: a family name such as GPT-5.x names no model, so ID extraction skips it; concrete IDs stay graded against the catalog. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-descriptors.json | 1 + .../external-held-gateway.test.ts | 15 ++++++++++++++- .../chat-catalog-domains/external-mcp-failure.ts | 9 ++++++++- .../model-option-catalog.test.ts | 3 ++- .../chat-catalog-domains/model-option-catalog.ts | 4 +++- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 06aa5932d26..1552cfd5dc7 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -2626,6 +2626,7 @@ "../../ui/src/pages/apps/Browse.tsx", "../../ui/src/features/connections/ConnectionSetupFlow.tsx", "../../ui/src/pages/apps/app-connect-policy.ts", + "../../ui/src/components/CoordinatorSidebar.tsx", "chat-catalog-domains/external-mcp-failure.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", diff --git a/tests/runner-e2e/chat-catalog-domains/external-held-gateway.test.ts b/tests/runner-e2e/chat-catalog-domains/external-held-gateway.test.ts index fac6d45b403..d4abbab4f89 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-held-gateway.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-held-gateway.test.ts @@ -6,13 +6,15 @@ import { } from "./external-mcp-failure.js"; import { scopedSessionFailureDiagnostic } from "./external-gateway-client.js"; import type { ExternalObservation } from "./external-support.js"; -function pageFixture(missing?: string) { +function pageFixture(missing?: string, connectorsInFlyout = false) { const actions: string[] = []; const page = { getByRole: (role: string, options?: { name?: string }) => { const name = options?.name ?? ""; const locator = { filter: () => locator, + first: () => locator, + isVisible: async () => !(connectorsInFlyout && name === "Connectors" && !actions.includes("More workspace pages")), getByRole: (_r: string, o: { name: string }) => ({ click: async () => { if (o.name === missing) throw new Error("Missing real control"); @@ -48,6 +50,17 @@ describe("finite held external diagnostics and visible entry correction", () => "actual /RUN/apps/byo", ]); }); + it("opens the Coordinator sidebar's More flyout when Connectors is not listed directly", async () => { + const { page, actions } = pageFixture(undefined, true); + await enterGenericMcpThroughCatalog(page, "RUN"); + expect(actions).toEqual([ + "More workspace pages", + "Connectors", + "Connect", + "Connect your own MCP server", + "actual /RUN/apps/byo", + ]); + }); it("does not rescue a missing catalog entry with direct navigation", async () => { for (const name of [ "Connectors", diff --git a/tests/runner-e2e/chat-catalog-domains/external-mcp-failure.ts b/tests/runner-e2e/chat-catalog-domains/external-mcp-failure.ts index 39a807e8453..2dc2d031b17 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-mcp-failure.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-mcp-failure.ts @@ -16,7 +16,13 @@ export async function enterGenericMcpThroughCatalog( page: Page, prefix: string, ) { - await page.getByRole("link", { name: "Connectors", exact: true }).click(); + // The Coordinator sidebar keeps Connectors under its More flyout; the + // classic sidebar lists it directly. Use whichever the person sees. + const direct = page.getByRole("link", { name: "Connectors", exact: true }); + if (!(await direct.first().isVisible())) { + await page.getByRole("button", { name: "More workspace pages", exact: true }).click(); + } + await direct.first().click(); const card = page .getByRole("listitem") .filter({ @@ -112,6 +118,7 @@ export const mcpFailureExternalCases = [ "../../ui/src/pages/apps/Browse.tsx", "../../ui/src/features/connections/ConnectionSetupFlow.tsx", "../../ui/src/pages/apps/app-connect-policy.ts", + "../../ui/src/components/CoordinatorSidebar.tsx", "chat-catalog-domains/external-mcp-failure.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", diff --git a/tests/runner-e2e/chat-catalog-domains/model-option-catalog.test.ts b/tests/runner-e2e/chat-catalog-domains/model-option-catalog.test.ts index a69069907e6..0ae028120eb 100644 --- a/tests/runner-e2e/chat-catalog-domains/model-option-catalog.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/model-option-catalog.test.ts @@ -1,6 +1,6 @@ import {it, expect} from "vitest"; import {gradeModelOptions} from "./runtime-runs-adapters.js"; -import {openRouterCatalogScope, modelChoiceMatches, publicOpenRouterCatalogEndpoint} from "./model-option-catalog.js"; +import {openRouterCatalogScope, modelChoiceMatches, observedModelChoices, publicOpenRouterCatalogEndpoint} from "./model-option-catalog.js"; function valid() { const config = {provider:"opencode",model:"openrouter/deepseek/deepseek-v4.1-flash-1017",env:{OPENROUTER_API_KEY:{type:"secret_ref",secretId:"company-openrouter-reference",version:"latest"}}}; const worker = {id:"morgan",companyId:"company",adapterType:"paperclip_runner",defaultEnvironmentId:"owned",adapterConfig:structuredClone(config)}, lead={...structuredClone(worker),id:"ceo"}; @@ -9,6 +9,7 @@ function valid() { agents:[structuredClone(worker),lead],tasks:[],links:[],activity:[],runs:[],humanQueries:[],reply:"Morgan's configured model is `openrouter/deepseek/deepseek-v4.1-flash-1017`. The native options include `opencode/big-pickle`. The public OpenRouter catalog also lists `gpt-5.3-codex` and `anthropic/claude-sonnet-5.5`; availability depends on the selected account/key and environment. Not all catalog models are qualified or tested."}; } it("accepts actually catalogued options with explicit current-account limits, without testing another model",()=>expect(gradeModelOptions(valid()).passed).toBe(true)); +it("treats a family name such as GPT-5.x as prose, not a model ID",()=>{const o=valid();o.reply+=" Coding highlights: the GPT-5.x Codex family, such as `openai/gpt-5.3-codex`.";expect(gradeModelOptions(o).passed).toBe(true);expect(observedModelChoices("GPT-5.x and gpt-5.3-codex")).toEqual(["gpt-5.3-codex"])}); it("accepts the same options reworded without caveat keywords",()=>{const o=valid();o.reply="Today Morgan runs `openrouter/deepseek/deepseek-v4.1-flash-1017`. Built in: `opencode/big-pickle`. OpenRouter also lists `gpt-5.3-codex` and `anthropic/claude-sonnet-5.5`, but its list is not a promise that the key works for each.";expect(gradeModelOptions(o).passed).toBe(true)}); it("uses the existing company-scoped public metadata endpoint",()=>expect(publicOpenRouterCatalogEndpoint("company")).toBe("/api/companies/company/adapters/opencode_local/models?provider=openrouter")); it.each(["gpt-99-ultra","openrouter/openai/not-a-real-model","wrong-vendor/gpt-5.3-codex","anthropic/gpt-5.3-codex"])("rejects invented/out-of-catalog or wrong-vendor choice %s",id=>{const o=valid();o.reply+=` Another option is \`${id}\`.`;expect(gradeModelOptions(o).passed).toBe(false)}); diff --git a/tests/runner-e2e/chat-catalog-domains/model-option-catalog.ts b/tests/runner-e2e/chat-catalog-domains/model-option-catalog.ts index dc6aa6f2f4a..614a0813bd2 100644 --- a/tests/runner-e2e/chat-catalog-domains/model-option-catalog.ts +++ b/tests/runner-e2e/chat-catalog-domains/model-option-catalog.ts @@ -27,7 +27,9 @@ export function publicOpenRouterCatalogEndpoint(companyId: string) { } function normalized(id: string) { return id.toLowerCase().replace(/^openrouter\//, ""); } export function observedModelChoices(reply: string) { - const vendor = [...reply.matchAll(/\b(?:gpt-\d[\w.-]*|o[134](?:-[\w.-]+)?|claude-(?:sonnet|opus|haiku)-[\w.-]+)/gi)].map(m => m[0].replace(/[.,;:!?]+$/, "")); + // A family name such as "GPT-5.x" names no model; only concrete IDs are graded. + const vendor = [...reply.matchAll(/\b(?:gpt-\d[\w.-]*|o[134](?:-[\w.-]+)?|claude-(?:sonnet|opus|haiku)-[\w.-]+)/gi)].map(m => m[0].replace(/[.,;:!?]+$/, "")) + .filter(id => !/(?:^|[-.])x(?:$|[-.])/i.test(id)); const code = [...reply.matchAll(/`([^`\n]+)`/g)].map(m => m[1]!).filter(s => /^[\w.-]+\/[\w./:+-]+$/.test(s) && !s.includes("://")); const plainOpenRouter = [...reply.matchAll(/(? m[0].replace(/[.,;:!?]+$/, "")); return [...new Set([...vendor, ...code, ...plainOpenRouter])];