From 582685e11bbf3e19d49d2f82d7a01cf5be1b4c66 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 9 Nov 2026 02:24:24 -0600 Subject: [PATCH] test(evals): wait for secure-key's rendered setup link before following it Run 42583237179 replied with [Connect your own MCP server](/COM/apps/byo), but act read the comment anchors once and compared the origin to the raw baseURL. It now polls for the rendered link for up to 30s and compares normalized origins. Co-Authored-By: Claude Opus 5.5 --- .../chat-catalog-domains/external-secure-key.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/external-secure-key.ts b/tests/runner-e2e/chat-catalog-domains/external-secure-key.ts index eaee0212bc0..7a2ca1ea420 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-secure-key.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-secure-key.ts @@ -1,3 +1,4 @@ +import { expect } from "@playwright/test"; import { uploadFactualQueryAttachment, verifyChatQueryAttachments, type ChatQueryAttachmentReceipt, type ChatQueryHumanComment } from "../chat-query-attachments.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import type { ToolConnection } from "../../../packages/shared/src/types/tool-access.js"; @@ -53,8 +54,11 @@ export const secureKeyExternalCases = [definition("connection-setup.secure-api-k sourceFiles: ["chat-catalog-domains/external-secure-key.ts", "chat-query-attachments.ts", "../../ui/src/App.tsx", "../../ui/src/pages/apps/app-connect-policy.ts", "../../ui/src/features/connections/ConnectionSetupFlow.tsx", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-secret-scan.ts", "run-observations.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/connection-setup.yaml"], async act(ctx, state) { await waitForReply(ctx); const provider = fixtures.get(ctx); if (!provider) throw new Error("Secure provider fixture missing"); - const links = await ctx.input.page.locator(`[id^="comment-"] a`).evaluateAll(anchors => anchors.map(anchor => (anchor as HTMLAnchorElement).href)); - const target = links.find(href => { try { const url = new URL(href); return url.origin === ctx.input.api.baseURL && url.pathname === state.setupPath; } catch { return false; } }); + // The saved reply can arrive before it renders; wait for the rendered link instead of reading anchors once. + const renderedTarget = async () => (await ctx.input.page.locator(`[id^="comment-"] a`).evaluateAll(anchors => anchors.map(anchor => (anchor as HTMLAnchorElement).href))) + .find(href => { try { const url = new URL(href); return url.origin === new URL(ctx.input.api.baseURL).origin && url.pathname === state.setupPath; } catch { return false; } }); + let target: string | undefined; + await expect.poll(async () => (target = await renderedTarget()) !== undefined, { timeout: 30_000 }).toBe(true).catch(() => undefined); if (!target) throw new Error("The model did not link the intended secure setup entry"); const receipt = await privateBrowserEntry({ mode: "secure-key", baseURL: ctx.input.api.baseURL, setupURL: target, providerURL: provider.url, credential: provider.token, cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL) });