From b90f2f2bf77933e63c0c9450a8180ae4f07277f4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 13:25:01 -0600 Subject: [PATCH] test(evals): wait for the app before opening quick search; grade the actual palette shortcut The independent observer pressed Cmd/Ctrl+K right after domcontentloaded, before the app mounted the palette listener, so the palette never opened (run 42404426034). Wait for the shell and retry the shortcut. The reply check accepted any 'command ... k' or a bare 'command palette' mention. Require the shortcut itself (a modifier followed by K, any common spelling), which is the fact the person needs; bump the oracle version. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../identity-navigation-preferences.ts | 23 ++++++++++++++----- .../chat-catalog-domains/identity.test.ts | 11 +++++++++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 81bf0a8d5f2..b4a90ddc409 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -3642,7 +3642,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.identity.quick-search.v2", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-navigation-preferences.ts b/tests/runner-e2e/chat-catalog-domains/identity-navigation-preferences.ts index 4a544de6842..cb9282c4fa9 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-navigation-preferences.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-navigation-preferences.ts @@ -4,6 +4,19 @@ import { checks, entity, hasLink, project, task, verifyInternalResult } from "./ import { identityDefinition, identityObservation, waitForReply, probeAgentPermission } from "./identity-support.js"; import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity-auth.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +// The palette shortcut is a fixed product fact the person needs to act on, so it is graded as a literal key combination: +// a modifier immediately followed by K, allowing the usual spellings (Cmd+K, ⌘K, Ctrl-K, `Ctrl` + `K`, ⌘K). +const paletteShortcut = /(?:⌘|\b(?:cmd|command|ctrl|control|mod)\b)(?:[\s`*+\-–/]|<\/?kbd>)*k\b/i; +async function openQuickSearch(page: ChatCatalogContext["input"]["page"]) { + const input = page.getByPlaceholder("Search tasks, agents, projects..."); + // A freshly loaded page fires keydown before the app mounts the palette listener; wait for the shell and retry the shortcut. + await expect(page.getByRole("button", { name: "Global search" }).first()).toBeVisible({ timeout: 30_000 }); + await expect(async () => { + if (!await input.isVisible()) await page.keyboard.press("ControlOrMeta+k"); + await expect(input).toBeVisible({ timeout: 2_000 }); + }).toPass({ timeout: 30_000 }); + return input; +} const searchObservers = new WeakMap>>(); export function gradeNavigation(value: unknown) { @@ -21,7 +34,7 @@ export function gradeNavigation(value: unknown) { c.add("personal-preferences-not-agent-mutated", o.state.agentPreferenceReadStatus === 403 && o.state.preferencesBefore !== undefined && JSON.stringify(o.state.preferencesBefore) === JSON.stringify(o.state.preferencesAfterReply), "The actual agent cannot impersonate the person's preferences, and the response does not change saved order or browser favorites/recents."); } else if (o.scenarioId === "navigation-preferences.quick-search") { - c.add("documented-palette-entry", /(?:command|cmd|ctrl|control|⌘).*k|quick\s+search|command\s+palette/i.test(o.reply), "The answer identifies the actual quick-search entry point."); + c.add("documented-palette-entry", paletteShortcut.test(o.reply), "The answer gives the actual quick-search shortcut (Cmd/Ctrl+K) the person can press without leaving their page."); c.add("browser-search-transition", o.state.paletteTaskVisible === true && o.state.searchTaskVisible === true && o.state.searchQuery === "Launch" && o.state.searchStatus === "backlog", "The human opens the palette, finds the fixture task, and carries its query and chosen status filter into full search."); c.add("independent-search-audience", o.state.observerPublicVisible === true && o.state.observerPrivatePaletteHidden === true && o.state.observerPrivateSearchHidden === true, @@ -71,12 +84,11 @@ export const navigationIdentityCases = [ return { ...state, privateTask, observerId: observer.id }; } catch (error) { await observer.dispose(); searchObservers.delete(ctx); throw error; } }, gradeNavigation, { - authentication: "human-session", + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.identity.quick-search.v2", sourceFiles: ["chat-catalog-domains/identity-navigation-preferences.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"], async act(ctx, state) { await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const source = ctx.input.page.url(); - await ctx.input.page.keyboard.press("ControlOrMeta+k"); - await ctx.input.page.getByPlaceholder("Search tasks, agents, projects...").fill("Launch status:backlog"); + await (await openQuickSearch(ctx.input.page)).fill("Launch status:backlog"); await expect(ctx.input.page.getByRole("dialog").getByText(entity(state, "task").title!, { exact: true })).toBeVisible(); state.paletteTaskVisible = true; await ctx.input.page.getByTestId("command-search-all").click(); @@ -87,8 +99,7 @@ export const navigationIdentityCases = [ const page = await observer.context.newPage(); try { await page.goto(`/${ctx.input.fixtures.company.issuePrefix}/projects/${entity(state, "project").id}`, { waitUntil: "domcontentloaded" }); - await page.keyboard.press("ControlOrMeta+k"); - await page.getByPlaceholder("Search tasks, agents, projects...").fill("Launch status:backlog"); + await (await openQuickSearch(page)).fill("Launch status:backlog"); const palette = page.getByRole("dialog"); await expect(palette.getByText(entity(state, "task").title!, { exact: true })).toBeVisible(); await expect(palette.getByText(entity(state, "privateTask").title!, { exact: true })).toHaveCount(0); state.observerPrivatePaletteHidden = true; diff --git a/tests/runner-e2e/chat-catalog-domains/identity.test.ts b/tests/runner-e2e/chat-catalog-domains/identity.test.ts index 10fe4166583..54653ade43a 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity.test.ts @@ -123,6 +123,17 @@ describe("identity catalog independent oracle calibration", () => { const observation = valid("navigation-preferences.quick-search"); observation.state.observerPrivatePaletteHidden = false; expect(gradeNavigation(observation).passed).toBe(false); }); + it("quick search accepts any spelling of the palette shortcut and rejects answers without it", () => { + const observation = valid("navigation-preferences.quick-search"); + for (const reply of ["Press **⌘K** (or Ctrl+K on Windows) and type Launch.", "Hit `Ctrl` + `K` anywhere to open search.", "Use ⌘K, then pick Search all.", + "Open the command palette with Cmd/Ctrl-K and type \"Launch\".", "Command-K opens quick search from any page."]) { + observation.reply = reply; expect(gradeNavigation(observation).passed, reply).toBe(true); + } + for (const reply of ["Keep it in chat: just ask me to show Launch tasks and I'll list them here.", "Use the command palette to find Launch tasks.", + "Use the task list command to filter by the Launch project.", "Press Ctrl+Shift+F to search."]) { + observation.reply = reply; expect(gradeNavigation(observation).passed, reply).toBe(false); + } + }); it("requires native provider question identity and answer-bearing creation provenance", () => { const o = valid("questions-and-approvals.necessary-question"); const original = o.state.original as Interaction; o.runs = [o.runs[0]!]; original.payload.runtimeRequestId = "request-7"; original.idempotencyKey = "paperclip-runner-question:run:request-7";