From 200e7fc3ac590d17d264ddf6ea24e0b3f745e344 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 03:20:56 -0500 Subject: [PATCH] fix(evals): bind held UI journeys to actual controls --- .../chat-catalog-domains/identity-company-lifecycle.ts | 2 ++ .../identity-navigation-preferences.ts | 3 ++- .../identity-questions-and-approvals.ts | 8 ++++---- tests/runner-e2e/chat-catalog-domains/runtime-agents.ts | 6 ++++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts index da16dc98c03..bef720c6549 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts @@ -48,6 +48,8 @@ export const companyLifecycleIdentityCases: readonly ChatCatalogCase[] = [ projectId: p.id, assigneeUserId: session.user.id, status: "backlog" }); const members = await Promise.all([ctx.input.fixtures.company.id, other.id].map(id => ctx.input.api.get<{ members: Array<{ principalId: string }> }>(`/api/companies/${id}/members`))); const denied = await readAgentResource(ctx, `/api/companies/${other.id}/issues`); + // Public API fixture creation must be reflected in the browser company inventory. + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); return { otherCompany: other, foreignProject: p, foreignTask: work, foreignMarker, foreignPrefix: other.issuePrefix, humanOwnsBoth: members.every(result => result.members.some(member => member.principalId === session.user.id)), agentForeignReadStatus: denied.status }; }, gradeCompanyLifecycle, { diff --git a/tests/runner-e2e/chat-catalog-domains/identity-navigation-preferences.ts b/tests/runner-e2e/chat-catalog-domains/identity-navigation-preferences.ts index 234334f73ce..4e43434ef9d 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-navigation-preferences.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-navigation-preferences.ts @@ -48,7 +48,8 @@ export const navigationIdentityCases = [ await expect(ctx.input.page.getByRole("heading", { name: "Launch", exact: true }).first()).toBeVisible(); await ctx.input.page.getByRole("button", { name: /^Filter/ }).first().click(); for (const status of ["Backlog", "Todo", "In Progress", "In Review", "Blocked"]) { - await ctx.input.page.getByRole("checkbox", { name: status, exact: true }).setChecked(true); + const row = ctx.input.page.locator("label").filter({ has: ctx.input.page.getByText(status, { exact: true }) }); + await row.getByRole("checkbox").setChecked(true); } await ctx.input.page.keyboard.press("Escape"); await expect(ctx.input.page.getByText(entity(state, "task").title!, { exact: true }).first()).toBeVisible(); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts index 649cebb46c2..e67fddcf012 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts @@ -185,8 +185,8 @@ export const questionIdentityCases = [ expectedRunCount: 2, observe: observeQuestion, async act(ctx, state) { await waitForReply(ctx); - const row = ctx.input.page.getByTestId("task-chat-unanswered-question").filter({ hasText: "Launch page details" }).first(); - if (!(await ctx.input.page.getByTestId("task-chat-composer-takeover").isVisible())) await row.click(); + const row = ctx.input.page.locator(`[id=${JSON.stringify(`interaction-${entity(state, "original").id}`)}]`); + await row.click(); const form = ctx.input.page.getByTestId("task-chat-composer-takeover"); await form.getByRole("button", { name: "Next", exact: true }).click(); state.validationObserved = await form.getByText(/required|choose|select an option/i).first().isVisible(); @@ -212,8 +212,8 @@ export const questionIdentityCases = [ async act(ctx, state) { await waitForReply(ctx); const page = ctx.input.page; const form = page.getByTestId("task-chat-composer-takeover"); - const historical = page.getByTestId("task-chat-unanswered-question").filter({ hasText: "Launch page details" }).first(); - if (!(await form.isVisible())) await historical.click(); + const historical = page.locator(`[id=${JSON.stringify(`interaction-${entity(state, "original").id}`)}]`); + await historical.click(); await form.getByTestId("question-text-answer-composer").locator('[contenteditable="true"],textarea').first().fill(String(state.answer)); await form.getByRole("button", { name: /^Dismiss / }).click(); await page.reload({ waitUntil: "domcontentloaded" }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts index 5f521477e7d..1c4f50f5a02 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts @@ -107,9 +107,11 @@ export const runtimeAgentsCases = [ const source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/agents/all`, { waitUntil: "domcontentloaded" }); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/agents`, { waitUntil: "domcontentloaded" }); const toggle = ctx.input.page.getByRole("button", { name: "Org chart view", exact: true }); - if (await toggle.isVisible()) await toggle.click(); - else await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/org`, { waitUntil: "domcontentloaded" }); + await expect(toggle).toBeVisible(); + await toggle.click(); + await expect(toggle).toHaveAttribute("aria-pressed", "true"); const morgan = ctx.input.page.locator("[data-org-card]").filter({ hasText: "Morgan" }); const riley = ctx.input.page.locator("[data-org-card]").filter({ hasText: "Riley" }); await expect(morgan).toBeVisible(); await expect(riley).toBeVisible();