fix(evals): bind held UI journeys to actual controls

This commit is contained in:
Dotta committed 2026-10-27 03:20:56 -05:00
1 parent 983f15394e
commit 200e7fc3ac
4 files changed
+12 -7

No files matched your search

@@ -48,6 +48,8 @@ export const companyLifecycleIdentityCases: readonly ChatCatalogCase[] = [
projectId: p.id, assigneeUserId: session.user.id, status: "backlog" });
const members = await Promise.all([ctx.input.fixtures.company.id, other.id].map(id => ctx.input.api.get<{ members: Array<{ principalId: string }> }>(`/api/companies/${id}/members`)));
const denied = await readAgentResource(ctx, `/api/companies/${other.id}/issues`);
// Public API fixture creation must be reflected in the browser company inventory.
await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
return { otherCompany: other, foreignProject: p, foreignTask: work, foreignMarker, foreignPrefix: other.issuePrefix,
humanOwnsBoth: members.every(result => result.members.some(member => member.principalId === session.user.id)), agentForeignReadStatus: denied.status };
}, gradeCompanyLifecycle, {
@@ -48,7 +48,8 @@ export const navigationIdentityCases = [
await expect(ctx.input.page.getByRole("heading", { name: "Launch", exact: true }).first()).toBeVisible();
await ctx.input.page.getByRole("button", { name: /^Filter/ }).first().click();
for (const status of ["Backlog", "Todo", "In Progress", "In Review", "Blocked"]) {
await ctx.input.page.getByRole("checkbox", { name: status, exact: true }).setChecked(true);
const row = ctx.input.page.locator("label").filter({ has: ctx.input.page.getByText(status, { exact: true }) });
await row.getByRole("checkbox").setChecked(true);
}
await ctx.input.page.keyboard.press("Escape");
await expect(ctx.input.page.getByText(entity(state, "task").title!, { exact: true }).first()).toBeVisible();
@@ -185,8 +185,8 @@ export const questionIdentityCases = [
expectedRunCount: 2, observe: observeQuestion,
async act(ctx, state) {
await waitForReply(ctx);
const row = ctx.input.page.getByTestId("task-chat-unanswered-question").filter({ hasText: "Launch page details" }).first();
if (!(await ctx.input.page.getByTestId("task-chat-composer-takeover").isVisible())) await row.click();
const row = ctx.input.page.locator(`[id=${JSON.stringify(`interaction-${entity(state, "original").id}`)}]`);
await row.click();
const form = ctx.input.page.getByTestId("task-chat-composer-takeover");
await form.getByRole("button", { name: "Next", exact: true }).click();
state.validationObserved = await form.getByText(/required|choose|select an option/i).first().isVisible();
@@ -212,8 +212,8 @@ export const questionIdentityCases = [
async act(ctx, state) {
await waitForReply(ctx); const page = ctx.input.page;
const form = page.getByTestId("task-chat-composer-takeover");
const historical = page.getByTestId("task-chat-unanswered-question").filter({ hasText: "Launch page details" }).first();
if (!(await form.isVisible())) await historical.click();
const historical = page.locator(`[id=${JSON.stringify(`interaction-${entity(state, "original").id}`)}]`);
await historical.click();
await form.getByTestId("question-text-answer-composer").locator('[contenteditable="true"],textarea').first().fill(String(state.answer));
await form.getByRole("button", { name: /^Dismiss / }).click();
await page.reload({ waitUntil: "domcontentloaded" });
@@ -107,9 +107,11 @@ export const runtimeAgentsCases = [
const source = ctx.input.page.url();
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/agents/all`, { waitUntil: "domcontentloaded" });
await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/agents`, { waitUntil: "domcontentloaded" });
const toggle = ctx.input.page.getByRole("button", { name: "Org chart view", exact: true });
if (await toggle.isVisible()) await toggle.click();
else await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/org`, { waitUntil: "domcontentloaded" });
await expect(toggle).toBeVisible();
await toggle.click();
await expect(toggle).toHaveAttribute("aria-pressed", "true");
const morgan = ctx.input.page.locator("[data-org-card]").filter({ hasText: "Morgan" });
const riley = ctx.input.page.locator("[data-org-card]").filter({ hasText: "Riley" });
await expect(morgan).toBeVisible(); await expect(riley).toBeVisible();