From 7d700f43e93e89e4c196f8799b0aa3cef41d20da Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 6 Oct 2026 13:14:18 -0500 Subject: [PATCH] fix(evals): create everyday tasks through current composer Reuse response-based task identity, preserve the exact prompt and chosen project/assignee, and calibrate the browser setup with paused agents. Co-Authored-By: Paperclip --- tests/e2e/plan-task-fixture.spec.ts | 38 ++++++++++++++++++++++++++++- tests/runner-e2e/everyday-flow.ts | 19 ++++----------- tests/runner-e2e/plan-task-ui.ts | 17 +++++++++++-- 3 files changed, 57 insertions(+), 17 deletions(-) diff --git a/tests/e2e/plan-task-fixture.spec.ts b/tests/e2e/plan-task-fixture.spec.ts index 1dec59fcc4..9635b0d5ac 100644 --- a/tests/e2e/plan-task-fixture.spec.ts +++ b/tests/e2e/plan-task-fixture.spec.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import { expect, test } from "@playwright/test"; -import { createPlanTaskThroughUi } from "../runner-e2e/plan-task-ui.js"; +import { createPlanTaskThroughUi, createTaskFromPromptThroughUi } from "../runner-e2e/plan-task-ui.js"; test("planning fixture creates the exact assignment through the current composer without providers", async ({ page, request }) => { const companyResponse = await request.post("/api/companies", { data: { name: `Planning setup ${randomUUID()}` } }); @@ -30,3 +30,39 @@ test("planning fixture creates the exact assignment through the current composer expect((await request.delete(`/api/companies/${company.id}`)).ok()).toBe(true); } }); + + +test("everyday fixture preserves prompt, selected project and assignee without providers", async ({ page, request }) => { + const companyResponse = await request.post("/api/companies", { data: { name: `Everyday setup ${randomUUID()}` } }); + expect(companyResponse.ok()).toBe(true); + const company = await companyResponse.json(); + try { + let lead: { id: string; name: string } | undefined; + for (const name of ["A Different Owner", "Everyday Fixture Lead"]) { + const response = await request.post(`/api/companies/${company.id}/agents`, { data: { + name, role: "general", adapterType: "process", adapterConfig: { command: "false" }, + runtimeConfig: { heartbeat: { enabled: false } }, + } }); + expect(response.ok()).toBe(true); + lead = await response.json(); + expect((await request.post(`/api/agents/${lead!.id}/pause`)).ok()).toBe(true); + } + let project: { id: string; name: string } | undefined; + for (const name of ["A Different Project", "Everyday Fixture Project"]) { + const response = await request.post(`/api/companies/${company.id}/projects`, { data: { name } }); + expect(response.ok()).toBe(true); + project = await response.json(); + } + const prompt = "Delegate the implementation to an engineer, review its delivered files, and finish after the revised result is ready."; + const created = await createTaskFromPromptThroughUi({ page, companyId: company.id, issuePrefix: company.issuePrefix, + agentName: lead!.name, projectName: project!.name, prompt }); + const issue = await (await request.get(`/api/issues/${created.id}`)).json(); + expect(issue).toMatchObject({ id: created.id, companyId: company.id, assigneeAgentId: lead!.id, + projectId: project!.id, description: prompt, workMode: "standard" }); + const runs = await request.get(`/api/companies/${company.id}/heartbeat-runs?limit=100`); + expect(runs.ok()).toBe(true); + expect(await runs.json()).toEqual([]); + } finally { + expect((await request.delete(`/api/companies/${company.id}`)).ok()).toBe(true); + } +}); diff --git a/tests/runner-e2e/everyday-flow.ts b/tests/runner-e2e/everyday-flow.ts index b9f23d1f47..22f9734815 100644 --- a/tests/runner-e2e/everyday-flow.ts +++ b/tests/runner-e2e/everyday-flow.ts @@ -13,7 +13,8 @@ import { } from "./everyday-delivery.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; -import { createTaskThroughUi, submitTaskReply } from "./user-actions.js"; +import { submitTaskReply } from "./user-actions.js"; +import { createTaskFromPromptThroughUi } from "./plan-task-ui.js"; import { waitForTaskChatRendered } from "./continuation-screenshot.js"; import { hasPersistedSource, isSavedSourceCheckpoint } from "./everyday-interruption.js"; import { setupAggregatorFixture } from "./aggregator-fixture.js"; @@ -667,25 +668,15 @@ export async function runEverydayFlow(input: Input) { if (state.connections.length) throw new Error("New-connection story requires an unconnected company"); } - await createTaskThroughUi({ + const created = await createTaskFromPromptThroughUi({ page, + companyId: fixtures.company.id, issuePrefix: prefix, agentName: fixtures.agent.name, - title: execution.task.buildTitle(nonce), prompt: ev.prompt, - workMode: "standard", projectName: project?.name, }); - parent = await pollUntil({ - label: "browser-created story task", - deadlineAt: Date.now() + 30_000, - load: () => - api.get(`/api/companies/${fixtures.company.id}/issues`), - accept: (rows) => - rows.some((i) => i.title === execution.task.buildTitle(nonce)), - }).then((rows) => - rows.find((i) => i.title === execution.task.buildTitle(nonce))!, - ); + parent = await api.get(`/api/issues/${created.id}`); input.observe(parent!, []); note("task-submitted", { issueId: parent!.id }); await openParent(); diff --git a/tests/runner-e2e/plan-task-ui.ts b/tests/runner-e2e/plan-task-ui.ts index b1e65bff16..0b8aa54803 100644 --- a/tests/runner-e2e/plan-task-ui.ts +++ b/tests/runner-e2e/plan-task-ui.ts @@ -3,8 +3,8 @@ import { expect, type Page } from "@playwright/test"; /** The current composer creates a task from its description and assigns its * title later. Capture the actual public create response; never guess its ID * from a title or mutate the issue behind the browser. */ -export async function createPlanTaskThroughUi(input: { - page: Page; companyId: string; issuePrefix: string; agentName: string; prompt: string; +export async function createTaskFromPromptThroughUi(input: { + page: Page; companyId: string; issuePrefix: string; agentName: string; prompt: string; projectName?: string; }): Promise<{ id: string }> { const { page } = input; await page.goto(`/${encodeURIComponent(input.issuePrefix)}/issues`, { waitUntil: "domcontentloaded" }); @@ -18,6 +18,14 @@ export async function createPlanTaskThroughUi(input: { await expect(option).toHaveCount(1); await option.click(); await expect(assignee).toContainText(input.agentName); + if (input.projectName) { + const project = dialog.getByRole("group", { name: "Task project and worktrees", exact: true }).getByRole("button").first(); + // Focus opens this selector without toggling an already-open popover closed. + await project.focus(); + await page.getByPlaceholder("Search projects...", { exact: true }).fill(input.projectName); + await page.getByRole("button", { name: input.projectName, exact: true }).click(); + await expect(project).toContainText(input.projectName); + } const responsePromise = page.waitForResponse(response => response.request().method() === "POST" && new URL(response.url()).pathname === `/api/companies/${input.companyId}/issues`); await dialog.getByRole("button", { name: "Create task", exact: true }).click(); @@ -25,5 +33,10 @@ export async function createPlanTaskThroughUi(input: { if (!response.ok()) throw new Error(`Browser task creation failed: ${response.status()}`); const issue = await response.json(); if (typeof issue.id !== "string" || issue.companyId !== input.companyId) throw new Error("Browser created an unexpected issue identity"); + if (issue.description !== input.prompt || issue.workMode !== "standard") { + throw new Error("Browser task creation changed the requested prompt or work mode"); + } return { id: issue.id }; } + +export const createPlanTaskThroughUi = createTaskFromPromptThroughUi;