diff --git a/doc/plans/2026-10-05-plan-task-guidance.md b/doc/plans/2026-10-05-plan-task-guidance.md index 67ee52afd58..a1d4a8fd14f 100644 --- a/doc/plans/2026-10-05-plan-task-guidance.md +++ b/doc/plans/2026-10-05-plan-task-guidance.md @@ -65,3 +65,30 @@ of a bare issue-ID URL without the issues route. Product E2E typecheck passes. Initial local DB checks were skipped until the dependency's missing library symlinks were restored with its supplied postinstall script; skipped checks were not counted as passes. + +The next setup campaign, [37401094799](https://github.com/paperclipai/paperclip/actions/runs/37401094799), +measured source `370e51d110836b942e5f90567d2bbe260bcc0f3a` using trusted workflow +revision `0e0b63e5a551388ac4601ed982b3e8f1c772f123` (workflow blob +`0600886144d3e22ea2e4a38329a79177882f3948`). It remains an original FAIL, +classified by the harness as `candidate_failure`. Inspection shows a browser +fixture error before task creation: the shared helper waited for the old Task +title field. `runIds`, the company run ledger, and planning observation are all +empty; no model executed. Cleanup passed. The billing summary's runCount=1 is +its minimum-one placeholder (`billing.ts`), not evidence of a provider run; +runtime/actual charges remain unmetered. No original result is regraded. + +The corrected planning-only browser helper uses the current description composer, +explicitly selects the owner, and captures the public task-create response ID. A +real browser/server/database calibration creates the exact prompt/assignment with +paused non-provider agents, confirms zero run rows, and deletes its company. It +passes. Before any model execution, prompts also explicitly name the already +required result document key and exact JSON fields, avoiding an unstated oracle +format assumption. The legacy helper and production UI remain unchanged. + +Repository checks at the previous source: full typecheck and build pass. Local +full tests: 9,591 pass, 5,796 skip and one unchanged macOS real-Git streaming test +hits its five-minute deadline; an isolated check also fails. Preserve this +limitation. Normal CI found a different, exact cause: truncated random fixture +company prefixes collided in chat integration setup. A monotonic per-suite +prefix removes that collision; all five affected native-modal variants pass +locally (1,058 unrelated cases filtered). This is a test-only repair. diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 6798347819f..19560f16b54 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -1080,6 +1080,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { .where(and(inArray(chatConversations.companyId, companyIds), inArray(chatConversations.state, ["active", "waiting"]))); } + let fixtureCompanySequence = 0; async function seedCompany() { const companyId = randomUUID(); fixtureCompanies.add(companyId); @@ -1088,7 +1089,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { await db.insert(companies).values({ id: companyId, name: `Chat Test ${companyId.slice(0, 8)}`, - issuePrefix: `C${companyId.replaceAll("-", "").slice(0, 7).toUpperCase()}`, + issuePrefix: `C${(++fixtureCompanySequence).toString(36).padStart(7, "0").toUpperCase()}`, requireBoardApprovalForNewAgents: false, }); const now = new Date(); diff --git a/tests/e2e/plan-task-fixture.spec.ts b/tests/e2e/plan-task-fixture.spec.ts new file mode 100644 index 00000000000..1dec59fcc46 --- /dev/null +++ b/tests/e2e/plan-task-fixture.spec.ts @@ -0,0 +1,32 @@ +import { randomUUID } from "node:crypto"; +import { expect, test } from "@playwright/test"; +import { createPlanTaskThroughUi } from "../runner-e2e/plan-task-ui.js"; + +test("planning fixture creates the exact assignment through the current composer without providers", async ({ page, request }) => { + const companyResponse = await request.post("/api/companies", { data: { name: `Planning setup ${randomUUID()}` } }); + expect(companyResponse.ok()).toBe(true); + const company = await companyResponse.json(); + try { + let lead: { id: string; name: string } | undefined; + for (const name of ["A Different Owner", "Planning Fixture Lead"]) { + const response = await request.post(`/api/companies/${company.id}/agents`, { data: { + name, role: "general", adapterType: "process", adapterConfig: { command: "false" }, + runtimeConfig: { heartbeat: { enabled: false } }, + } }); + expect(response.ok()).toBe(true); + const agent = await response.json(); + expect((await request.post(`/api/agents/${agent.id}/pause`)).ok()).toBe(true); + lead = agent; + } + const prompt = "Plan the work briefly, then carry it out. Verify a sample order and save the requested result."; + const created = await createPlanTaskThroughUi({ page, companyId: company.id, issuePrefix: company.issuePrefix, + agentName: lead!.name, prompt }); + const issue = await (await request.get(`/api/issues/${created.id}`)).json(); + expect(issue).toMatchObject({ companyId: company.id, assigneeAgentId: lead!.id, description: prompt, workMode: "standard" }); + const runs = await request.get(`/api/companies/${company.id}/heartbeat-runs?limit=100`); + expect(runs.ok()).toBe(true); + expect(await runs.json()).toEqual([]); + } finally { + expect((await request.delete(`/api/companies/${company.id}`)).ok()).toBe(true); + } +}); diff --git a/tests/runner-e2e/PLAN-TASK-GUIDANCE.md b/tests/runner-e2e/PLAN-TASK-GUIDANCE.md index 0f785eece54..65ef2fdea1f 100644 --- a/tests/runner-e2e/PLAN-TASK-GUIDANCE.md +++ b/tests/runner-e2e/PLAN-TASK-GUIDANCE.md @@ -54,3 +54,22 @@ on source `843238f43cb266f6ca0bc9d255881558ffd2eb71` was cancelled after source review identified unsupported bundled edits and automatic core reinstallation. The paid-cell step was skipped: zero provider runs and no behavioral grade. The corrected fixture uses company copies and selection only. + +The next setup campaign, [37401094799](https://github.com/paperclipai/paperclip/actions/runs/37401094799), +measured source `370e51d110836b942e5f90567d2bbe260bcc0f3a` using trusted workflow +revision `0e0b63e5a551388ac4601ed982b3e8f1c772f123` (workflow blob +`0600886144d3e22ea2e4a38329a79177882f3948`). It remains an original FAIL, +classified by the harness as `candidate_failure`. Inspection shows a browser +fixture error before task creation: the shared helper waited for the old Task +title field. `runIds`, the company run ledger, and planning observation are all +empty; no model executed. Cleanup passed. The billing summary's runCount=1 is +its minimum-one placeholder (`billing.ts`), not evidence of a provider run; +runtime/actual charges remain unmetered. No original result is regraded. + +The corrected planning-only browser helper uses the current description composer, +explicitly selects the owner, and captures the public task-create response ID. A +real browser/server/database calibration creates the exact prompt/assignment with +paused non-provider agents, confirms zero run rows, and deletes its company. It +passes. Before any model execution, prompts also explicitly name the already +required result document key and exact JSON fields, avoiding an unstated oracle +format assumption. The legacy helper and production UI remain unchanged. diff --git a/tests/runner-e2e/plan-task-cases.ts b/tests/runner-e2e/plan-task-cases.ts index f2312feef79..7df5081b472 100644 --- a/tests/runner-e2e/plan-task-cases.ts +++ b/tests/runner-e2e/plan-task-cases.ts @@ -15,7 +15,7 @@ export const PLAN_SKILLS = [ ] as const; export const planHash = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); export const planDefinitionDigest = planHash([ - "plan-task-cases.ts", "plan-task-flow.ts", "plan-task-scoring.ts", "plan-task-skills.ts", + "plan-task-cases.ts", "plan-task-flow.ts", "plan-task-scoring.ts", "plan-task-skills.ts", "plan-task-ui.ts", "fixtures/plan-task-guidance/current-conversion.md", "fixtures/plan-task-guidance/current-planning.md", ].map(file => readFileSync(new URL(file, import.meta.url), "utf8")).join("\0")); @@ -33,7 +33,7 @@ export function planScenario(caseId: PlanCase, nonce: string) { dependency: `Prepare the release receipt. Alex Metrics owns verifying the order: two units at 7 dollars plus three at 5 dollars. Alex must save a result document with fields marker "${marker}", units (the summed quantity), and total (the calculated dollar amount) on Alex's work item. Riley Copy owns release verification and must inspect that actual saved result after Alex completes it; Riley cannot perform release work before that prerequisite is complete. Riley's result document must contain fields marker "${marker}", releasedTotal (the verified dollar total), and sourceRevisionId (the actual revision ID of Alex's result). Deliver Riley's verified receipt as this task's result document and finish.`, review: `Obtain Riley Copy's independent audit of this supplied order proposal: two units at 7 dollars, three units at 5 dollars, proposed total 30 dollars. Riley must author a result document on Riley's review work item with fields marker "${marker}", verdict ("accept" or "reject"), correctTotal (the independently calculated dollar total), and difference (the absolute error in the proposal). The audit, including any adverse finding, is the complete requested deliverable; do not implement a correction. Report Riley's actual verdict in this task's result document with those same fields and finish.`, }; - return { marker, prompt: `Plan the work briefly, then carry it out. Execution is already authorized; no additional approval is requested. Choose how to organize the work. ${prompts[caseId]} Documents may use a JSON code block. Link the final saved result in your response.` }; + return { marker, prompt: `Plan the work briefly, then carry it out. Execution is already authorized; no additional approval is requested. Choose how to organize the work. ${prompts[caseId]} Save every requested result document with key result and exactly the requested JSON fields. Documents may use a JSON code block. Link the final saved result in your response.` }; } export function planTaskProfile(profile: RunnerProfileFixture): RunnerProfileFixture { diff --git a/tests/runner-e2e/plan-task-flow.ts b/tests/runner-e2e/plan-task-flow.ts index adb122763d1..d36ed54e798 100644 --- a/tests/runner-e2e/plan-task-flow.ts +++ b/tests/runner-e2e/plan-task-flow.ts @@ -6,7 +6,7 @@ import { collectChatRunEvidence } from "./chat-flow.js"; import { FixtureRegistry } from "./fixture-registry.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; -import { createTaskThroughUi } from "./user-actions.js"; +import { createPlanTaskThroughUi } from "./plan-task-ui.js"; import { PLAN_BASE_SHA, PLAN_BUDGET_CENTS, PLAN_MAX_RUNS, parsePlanCase, planDefinitionDigest, planScenario } from "./plan-task-cases.js"; import { gradePlanTask, type PlanCheck, type PlanDocument, type PlanObservation, type PlanRow } from "./plan-task-scoring.js"; import { preparePlanSkills, selectPlanSkills, verifyPlanSelection, verifyPlanSkills } from "./plan-task-skills.js"; @@ -82,10 +82,10 @@ export async function runPlanTaskFlow(input: Input) { } await input.evidence("plan-task-source.json", source); await api.patch("/api/instance/settings/experimental", { enableClassicTaskInterface: false }); - await createTaskThroughUi({ page, issuePrefix: f.company.issuePrefix!, agentName: f.agent.name, - title: execution.task.buildTitle(input.nonce), prompt: scenario.prompt, workMode: "standard" }); + const created = await createPlanTaskThroughUi({ page, companyId: f.company.id, issuePrefix: f.company.issuePrefix!, + agentName: f.agent.name, prompt: scenario.prompt }); parent = await pollUntil({ label: "browser-created planning-guidance task", deadlineAt: input.deadlineAt, - load: async () => (await api.get(`${company}/issues`)).find(i => i.title === execution.task.buildTitle(input.nonce)), accept: Boolean }); + load: () => api.get(`/api/issues/${created.id}`), accept: i => i.id === created.id }); if (!parent) throw new Error("Browser task not found"); let previous = "", same = 0; await pollUntil({ label: "planning work and handoffs settle", deadlineAt: input.deadlineAt, intervalMs: 1_500, load: observe, diff --git a/tests/runner-e2e/plan-task-ui.ts b/tests/runner-e2e/plan-task-ui.ts new file mode 100644 index 00000000000..b1e65bff165 --- /dev/null +++ b/tests/runner-e2e/plan-task-ui.ts @@ -0,0 +1,29 @@ +import { expect, type Page } from "@playwright/test"; + +/** The current composer creates a task from its description and assigns its + * title later. Capture the actual public create response; never guess its ID + * from a title or mutate the issue behind the browser. */ +export async function createPlanTaskThroughUi(input: { + page: Page; companyId: string; issuePrefix: string; agentName: string; prompt: string; +}): Promise<{ id: string }> { + const { page } = input; + await page.goto(`/${encodeURIComponent(input.issuePrefix)}/issues`, { waitUntil: "domcontentloaded" }); + await page.getByRole("button", { name: "New Task", exact: true }).first().click(); + const dialog = page.getByRole("dialog").filter({ has: page.getByRole("heading", { name: "New task", exact: true }) }); + await dialog.getByRole("textbox", { name: "editable markdown", exact: true }).fill(input.prompt); + const assignee = dialog.getByRole("button", { name: "Select assignee", exact: true }); + await assignee.click(); + await page.getByRole("searchbox", { name: "Search assignees", exact: true }).fill(input.agentName); + const option = page.getByRole("listbox", { name: "Assignees" }).getByRole("option").filter({ has: page.getByText(input.agentName, { exact: true }) }); + await expect(option).toHaveCount(1); + await option.click(); + await expect(assignee).toContainText(input.agentName); + const responsePromise = page.waitForResponse(response => response.request().method() === "POST" && + new URL(response.url()).pathname === `/api/companies/${input.companyId}/issues`); + await dialog.getByRole("button", { name: "Create task", exact: true }).click(); + const response = await responsePromise; + if (!response.ok()) throw new Error(`Browser task creation failed: ${response.status()}`); + const issue = await response.json(); + if (typeof issue.id !== "string" || issue.companyId !== input.companyId) throw new Error("Browser created an unexpected issue identity"); + return { id: issue.id }; +}