From 3e1c87117ea2ff31759d9c67d6e5b796a9dad8d1 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 28 Oct 2026 11:03:18 -0500 Subject: [PATCH] fix(evals): seed active prerequisite tasks with valid owners --- .../internal-inbox-triage.ts | 12 +++++++-- .../internal-seed-assignees.test.ts | 26 +++++++++++++++++++ .../internal-task-inventory.ts | 16 ++++++++---- 3 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/internal-seed-assignees.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/internal-inbox-triage.ts b/tests/runner-e2e/chat-catalog-domains/internal-inbox-triage.ts index ffd0a561f63..8927b583464 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-inbox-triage.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-inbox-triage.ts @@ -89,6 +89,14 @@ export function gradeInboxTriage(value: unknown) { } else c.add("implemented-scenario", false, "This oracle does not implement this scenario."); return c.result(); } +export function inboxTriageControlTaskSeeds(personId: string) { + return [ + { title: "Cancelled experiment", status: "cancelled" }, + { title: "Active launch work", status: "in_progress", assigneeUserId: personId }, + { title: "Launch review", status: "in_review" }, + { title: "Unanswered launch decision", status: "backlog" }, + ]; +} function triageCase(scenarioId: string): ChatCatalogCase { let other: Awaited> | undefined; let observe: (ctx: ChatCatalogContext, state: Record) => Promise; @@ -103,8 +111,8 @@ function triageCase(scenarioId: string): ChatCatalogCase { const p = await project(ctx); const originalTasks: Entity[] = []; for (let index = 0; index < (scenarioId === "inbox-search.archive-done" ? 501 : 3); index++) originalTasks.push(await task(ctx, `Done launch preparation ${String(index + 1).padStart(3, "0")}`, p.id, { status: "done" })); - for (const [title, status] of [["Cancelled experiment", "cancelled"], ["Active launch work", "in_progress"], ["Launch review", "in_review"], ["Unanswered launch decision", "backlog"]]) - originalTasks.push(await task(ctx, title!, p.id, { status })); + for (const { title, ...fields } of inboxTriageControlTaskSeeds(session.user.id)) + originalTasks.push(await task(ctx, title, p.id, fields)); const unanswered = originalTasks.at(-1)!; const question = await ctx.input.api.post(`/api/issues/${unanswered.id}/interactions`, createIssueThreadInteractionSchema.parse({ kind: "ask_user_questions", title: "Unanswered launch decision", continuationPolicy: "none", resolverPolicy: "human_only", addresseeUserId: session.user.id, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-seed-assignees.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-seed-assignees.test.ts new file mode 100644 index 00000000000..9ca3df668c9 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-seed-assignees.test.ts @@ -0,0 +1,26 @@ +import { expect, it } from "vitest"; +import { createIssueSchema } from "../../../packages/shared/src/validators/issue.js"; +import { inboxTriageControlTaskSeeds } from "./internal-inbox-triage.js"; +import { openInventoryTaskSeed } from "./internal-task-inventory.js"; +const projectId = "11111111-1111-4111-8111-111111111111"; +const workerId = "22222222-2222-4222-8222-222222222222"; +const secondWorkerId = "33333333-3333-4333-8333-333333333333"; +const personId = "real-authenticated-fixture-person"; +it("gives the retained active inbox control its real human owner without changing statuses", () => { + const seeds = inboxTriageControlTaskSeeds(personId).map(seed => createIssueSchema.parse({ ...seed, projectId })); + expect(seeds.map(seed => seed.status)).toEqual(["cancelled", "in_progress", "in_review", "backlog"]); + const active = seeds.find(seed => seed.status === "in_progress")!; + expect(active.assigneeUserId).toBe(personId); + expect(active.assigneeAgentId).toBeUndefined(); + expect(seeds.filter(seed => seed.status !== "in_progress").every(seed => !seed.assigneeAgentId && !seed.assigneeUserId)).toBe(true); +}); +it("keeps all 501 inventory rows valid with both owners and genuine unassigned Todo work", () => { + const seeds = Array.from({ length: 501 }, (_, index) => createIssueSchema.parse({ ...openInventoryTaskSeed(index, workerId, secondWorkerId), projectId })); + expect(new Set(seeds.map(seed => seed.title)).size).toBe(501); + expect(seeds.filter(seed => seed.status === "in_progress").every(seed => [workerId, secondWorkerId].includes(seed.assigneeAgentId!))).toBe(true); + expect(seeds.filter(seed => !seed.assigneeAgentId).length).toBe(167); + expect(seeds.some(seed => !seed.assigneeAgentId && seed.status === "todo")).toBe(true); + expect(seeds.some(seed => !seed.assigneeAgentId && seed.status === "blocked")).toBe(true); + for (const owner of [workerId, secondWorkerId]) for (const status of ["blocked", "in_progress", "todo"]) + expect(seeds.some(seed => seed.assigneeAgentId === owner && seed.status === status)).toBe(true); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-task-inventory.ts b/tests/runner-e2e/chat-catalog-domains/internal-task-inventory.ts index 32085c44ff5..48ce716c95b 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-task-inventory.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-task-inventory.ts @@ -41,6 +41,15 @@ export function gradeOpenInventory(value: unknown) { c.add("read-only-audit", !o.activity.some(row => row.agentId === o.leadId && row.action !== "issue.read_marked"), "Organizing the answer creates no control-plane work changes."); return c.result(); } +export function openInventoryTaskSeed(index: number, workerId: string, secondWorkerId: string) { + const owner = index % 3 === 0 ? workerId : index % 3 === 1 ? secondWorkerId : null; + return { + title: `Launch work ${String(index + 1).padStart(3, "0")}`, + status: index % 7 === 0 ? "blocked" : index % 2 === 0 && owner ? "in_progress" : "todo", + assigneeAgentId: owner, + description: "Existing launch work. This inventory fixture records current state and must not trigger execution.", + }; +} function openInventory(): ChatCatalogCase { const base = makeDefinition("tasks.organize-open-work", async ctx => { const p = await project(ctx); const other = await project(ctx, "Experiments"); const worker = await fixtureWorker(ctx); @@ -55,11 +64,8 @@ function openInventory(): ChatCatalogCase { } const originalTasks: Entity[] = []; for (let index = 0; index < 501; index++) { - const owner = index % 3 === 0 ? worker.id : index % 3 === 1 ? secondWorker.id : null; - originalTasks.push(await task(ctx, `Launch work ${String(index + 1).padStart(3, "0")}`, p.id, { - status: index % 7 === 0 ? "blocked" : index % 2 === 0 ? "in_progress" : "todo", assigneeAgentId: owner, - description: "Existing launch work. This inventory fixture records current state and must not trigger execution.", - })); + const { title, ...fields } = openInventoryTaskSeed(index, worker.id, secondWorker.id); + originalTasks.push(await task(ctx, title, p.id, fields)); } originalTasks.push(await task(ctx, "Completed launch work", p.id, { status: "done" }), await task(ctx, "Cancelled launch work", p.id, { status: "cancelled" }), await task(ctx, "Unrelated experiment", other.id));