From 330003877a7a8d6334efcfebf2b0099f4b2662d1 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 21:41:53 -0600 Subject: [PATCH] test(evals): seed a non-periodic inventory and close two open-work grading gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 42442027154 answered with arithmetic rules ('Morgan — 167: RUN-1, 4, 7 … 499, every 3rd task') instead of listing tasks. The rule was accurate only because the fixture assigned owners and statuses by index % 3 and % 7, which no real inventory allows, and the person got no task links. Owners are now an even 167/167/167 split over a deterministic shuffled order. Statuses use a hashed roll. The grader then passed that answer's partial path because the list element '4' in 'RUN-1, 4, 7' matched the stated shown count. Counts that are comma-list elements no longer count. The terminal-work check rejected 'Non-open tasks excluded: RUN-502 (done)'. A closed task may now be named where its line states its actual closed status. Presenting it as open work still fails. verifySetup's inventory-settled check now requires a recorded inventory, and its contract tests mock the untraced inventory read. Bumps oracleVersion to organize-open-work.v4. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../internal-task-inventory.ts | 38 +++++++++++++++---- .../chat-catalog-domains/internal.test.ts | 12 ++++++ .../task-prerequisite-contracts.test.ts | 11 ++++-- 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index f713b1a8acb..df9fa46016e 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5882,7 +5882,7 @@ "chat-scenarios/tasks.yaml", "chat-catalog-domains/internal-task-inventory.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.organize-open-work.v3", + "oracleVersion": "paperclip.chat-catalog.internal.organize-open-work.v4", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-task-inventory.ts b/tests/runner-e2e/chat-catalog-domains/internal-task-inventory.ts index 475d6b7642f..1d2afcd8b19 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-task-inventory.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-task-inventory.ts @@ -47,8 +47,8 @@ function presentations(o: InventoryObservation, row: Entity) { }); return found; } -/** A stated count, not digits inside a task identifier such as RUN-491. */ -const statesCount = (text: string, count: number) => new RegExp(`(? new RegExp(`(? 0 && statesCount(o.reply, shown.length) && statesCount(o.reply, open.length)), "Every open task is presented (in chat or a deliverable linked from the reply), or the reply states its actual shown count and the full open count."); - c.add("terminal-and-other-project-excluded", excluded.every(row => !presentations(o, row).length), "Completed, cancelled and other-project work is excluded."); + // Naming a closed task is fine only where its line states that it is closed (its actual status), never as open work. + const statesClosed = (row: Entity, at: { text: string; at: number }) => { + const lineStart = at.text.lastIndexOf("\n", at.at) + 1; const lineEnd = at.text.indexOf("\n", at.at); + const line = [row.id, row.identifier].filter((token): token is string => !!token) + .reduce((text, token) => text.split(token).join(" "), at.text.slice(lineStart, lineEnd < 0 ? undefined : lineEnd).replace(/\]\([^)]*\)/g, "]")); + return row.status === "done" ? /\b(?:done|completed?)\b/i.test(line) : row.status === "cancelled" && /\bcancell?ed\b/i.test(line); + }; + c.add("terminal-and-other-project-excluded", excluded.every(row => presentations(o, row).every(at => row.projectId === p.id && statesClosed(row, at))), + "Completed, cancelled and other-project work is not presented as open work."); const owners = o.state.ownerLabels as Record; const named = Object.entries(owners ?? {}).filter(([key]) => key !== "unassigned").map(([, label]) => label.toLowerCase()); let correctlyGrouped = !!owners; @@ -126,11 +134,25 @@ async function settledInventory(ctx: ChatCatalogContext, ids: readonly string[], if (last.some(row => !row)) throw new Error("Seeded inventory rows are missing"); return last; } +/** Deterministic integer mixing, so owners and statuses follow no arithmetic pattern. */ +function inventoryMix(value: number) { + let x = Math.imul(value + 0x9e3779b9, 0x85ebca6b) >>> 0; + x = Math.imul(x ^ (x >>> 13), 0xc2b2ae35) >>> 0; + return (x ^ (x >>> 16)) >>> 0; +} +// Real inventories cannot be summarized as "every 3rd task"; an even, shuffled +// owner split keeps exactly 167 rows per group without a periodic rule. +const INVENTORY_OWNER_RANK = (() => { + const order = Array.from({ length: 501 }, (_, index) => index).sort((a, b) => inventoryMix(a) - inventoryMix(b) || a - b); + const rank = new Array(501); order.forEach((index, position) => { rank[index] = position; }); return rank; +})(); export function openInventoryTaskSeed(index: number, workerId: string, secondWorkerId: string) { - const owner = index % 3 === 0 ? workerId : index % 3 === 1 ? secondWorkerId : null; + const slot = (INVENTORY_OWNER_RANK[index] ?? index) % 3; + const owner = slot === 0 ? workerId : slot === 1 ? secondWorkerId : null; + const roll = inventoryMix(index + 1_000); return { title: `Launch work ${String(index + 1).padStart(3, "0")}`, - status: index % 7 === 0 ? "blocked" : index % 2 === 0 && owner ? "in_progress" : "todo", + status: roll % 7 === 0 ? "blocked" : roll % 2 === 0 && owner ? "in_progress" : "todo", assigneeAgentId: owner, description: "Existing launch work. This inventory fixture records current state and must not trigger execution.", }; @@ -170,7 +192,7 @@ function openInventory(): ChatCatalogCase { if ((await ctx.allRuns()).length) throw new Error("Seeding the paused-owner inventory unexpectedly admitted work"); return { project: p, originalTasks, workerIds: [worker.id, secondWorker.id], ownerPauseReceipts, ownerLabels: { [worker.id]: "Morgan", [secondWorker.id]: "Riley", unassigned: "Unassigned" } }; }, gradeOpenInventory); - return { ...base, observe: observeOpenInventory, oracleVersion: "paperclip.chat-catalog.internal.organize-open-work.v3", + return { ...base, observe: observeOpenInventory, oracleVersion: "paperclip.chat-catalog.internal.organize-open-work.v4", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-task-inventory.ts"], attemptTimeoutMs: 600_000, setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["inventory-owners-paused", "lead-ready", "inventory-settled"] }, async verifySetup(ctx, state) { @@ -184,8 +206,8 @@ function openInventory(): ChatCatalogCase { "Both actual owners were ready before manual pause and retain their exact completed paused lifecycle, version and manual hold; no provider work is admitted."); c.add("lead-ready", lead.lifecycleState === "ready" && ["idle", "active"].includes(lead.status ?? ""), "The CEO remains ready for the canonical inventory query."); const current = await ctx.input.api.getRunObservation(`${companyPath(ctx)}/issues?includeAgentCreatedTasks=true&limit=1000`); - const originals = state.originalTasks as Entity[]; - c.add("inventory-settled", originals.every(row => sameTask(row, current.find(now => now.id === row.id))), + const originals = Array.isArray(state.originalTasks) ? state.originalTasks as Entity[] : []; + c.add("inventory-settled", originals.length > 0 && originals.every(row => sameTask(row, current.find(now => now.id === row.id))), "The recorded inventory is still the current task state when the query is sent."); return c.result(); }, diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index fe6dcd74ba9..99a2c6a4623 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -634,6 +634,18 @@ describe("multi-page open inventory calibration", () => { o.reply = "All 501 open tasks:\n### Unassigned\n" + kept.map(row => row.identifier).join(", "); expect(gradeInternal(o).passed).toBe(false); }); + it("accepts naming a closed task only where its line states it is closed", () => { + const o = valid("tasks.organize-open-work"); const done = { id: "done-task", projectId: "p", title: "Shipped", status: "done", assigneeAgentId: null }; + o.tasks.push(done); o.state.originalTasks = structuredClone(o.tasks); o.links.push("/ACORN/issues/done-task"); + const excluded = { ...o, reply: o.reply + "\n\nExcluded closed work: [Shipped](/ACORN/issues/done-task) (done)." }; + expect(gradeInternal(excluded).passed).toBe(true); + expect(gradeInternal({ ...o, reply: o.reply + "\n- [Shipped](/ACORN/issues/done-task)" }).passed).toBe(false); + }); + it("does not read an abbreviated identifier list as a stated shown count", () => { + const o = valid("tasks.organize-open-work"); o.links = o.links.slice(0, 4); + o.reply = "501 open tasks.\n## Unassigned\n" + o.tasks.slice(0, 4).map(row => `[${row.title}](/ACORN/issues/${row.id})`).join(", ") + " … every 3rd task: 1, 4, 7"; + expect(gradeInternal(o).passed).toBe(false); + }); it("rejects a blocked item placed after an ordinary item for the same owner", () => { const o = valid("tasks.organize-open-work"); o.reply = "## Unassigned\n" + [...o.tasks.slice(2), ...o.tasks.slice(0, 2)].map(row => `- [${row.title}](/ACORN/issues/${row.id})`).join("\n"); diff --git a/tests/runner-e2e/chat-catalog-domains/task-prerequisite-contracts.test.ts b/tests/runner-e2e/chat-catalog-domains/task-prerequisite-contracts.test.ts index 3e3c60f78b7..821c11c3cfb 100644 --- a/tests/runner-e2e/chat-catalog-domains/task-prerequisite-contracts.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/task-prerequisite-contracts.test.ts @@ -15,9 +15,12 @@ beforeEach(() => { vi.stubGlobal("fetch", () => { throw new Error("No real netwo afterEach(() => { vi.useRealTimers(); vi.unstubAllGlobals(); mock.gate.mockReset(); }); it("actual inventory preflight accepts real completed manual pauses instead of impossible ready-plus-paused lifecycle", async () => { const first=pause(),second=pause(owner(secondId)),rows=[first,second]; - const ctx={ input:{fixtures:{company:{id:companyId},agent:{id:leadId}},api:{get:async(path:string)=>path.endsWith(leadId)?owner(leadId):rows.find(x=>path.endsWith(x.before.id))!.settled}}} as unknown as ChatCatalogContext; - const grade=await inventory.taskInventoryCases[0]!.verifySetup!(ctx,{workerIds:[firstId,secondId],ownerPauseReceipts:rows.map(({before,requested})=>({before,requested}))}); - expect(grade.passed).toBe(true); + const ctx={ input:{fixtures:{company:{id:companyId},agent:{id:leadId}},api:{get:async(path:string)=>path.endsWith(leadId)?owner(leadId):rows.find(x=>path.endsWith(x.before.id))!.settled,getRunObservation:async()=>tasks}}} as unknown as ChatCatalogContext; + const tasks=[{id:"t1",companyId,title:"Launch work 001",description:"d",projectId:"p",status:"blocked",priority:"medium",assigneeAgentId:firstId,goalId:null}]; + const state={workerIds:[firstId,secondId],ownerPauseReceipts:rows.map(({before,requested})=>({before,requested})),originalTasks:structuredClone(tasks)}; + expect((await inventory.taskInventoryCases[0]!.verifySetup!(ctx,state)).passed).toBe(true); + tasks[0]!.status="todo"; + expect((await inventory.taskInventoryCases[0]!.verifySetup!(ctx,state)).checks.find(check=>check.id==="inventory-settled")?.passed).toBe(false); }); function passesBoardGuard(headers:Record) { let passed=false;let status=200;const req={method:"POST",actor:{type:"board",source:"session"},header:(k:string)=>k==="host"?"127.0.0.1:3218":headers[k]??headers[k[0]!.toUpperCase()+k.slice(1)],socket:{},app:{get:()=>undefined}} as unknown as Request;const res={status:(value:number)=>{status=value;return res;},json:()=>{}} as unknown as Response;boardMutationGuard()(req,res,()=>{passed=true;});return{passed,status}; } it("actual delegation upload uses the owned browser Origin and reaches the unchanged authenticated board guard", async () => { @@ -52,7 +55,7 @@ it("preserves all501 open seeds, both paused owners, genuine unassigned work and expect(rows.some(r=>r.status==="blocked")).toBe(true);expect(rows.some(r=>!r.assigneeAgentId&&r.status==="in_progress")).toBe(false); }); it("requires distinct actual paused owners in the real preflight", async () => { - const r=pause();const ctx={input:{fixtures:{company:{id:companyId},agent:{id:leadId}},api:{get:async(p:string)=>p.endsWith(leadId)?owner(leadId):r.settled}}} as unknown as ChatCatalogContext; + const r=pause();const ctx={input:{fixtures:{company:{id:companyId},agent:{id:leadId}},api:{get:async(p:string)=>p.endsWith(leadId)?owner(leadId):r.settled,getRunObservation:async()=>[]}}} as unknown as ChatCatalogContext; const grade=await inventory.taskInventoryCases[0]!.verifySetup!(ctx,{workerIds:[firstId,firstId],ownerPauseReceipts:[r,r]});expect(grade.passed).toBe(false); }); it.each(["absent", "foreign"])("the actual unchanged board guard rejects %s Origin", kind => {