test(evals): treat a bold-labelled line as its own owner group

Run 42443802148 wrote '**Unassigned — 18 blocked:** RUN-18, …' directly
below '**Morgan — 166 blocked:** …'. The grouping check joined each item's
line with the nearest earlier label, so those unassigned items were
judged to sit under Morgan. A line that opens with its own label is now
its own group. Bumps oracleVersion to organize-open-work.v5.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-08 02:50:39 -06:00
1 parent 330003877a
commit 8524f209ab
3 files changed
+14 -3

No files matched your search

@@ -5882,7 +5882,7 @@
"chat-scenarios/tasks.yaml",
"chat-catalog-domains/internal-task-inventory.ts"
],
"oracleVersion": "paperclip.chat-catalog.internal.organize-open-work.v4",
"oracleVersion": "paperclip.chat-catalog.internal.organize-open-work.v5",
"coverageGaps": [],
"expectedRunCount": 1,
"turnTimeoutMs": 120000,
@@ -52,7 +52,9 @@ const statesCount = (text: string, count: number) => new RegExp(`(?<![\\w-]|,\\s
function groupContext(text: string, at: number) {
const lineStart = text.lastIndexOf("\n", at) + 1; const lineEnd = text.indexOf("\n", at);
const line = text.slice(lineStart, lineEnd < 0 ? undefined : lineEnd);
const prior = text.slice(0, lineStart).split("\n").reverse().find(line => /^\s*(?:#{1,6}\s+|\*\*)/.test(line)) ?? "";
const label = (candidate: string) => /^\s*(?:#{1,6}\s+|\*\*)/.test(candidate);
// A line that opens with its own group label ("**Unassigned — 18 blocked:** RUN-18, …") is its own group.
const prior = label(line) ? "" : text.slice(0, lineStart).split("\n").reverse().find(label) ?? "";
return `${line} ${prior}`.toLowerCase();
}
export function gradeOpenInventory(value: unknown) {
@@ -192,7 +194,7 @@ function openInventory(): ChatCatalogCase {
if ((await ctx.allRuns()).length) throw new Error("Seeding the paused-owner inventory unexpectedly admitted work");
return { project: p, originalTasks, workerIds: [worker.id, secondWorker.id], ownerPauseReceipts, ownerLabels: { [worker.id]: "Morgan", [secondWorker.id]: "Riley", unassigned: "Unassigned" } };
}, gradeOpenInventory);
return { ...base, observe: observeOpenInventory, oracleVersion: "paperclip.chat-catalog.internal.organize-open-work.v4",
return { ...base, observe: observeOpenInventory, oracleVersion: "paperclip.chat-catalog.internal.organize-open-work.v5",
sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-task-inventory.ts"], attemptTimeoutMs: 600_000,
setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["inventory-owners-paused", "lead-ready", "inventory-settled"] },
async verifySetup(ctx, state) {
@@ -646,6 +646,15 @@ describe("multi-page open inventory calibration", () => {
o.reply = "501 open tasks.\n## Unassigned\n" + o.tasks.slice(0, 4).map(row => `[${row.title}](/ACORN/issues/${row.id})`).join(", ") + " … every 3rd task: 1, 4, 7";
expect(gradeInternal(o).passed).toBe(false);
});
it("treats a bold-labelled line as its own owner group, not the group above it", () => {
const o = valid("tasks.organize-open-work"); o.state.ownerLabels = { morgan: "Morgan", unassigned: "Unassigned" };
const link = (row: { id: string; title?: string }) => `[${row.title}](/ACORN/issues/${row.id})`;
o.reply = "**Morgan — 0 blocked:** none\n**Unassigned — 2 blocked:** " + o.tasks.slice(0, 2).map(link).join(", ")
+ "\n**Unassigned — todo:** " + o.tasks.slice(2).map(link).join(", ");
expect(gradeInternal(o).passed).toBe(true);
o.reply = "**Morgan — 2 blocked:** " + o.tasks.slice(0, 2).map(link).join(", ") + "\n- " + o.tasks.slice(2).map(link).join(", ");
expect(gradeInternal(o).passed).toBe(false);
});
it("rejects a blocked item placed after an ordinary item for the same owner", () => {
const o = valid("tasks.organize-open-work");
o.reply = "## Unassigned\n" + [...o.tasks.slice(2), ...o.tasks.slice(0, 2)].map(row => `- [${row.title}](/ACORN/issues/${row.id})`).join("\n");