test(evals): accept a linked conversation document as the open-work inventory

Run 42456103697 answered with a summary table and linked
/RUN/issues/RUN-505#document-launch_open_work, a document the CEO wrote on
the conversation in its run. It lists all 501 open tasks under "## Riley",
"## Morgan" and "## Unassigned", each with "**Blocked (n)**" before
"**Todo (n)**"; an offline check matched every task to its settled owner
and status. The grader only read attachments, so it saw no deliverable.

The observer now also reads a document on the conversation that the reply
links, when its current revision was written by the lead in this
conversation's run. Grouping walks enclosing labels of higher rank (bold
under heading) until one names an owner, so an owner heading with status
sub-labels groups correctly while a bold owner line stays its own group.
Bumps oracleVersion to organize-open-work.v6.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -5882,7 +5882,7 @@
       "chat-scenarios/tasks.yaml",
       "chat-catalog-domains/internal-task-inventory.ts"
     ],
-    "oracleVersion": "paperclip.chat-catalog.internal.organize-open-work.v5",
+    "oracleVersion": "paperclip.chat-catalog.internal.organize-open-work.v6",
     "coverageGaps": [],
     "expectedRunCount": 1,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/internal-task-inventory.ts

@@ -3,7 +3,7 @@ import { remainingIdentityObservationBudget } from "./identity-support.js";
 import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
 import { makeDefinition, fixtureWorker } from "./internal-definition.js";
 import { checks, companyPath, entity, hasLink, observation, observeInternal, project, sameTask, task, type Entity, type InternalObservation } from "./internal-support.js";
-import type { IssueAttachment } from "../../../packages/shared/src/types/issue.js";
+import type { DocumentRevision, IssueAttachment } from "../../../packages/shared/src/types/issue.js";
 
 interface InventoryOwner extends Entity { companyId?: string; lifecycleState?: string; lifecycleVersion?: number; lifecycleHolds?: string[]; lifecycleError?: unknown }
 export interface InventoryOwnerPause { before: InventoryOwner; requested: InventoryOwner }
@@ -49,13 +49,24 @@ function presentations(o: InventoryObservation, row: Entity) {
 }
 /** A stated count: not digits inside a task identifier (RUN-491) or an element of a number list (RUN-1, 4, 7). */
 const statesCount = (text: string, count: number) => new RegExp(`(?<![\\w-]|,\\s?)${count}(?![\\w-]|,)`).test(text);
-function groupContext(text: string, at: number) {
+/**
+ * The labels an item sits under: its own line, then each enclosing label of higher rank (a bold line
+ * outranks plain lines, a heading outranks bold lines and deeper headings). The walk stops at the
+ * first label naming an owner, so "## Riley" then "**Blocked (167)**: RUN-1" groups RUN-1 under Riley,
+ * while "**Unassigned — 18 blocked:** RUN-18" below "**Morgan — 166 blocked:**" stays its own group.
+ */
+function groupContext(text: string, at: number, ownerLabels: readonly string[]) {
   const lineStart = text.lastIndexOf("\n", at) + 1; const lineEnd = text.indexOf("\n", at);
   const line = text.slice(lineStart, lineEnd < 0 ? undefined : lineEnd);
-  const label = (candidate: string) => /^\s*(?:#{1,6}\s+|\*\*)/.test(candidate);
-  // A line that opens with its own group label ("**Unassigned — 18 blocked:** RUN-18, …") is its own group.
-  const prior = label(line) ? "" : text.slice(0, lineStart).split("\n").reverse().find(label) ?? "";
-  return `${line} ${prior}`.toLowerCase();
+  const rank = (candidate: string) => { const heading = /^\s*(#{1,6})\s+/.exec(candidate);
+    return heading ? heading[1]!.length : /^\s*\*\*/.test(candidate) ? 7 : 8; };
+  const namesOwner = (candidate: string) => ownerLabels.some(label => candidate.toLowerCase().includes(label));
+  const chain = [line]; let current = rank(line);
+  for (const prior of text.slice(0, lineStart).split("\n").reverse()) {
+    if (namesOwner(chain[chain.length - 1]!) || current === 1) break;
+    if (rank(prior) < current) { chain.push(prior); current = rank(prior); }
+  }
+  return chain.join(" ").toLowerCase();
 }
 export function gradeOpenInventory(value: unknown) {
   const c = checks(); const o = observation(value) as InventoryObservation | undefined;
@@ -84,9 +95,10 @@ export function gradeOpenInventory(value: unknown) {
     "Completed, cancelled and other-project work is not presented as open work.");
   const owners = o.state.ownerLabels as Record<string, string>;
   const named = Object.entries(owners ?? {}).filter(([key]) => key !== "unassigned").map(([, label]) => label.toLowerCase());
+  const allLabels = Object.values(owners ?? {}).map(label => label.toLowerCase());
   let correctlyGrouped = !!owners;
   for (const row of shown) for (const at of placed.get(row.id)!) {
-    const context = groupContext(at.text, at.at); const key = row.assigneeAgentId ?? "unassigned";
+    const context = groupContext(at.text, at.at, allLabels); const key = row.assigneeAgentId ?? "unassigned";
     const owner = owners?.[key]?.toLowerCase();
     if (!owner) correctlyGrouped = false;
     else if (key === "unassigned" ? named.some(label => context.includes(label)) : !context.includes(owner)) correctlyGrouped = false;
@@ -117,6 +129,20 @@ export async function observeOpenInventory(ctx: ChatCatalogContext, state: Recor
     if (!href || file.createdByAgentId !== o.leadId || !runIds.has(file.originatingRunId ?? "") || file.byteSize > 1024 * 1024) continue;
     deliverables.push({ id: file.id, href, body: (await ctx.input.api.getAttachmentContent(file.id)).toString("utf8") });
   }
+  // A document on this conversation, linked from the reply as /<prefix>/issues/<conversation>#document-<key>,
+  // counts when its current revision was written by the answering agent in this conversation's run.
+  if (source) for (const href of o.links) {
+    const url = new URL(href, o.origin); const key = /^#document-(.+)$/.exec(url.hash)?.[1];
+    const parts = url.pathname.split("/").map(decodeURIComponent);
+    if (url.origin !== new URL(o.origin).origin || !key || parts.length !== 4 || parts[2] !== "issues"
+      || ![source.id, source.identifier].includes(parts[3])) continue;
+    const endpoint = `/api/issues/${source.id}/documents/${encodeURIComponent(decodeURIComponent(key))}`;
+    const doc = await ctx.input.api.get<{ id: string; latestRevisionId?: string | null }>(endpoint).catch(() => undefined);
+    const revisions = doc ? await ctx.input.api.get<DocumentRevision[]>(`${endpoint}/revisions`).catch(() => []) : [];
+    const latest = revisions.find(revision => revision.id === doc?.latestRevisionId);
+    if (!doc || !latest || latest.createdByAgentId !== o.leadId || !runIds.has(latest.createdByRunId ?? "") || latest.body.length > 1024 * 1024) continue;
+    deliverables.push({ id: doc.id, href, body: latest.body });
+  }
   return { ...o, sourceIssueId: source?.id, deliverables };
 }
 async function settledInventory(ctx: ChatCatalogContext, ids: readonly string[], deadlineAt: number) {
@@ -194,7 +220,7 @@ function openInventory(): ChatCatalogCase {
     if ((await ctx.allRuns()).length) throw new Error("Seeding the paused-owner inventory unexpectedly admitted work");
     return { project: p, originalTasks, workerIds: [worker.id, secondWorker.id], ownerPauseReceipts, ownerLabels: { [worker.id]: "Morgan", [secondWorker.id]: "Riley", unassigned: "Unassigned" } };
   }, gradeOpenInventory);
-  return { ...base, observe: observeOpenInventory, oracleVersion: "paperclip.chat-catalog.internal.organize-open-work.v5",
+  return { ...base, observe: observeOpenInventory, oracleVersion: "paperclip.chat-catalog.internal.organize-open-work.v6",
     sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-task-inventory.ts"], attemptTimeoutMs: 600_000,
     setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["inventory-owners-paused", "lead-ready", "inventory-settled"] },
     async verifySetup(ctx, state) {

tests/runner-e2e/chat-catalog-domains/internal.test.ts

@@ -605,6 +605,41 @@ describe("multi-page open inventory calibration", () => {
     const unlinked = asFile(fileBody(o)); unlinked.reply = "501 open tasks; see the attached file.";
     expect(gradeInternal(unlinked).passed).toBe(false);
   });
+  describe("an inventory document on the conversation, sectioned by owner then status", () => {
+    const docHref = "/ACORN/issues/ACORN-505#document-launch_open_work";
+    const owned = () => {
+      const o = valid("tasks.organize-open-work");
+      o.tasks.forEach((row, index) => { row.identifier = `ACORN-${index + 1}`; row.assigneeAgentId = index % 3 === 0 ? "riley" : index % 3 === 1 ? "morgan" : null;
+        row.status = index % 5 === 0 ? "blocked" : "todo"; });
+      o.state.originalTasks = structuredClone(o.tasks); o.state.ownerLabels = { riley: "Riley", morgan: "Morgan", unassigned: "Unassigned" };
+      return o;
+    };
+    const ids = (o: ReturnType<typeof valid>, owner: string | null, blocked: boolean) =>
+      o.tasks.filter(row => row.assigneeAgentId === owner && (row.status === "blocked") === blocked).map(row => row.identifier).join(", ");
+    const sectioned = (o: ReturnType<typeof valid>, heading: (label: string) => string, status: (label: string) => string) =>
+      "# Open Launch work\n\n" + ([["riley", "Riley"], ["morgan", "Morgan"], [null, "Unassigned"]] as const).map(([owner, label]) =>
+        `${heading(label)}\n\n${status("Blocked")}: ${ids(o, owner, true)}\n${status("Todo")}: ${ids(o, owner, false)}`).join("\n\n");
+    const asDoc = (o: ReturnType<typeof valid>, body: string) => {
+      o.links = [docHref]; o.reply = `501 open tasks by owner, blocked first. Every task is listed in [Open Launch work by owner](${docHref}).`;
+      return Object.assign(o, { deliverables: [{ id: "doc", href: docHref, body }] });
+    };
+    it("accepts owner headings with bold status sub-labels, and reworded heading levels", () => {
+      const o = owned();
+      expect(gradeInternal(asDoc(o, sectioned(o, label => `## ${label} — 167 open`, label => `**${label}**`))).passed).toBe(true);
+      const reworded = owned();
+      expect(gradeInternal(asDoc(reworded, sectioned(reworded, label => `### Owner: ${label}`, label => `#### ${label} items`))).passed).toBe(true);
+    });
+    it("rejects a task placed under another owner's heading", () => {
+      const o = owned(); const body = sectioned(o, label => `## ${label}`, label => `**${label}**`);
+      expect(gradeInternal(asDoc(o, body.replace("## Unassigned", "## Morgan (continued)"))).passed).toBe(false);
+    });
+    it("rejects blocked work listed after other work in an owner section", () => {
+      const o = owned();
+      const late = ([["riley", "Riley"], ["morgan", "Morgan"], [null, "Unassigned"]] as const).map(([owner, label]) =>
+        `## ${label}\n\n**Todo**: ${ids(o, owner, false)}\n**Blocked**: ${ids(o, owner, true)}`).join("\n\n");
+      expect(gradeInternal(asDoc(o, late)).passed).toBe(false);
+    });
+  });
   it("rejects unassigned work presented under a named owner in a deliverable", () => {
     const o = asFile("## Morgan\n" + valid("tasks.organize-open-work").tasks.map(row => `- ${row.id}`).join("\n"));
     o.state.ownerLabels = { morgan: "Morgan", unassigned: "Unassigned" };