test(evals): grade decision listing by source links and section structure

The show-decisions grader read only the line that named each item and
required words such as "you" or "answered" there. A correct reply that
put "waiting on you" in its list heading failed. Read each item with its
heading, require the exact approval and task links, require Riley to be
named beyond the setup titles, and check that Riley's question and the
answered request are listed apart from the person's own decision.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -3402,7 +3402,7 @@
       "chat-catalog-domains/identity-auth.ts",
       "../../packages/shared/src/validators/issue.ts"
     ],
-    "oracleVersion": "paperclip.chat-catalog.internal.v1",
+    "oracleVersion": "paperclip.chat-catalog.decision-search.v2",
     "coverageGaps": [],
     "expectedRunCount": 1,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/internal-decision-search.test.ts

@@ -1,8 +1,8 @@
 import { describe, expect, it } from "vitest";
 import { gradeDecisionSearch } from "./internal-decision-search.js";
 export function validDecisionSearch() {
-  const t = { id: "question-task", title: "Riley launch question", status: "backlog" }; const history = { id: "history-task", title: "Completed launch information", status: "backlog" };
-  const hiring = { id: "hire", companyId: "company", status: "pending", decidedByUserId: null }; const pendingQuestion = { id: "question", companyId: "company", issueId: t.id, status: "pending", addresseeUserId: "riley", effectiveResolverPolicy: "human_only" };
+  const t = { id: "question-task", identifier: "RUN-1", title: "Riley launch question", status: "backlog" }; const history = { id: "history-task", identifier: "RUN-2", title: "Completed launch information", status: "backlog" };
+  const hiring = { id: "hire", companyId: "company", status: "pending", decidedByUserId: null }; const pendingQuestion = { id: "question", title: "Riley: choose the launch audience", companyId: "company", issueId: t.id, status: "pending", addresseeUserId: "riley", effectiveResolverPolicy: "human_only" };
   const completedQuestion = { id: "completed", status: "answered", resolvedByUserId: "owner", result: { answers: ["received"] } };
   return { scenarioId: "inbox-search.show-decisions", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(t), historyTask: structuredClone(history), hiring: structuredClone(hiring), pendingQuestion: structuredClone(pendingQuestion), completedQuestion: structuredClone(completedQuestion), ownerId: "owner", rileyId: "riley" },
     tasks: [t, history], projects: [], goals: [], comments: [], activity: [] as Array<{ agentId: string; action: string }>, documents: [], revisions: [],
@@ -19,6 +19,32 @@ describe("decision source and resolver calibration", () => {
   it("rejects wrong source or foreign links", () => { const o = validDecisionSearch(); o.links[0] = "https://foreign.test/RUN/approvals/hire"; expect(gradeDecisionSearch(o).passed).toBe(false); });
   it("rejects universal resolution claims", () => { const o = validDecisionSearch(); o.reply += "\nYou can approve every visible item."; expect(gradeDecisionSearch(o).passed).toBe(false); });
   it("rejects changed completed answer and board-feed privilege substitution", () => { const o = validDecisionSearch(); o.completedQuestion.result.answers = ["changed"]; expect(gradeDecisionSearch(o).passed).toBe(false); const admin = validDecisionSearch(); admin.permissionStatus = 200; expect(gradeDecisionSearch(admin).passed).toBe(false); });
+  // Shape of a real reply (runner-opencode, run 42404417353) once the approval route is available: the
+  // "waiting on you" meaning sits in the list heading, and the answered request is listed apart as not needing a decision.
+  const sectioned = [
+    "One decision is waiting on you:", "",
+    "- **[Hire approval: \"Launch designer\"](/RUN/approvals/hire)** — pending `hire_agent` approval (role: designer).", "",
+    "Pending but not on you:",
+    "- [RUN-1](/RUN/issues/RUN-1) holds an open question, \"Riley: choose the launch audience,\" addressed to Riley (operator), not your account.",
+    "- [RUN-1](/RUN/issues/RUN-1) and [RUN-2](/RUN/issues/RUN-2) are unassigned backlog tasks with no decisions required yet.",
+  ].join("\n");
+  const withReply = (reply: string) => { const o = validDecisionSearch(); o.reply = reply; o.links = ["/RUN/approvals/hire", "/RUN/issues/RUN-1", "/RUN/issues/RUN-2"]; return o; };
+  it("accepts a sectioned reply whose headings carry who each item waits on", () => expect(gradeDecisionSearch(withReply(sectioned)).passed).toBe(true));
+  it("accepts a reworded sectioned reply", () => expect(gradeDecisionSearch(withReply([
+    "Needs your call", "1. [Launch designer](/RUN/approvals/hire) hire request.", "",
+    "Someone else's", "1. [RUN-1](/RUN/issues/RUN-1): only Riley can respond to it.", "",
+    "Done already", "1. [RUN-2](/RUN/issues/RUN-2): you replied to the brief question earlier; nothing left to do.",
+  ].join("\n"))).passed).toBe(true));
+  it("rejects listing every item as the person's own decision", () => expect(gradeDecisionSearch(withReply([
+    "Three decisions are waiting on you:",
+    "- [Launch designer](/RUN/approvals/hire) hire approval.",
+    "- [RUN-1](/RUN/issues/RUN-1): Riley's launch audience question.",
+    "- [RUN-2](/RUN/issues/RUN-2): Completed launch information.",
+  ].join("\n"))).checks.filter(check => !check.passed).map(check => check.id)).toEqual(["addressed-question-is-not-everyones-decision", "completed-request-separated-from-pending"]));
+  it("rejects an unlinked approval", () => { const o = withReply(sectioned.replace("[Hire approval: \"Launch designer\"](/RUN/approvals/hire)", "Hire approval: \"Launch designer\"")); o.links = o.links.slice(1); expect(gradeDecisionSearch(o).passed).toBe(false); });
+  it("rejects a question whose resolver is named only by the setup titles", () => expect(gradeDecisionSearch(withReply(sectioned.replace("addressed to Riley (operator), not your account", "still open"))).passed).toBe(false));
+  it("rejects attributing the hiring approval to Riley", () => expect(gradeDecisionSearch(withReply(sectioned.replace("pending `hire_agent` approval", "pending approval for Riley"))).passed).toBe(false));
+  it("rejects presenting the answered request as awaiting the person", () => expect(gradeDecisionSearch(withReply(sectioned.replace("are unassigned backlog tasks with no decisions required yet", "still need your answer"))).passed).toBe(false));
   it.each(["missing-resolver-feed", "missing-principal", "foreign-feed", "borrowed-user", "wrong-source", "wrong-task", "wrong-resolver", "wrong-policy", "current-user-leak", "foreign-question"])("rejects incorrect independent audience custody: %s", kind => {
     const o = validDecisionSearch();
     if (kind === "missing-resolver-feed") Object.assign(o, { resolverAttention: undefined });

tests/runner-e2e/chat-catalog-domains/internal-decision-search.ts

@@ -11,10 +11,25 @@ interface AttentionFeed { companyId?: string; items?: Array<{ subject?: { id?: s
 function approvalLink(o: InternalObservation, id: string) {
   return o.links.some(href => { try { const u = new URL(href, o.origin); return u.origin === o.origin && u.pathname.split("/").slice(-2).join("/") === `approvals/${id}`; } catch { return false; } });
 }
+/**
+ * The reply's statement about one source: every sentence or list item that names it, read together with the
+ * list heading that introduces it ("Waiting on you:"), so meaning carried by the heading is not lost. `sections`
+ * identifies which heading or sentence each mention sits under, so the grader can check separation structurally.
+ */
 function statement(o: InternalObservation, source: Entity) {
   const names = [source.title, source.identifier, source.id].filter((name): name is string => !!name);
-  return o.reply.split(/\n|(?<=[.!?])\s+/).filter(part => names.some(name => part.includes(name))).join(" ");
+  const units: Array<{ own: string; text: string; section: string }> = []; let lead: { text: string; section: string } | undefined;
+  o.reply.split("\n").forEach((line, index) => {
+    if (!line.trim()) return;
+    if (/^\s*(?:[-*+]|\d+[.)])\s/.test(line)) { units.push({ own: line, text: `${lead?.text ?? ""}\n${line}`, section: lead?.section ?? `line-${index}` }); return; }
+    line.split(/(?<=[.!?])\s+/).forEach((part, n) => units.push({ own: part, text: part, section: `line-${index}-${n}` }));
+    lead = { text: line, section: `lead-${index}` };
+  });
+  const hits = units.filter(unit => names.some(name => unit.own.includes(name)));
+  return { text: hits.map(unit => unit.text).join("\n"), sections: new Set(hits.map(unit => unit.section)) };
 }
+function apart(a: Set<string>, b: Set<string>) { return a.size > 0 && b.size > 0 && [...a].every(section => !b.has(section)); }
+function without(text: string, literals: readonly (string | undefined)[]) { return literals.reduce<string>((rest, literal) => literal ? rest.split(literal).join(" ") : rest, text); }
 export function gradeDecisionSearch(value: unknown) {
   const c = checks(); const o = observation(value) as DecisionObservation | undefined;
   c.add("actual-source-decision-state", !!o && !!o.hiring && !!o.pendingQuestion && !!o.completedQuestion && o.visibleAttention !== undefined, "Actual hiring, addressed question, answered history and human-visible attention records are required.");
@@ -36,17 +51,19 @@ export function gradeDecisionSearch(value: unknown) {
     && [t, history].every(before => !!before && sameTask(before, o.tasks.find(row => row.id === before.id)))
     && !o.activity.some(row => row.agentId === o.leadId && /approval\.(?:approved|rejected)|issue\.interaction_(?:answered|accepted|rejected|cancelled)/.test(row.action ?? "")),
     "Listing preserves the pending hiring decision, Riley's pending question and the actual already-answered human history.");
-  const hiringText = statement(o, { id: hire.id, title: "Launch designer" }); const questionText = statement(o, t); const historyText = statement(o, history);
-  c.add("pending-hire-source-and-human-resolver", approvalLink(o, hire.id) && /\b(?:pending|await|approv|decision)\b/i.test(hiringText)
-    && /\b(?:you|human|board|operator|Alex)\b/i.test(hiringText), "The hiring decision links its exact approval and identifies the current human/operator approval step.");
-  c.add("addressed-question-is-not-everyones-decision", !!t && hasLink(o, "issues", t) && /\bRiley\b/.test(questionText)
-    && /\b(?:answer|respond|resolver|addressed|waiting|pending)\b/i.test(questionText)
-    && !/\b(?:you|Alex)\s+(?:can|should|must)\s+(?:answer|resolve|approve)\b/i.test(questionText),
-    "The visible question links its actual task and identifies Riley, without offering Alex permission to resolve it.");
-  c.add("completed-request-separated-from-pending", !!history && hasLink(o, "issues", history)
-    && /\b(?:answered|completed|resolved|history|no action|nothing.*waiting)\b/i.test(historyText)
-    && !/\b(?:still|currently)\s+(?:pending|waiting|unanswered)\b/i.test(historyText),
-    "The informational completed request is linked as actual history, not another unresolved decision.");
+  const hiring = statement(o, { id: hire.id, title: "Launch designer" }); const question = statement(o, t); const historyStatement = statement(o, history);
+  const ownTitles = [t.title, t.identifier, pending.title];
+  c.add("pending-hire-source-and-human-resolver", approvalLink(o, hire.id) && hiring.sections.size > 0
+    && !/\bRiley\b/.test(hiring.text) && !/\b(?:was|been|is|has)\s+(?:already\s+)?(?:approved|rejected)\b/i.test(hiring.text),
+    "The hiring decision links its exact approval, is not attributed to Riley and is not reported as already decided.");
+  c.add("addressed-question-is-not-everyones-decision", !!t && hasLink(o, "issues", t) && /\bRiley\b/.test(without(question.text, ownTitles))
+    && apart(question.sections, hiring.sections)
+    && !/\b(?:you|Alex)\s+(?:can|should|must)\s+(?:answer|resolve|approve)\b/i.test(question.text),
+    "The visible question links its actual task, names Riley as its resolver beyond the setup titles, is listed apart from the person's own decision and is not offered to Alex.");
+  c.add("completed-request-separated-from-pending", !!history && hasLink(o, "issues", history) && apart(historyStatement.sections, hiring.sections)
+    && !/\b(?:still|currently)\s+(?:pending|waiting|unanswered)\b/i.test(historyStatement.text)
+    && !/\b(?:needs?|awaits?|awaiting|requires?)\s+your\s+(?:answer|response|decision|input|approval)\b/i.test(historyStatement.text),
+    "The informational completed request is linked and kept apart from the person's pending decision, without being described as awaiting them.");
   c.add("board-feed-permission-boundary", o.permissionStatus === 403 && !/\b(?:you can|you may|you should)\s+(?:approve|resolve|answer)\s+(?:all|every)\b/i.test(o.reply),
     "The independent agent cannot read the board-only attention aggregation, and the reply does not grant universal resolver authority.");
   return c.result();
@@ -74,7 +91,7 @@ function showDecisions(): ChatCatalogCase {
       return { project: p, task: t, historyTask: history, ownerId, rileyId: riley.id, pendingQuestion, completedQuestion, hiring };
     } catch (error) { await riley.dispose(); riley = undefined; throw error; }
   }, gradeDecisionSearch);
-  return { ...base, authentication: "human-session", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-decision-search.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "../../packages/shared/src/validators/issue.ts"],
+  return { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.decision-search.v2", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-decision-search.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "../../packages/shared/src/validators/issue.ts"],
     async observe(ctx, state): Promise<DecisionObservation> {
       if (!key || !riley || riley.id !== state.rileyId || riley.id === state.ownerId) throw new Error("Missing distinct authenticated decision resolver or attempt-private probe");
       const o = await observeInternal(ctx, state); const root = `/api/companies/${o.companyId}`;