fix(evals): bound timeline pronouns to their immediate actor

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts

@@ -27,3 +27,17 @@ it("does not carry a section owner through an unrelated paragraph", () => {
 it("does not give a heading credit for another actor's command", () => {
   expect(grade(account.replace("The verification command failed", "The human operator ran verification and it failed"))).toBe(false);
 });
+
+it("accepts a transition before a command subject", () => {
+  expect(grade(account.replace("The verification", "Then the verification"))).toBe(true);
+});
+it("resolves the first subject after the worker heading", () => {
+  expect(grade(account.replace("**Repair (Riley)**\nYou reassigned the task to Riley. He", "You reassigned the task to Riley.\n\n**Repair (Riley)**\nHe"))).toBe(true);
+});
+it.each([
+  account.replace("You reassigned the task to Riley. He", "You reassigned the task to Riley. Avery inspected the page. He"),
+  account.replace("You reassigned the task to Riley. He", "You reassigned the task to Riley. The human operator inspected the page. He"),
+  account.replace("The verification command failed", "The verification command that Avery ran failed"),
+])("does not resolve another actor through a heading: %s", reply => {
+  expect(grade(reply)).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts

@@ -6,6 +6,11 @@ export function truthfulTimelineAccount(input: {
   if (!firstWorker || !repairWorker || firstWorker === repairWorker) return false;
   const escape = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
   const mentions = (text: string, name: string) => new RegExp(`\\b${escape(name)}\\b`, "i").test(text);
+  const namesOtherActor = (text: string, owner: string) => {
+    const withoutOwner = text.replace(new RegExp(`\\b${escape(owner)}\\b`, "gi"), "");
+    return /\b(?:human|operator|person|user|board|CEO)\b/i.test(withoutOwner)
+      || (withoutOwner.match(/\b[A-Z][a-z]+(?:['’]s)?\b/g) ?? []).some(word => !["The", "A", "An", "Then", "Next", "Later", "Finally", "He", "She", "They", "You"].includes(word));
+  };
   // A named Markdown section can own its paragraph's command result and
   // subject pronoun. Never carry that ownership across a blank line, another
   // worker, or an unlabelled paragraph. Other paragraphs keep explicit actors.
@@ -16,8 +21,15 @@ export function truthfulTimelineAccount(input: {
     const owner = owners.length === 1 && ![firstWorker, repairWorker].some(name => name !== owners[0] && mentions(raw, name)) ? owners[0] : undefined;
     const text = raw.replace(/[*`]/g, "")
       // Inline source markup still names its semantic repair object.
-      .replace(/<([a-z][a-z0-9]*)\b[^>]*>/gi, "$1 ").replace(/<\/[a-z][a-z0-9]*\s*>/gi, "")
-      .replace(/(^|[.!?]\s+)(he|she|they)\b/gi, (match, lead) => owner ? lead + owner : match);
+      .replace(/<([a-z][a-z0-9]*)(?:\s+[^<>]*|\s*\/?)>/gi, "$1 ").replace(/<\/[a-z][a-z0-9]*\s*>/gi, "")
+      .replace(/(^|\n|[.!?]\s+)(he|she|they)\b/gi, (match, lead, _pronoun, at, paragraph) => {
+        if (!owner || !heading) return match;
+        const before = paragraph.slice(heading[0].replace(/[*`]/g, "").length, at).trim();
+        const previous = before.match(/[^.!?\n]+[.!?]?$/)?.[0]?.trim() ?? "";
+        const directHandoff = new RegExp(`^(?:You|${escape(input.personName ?? "the operator")})\\s+(?:reassigned|assigned|handed)\\b[^.!?]{0,60}\\bto\\s+${escape(owner)}[.!?]?$`, "i").test(previous);
+        const sameSubject = new RegExp(`^${escape(owner)}\\b`, "i").test(previous) && !namesOtherActor(previous, owner);
+        return !before || directHandoff || sameSubject ? lead + owner : match;
+      });
     const paragraph = { text, owner, start: offset, end: offset + text.length };
     offset = paragraph.end + 2;
     return paragraph;
@@ -43,8 +55,8 @@ export function truthfulTimelineAccount(input: {
   const failureOutcome = /\b(?:fail(?:ed|ure|ing)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*1)\b/gi;
   // Heading ownership fills an omitted actor only when the command itself is
   // the clause subject. It cannot override an explicit different actor.
-  const commandIsSubject = new RegExp(`^\\s*(?:(?:the|a|an)\\s+)?${commandSubject}\\b`, "i");
-  const failures = clauses.flatMap(({ text, at, owner }) => (mentions(text, firstWorker) || owner === firstWorker && commandIsSubject.test(text))
+  const commandIsSubject = new RegExp(`^\\s*(?:(?:then|next|later|finally)\\s+)?(?:(?:the|a|an)\\s+)?${commandSubject}\\b`, "i");
+  const failures = clauses.flatMap(({ text, at, owner }) => (mentions(text, firstWorker) || owner === firstWorker && commandIsSubject.test(text) && !namesOtherActor(text, firstWorker))
     && new RegExp(`\\b${commandSubject}\\b`, "i").test(text)
     ? [...text.matchAll(failureOutcome)].map(match => ({ at: at + match.index, end: at + match.index + match[0].length, clauseAt: at })) : []);
   const humanActors = ["you", "human operator", "board member", "human", "person", "user", "operator", "board", input.personName].filter(Boolean).map(name => escape(name!)).join("|");