fix(evals): retain contrary worker scope for unnamed attempts

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts

@@ -20,8 +20,18 @@ it.each([
 ])("retains consequential actor and outcome requirements: %s", reply => {
   expect(grade(reply)).toBe(false);
 });
-it("does not infer the initial worker over an explicit contrary actor in another paragraph", () => {
-  expect(grade(account.replace("The verification command failed", "\n\nRiley ran verification and it failed"))).toBe(false);
+it("does not infer the initial worker over a contrary possessive within the command clause", () => {
+  const reply = account.replace("**First attempt (Morgan)**\nMorgan built the page. ", "")
+    .replace("The verification command failed", "The verification command failed on Riley's attempt");
+  expect(grade(reply)).toBe(false);
+});
+it("keeps a contrary worker heading as invalidation across blank lines", () => {
+  const reply = account.replace("First attempt (Morgan)", "Initial attempt (Riley)")
+    .replace("Morgan built the page. ", "Riley built the page.\n\n");
+  expect(grade(reply)).toBe(false);
+});
+it("keeps a legitimate anonymous command description across a blank line", () => {
+  expect(grade(account.replace("The verification", "\n\nThe verification"))).toBe(true);
 });
 
 it("does not give a heading credit for another actor's command", () => {

tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts

@@ -16,7 +16,8 @@ export function truthfulTimelineAccount(input: {
   const namesOtherActor = (text: string, owner: string) => {
     const withoutOwner = text.replace(new RegExp(`\\b${escape(owner)}\\b`, "gi"), "");
     return /\b(?:human|operator|person|user|board|CEO)\b/i.test(withoutOwner)
-      || /^\s*[A-Z][a-z]+['’]s\b/.test(withoutOwner)
+      || [...withoutOwner.matchAll(/\b([A-Z][\p{L}'-]+)['’]s\b/gu)].some(match =>
+        !calendarWords.has(match[1]!.toLowerCase()) && !/^(?:today|tomorrow|yesterday)$/i.test(match[1]!))
       || /\b(?:another|different|other)\s+(?:worker|agent|person|engineer|reviewer|operator)\b/i.test(withoutOwner)
       || /\b(?:run|ran|executed|performed)\s+by\s+[A-Z][a-z]+\b|\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\s+(?:ran|executed|performed)\b/.test(withoutOwner)
       || /(?:^|[.!?]\s+|\b(?:that|which|while|when)\s+)(?!(?:The|A|An|Then|Next|Later|Finally|He|She|They|You)\b)[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\s+(?:[a-z]+(?:ed|s)|ran|built|made|did|had|was|is)\b/.test(withoutOwner);
@@ -60,12 +61,14 @@ export function truthfulTimelineAccount(input: {
     });
   };
   let offset = 0;
+  let activeHeadingWorkers: string[] = [];
   const paragraphs = reply.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
     .replace(/^([ \t]*(?:#{1,6} +[^\n]+|\*\*[^\n*]+\*\*))[ \t]*\n[ \t]*\n(?=\S)/gm, "$1\n")
     .replace(/^([ \t]*\*\*[^\n*]+\*\*)[ \t]+(?=\S)/gm, "$1\n")
     .split(/\n\s*\n/).map(raw => {
     const heading = /^\s*(?:#{1,6} +([^\n]+)|\*\*([^\n]+)\*\*)[ \t]*\n/.exec(raw);
     const owners = [firstWorker, repairWorker].filter(name => mentions(heading?.[1] ?? heading?.[2] ?? "", name));
+    if (heading) activeHeadingWorkers = owners;
     const owner = owners.length === 1 && ![firstWorker, repairWorker].some(name => name !== owners[0] && mentions(raw, name)) ? owners[0] : undefined;
     const resolvedPronouns = new Set<number>();
     const headingPrefix = heading?.[0] ?? "";
@@ -85,7 +88,7 @@ export function truthfulTimelineAccount(input: {
         resolvedPronouns.add(at + lead.length);
         return lead + owner;
       });
-    const paragraph = { text, owner, namedWorkerHeading: owners.length > 0, start: offset, end: offset + text.length };
+    const paragraph = { text, owner, contraryWorkerHeading: activeHeadingWorkers.some(name => name !== firstWorker), start: offset, end: offset + text.length };
     offset = paragraph.end + 2;
     return paragraph;
   });
@@ -93,7 +96,7 @@ export function truthfulTimelineAccount(input: {
   // Search milestone clauses, not the first occurrence of a worker or outcome.
   // A summary or heading may mention the repair before the ordered account.
   const clauses = [...body.matchAll(new RegExp(`${withinClause}+(?:[!?]|\\.(?=\\s|$)|(?=\\n|$))`, "g"))]
-    .map(match => { const paragraph = paragraphs.find(row => row.start <= match.index && row.end > match.index); return { text: match[0], at: match.index, owner: paragraph?.owner, namedWorkerHeading: paragraph?.namedWorkerHeading, paragraphStart: paragraph?.start }; });
+    .map(match => { const paragraph = paragraphs.find(row => row.start <= match.index && row.end > match.index); return { text: match[0], at: match.index, owner: paragraph?.owner, contraryWorkerHeading: paragraph?.contraryWorkerHeading, paragraphStart: paragraph?.start }; });
   // A command noun can have ordinary modifiers, but never absorb a new event,
   // actor, negation or clause boundary while connecting this pause to failure.
   const commandBoundaries = [
@@ -108,7 +111,7 @@ export function truthfulTimelineAccount(input: {
 
   const failureOutcome = /\b(?:fail(?:ed|ure|ing)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*1)\b/gi;
   const commandIsSubject = new RegExp(`^\\s*(?:(?:then|next|later|finally)\\s+)?(?:(?:the|a|an)\\s+)?${commandSubject}\\b`, "i");
-  const failures = clauses.flatMap(({ text, at, owner, namedWorkerHeading }) => (mentions(text, firstWorker) || (owner === firstWorker || !namedWorkerHeading) && commandIsSubject.test(text) && !namesOtherActor(text.replace(commandIsSubject, ""), firstWorker))
+  const failures = clauses.flatMap(({ text, at, contraryWorkerHeading }) => (mentions(text, firstWorker) || !contraryWorkerHeading && commandIsSubject.test(text) && !namesOtherActor(text.replace(commandIsSubject, ""), firstWorker))
     && new RegExp(`\\b${commandSubject}\\b`, "i").test(text)
     ? [...text.matchAll(failureOutcome)].map(match => ({ at: at + match.index, end: at + match.index + match[0].length, clauseAt: at })) : []);
   const handoffs = clauses.flatMap(({ text, at }) => handoffMatches(text).map(match => ({ at: at + match.index })));

tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts

@@ -392,7 +392,7 @@ it("accepts a recorded failed attempt without requiring every worker's name", ()
 it.each([
   unnamedAttempt.replace("You reassigned the task to Riley. ", ""),
   unnamedAttempt.replace("the check passed", "the check failed"),
-  unnamedAttempt.replace("The verification command failed", "Riley ran verification and it failed"),
+  unnamedAttempt.replace("The verification command failed", "The verification command failed on Riley's attempt"),
 ])("keeps missing handoff, failed repair and explicit wrong initial actor rejected: %s", reply => {
   const o = fixture(); o.reply = reply;
   expect(gradeWorkTimeline(o).passed).toBe(false);