From 3a9c3fa8cc6a766815e9062bb041a344119694bb Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 13:36:42 -0600 Subject: [PATCH] fix(evals): recognize successor takeover in timeline context --- .../internal-timeline-narrative.test.ts | 15 ++++++++++++ .../internal-timeline-narrative.ts | 24 ++++++++++++------- ...internal-timeline-native-milestone.test.ts | 6 ++++- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts index 6764835d8a1..f6fce200f27 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts @@ -129,3 +129,18 @@ it.each([ ])("does not borrow another actor, paragraph or unrelated result for a repair: %s", reply => { expect(grade(reply)).toBe(false); }); + +const coordinatedAccount = bulletedAccount + .replace("The task was reassigned to Riley.\n", "") + .replace("- Riley added the label while keeping the heading.\n- Verification on the saved bytes passed", "- Riley took the same task, added the label while keeping the heading, and re-ran the check on the saved bytes. It passed"); +it("retains the successor subject through coordinated repair and its nearest command result", () => { + expect(grade(coordinatedAccount)).toBe(true); +}); +it.each([ + coordinatedAccount.replace("and re-ran", "and Morgan re-ran"), + coordinatedAccount.replace("It passed", "Morgan inspected another task. It passed"), + coordinatedAccount.replace("It passed", "\n\nIt passed"), + coordinatedAccount.replace("added the label", "did not add the label"), +])("keeps coordinated command ownership and adjacency bounded: %s", reply => { + expect(grade(reply)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts index 15481798489..f7bcbdaad40 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts @@ -23,6 +23,7 @@ export function truthfulTimelineAccount(input: { const passiveAction = `\\b(?:was|got|had been|has been)\\s+${assignment}`; const handoffToRepairWorker = new RegExp(`${actorAction}${withinClause}{0,60}${target}|${passiveAction}${withinClause}{0,60}${target}${withinClause}{0,40}\\bby\\s+(?:(?:the|a|an)\\s+)?${humanActor}`, "i"); const passiveHandoffToRepairWorker = new RegExp(`${passiveAction}${withinClause}{0,60}${target}`, "gi"); + const successorOwnership = new RegExp(`\\b${escape(repairWorker)}\\s+took\\s+(?:over(?:\\s+(?:the\\s+)?(?:task|work))?|the\\s+same\\s+(?:task|work))\\b`, "gi"); const differentNamedPerson = (text: string) => { const namedRole = /^\s*(?:Then\s+)?([A-Z][\p{L}'-]+(?:\s+[A-Z][\p{L}'-]+)*)\s*,?\s*(?:the\s+)?(?:human|person|user|operator)\b/u.exec(text); return !!namedRole && !["The", "A", "An", input.personName].some(name => name?.toLowerCase() === namedRole[1]!.toLowerCase()) @@ -33,7 +34,7 @@ export function truthfulTimelineAccount(input: { const actorUnspecified = [...text.matchAll(passiveHandoffToRepairWorker)].filter(match => !/\bby\b/i.test(text.slice(match.index + match[0].length)) && !/\b(?:automatically|autonomously|self-assigned)\b/i.test(text)); - return [...text.matchAll(new RegExp(handoffToRepairWorker.source, "gi")), ...actorUnspecified].filter(match => { + return [...text.matchAll(new RegExp(handoffToRepairWorker.source, "gi")), ...actorUnspecified, ...text.matchAll(successorOwnership)].filter(match => { // An explicit other person's role must not become the current human just // because earlier milestones share this sentence. if (!/^(?:human|person|user|operator|board)\b/i.test(match[0])) return true; @@ -101,7 +102,7 @@ export function truthfulTimelineAccount(input: { "(?:completed|finished|performed|applied|implemented)\\s+(?:(?:a|the|needed|required|label|accessibility)\\s+){0,4}(?:repairs?|fix(?:es)?)", "(?:added|inserted|supplied|provided|restored)\\s+(?:(?:a|the|missing|real|email|input|proper|accessible)\\s+){0,5}label", ].join("|"); - const repairAction = new RegExp(`\\b${escape(repairWorker)}\\b(?:${repairGap}${repairModifier}){0,3}${repairGap}(?:${repairVerb})\\b|\\b(?:was|got|had been|has been)\\s+(?:repaired|fixed|completed|added|inserted|supplied|provided|restored)\\b${withinClause}{0,40}\\bby\\s+${escape(repairWorker)}\\b`, "gi"); + const repairAction = new RegExp(`\\b${escape(repairWorker)}\\b(?:${repairGap}${repairModifier}){0,3}${repairGap}(?:${repairVerb})\\b|\\b(?:was|got|had been|has been)\\s+(?:repaired|fixed|completed|added|inserted|supplied|provided|restored)\\b${withinClause}{0,40}\\bby\\s+${escape(repairWorker)}\\b|${successorOwnership.source}\\s*,\\s*(?:${repairVerb})\\b`, "gi"); const repairSuccess = /\b(?:pass(?:ed)?|succeed(?:ed)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*0)\b/i; const successModifiers = "(?:(?:has|had|have|then|now|finally|successfully)\\s+){0,3}"; const commandTail = `${commandSubject}\\s+${successModifiers}$`; @@ -119,6 +120,7 @@ export function truthfulTimelineAccount(input: { // repair account. No intervening sentence or earlier/unrelated result joins. const adjacentVerification = new RegExp(`^\\s*(?:(?:the|a|an)\\s+|${escape(repairWorker)}['’]s\\s+)?${commandSubject}\\s+${successModifiers}(?:${repairSuccess.source})(?:\\s+with\\s+exit(?:\\s+(?:code|status))?\\s*[:=]?\\s*0)?\\s*[.!]?\\s*$`, "i"); const contextualCommandLead = new RegExp(`^\\s*(?:(?:the|a|an)\\s+)?${commandSubject}(?:[ \\t]+${commandModifier}){0,5}\\s+${successModifiers}$`, "i"); + const coordinatedCommand = new RegExp(`(?:,\\s*(?:and\\s+)?|\\band\\s+)(?:re-?ran|ran|executed|performed)\\s+(?:(?:the|a|an)\\s+)?${commandSubject}(?:[ \\t]+${commandModifier}){0,5}\\s*[.!]?\\s*$`, "i"); const repairObjectTail = new RegExp(`^(?:[ \\t]+${commandModifier}){0,6}\\s*[.!]?\\s*$`, "i"); const passiveRepairSubject = new RegExp(`^\\s*(?:${commandModifier}[ \\t]+){1,6}$`, "i"); const deniedRepairObject = /^\s*(?:[a-z]+ly\s+){0,2}(?:(?:no|none|neither|nothing)\b|zero(?=\s|[.!?]|$))/i; @@ -137,13 +139,16 @@ export function truthfulTimelineAccount(input: { const assertedRepair = /^\s*(?:(?:then|later|finally)\s+)?$/i.test(lead) || passiveRepair && passiveRepairSubject.test(lead); const nextClause = clauses[clauseIndex + 1]; - const contextualOutcome = next && repairSuccess.exec(next); - const contextualResult = owner === repairWorker && nextClause?.owner === repairWorker - && paragraphStart === nextClause.paragraphStart && !!contextualOutcome - && contextualCommandLead.test(next.slice(0, contextualOutcome.index)) - && successIsOwned(next, contextualOutcome.index) + const sameWorkerParagraph = owner === repairWorker && nextClause?.owner === repairWorker + && paragraphStart === nextClause.paragraphStart; + const nextCommand = next && sameWorkerParagraph && coordinatedCommand.test(text.slice(end)) + ? next.replace(/^\s*it\b/i, "check") : next; + const contextualOutcome = nextCommand && repairSuccess.exec(nextCommand); + const contextualResult = sameWorkerParagraph && !!contextualOutcome + && contextualCommandLead.test(nextCommand.slice(0, contextualOutcome.index)) + && successIsOwned(nextCommand, contextualOutcome.index) && !new RegExp(failureOutcome.source, "i").test(next) - && !namesOtherActor(text + " " + next, repairWorker) && !mentions(text + " " + next, firstWorker); + && !namesOtherActor(text + " " + nextCommand, repairWorker) && !mentions(text + " " + next, firstWorker); if (next && assertedRepair && !deniedRepairObject.test(passiveRepair ? lead : text.slice(end)) && (repairObjectTail.test(text.slice(end)) || contextualResult) && !text.trimEnd().endsWith("?") && !next.trimEnd().endsWith("?") && (adjacentVerification.test(next) || contextualResult) @@ -154,7 +159,8 @@ export function truthfulTimelineAccount(input: { return [...prefix.matchAll(new RegExp(repairSuccess.source, "gi"))].some(success => commandLead.test(prefix.slice(0, success.index)) && successIsOwned(text, success.index) && causedByRepair.test(prefix.slice(success.index + success[0].length))); - }).map(match => ({ at: at + match.index }))); + }).map(match => ({ at: at + match.index + + (new RegExp(`^${successorOwnership.source}`, "i").test(match[0]) ? match[0].indexOf(",") + 1 : 0) }))); const pauseCausedByFailure = new RegExp(`\\b(?:after|because)\\s+(?:(?:the|his|her|their|its|${escape(firstWorker)}['’]s)\\s+)?${commandTail}`, "i"); const failureAfterPause = new RegExp(`\\bafter\\s+(?:(?:the|his|her|their|its|a)\\s+)?(?:task|work|it|${escape(firstWorker)})(?:\\s+(?:was|got|had been|has been))?\\s*$`, "i"); const ordered = failures.some(failure => handoffs.some(handoff => handoff.at > failure.at diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts index 93779007197..c029bd77469 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -42,11 +42,15 @@ it("accepts an ordered recorded reassignment without an unrequested human identi o.reply = o.reply.replace("Then a human reassigned the task to Riley.", "Then the task was reassigned to Riley."); expect(gradeWorkTimeline(o).passed).toBe(true); }); +it("accepts the successor's natural ownership transition in the ordered account", () => { + const o = fixture(); + o.reply = o.reply.replace("Then a human reassigned the task to Riley.", "Then Riley took over the task."); + expect(gradeWorkTimeline(o).passed).toBe(true); +}); it.each([ "Then the task was reassigned to Riley by Morgan.", "Then the task was reassigned to Riley by Jordan.", "Then the task was reassigned to Morgan.", - "Then Riley took the same task.", "Then the task was reassigned to Riley automatically.", ])("rejects false or missing handoff provenance: %s", handoff => { const o = fixture();