From 6d6dc6e39716aade9f9e313e6c88c5efd458b5d3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 14:06:06 -0600 Subject: [PATCH] fix(evals): retain correlative success and known-source takeover --- .../chat-catalog-domains/internal-timeline-narrative.ts | 6 ++++-- .../internal-timeline-native-milestone.test.ts | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts index c06f57fa528..406f8a74ef2 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts @@ -30,7 +30,7 @@ export function truthfulTimelineAccount(input: { const passiveAction = `\\b(?:was|got|had been|has been)\\s+${assignment}`; const handoffToRepairWorker = new RegExp(`${actorAction}${withinClause}{0,60}${target}|${passiveAction}${withinClause}{0,60}${target}${withinClause}{0,40}\\bby\\s+(?:(?:the|a|an)\\s+)?${humanActor}`, "i"); const passiveHandoffToRepairWorker = new RegExp(`${passiveAction}${withinClause}{0,60}${target}`, "gi"); - const successorOwnership = new RegExp(`\\b${escape(repairWorker)}\\s+took\\s+(?:over\\s+(?:the\\s+)?(?:same\\s+)?(?:task|work)|over(?=\\s*(?:[,;.!?\\n]|$))|the\\s+same\\s+(?:task|work))\\b`, "gi"); + const successorOwnership = new RegExp(`\\b${escape(repairWorker)}\\s+took\\s+(?:over\\s+(?:the\\s+)?(?:same\\s+)?(?:task|work)|over\\s+from\\s+${escape(firstWorker)}|over(?=\\s*(?:[,;.!?\\n]|$))|the\\s+same\\s+(?:task|work))\\b`, "gi"); const differentNamedPerson = (text: string) => { const namedRole = /^\s*(?:Then\s+)?([A-Z][\p{L}'-]+(?:\s+[A-Z][\p{L}'-]+)*)\s*,?\s*(?:the\s+)?(?:human|person|user|operator)\b/u.exec(text); return !!namedRole && !["The", "A", "An", input.personName].some(name => name?.toLowerCase() === namedRole[1]!.toLowerCase()) @@ -134,9 +134,11 @@ export function truthfulTimelineAccount(input: { const successIsOwned = (text: string, at: number) => { const before = text.slice(0, at); const resultStart = [...before.matchAll(/\b(?:verification|verify_launch_html|check|test|command)\b/gi)].at(-1)?.index ?? at; + const correlative = /\bnot\s+only\b[^.!?\n]*\bbut\b/i.test(text.slice(resultStart)); + const assertionPrefix = currentResult(before.slice(resultStart)).replace(/\bnot\s+only\b/gi, match => correlative ? "" : match); return (!explicitOwner.test(before) || repairOwner.test(before)) && !firstWorkerCommand.test(before) && !nonzeroExit(text.slice(resultStart)) - && !/\b(?:not|never|no)\b|n['’]t\b/i.test(currentResult(before.slice(resultStart))); + && !/\b(?:not|never|no)\b|n['’]t\b/i.test(assertionPrefix); }; const causedByRepair = /^\s+(?:with\s+exit(?:\s+(?:code|status))?\s*[:=]?\s*0\s+)?(?:(?:only|immediately|just)\s+)?(?:after|once)(?:\s+(?:the|a|missing|email|real|label|input|task|repair)){0,6}\s*$/i; const repairAfterSuccess = new RegExp(`\\bafter\\s+(?:(?:the|a|his|her|their|its)\\s+)?${commandTail}`, "i"); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts index b66ecfe4c8e..a9e2529a62e 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -369,3 +369,11 @@ it.each([ o.reply = `Morgan's verification failed and work paused. ${handoff} ${repair}`; expect(gradeWorkTimeline(o).passed).toBe(expected); }); + +it.each([ + "Morgan's verification failed and work paused. Riley took the same task. Riley fixed the label and the check not only passed but exited 0.", + "Morgan's verification failed and work paused. Riley took over from Morgan. Riley fixed the label and the check passed.", +])("accepts the reviewed correlative assertion or known-source takeover: %s", reply => { + const o = fixture(); o.reply = reply; + expect(gradeWorkTimeline(o).passed).toBe(true); +});