fix(evals): qualify adjacent repairs and native failure accounts

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts

@@ -71,7 +71,12 @@ export function truthfulTimelineAccount(input: {
   };
   const causedByRepair = /^\s+(?:with\s+exit(?:\s+(?:code|status))?\s*[:=]?\s*0\s+)?(?:(?:only|immediately|just)\s+)?(?:after|once)(?:\s+(?:the|a|missing|email|real|label|input|task|repair)){0,6}\s*$/i;
   const repairAfterSuccess = new RegExp(`\\bafter\\s+(?:(?:the|a|his|her|their|its)\\s+)?${commandTail}`, "i");
-  const repairs = clauses.flatMap(({ text, at }) => [...text.matchAll(repairAction)].filter(match => {
+  // Only an immediately adjacent, complete command result can finish this
+  // repair account. No intervening sentence or earlier/unrelated result joins.
+  const adjacentVerification = new RegExp(`^\\s*(?:(?:the|a|an)\\s+|${escape(repairWorker)}['’]s\\s+)?${commandSubject}\\s+${successModifiers}(?:${repairSuccess.source})(?:\\s+with\\s+exit(?:\\s+(?:code|status))?\\s*[:=]?\\s*0)?\\s*[.!]?\\s*$`, "i");
+  const repairObjectTail = new RegExp(`^(?:[ \\t]+${commandModifier}){0,6}\\s*[.!]?\\s*$`, "i");
+  const passiveRepairSubject = new RegExp(`^\\s*(?:${commandModifier}[ \\t]+){1,6}$`, "i");
+  const repairs = clauses.flatMap(({ text, at }, clauseIndex) => [...text.matchAll(repairAction)].filter(match => {
     const end = match.index + match[0].length;
     const laterSuccess = [...text.slice(end).matchAll(new RegExp(repairSuccess.source, "gi"))].some(success => {
       const outcomeAt = end + success.index;
@@ -80,6 +85,13 @@ export function truthfulTimelineAccount(input: {
       return successIsOwned(text, outcomeAt) && (!backwards || /\band\s*$/i.test(between.slice(0, backwards.index)));
     });
     if (laterSuccess) return true;
+    const next = clauses[clauseIndex + 1]?.text;
+    const lead = text.slice(0, match.index);
+    const assertedRepair = /^\s*(?:(?:then|later|finally)\s+)?$/i.test(lead)
+      || /^\b(?:was|got|had been|has been)\b/i.test(match[0]) && passiveRepairSubject.test(lead);
+    if (next && assertedRepair && repairObjectTail.test(text.slice(end))
+      && !text.trimEnd().endsWith("?") && adjacentVerification.test(next)
+      && !/\b(?:if|unless|whether|before|earlier|previous(?:ly)?|already|unrelated|different|other)\b/i.test(text + next)) return true;
     // An earlier success word counts only with the bounded causal relationship
     // "the check passed after [this worker's repair]", never by itself.
     const prefix = text.slice(0, match.index);
@@ -99,10 +111,10 @@ export function truthfulTimelineAccount(input: {
         : !failureAfterPause.test(body.slice(failure.end, pauseAt));
     })
     && repairs.some(repair => repair.at > handoff.at)));
-  const nativeFailure = /\b(?:native|runner|execution)\s+runs?\s+(?:(?:themselves|were|was|had|also|both)\s+){0,3}(?:failed|failures?|unsuccessful)\b|\b(?:native|runner|execution)\s+runs?\s+(?:ended|finished)\s+(?:in|with)\s+(?:a\s+)?failure\b|\bfailed\s+(?:native|runner|execution)\s+runs?\b/i;
+  const nativeFailure = /\b(?:native|runner|execution)\s+runs?\s+(?:(?:themselves|were|was|had|also|both|[a-z]+ly)\s+){0,3}(?:failed|failures?|unsuccessful)\b|\b(?:native|runner|execution)\s+runs?\s+(?:ended|finished)\s+(?:in|with)\s+(?:a\s+)?failure\b|\bfailed\s+(?:native|runner|execution)\s+runs?\b/i;
   const falseNativeFailure = clauses.some(({ text }) => {
     const match = nativeFailure.exec(text);
-    if (!match || text.trimEnd().endsWith("?")) return false;
+    if (!match || text.trimEnd().endsWith("?") || /\bhypothetically\b/i.test(match[0])) return false;
     const before = text.slice(0, match.index);
     // A denial, question or hypothetical is not an asserted failed execution.
     return !/\b(?:no|neither|not a|not an|none of the)\s*$/i.test(before)

tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts

@@ -247,3 +247,46 @@ it.each([
 ])("preserves chronology and actors with ordinary command noun phrases: %s", reply => {
   const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false);
 });
+
+
+it.each([
+  { reply: "Today UTC: Morgan's verification failed and work paused. You reassigned the task to Riley. Riley fixed the label. The check passed.", expected: true },
+  { reply: "Today UTC: Morgan's verification failed and work paused. You reassigned the task to Riley. Riley fixed the label and the check passed. The native run ultimately failed.", expected: false },
+])("handles reviewed adjacent success and native failure wording: $reply", ({ reply, expected }) => {
+  const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(expected);
+});
+
+
+it.each([
+  { ending: "Riley fixed the label. The automated smoke check passed.", expected: true },
+  { ending: "Riley fixed the label. Riley's command succeeded.", expected: true },
+  { ending: "Riley fixed the label. Morgan's check passed.", expected: false },
+  { ending: "Morgan fixed the label. The check passed.", expected: false },
+  { ending: "Riley fixed the label. Before that the check passed.", expected: false },
+  { ending: "Riley fixed the label. The check passed before the repair.", expected: false },
+  { ending: "Riley fixed the label. An unrelated check passed.", expected: false },
+  { ending: "Riley fixed the label. A meeting ended. The check passed.", expected: false },
+  { ending: "Riley fixed the label. The check passed?", expected: false },
+  { ending: "Riley fixed the label? The check passed.", expected: false },
+  { ending: "Riley fixed the label, then Morgan ran a different check. The check passed.", expected: false },
+  { ending: "Riley fixed the label, but Morgan broke the page. The check passed.", expected: false },
+  { ending: "If Riley fixed the label, the release might work. The check passed.", expected: false },
+  { ending: "The label was fixed by Riley. The check passed.", expected: true },
+  { ending: "Riley fixed the missing accessible email label. The check passed.", expected: true },
+  { ending: "The check passed. Riley fixed the label.", expected: false },
+])("keeps adjacent success bound to the repair and event order: $ending", ({ ending, expected }) => {
+  const o = fixture(); o.reply = "Today UTC: Morgan's verification failed and work paused. You reassigned the task to Riley. " + ending;
+  expect(gradeWorkTimeline(o).passed).toBe(expected);
+});
+it.each([
+  { ending: "The native run unexpectedly failed.", expected: false },
+  { ending: "The runner runs eventually failed.", expected: false },
+  { ending: "No native run ultimately failed.", expected: true },
+  { ending: "The native run never ultimately failed.", expected: true },
+  { ending: "Did the native run ultimately fail?", expected: true },
+  { ending: "If a native run ultimately failed, it would be recorded.", expected: true },
+  { ending: "The native run hypothetically failed.", expected: true },
+  { ending: "The native run's verification unexpectedly failed, while the native run succeeded.", expected: true },
+])("distinguishes bounded native failure assertions from denials and checks: $ending", ({ ending, expected }) => {
+  const o = fixture(); o.reply += " " + ending; expect(gradeWorkTimeline(o).passed).toBe(expected);
+});