diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts index 08a101bd058..4bbc7401237 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts @@ -71,7 +71,12 @@ export function truthfulTimelineAccount(input: { }; const causedByRepair = /^\s+(?:with\s+exit(?:\s+(?:code|status))?\s*[:=]?\s*0\s+)?(?:(?:only|immediately|just)\s+)?(?:after|once)(?:\s+(?:the|a|missing|email|real|label|input|task|repair)){0,6}\s*$/i; const repairAfterSuccess = new RegExp(`\\bafter\\s+(?:(?:the|a|his|her|their|its)\\s+)?${commandTail}`, "i"); - const repairs = clauses.flatMap(({ text, at }) => [...text.matchAll(repairAction)].filter(match => { + // Only an immediately adjacent, complete command result can finish this + // repair account. No intervening sentence or earlier/unrelated result joins. + const adjacentVerification = new RegExp(`^\\s*(?:(?:the|a|an)\\s+|${escape(repairWorker)}['’]s\\s+)?${commandSubject}\\s+${successModifiers}(?:${repairSuccess.source})(?:\\s+with\\s+exit(?:\\s+(?:code|status))?\\s*[:=]?\\s*0)?\\s*[.!]?\\s*$`, "i"); + const repairObjectTail = new RegExp(`^(?:[ \\t]+${commandModifier}){0,6}\\s*[.!]?\\s*$`, "i"); + const passiveRepairSubject = new RegExp(`^\\s*(?:${commandModifier}[ \\t]+){1,6}$`, "i"); + const repairs = clauses.flatMap(({ text, at }, clauseIndex) => [...text.matchAll(repairAction)].filter(match => { const end = match.index + match[0].length; const laterSuccess = [...text.slice(end).matchAll(new RegExp(repairSuccess.source, "gi"))].some(success => { const outcomeAt = end + success.index; @@ -80,6 +85,13 @@ export function truthfulTimelineAccount(input: { return successIsOwned(text, outcomeAt) && (!backwards || /\band\s*$/i.test(between.slice(0, backwards.index))); }); if (laterSuccess) return true; + const next = clauses[clauseIndex + 1]?.text; + const lead = text.slice(0, match.index); + const assertedRepair = /^\s*(?:(?:then|later|finally)\s+)?$/i.test(lead) + || /^\b(?:was|got|had been|has been)\b/i.test(match[0]) && passiveRepairSubject.test(lead); + if (next && assertedRepair && repairObjectTail.test(text.slice(end)) + && !text.trimEnd().endsWith("?") && adjacentVerification.test(next) + && !/\b(?:if|unless|whether|before|earlier|previous(?:ly)?|already|unrelated|different|other)\b/i.test(text + next)) return true; // An earlier success word counts only with the bounded causal relationship // "the check passed after [this worker's repair]", never by itself. const prefix = text.slice(0, match.index); @@ -99,10 +111,10 @@ export function truthfulTimelineAccount(input: { : !failureAfterPause.test(body.slice(failure.end, pauseAt)); }) && repairs.some(repair => repair.at > handoff.at))); - const nativeFailure = /\b(?:native|runner|execution)\s+runs?\s+(?:(?:themselves|were|was|had|also|both)\s+){0,3}(?:failed|failures?|unsuccessful)\b|\b(?:native|runner|execution)\s+runs?\s+(?:ended|finished)\s+(?:in|with)\s+(?:a\s+)?failure\b|\bfailed\s+(?:native|runner|execution)\s+runs?\b/i; + const nativeFailure = /\b(?:native|runner|execution)\s+runs?\s+(?:(?:themselves|were|was|had|also|both|[a-z]+ly)\s+){0,3}(?:failed|failures?|unsuccessful)\b|\b(?:native|runner|execution)\s+runs?\s+(?:ended|finished)\s+(?:in|with)\s+(?:a\s+)?failure\b|\bfailed\s+(?:native|runner|execution)\s+runs?\b/i; const falseNativeFailure = clauses.some(({ text }) => { const match = nativeFailure.exec(text); - if (!match || text.trimEnd().endsWith("?")) return false; + if (!match || text.trimEnd().endsWith("?") || /\bhypothetically\b/i.test(match[0])) return false; const before = text.slice(0, match.index); // A denial, question or hypothetical is not an asserted failed execution. return !/\b(?:no|neither|not a|not an|none of the)\s*$/i.test(before) diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts index 479a925fcc5..a2a43eac774 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -247,3 +247,46 @@ it.each([ ])("preserves chronology and actors with ordinary command noun phrases: %s", reply => { const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false); }); + + +it.each([ + { reply: "Today UTC: Morgan's verification failed and work paused. You reassigned the task to Riley. Riley fixed the label. The check passed.", expected: true }, + { reply: "Today UTC: Morgan's verification failed and work paused. You reassigned the task to Riley. Riley fixed the label and the check passed. The native run ultimately failed.", expected: false }, +])("handles reviewed adjacent success and native failure wording: $reply", ({ reply, expected }) => { + const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(expected); +}); + + +it.each([ + { ending: "Riley fixed the label. The automated smoke check passed.", expected: true }, + { ending: "Riley fixed the label. Riley's command succeeded.", expected: true }, + { ending: "Riley fixed the label. Morgan's check passed.", expected: false }, + { ending: "Morgan fixed the label. The check passed.", expected: false }, + { ending: "Riley fixed the label. Before that the check passed.", expected: false }, + { ending: "Riley fixed the label. The check passed before the repair.", expected: false }, + { ending: "Riley fixed the label. An unrelated check passed.", expected: false }, + { ending: "Riley fixed the label. A meeting ended. The check passed.", expected: false }, + { ending: "Riley fixed the label. The check passed?", expected: false }, + { ending: "Riley fixed the label? The check passed.", expected: false }, + { ending: "Riley fixed the label, then Morgan ran a different check. The check passed.", expected: false }, + { ending: "Riley fixed the label, but Morgan broke the page. The check passed.", expected: false }, + { ending: "If Riley fixed the label, the release might work. The check passed.", expected: false }, + { ending: "The label was fixed by Riley. The check passed.", expected: true }, + { ending: "Riley fixed the missing accessible email label. The check passed.", expected: true }, + { ending: "The check passed. Riley fixed the label.", expected: false }, +])("keeps adjacent success bound to the repair and event order: $ending", ({ ending, expected }) => { + const o = fixture(); o.reply = "Today UTC: Morgan's verification failed and work paused. You reassigned the task to Riley. " + ending; + expect(gradeWorkTimeline(o).passed).toBe(expected); +}); +it.each([ + { ending: "The native run unexpectedly failed.", expected: false }, + { ending: "The runner runs eventually failed.", expected: false }, + { ending: "No native run ultimately failed.", expected: true }, + { ending: "The native run never ultimately failed.", expected: true }, + { ending: "Did the native run ultimately fail?", expected: true }, + { ending: "If a native run ultimately failed, it would be recorded.", expected: true }, + { ending: "The native run hypothetically failed.", expected: true }, + { ending: "The native run's verification unexpectedly failed, while the native run succeeded.", expected: true }, +])("distinguishes bounded native failure assertions from denials and checks: $ending", ({ ending, expected }) => { + const o = fixture(); o.reply += " " + ending; expect(gradeWorkTimeline(o).passed).toBe(expected); +});