From d113cd3fde393230a343e13cb4672259f67b183a Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 10:04:45 -0600 Subject: [PATCH] test(activity): accept bounded command descriptions in timeline grammar Co-Authored-By: Paperclip --- .../internal-timeline-narrative.ts | 15 ++++++-- ...internal-timeline-native-milestone.test.ts | 36 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts index e4ce1292c8a..08a101bd058 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts @@ -12,9 +12,21 @@ export function truthfulTimelineAccount(input: { const withinClause = "(?:[^.!?\\n]|\\.(?=\\S))"; const clauses = [...body.matchAll(new RegExp(`${withinClause}+(?:[!?]|\\.(?=\\s|$)|$)`, "g"))] .map(match => ({ text: match[0], at: match.index })); + // A command noun can have ordinary modifiers, but never absorb a new event, + // actor, negation or clause boundary while connecting this pause to failure. + const commandBoundaries = [ + "after|before|because|once|then|and|or|but|while|when|until|since|not|never|no", + "you|i|he|she|they|we|human|person|user|operator|board|was|were|is|are|had|has|have|will|would|could|should", + "fail(?:ed|ure|ing)?|pass(?:ed)?|succeed(?:ed)?|exit(?:ed)?|paus(?:e|ed)|block(?:ed)?|stop(?:ped)?", + "(?:re)?assign(?:ed|ment)?|hand(?:ed|off)?|fix(?:ed)?|repair(?:ed)?|watch(?:ed)?|ask(?:ed)?|ran|run|execut(?:e|ed)", + ...[firstWorker, repairWorker, input.personName].filter(Boolean).flatMap(name => name!.split(/\s+/).map(escape)), + ].join("|"); + const commandModifier = `(?!(?:${commandBoundaries})\\b)[a-z][a-z0-9_-]*`; + const commandSubject = `(?:${commandModifier}[ \\t]+){0,3}(?:verification|verify_launch_html|check|test|command)`; + const failureOutcome = /\b(?:fail(?:ed|ure|ing)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*1)\b/gi; const failures = clauses.flatMap(({ text, at }) => mentions(text, firstWorker) - && /\b(?:verification|verify_launch_html|check|test|command)\b/i.test(text) + && new RegExp(`\\b${commandSubject}\\b`, "i").test(text) ? [...text.matchAll(failureOutcome)].map(match => ({ at: at + match.index, end: at + match.index + match[0].length, clauseAt: at })) : []); const humanActors = ["you", "human operator", "board member", "human", "person", "user", "operator", "board", input.personName].filter(Boolean).map(name => escape(name!)).join("|"); const humanActor = `\\b(?:${humanActors})\\b`; @@ -47,7 +59,6 @@ export function truthfulTimelineAccount(input: { ].join("|"); const repairAction = new RegExp(`\\b${escape(repairWorker)}\\b(?:${repairGap}${repairModifier}){0,3}${repairGap}(?:${repairVerb})\\b|\\b(?:was|got|had been|has been)\\s+(?:repaired|fixed|completed|added|inserted|supplied|provided|restored)\\b${withinClause}{0,40}\\bby\\s+${escape(repairWorker)}\\b`, "gi"); const repairSuccess = /\b(?:pass(?:ed)?|succeed(?:ed)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*0)\b/i; - const commandSubject = "(?:verification|verify_launch_html|check|test|command)"; const successModifiers = "(?:(?:has|had|have|then|now|finally|successfully)\\s+){0,3}"; const commandTail = `${commandSubject}\\s+${successModifiers}$`; const commandLead = new RegExp(`\\b${commandTail}`, "i"); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts index 50808a74cf7..479a925fcc5 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -211,3 +211,39 @@ it.each([ ])("rejects reversed or unrelated pause chronology: %s", reply => { const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false); }); + + +it.each([ + "Today UTC: Morgan paused the task after the verification command failed. You reassigned the task to Riley. Riley fixed the label and the check passed.", + "Today UTC: Morgan was blocked because the accessibility check failed. You reassigned the task to Riley. Riley fixed the label and the check passed.", + "Today UTC: Morgan paused the task after the email label check failed. You reassigned the task to Riley. Riley fixed the label and the accessibility check passed.", + "Today UTC: Morgan was blocked because the accessible heading test failed. You reassigned the task to Riley. Riley fixed the label and the email label check passed.", + "Today UTC: Morgan paused the task after verify_launch_html failed. You reassigned the task to Riley. Riley fixed the label and verify_launch_html passed.", +])("accepts bounded command descriptions linking a pause to failure: %s", reply => { + const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(true); +}); +it.each([ + "Today UTC: Morgan paused the task before the verification command failed. You reassigned the task to Riley. Riley fixed the label and the check passed.", + "Today UTC: Morgan was blocked before the accessibility check failed. You reassigned the task to Riley. Riley fixed the label and the check passed.", + "Today UTC: Morgan paused the task after the unrelated meeting failed while a verification command was available. You reassigned the task to Riley. Riley fixed the label and the check passed.", + "Today UTC: Morgan paused the task after the email label check was scheduled and an unrelated meeting failed. You reassigned the task to Riley. Riley fixed the label and the check passed.", +])("rejects reversed pause order with bounded command descriptions: %s", reply => { + const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false); +}); + + +it.each([ + "Today UTC: Morgan paused the task after the automated smoke check failed. You reassigned the task to Riley. Riley fixed the label and the automated smoke check passed.", + "Today UTC: Morgan was blocked because the build validation command failed. You reassigned the task to Riley. Riley fixed the label and the build validation command passed.", +])("accepts ordinary bounded command noun phrases: %s", reply => { + const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(true); +}); +it.each([ + "Today UTC: Morgan paused the task before the automated smoke check failed. You reassigned the task to Riley. Riley fixed the label and the check passed.", + "Today UTC: Morgan was blocked before the build validation command failed. You reassigned the task to Riley. Riley fixed the label and the check passed.", + "Today UTC: Morgan paused the task after the automated smoke check failed. You watched while Morgan reassigned the task to Riley. Riley fixed the label and the check passed.", + "Today UTC: Morgan paused the task after the build validation command failed. You asked Morgan to reassign the task to Riley. Riley fixed the label and the check passed.", + "Today UTC: Morgan paused the task after the build validation command failed. You reassigned the task to Riley. Morgan fixed the label and the check passed.", +])("preserves chronology and actors with ordinary command noun phrases: %s", reply => { + const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false); +});