From a45fefca6b598fe9f023a78a33adc0b771779173 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 09:36:33 -0600 Subject: [PATCH] test(activity): preserve bounded causal outcomes and file tokens Co-Authored-By: Paperclip --- .../internal-timeline-narrative.ts | 40 ++++++++++++++++--- ...internal-timeline-native-milestone.test.ts | 20 ++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts index b115bf4e393..ee88a36d8e4 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts @@ -9,7 +9,9 @@ export function truthfulTimelineAccount(input: { const mentions = (text: string, name: string) => new RegExp(`\\b${escape(name)}\\b`, "i").test(text); // Search milestone clauses, not the first occurrence of a worker or outcome. // A summary or heading may mention the repair before the ordered account. - const clauses = [...body.matchAll(/[^.!?\n]+(?:[.!?]|$)/g)].map(match => ({ text: match[0], at: match.index })); + const withinClause = "(?:[^.!?\\n]|\\.(?=\\S))"; + const clauses = [...body.matchAll(new RegExp(`${withinClause}+(?:[!?]|\\.(?=\\s|$)|$)`, "g"))] + .map(match => ({ text: match[0], at: match.index })); const failureOutcome = /\b(?:fail(?:ed|ure|ing)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*1)\b/gi; const failures = clauses.flatMap(({ text, at }) => mentions(text, firstWorker) && /\b(?:verification|verify_launch_html|check|test|command)\b/i.test(text) @@ -20,7 +22,7 @@ export function truthfulTimelineAccount(input: { const target = `\\bto\\s+${escape(repairWorker)}\\b`; const actorAction = `${humanActor}(?:\\s+(?:then|later|now|personally|explicitly|actually|had|has|have|also)){0,3}\\s+${assignment}`; const passiveAction = `\\b(?:was|got|had been|has been)\\s+${assignment}`; - const handoffToRepairWorker = new RegExp(`${actorAction}[^.!?\\n]{0,60}${target}|${passiveAction}[^.!?\\n]{0,60}${target}[^.!?\\n]{0,40}\\bby\\s+(?:(?:the|a|an)\\s+)?${humanActor}`, "i"); + const handoffToRepairWorker = new RegExp(`${actorAction}${withinClause}{0,60}${target}|${passiveAction}${withinClause}{0,60}${target}${withinClause}{0,40}\\bby\\s+(?:(?:the|a|an)\\s+)?${humanActor}`, "i"); const differentNamedPerson = (text: string) => { const namedRole = /^\s*(?:Then\s+)?([A-Z][\p{L}'-]+(?:\s+[A-Z][\p{L}'-]+)*)\s*,?\s*(?:the\s+)?(?:human|person|user|operator)\b/u.exec(text); return !!namedRole && !["The", "A", "An", input.personName].some(name => name?.toLowerCase() === namedRole[1]!.toLowerCase()) @@ -43,11 +45,37 @@ export function truthfulTimelineAccount(input: { "(?:completed|finished|performed|applied|implemented)\\s+(?:(?:a|the|needed|required|label|accessibility)\\s+){0,4}(?:repairs?|fix(?:es)?)", "(?:added|inserted|supplied|provided|restored)\\s+(?:(?:a|the|missing|real|email|input|proper|accessible)\\s+){0,5}label", ].join("|"); - const repairAction = new RegExp(`\\b${escape(repairWorker)}\\b(?:${repairGap}${repairModifier}){0,3}${repairGap}(?:${repairVerb})\\b|\\b(?:was|got|had been|has been)\\s+(?:repaired|fixed|completed|added|inserted|supplied|provided|restored)\\b[^.!?\\n]{0,40}\\bby\\s+${escape(repairWorker)}\\b`, "gi"); + const repairAction = new RegExp(`\\b${escape(repairWorker)}\\b(?:${repairGap}${repairModifier}){0,3}${repairGap}(?:${repairVerb})\\b|\\b(?:was|got|had been|has been)\\s+(?:repaired|fixed|completed|added|inserted|supplied|provided|restored)\\b${withinClause}{0,40}\\bby\\s+${escape(repairWorker)}\\b`, "gi"); const repairSuccess = /\b(?:pass(?:ed)?|succeed(?:ed)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*0)\b/i; - const repairs = clauses.flatMap(({ text, at }) => [...text.matchAll(repairAction)] - .filter(match => repairSuccess.test(text.slice(match.index + match[0].length))) - .map(match => ({ at: at + match.index }))); + const commandSubject = "(?:verification|verify_launch_html|check|test|command)"; + const successModifiers = "(?:(?:has|had|have|then|now|finally|successfully)\\s+){0,3}"; + const commandTail = `${commandSubject}\\s+${successModifiers}$`; + const commandLead = new RegExp(`\\b${commandTail}`, "i"); + const explicitOwner = new RegExp(`['’]s\\s+${commandTail}`, "i"); + const repairOwner = new RegExp(`\\b${escape(repairWorker)}['’]s\\s+${commandTail}`, "i"); + const firstWorkerCommand = new RegExp(`\\b${escape(firstWorker)}\\s+(?:ran|executed|performed)\\s+(?:the\\s+)?${commandTail}`, "i"); + const successIsOwned = (text: string, at: number) => { + const before = text.slice(0, at); + return (!explicitOwner.test(before) || repairOwner.test(before)) && !firstWorkerCommand.test(before); + }; + const causedByRepair = /^\s+(?:with\s+exit(?:\s+(?:code|status))?\s*[:=]?\s*0\s+)?(?:(?:only|immediately|just)\s+)?after(?:\s+(?:the|a|missing|email|real|label|input|task|repair)){0,6}\s*$/i; + const repairAfterSuccess = new RegExp(`\\bafter\\s+(?:(?:the|a|his|her|their|its)\\s+)?${commandTail}`, "i"); + const repairs = clauses.flatMap(({ text, at }) => [...text.matchAll(repairAction)].filter(match => { + const end = match.index + match[0].length; + const laterSuccess = [...text.slice(end).matchAll(new RegExp(repairSuccess.source, "gi"))].some(success => { + const outcomeAt = end + success.index; + const between = text.slice(end, outcomeAt); + const backwards = repairAfterSuccess.exec(between); + return successIsOwned(text, outcomeAt) && (!backwards || /\band\s*$/i.test(between.slice(0, backwards.index))); + }); + if (laterSuccess) return true; + // An earlier success word counts only with the bounded causal relationship + // "the check passed after [this worker's repair]", never by itself. + const prefix = text.slice(0, match.index); + return [...prefix.matchAll(new RegExp(repairSuccess.source, "gi"))].some(success => + commandLead.test(prefix.slice(0, success.index)) && successIsOwned(text, success.index) + && causedByRepair.test(prefix.slice(success.index + success[0].length))); + }).map(match => ({ at: at + match.index }))); const ordered = failures.some(failure => handoffs.some(handoff => handoff.at > failure.at && /\b(?:paused|blocked|stopped)\b/i.test(body.slice(failure.clauseAt, handoff.at)) && repairs.some(repair => repair.at > handoff.at))); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts index 830bfd0b2f3..48cf82c908f 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -161,6 +161,26 @@ it.each([ const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false); }); +it.each([ + "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. The verification passed after Riley fixed the label.", + "Today UTC: Morgan verification failed and the task paused, you reassigned the task to Riley, and the check passed with exit code 0 after Riley added the missing label.", + "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. Riley's check passed after Riley completed the repair.", + "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. The check passed after the label was fixed by Riley.", + "Today UTC: Morgan's verification of `launch.html` failed and the task paused. You reassigned the task to Riley. Riley added the missing label in `launch.html` and the verification passed.", +])("keeps bounded causal success and owned file tokens valid: %s", reply => { + const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(true); +}); +it.each([ + "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. The verification passed before Riley fixed the label.", + "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. Riley fixed the label after the verification passed.", + "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. Morgan's verification passed after Riley fixed the label.", + "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. Riley fixed the label and Morgan's verification passed.", + "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. An unrelated earlier check passed, then Riley fixed the label.", + "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. The check passed after Morgan fixed the label.", +])("rejects success before repair, wrong success owner and unrelated earlier success: %s", reply => { + const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false); +}); + it("admits the selected live timeline metadata without invoking fixture callbacks", async () => { const descriptor = findChatCatalogCase("activity.work-timeline")!; const implementation = workTimelineCases[0]!;