fix(evals): accept corroborated implicit timeline worker transitions

This commit is contained in:
Dotta committed 2026-11-06 04:58:44 -06:00
1 parent d68c00a3d7
commit ef7c8a4e01
3 files changed
+52 -4

No files matched your search

@@ -279,3 +279,25 @@ it.each([
])("does not borrow omitted command subjects across a material boundary: %s", reply => {
expect(gradeBulletCommand(reply)).toBe(false);
});
const implicitTransitionAccount = `## Signup implementation — RUN-3
- Morgan built the page. The verification command returned exit 1. The task paused as blocked.
- Riley added the missing label and re-ran verification: exit 0.`;
const gradeImplicitTransition = (reply: string) => truthfulTimelineAccount({ reply, firstWorker: "Morgan", repairWorker: "Riley", taskIdentifier: "RUN-3" });
it("accepts a same-task named-worker transition without requiring an assignment phrase", () => {
expect(gradeImplicitTransition(implicitTransitionAccount)).toBe(true);
expect(gradeImplicitTransition(implicitTransitionAccount.replace("## Signup implementation — RUN-3", "2. **Signup implementation — RUN-3**"))).toBe(true);
});
it.each([
implicitTransitionAccount.replace("Riley added", "Riley repaired RUN-4 and added"),
implicitTransitionAccount.replace("- Riley", "\n## Separate work — RUN-4\n- Riley"),
implicitTransitionAccount.replace("The task paused as blocked.\n- Riley added the missing label and re-ran verification: exit 0.", "Riley added the missing label and re-ran verification: exit 0. The task paused as blocked."),
implicitTransitionAccount.replace("The task paused as blocked.", ""),
implicitTransitionAccount.replace("returned exit 1", "returned exit 0"),
implicitTransitionAccount.replace("Riley added", "Morgan added"),
implicitTransitionAccount.replace("Riley added", "Riley automatically added"),
implicitTransitionAccount.replace("Riley added", "Riley planned to add"),
implicitTransitionAccount.replace("exit 0", "exit 2"),
])("retains task, pause, failure, actor and chronology boundaries for an implicit transition: %s", reply => {
expect(gradeImplicitTransition(reply)).toBe(false);
});
@@ -211,16 +211,30 @@ export function truthfulTimelineAccount(input: {
+ (new RegExp(`^${successorOwnership.source}`, "i").test(match[0]) ? match[0].indexOf(",") + 1 : 0) })));
const pauseCausedByFailure = new RegExp(`\\b(?:after|because)\\s+(?:(?:the|his|her|their|its|${escape(firstWorker)}['’]s)\\s+)?${commandTail}`, "i");
const failureAfterPause = new RegExp(`\\bafter\\s+(?:(?:the|his|her|their|its|a)\\s+)?(?:task|work|it|${escape(firstWorker)})(?:\\s+(?:was|got|had been|has been))?\\s*$`, "i");
const ordered = failures.some(failure => handoffs.some(handoff => handoff.at > failure.at
&& [...body.slice(failure.clauseAt, handoff.at).matchAll(/\b(?:paused|blocked|stopped)\b/gi)].some(pause => {
const pauseBetween = (failure: typeof failures[number], end: number) =>
[...body.slice(failure.clauseAt, end).matchAll(/\b(?:paused|blocked|stopped)\b/gi)].some(pause => {
const pauseAt = failure.clauseAt + pause.index;
// Wording order can differ from event order, but an earlier pause needs
// an explicit causal link to this failure, not just the same clause.
return pauseAt < failure.at
? pauseCausedByFailure.test(body.slice(pauseAt + pause[0].length, failure.at))
: !failureAfterPause.test(body.slice(failure.end, pauseAt));
})
&& repairs.some(repair => repair.at > handoff.at)));
});
const explicitTransition = failures.some(failure => handoffs.some(handoff => handoff.at > failure.at
&& pauseBetween(failure, handoff.at) && repairs.some(repair => repair.at > handoff.at)));
const taskReference = input.taskIdentifier && new RegExp(`\\b${escape(input.taskIdentifier)}\\b`, "i");
const taskNamespace = input.taskIdentifier && /\d+$/.test(input.taskIdentifier)
? new RegExp(`\\b${escape(input.taskIdentifier.replace(/\d+$/, ""))}\\d+\\b`, "gi") : null;
const implicitTransition = !!taskReference && paragraphs.some(paragraph => {
if (!taskReference.test(paragraph.text.split("\n")[0]!)
|| new RegExp(assignment, "i").test(paragraph.text) || new RegExp(successorOwnership.source, "i").test(paragraph.text)
|| /\b(?:automatically|autonomously|self-assigned)\b|\b(?:another|different|other)\s+(?:task|issue|work)\b/i.test(paragraph.text)
|| taskNamespace && [...paragraph.text.matchAll(taskNamespace)].some(match => match[0].toLowerCase() !== input.taskIdentifier!.toLowerCase())) return false;
return failures.some(failure => failure.at >= paragraph.start && failure.at < paragraph.end
&& mentions(body.slice(paragraph.start, failure.at), firstWorker)
&& repairs.some(repair => repair.at > failure.at && repair.at < paragraph.end && pauseBetween(failure, repair.at)));
});
const ordered = explicitTransition || implicitTransition;
const nativeFailure = /\b(?:native|runner|execution)\s+runs?\s+(?:(?:themselves|were|was|had|also|both|[a-z]+ly)\s+){0,3}(?:failed|failures?|unsuccessful)\b|\b(?:native|runner|execution)\s+runs?\s+(?:ended|finished)\s+(?:in|with)\s+(?:a\s+)?failure\b|\bfailed\s+(?:native|runner|execution)\s+runs?\b/i;
const falseNativeFailure = clauses.some(({ text }) => {
const match = nativeFailure.exec(text);
@@ -430,3 +430,15 @@ it.each([
const o = fixture(); o.reply = reply;
expect(gradeWorkTimeline(o).passed).toBe(false);
});
it("corroborates a same-task implicit worker transition with the saved handoff and inspection links", () => {
const o = fixture();
o.reply = "## Signup implementation — RUN-3\nMorgan built the page. The verification command failed with exit 1 and the task paused as blocked. Riley added the missing label and verification exited 0.";
expect(gradeWorkTimeline(o).passed).toBe(true);
const withoutHandoff = { ...o, timelineEvents: o.timelineEvents.filter(event => event.id !== "handoff") };
expect(gradeWorkTimeline(withoutHandoff).passed).toBe(false);
const wrongAssignee = { ...o, timelineEvents: o.timelineEvents.map(event => event.id === "handoff"
? { ...event, details: { assigneeAgentId: "morgan" } } : event) };
expect(gradeWorkTimeline(wrongAssignee).passed).toBe(false);
expect(gradeWorkTimeline({ ...o, links: [] }).passed).toBe(false);
});