mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
fix(evals): accept corroborated implicit timeline worker transitions
This commit is contained in:
1 parent
d68c00a3d7
commit
ef7c8a4e01
3 files changed
+52
-4
No files matched your search
@@ -279,3 +279,25 @@ it.each([
|
||||
])("does not borrow omitted command subjects across a material boundary: %s", reply => {
|
||||
expect(gradeBulletCommand(reply)).toBe(false);
|
||||
});
|
||||
|
||||
const implicitTransitionAccount = `## Signup implementation — RUN-3
|
||||
- Morgan built the page. The verification command returned exit 1. The task paused as blocked.
|
||||
- Riley added the missing label and re-ran verification: exit 0.`;
|
||||
const gradeImplicitTransition = (reply: string) => truthfulTimelineAccount({ reply, firstWorker: "Morgan", repairWorker: "Riley", taskIdentifier: "RUN-3" });
|
||||
it("accepts a same-task named-worker transition without requiring an assignment phrase", () => {
|
||||
expect(gradeImplicitTransition(implicitTransitionAccount)).toBe(true);
|
||||
expect(gradeImplicitTransition(implicitTransitionAccount.replace("## Signup implementation — RUN-3", "2. **Signup implementation — RUN-3**"))).toBe(true);
|
||||
});
|
||||
it.each([
|
||||
implicitTransitionAccount.replace("Riley added", "Riley repaired RUN-4 and added"),
|
||||
implicitTransitionAccount.replace("- Riley", "\n## Separate work — RUN-4\n- Riley"),
|
||||
implicitTransitionAccount.replace("The task paused as blocked.\n- Riley added the missing label and re-ran verification: exit 0.", "Riley added the missing label and re-ran verification: exit 0. The task paused as blocked."),
|
||||
implicitTransitionAccount.replace("The task paused as blocked.", ""),
|
||||
implicitTransitionAccount.replace("returned exit 1", "returned exit 0"),
|
||||
implicitTransitionAccount.replace("Riley added", "Morgan added"),
|
||||
implicitTransitionAccount.replace("Riley added", "Riley automatically added"),
|
||||
implicitTransitionAccount.replace("Riley added", "Riley planned to add"),
|
||||
implicitTransitionAccount.replace("exit 0", "exit 2"),
|
||||
])("retains task, pause, failure, actor and chronology boundaries for an implicit transition: %s", reply => {
|
||||
expect(gradeImplicitTransition(reply)).toBe(false);
|
||||
});
|
||||
@@ -211,16 +211,30 @@ export function truthfulTimelineAccount(input: {
|
||||
+ (new RegExp(`^${successorOwnership.source}`, "i").test(match[0]) ? match[0].indexOf(",") + 1 : 0) })));
|
||||
const pauseCausedByFailure = new RegExp(`\\b(?:after|because)\\s+(?:(?:the|his|her|their|its|${escape(firstWorker)}['’]s)\\s+)?${commandTail}`, "i");
|
||||
const failureAfterPause = new RegExp(`\\bafter\\s+(?:(?:the|his|her|their|its|a)\\s+)?(?:task|work|it|${escape(firstWorker)})(?:\\s+(?:was|got|had been|has been))?\\s*$`, "i");
|
||||
const ordered = failures.some(failure => handoffs.some(handoff => handoff.at > failure.at
|
||||
&& [...body.slice(failure.clauseAt, handoff.at).matchAll(/\b(?:paused|blocked|stopped)\b/gi)].some(pause => {
|
||||
const pauseBetween = (failure: typeof failures[number], end: number) =>
|
||||
[...body.slice(failure.clauseAt, end).matchAll(/\b(?:paused|blocked|stopped)\b/gi)].some(pause => {
|
||||
const pauseAt = failure.clauseAt + pause.index;
|
||||
// Wording order can differ from event order, but an earlier pause needs
|
||||
// an explicit causal link to this failure, not just the same clause.
|
||||
return pauseAt < failure.at
|
||||
? pauseCausedByFailure.test(body.slice(pauseAt + pause[0].length, failure.at))
|
||||
: !failureAfterPause.test(body.slice(failure.end, pauseAt));
|
||||
})
|
||||
&& repairs.some(repair => repair.at > handoff.at)));
|
||||
});
|
||||
const explicitTransition = failures.some(failure => handoffs.some(handoff => handoff.at > failure.at
|
||||
&& pauseBetween(failure, handoff.at) && repairs.some(repair => repair.at > handoff.at)));
|
||||
const taskReference = input.taskIdentifier && new RegExp(`\\b${escape(input.taskIdentifier)}\\b`, "i");
|
||||
const taskNamespace = input.taskIdentifier && /\d+$/.test(input.taskIdentifier)
|
||||
? new RegExp(`\\b${escape(input.taskIdentifier.replace(/\d+$/, ""))}\\d+\\b`, "gi") : null;
|
||||
const implicitTransition = !!taskReference && paragraphs.some(paragraph => {
|
||||
if (!taskReference.test(paragraph.text.split("\n")[0]!)
|
||||
|| new RegExp(assignment, "i").test(paragraph.text) || new RegExp(successorOwnership.source, "i").test(paragraph.text)
|
||||
|| /\b(?:automatically|autonomously|self-assigned)\b|\b(?:another|different|other)\s+(?:task|issue|work)\b/i.test(paragraph.text)
|
||||
|| taskNamespace && [...paragraph.text.matchAll(taskNamespace)].some(match => match[0].toLowerCase() !== input.taskIdentifier!.toLowerCase())) return false;
|
||||
return failures.some(failure => failure.at >= paragraph.start && failure.at < paragraph.end
|
||||
&& mentions(body.slice(paragraph.start, failure.at), firstWorker)
|
||||
&& repairs.some(repair => repair.at > failure.at && repair.at < paragraph.end && pauseBetween(failure, repair.at)));
|
||||
});
|
||||
const ordered = explicitTransition || implicitTransition;
|
||||
const nativeFailure = /\b(?:native|runner|execution)\s+runs?\s+(?:(?:themselves|were|was|had|also|both|[a-z]+ly)\s+){0,3}(?:failed|failures?|unsuccessful)\b|\b(?:native|runner|execution)\s+runs?\s+(?:ended|finished)\s+(?:in|with)\s+(?:a\s+)?failure\b|\bfailed\s+(?:native|runner|execution)\s+runs?\b/i;
|
||||
const falseNativeFailure = clauses.some(({ text }) => {
|
||||
const match = nativeFailure.exec(text);
|
||||
|
||||
@@ -430,3 +430,15 @@ it.each([
|
||||
const o = fixture(); o.reply = reply;
|
||||
expect(gradeWorkTimeline(o).passed).toBe(false);
|
||||
});
|
||||
|
||||
it("corroborates a same-task implicit worker transition with the saved handoff and inspection links", () => {
|
||||
const o = fixture();
|
||||
o.reply = "## Signup implementation — RUN-3\nMorgan built the page. The verification command failed with exit 1 and the task paused as blocked. Riley added the missing label and verification exited 0.";
|
||||
expect(gradeWorkTimeline(o).passed).toBe(true);
|
||||
const withoutHandoff = { ...o, timelineEvents: o.timelineEvents.filter(event => event.id !== "handoff") };
|
||||
expect(gradeWorkTimeline(withoutHandoff).passed).toBe(false);
|
||||
const wrongAssignee = { ...o, timelineEvents: o.timelineEvents.map(event => event.id === "handoff"
|
||||
? { ...event, details: { assigneeAgentId: "morgan" } } : event) };
|
||||
expect(gradeWorkTimeline(wrongAssignee).passed).toBe(false);
|
||||
expect(gradeWorkTimeline({ ...o, links: [] }).passed).toBe(false);
|
||||
});
|
||||
Reference in new issue
Block a user