fix(evals): recognize prospective task wording in approval cards

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-16 08:47:40 -05:00
1 parent 82087ac7e1
commit 94aefb1f38
2 files changed
+18 -1

No files matched your search

+4 -1
View File
@@ -73,7 +73,10 @@ function hasTaskProposal(text: string, confirmationCard = false) {
).join("\n");
return /\b(?:approve|accept|propose|proposed|proposing|proposal|suggest|suggested|suggesting|recommend|recommended|recommending|create|creating|set up)\b[\s\S]{0,160}\b(?:subtask|task)\b/i.test(affirmative) ||
/\b(?:subtask|task)\b[^.!?\n]{0,30}\bproposal\b/i.test(affirmative) ||
(confirmationCard && /\b(?:I|we)(?:['’]ll| will)\s+(?:save|attach|write|make|open|create)\b[^.!?]{0,160}\b(?:(?:new|child)\s+task|subtask)\b/i.test(affirmative));
(confirmationCard && (
/\b(?:I|we)(?:['’]ll| will)\s+(?:save|attach|write|make|open|create)\b[^.!?]{0,160}\b(?:(?:new|child)\s+task|subtask)\b/i.test(affirmative) ||
/\b(?:subtask|task)\s+(?:I|we)(?:['’]ll| will)\s+(?:create|open|set up)\b/i.test(affirmative)
));
}
export const activeRuns = (runs: Row[]) =>
+14
View File
@@ -434,6 +434,8 @@ describe("first-task fixtures and state grading", () => {
"I won't create a child task. Continue?",
"A new task was mentioned earlier. Continue?",
"Do you understand what a subtask is?",
"Here is the task I already created. Continue?",
"Here is the task I will not create. Continue?",
])("does not count incidental or declined work as a proposal: %s", (prompt) => {
const e = recording("clear-task-first-response");
e.checkpoints = e.checkpoints.slice(0, 2);
@@ -457,6 +459,18 @@ describe("first-task fixtures and state grading", () => {
expect(failed(e)).toEqual([]);
});
it.each(["create and complete", "create and run"])("recognizes the recorded task I will %s confirmation", (action) => {
const e = recording("clear-task-first-response");
e.checkpoints = e.checkpoints.slice(0, 2);
e.checkpoints[1].comments.pop();
e.checkpoints[1].interactions.push({
id: "proposal-card", kind: "request_confirmation", status: "pending",
title: "Confirm: Welcome note task",
payload: { prompt: `Here's the task I'll ${action}:\n\n**Write a welcome note for the garden club**\n- A two-sentence welcome note\n- Invites beginners to the free Saturday meetup\n- Includes the exact phrase: ${e.nonce}\n- Saved as a document attached to the task\n\nShall I go ahead?` },
});
expect(failed(e)).toEqual([]);
});
it("recognizes a Proposal heading introducing the task, as in the Claude verification recording", () => {
const e = recording("clear-task-first-response");
e.checkpoints = e.checkpoints.slice(0, 2);