From 94aefb1f38ce80dc1ad3d6557a65f102c0d31e4f Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 16 Sep 2026 08:47:40 -0500 Subject: [PATCH] fix(evals): recognize prospective task wording in approval cards Co-Authored-By: Paperclip --- tests/runner-e2e/first-task-scoring.ts | 5 ++++- tests/runner-e2e/first-task.test.ts | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/first-task-scoring.ts b/tests/runner-e2e/first-task-scoring.ts index e9f8b1b58b2..1ccb68fc4d7 100644 --- a/tests/runner-e2e/first-task-scoring.ts +++ b/tests/runner-e2e/first-task-scoring.ts @@ -73,7 +73,10 @@ function hasTaskProposal(text: string, confirmationCard = false) { ).join("\n"); return /\b(?:approve|accept|propose|proposed|proposing|proposal|suggest|suggested|suggesting|recommend|recommended|recommending|create|creating|set up)\b[\s\S]{0,160}\b(?:subtask|task)\b/i.test(affirmative) || /\b(?:subtask|task)\b[^.!?\n]{0,30}\bproposal\b/i.test(affirmative) || - (confirmationCard && /\b(?:I|we)(?:['’]ll| will)\s+(?:save|attach|write|make|open|create)\b[^.!?]{0,160}\b(?:(?:new|child)\s+task|subtask)\b/i.test(affirmative)); + (confirmationCard && ( + /\b(?:I|we)(?:['’]ll| will)\s+(?:save|attach|write|make|open|create)\b[^.!?]{0,160}\b(?:(?:new|child)\s+task|subtask)\b/i.test(affirmative) || + /\b(?:subtask|task)\s+(?:I|we)(?:['’]ll| will)\s+(?:create|open|set up)\b/i.test(affirmative) + )); } export const activeRuns = (runs: Row[]) => diff --git a/tests/runner-e2e/first-task.test.ts b/tests/runner-e2e/first-task.test.ts index 10990a86cec..06d3e7ac5bf 100644 --- a/tests/runner-e2e/first-task.test.ts +++ b/tests/runner-e2e/first-task.test.ts @@ -434,6 +434,8 @@ describe("first-task fixtures and state grading", () => { "I won't create a child task. Continue?", "A new task was mentioned earlier. Continue?", "Do you understand what a subtask is?", + "Here is the task I already created. Continue?", + "Here is the task I will not create. Continue?", ])("does not count incidental or declined work as a proposal: %s", (prompt) => { const e = recording("clear-task-first-response"); e.checkpoints = e.checkpoints.slice(0, 2); @@ -457,6 +459,18 @@ describe("first-task fixtures and state grading", () => { expect(failed(e)).toEqual([]); }); + it.each(["create and complete", "create and run"])("recognizes the recorded task I will %s confirmation", (action) => { + const e = recording("clear-task-first-response"); + e.checkpoints = e.checkpoints.slice(0, 2); + e.checkpoints[1].comments.pop(); + e.checkpoints[1].interactions.push({ + id: "proposal-card", kind: "request_confirmation", status: "pending", + title: "Confirm: Welcome note task", + payload: { prompt: `Here's the task I'll ${action}:\n\n**Write a welcome note for the garden club**\n- A two-sentence welcome note\n- Invites beginners to the free Saturday meetup\n- Includes the exact phrase: ${e.nonce}\n- Saved as a document attached to the task\n\nShall I go ahead?` }, + }); + expect(failed(e)).toEqual([]); + }); + it("recognizes a Proposal heading introducing the task, as in the Claude verification recording", () => { const e = recording("clear-task-first-response"); e.checkpoints = e.checkpoints.slice(0, 2);