mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-25 02:26:12 +02:00
fix(evals): recognize prospective task wording in approval cards
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
82087ac7e1
commit
94aefb1f38
2 files changed
+18
-1
No files matched your search
@@ -73,7 +73,10 @@ function hasTaskProposal(text: string, confirmationCard = false) {
|
||||
).join("\n");
|
||||
return /\b(?:approve|accept|propose|proposed|proposing|proposal|suggest|suggested|suggesting|recommend|recommended|recommending|create|creating|set up)\b[\s\S]{0,160}\b(?:subtask|task)\b/i.test(affirmative) ||
|
||||
/\b(?:subtask|task)\b[^.!?\n]{0,30}\bproposal\b/i.test(affirmative) ||
|
||||
(confirmationCard && /\b(?:I|we)(?:['’]ll| will)\s+(?:save|attach|write|make|open|create)\b[^.!?]{0,160}\b(?:(?:new|child)\s+task|subtask)\b/i.test(affirmative));
|
||||
(confirmationCard && (
|
||||
/\b(?:I|we)(?:['’]ll| will)\s+(?:save|attach|write|make|open|create)\b[^.!?]{0,160}\b(?:(?:new|child)\s+task|subtask)\b/i.test(affirmative) ||
|
||||
/\b(?:subtask|task)\s+(?:I|we)(?:['’]ll| will)\s+(?:create|open|set up)\b/i.test(affirmative)
|
||||
));
|
||||
}
|
||||
|
||||
export const activeRuns = (runs: Row[]) =>
|
||||
|
||||
@@ -434,6 +434,8 @@ describe("first-task fixtures and state grading", () => {
|
||||
"I won't create a child task. Continue?",
|
||||
"A new task was mentioned earlier. Continue?",
|
||||
"Do you understand what a subtask is?",
|
||||
"Here is the task I already created. Continue?",
|
||||
"Here is the task I will not create. Continue?",
|
||||
])("does not count incidental or declined work as a proposal: %s", (prompt) => {
|
||||
const e = recording("clear-task-first-response");
|
||||
e.checkpoints = e.checkpoints.slice(0, 2);
|
||||
@@ -457,6 +459,18 @@ describe("first-task fixtures and state grading", () => {
|
||||
expect(failed(e)).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(["create and complete", "create and run"])("recognizes the recorded task I will %s confirmation", (action) => {
|
||||
const e = recording("clear-task-first-response");
|
||||
e.checkpoints = e.checkpoints.slice(0, 2);
|
||||
e.checkpoints[1].comments.pop();
|
||||
e.checkpoints[1].interactions.push({
|
||||
id: "proposal-card", kind: "request_confirmation", status: "pending",
|
||||
title: "Confirm: Welcome note task",
|
||||
payload: { prompt: `Here's the task I'll ${action}:\n\n**Write a welcome note for the garden club**\n- A two-sentence welcome note\n- Invites beginners to the free Saturday meetup\n- Includes the exact phrase: ${e.nonce}\n- Saved as a document attached to the task\n\nShall I go ahead?` },
|
||||
});
|
||||
expect(failed(e)).toEqual([]);
|
||||
});
|
||||
|
||||
it("recognizes a Proposal heading introducing the task, as in the Claude verification recording", () => {
|
||||
const e = recording("clear-task-first-response");
|
||||
e.checkpoints = e.checkpoints.slice(0, 2);
|
||||
|
||||
Reference in new issue
Block a user