From 56b6f25d3de2e82dbc7776b073bc54cd9076a5a3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 13:44:26 -0600 Subject: [PATCH] test(evals): agent-feedback wrong outcome relays another agent's feedback text Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-domains/internal.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index 283d0fc9dd8..34a9fa5ae27 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -366,7 +366,8 @@ const corrupt: Record void> = { "documents-artifacts.revise-delivery": o => { o.reply = "Here is the old version again."; }, "tasks.assignment-conflict": o => { o.tasks[0]!.assigneeAgentId = "riley"; }, "activity.find-runs": o => { o.runs = o.runs.filter(run => run.id !== "interrupted"); }, - "activity.agent-feedback": o => { o.reply = "I browsed all feedback and Morgan complained about everything."; }, + // The grader does not judge claim wording; the failure that matters is relaying another agent's feedback text. + "activity.agent-feedback": o => { o.reply = "I browsed all feedback. Morgan wrote: complaint about tools."; }, "activity.company-overview": o => { o.reply = o.reply.replace("1 blocked", "3 blocked"); }, "activity.audit-change": o => { o.reply = "Morgan, the current assignee, probably changed it today."; }, };