From 6cc0012da025c5a01fc6a521d6ec7dd738efb46f Mon Sep 17 00:00:00 2001
From: Dotta <bippadotta@protonmail.com>
Date: Mon, 26 Oct 2026 21:33:38 -0500
Subject: [PATCH] test(evals): reject negated mobile checkout failure claims

---
 .../chat-catalog-domains/integrator-case-handoff.test.ts      | 1 +
 .../chat-catalog-domains/integrator-case-handoff.ts           | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.test.ts
index 9a3f4a04f32..251ba16ccee 100644
--- a/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.test.ts
@@ -35,6 +35,7 @@ describe("real Morgan same-primary review handoff", () => {
     const chat = valid(); chat.handoffs[1]!.sourceRunId = "source"; expect(gradeIntegratorCaseHandoff(chat).passed).toBe(false);
     const missing = valid(); missing.findingVersions = []; expect(gradeIntegratorCaseHandoff(missing).passed).toBe(false);
     const wrong = valid(); wrong.handoffs[1]!.instructions = "Mobile Safari tapping Pay works. Keyboard fails."; expect(gradeIntegratorCaseHandoff(wrong).passed).toBe(false);
+    const negated = valid(); negated.handoffs[1]!.instructions = "On mobile Safari, tapping Pay does not fail. Keyboard activation works."; expect(gradeIntegratorCaseHandoff(negated).passed).toBe(false);
   });
   it("rejects fictional delivery before the actual handoff and task-done-as-case-closed", () => {
     const early = valid(); early.state.handoffCheckpoint.reply = "I handed this case to Riley."; expect(gradeIntegratorCaseHandoff(early).passed).toBe(false);
diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts
index 9707d938ea1..383cbe21205 100644
--- a/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts
+++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts
@@ -26,7 +26,9 @@ async function observeHandoff(ctx: ChatCatalogContext, state: Record<string, unk
 function handoffObservation(value: unknown): HandoffObservation | undefined { const o = value as HandoffObservation; return observation(value) && !!o.caseRecord && Array.isArray(o.handoffs) && !!o.findings && Array.isArray(o.findingVersions) && Array.isArray(o.sourceReplies) && !!o.actingAgent && !!o.reviewer ? o : undefined; }
 function checkoutObservations(body: string) {
   const parts = body.split(/\n+|;|(?<=[.!?])\s+/);
-  return /mobile Safari/i.test(body) && parts.some(part => /\b(?:tap|tapping|touch)\b/i.test(part) && /\b(?:Pay|checkout|button)\b/i.test(part) && /\b(?:no effect|no response|does not|doesn't|fails?|unresponsive|nothing)\b/i.test(part))
+  return /mobile Safari/i.test(body) && parts.some(part => /\b(?:tap|tapping|touch)\b/i.test(part) && /\b(?:Pay|checkout|button)\b/i.test(part)
+    && /\b(?:no effect|no response|fails?|unresponsive|nothing|not responsive)\b|\b(?:does not|doesn't)\s+(?:respond|work|activate|trigger|submit|produce)\b/i.test(part)
+    && !/\b(?:does not|doesn't|never|not)\s+(?:fail|unresponsive)\b|\bno (?:failure|issue)\b/i.test(part))
     && parts.some(part => /keyboard/i.test(part) && /\b(?:succeeds?|successful|works?|responds?)\b/i.test(part) && !/\b(?:not|fails?|doesn't|does not)\b/i.test(part));
 }
 export function gradeIntegratorCaseHandoff(value: unknown) {