Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-05 21:02:34 +01:00
Code Issues Packages Projects Releases Wiki Activity

test(evals): reject negated mobile checkout failure claims

Browse Source
This commit is contained in:
Dotta committed 2026-10-26 21:33:38 -05:00
1 parent d1fa9efdaf
commit 6cc0012da0
2 files changed
+4 -1
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.test.ts
+1
View File
@@ -35,6 +35,7 @@ describe("real Morgan same-primary review handoff", () => {
const chat = valid(); chat.handoffs[1]!.sourceRunId = "source"; expect(gradeIntegratorCaseHandoff(chat).passed).toBe(false);
const missing = valid(); missing.findingVersions = []; expect(gradeIntegratorCaseHandoff(missing).passed).toBe(false);
const wrong = valid(); wrong.handoffs[1]!.instructions = "Mobile Safari tapping Pay works. Keyboard fails."; expect(gradeIntegratorCaseHandoff(wrong).passed).toBe(false);
const negated = valid(); negated.handoffs[1]!.instructions = "On mobile Safari, tapping Pay does not fail. Keyboard activation works."; expect(gradeIntegratorCaseHandoff(negated).passed).toBe(false);
});
it("rejects fictional delivery before the actual handoff and task-done-as-case-closed", () => {
const early = valid(); early.state.handoffCheckpoint.reply = "I handed this case to Riley."; expect(gradeIntegratorCaseHandoff(early).passed).toBe(false);
tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts
+3 -1
View File
@@ -26,7 +26,9 @@ async function observeHandoff(ctx: ChatCatalogContext, state: Record<string, unk
function handoffObservation(value: unknown): HandoffObservation | undefined { const o = value as HandoffObservation; return observation(value) && !!o.caseRecord && Array.isArray(o.handoffs) && !!o.findings && Array.isArray(o.findingVersions) && Array.isArray(o.sourceReplies) && !!o.actingAgent && !!o.reviewer ? o : undefined; }
function checkoutObservations(body: string) {
const parts = body.split(/\n+|;|(?<=[.!?])\s+/);
return /mobile Safari/i.test(body) && parts.some(part => /\b(?:tap|tapping|touch)\b/i.test(part) && /\b(?:Pay|checkout|button)\b/i.test(part) && /\b(?:no effect|no response|does not|doesn't|fails?|unresponsive|nothing)\b/i.test(part))
return /mobile Safari/i.test(body) && parts.some(part => /\b(?:tap|tapping|touch)\b/i.test(part) && /\b(?:Pay|checkout|button)\b/i.test(part)
&& /\b(?:no effect|no response|fails?|unresponsive|nothing|not responsive)\b|\b(?:does not|doesn't)\s+(?:respond|work|activate|trigger|submit|produce)\b/i.test(part)
&& !/\b(?:does not|doesn't|never|not)\s+(?:fail|unresponsive)\b|\bno (?:failure|issue)\b/i.test(part))
&& parts.some(part => /keyboard/i.test(part) && /\b(?:succeeds?|successful|works?|responds?)\b/i.test(part) && !/\b(?:not|fails?|doesn't|does not)\b/i.test(part));
}
export function gradeIntegratorCaseHandoff(value: unknown) {