From 674305549374c4b69a4bfe166d07cf11b8ab23c2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 21:46:22 -0600 Subject: [PATCH] test(evals): reject explicit missing target access reports Co-Authored-By: Paperclip --- .../chat-catalog-domains/external-app-test-native.test.ts | 2 ++ tests/runner-e2e/chat-catalog-domains/external-app-test.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-test-native.test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-test-native.test.ts index 902c1bfd3c7..eec7bd6ab1d 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-test-native.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-test-native.test.ts @@ -37,6 +37,8 @@ it.each([ "Morgan cannot read the page; the test returned an error.", "Morgan cannot read the sample page, although this test succeeded.", "Morgan's read test failed; access was not verified.", + "Morgan has no read access to the page, although the test passed.", + "Morgan doesn't have read access to the page, although the test passed.", ])("rejects contradictory access/test outcome despite retaining all real successful read proof: %s", reply => { const o = authorizedTestObservation(); o.reply = reply; o.phaseReplies[0]!.body = reply; expect(o.state.decisions[0]!.invocation.status).toBe("succeeded"); diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-test.ts index 62365817010..b8a99ac23af 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-test.ts @@ -19,7 +19,7 @@ function allowedScope(value: unknown, agentId: unknown, read: unknown, write: un } /** A test reports an observed access result; it does not qualify a future runtime. */ export function appTestReport(reply: string, marker: string) { - const deniedAccess = /\bMorgan(?:['’]s)?\b[^.!?;\n]{0,45}\b(?:cannot|can't|could not|couldn't|is unable to|was unable to|is not allowed to|does not have)\b[^.!?;\n]{0,35}\b(?:read|access)\b/i.test(reply); + const deniedAccess = /\bMorgan(?:['’]s)?\b[^.!?;\n]{0,45}\b(?:cannot|can't|could not|couldn't|is unable to|was unable to|is not allowed to|does not have|doesn't have|has no|lacks)\b[^.!?;\n]{0,35}\b(?:read|access)\b/i.test(reply); const failedTest = /\b(?:test|read|access check)\s+(?:(?:has|had|was|is|did|returned|result|request)\s+){0,3}(?:failed|denied|unsuccessful|an? error|not succeed|not successful)\b/i.test(reply); return /Morgan/i.test(reply) && /read|page|launch_read/i.test(reply) && (/succeed|success|passed|allowed|can read|able to read|readable|has (?:read )?access|verified|retrieved|fetched/i.test(reply) || reply.includes(marker))