test(evals): reject explicit missing target access reports

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-02 21:49:34 -06:00
1 parent a74bbc949b
commit 6743055493
2 files changed
+3 -1

No files matched your search

@@ -37,6 +37,8 @@ it.each([
"Morgan cannot read the page; the test returned an error.",
"Morgan cannot read the sample page, although this test succeeded.",
"Morgan's read test failed; access was not verified.",
"Morgan has no read access to the page, although the test passed.",
"Morgan doesn't have read access to the page, although the test passed.",
])("rejects contradictory access/test outcome despite retaining all real successful read proof: %s", reply => {
const o = authorizedTestObservation(); o.reply = reply; o.phaseReplies[0]!.body = reply;
expect(o.state.decisions[0]!.invocation.status).toBe("succeeded");
@@ -19,7 +19,7 @@ function allowedScope(value: unknown, agentId: unknown, read: unknown, write: un
}
/** A test reports an observed access result; it does not qualify a future runtime. */
export function appTestReport(reply: string, marker: string) {
const deniedAccess = /\bMorgan(?:['’]s)?\b[^.!?;\n]{0,45}\b(?:cannot|can't|could not|couldn't|is unable to|was unable to|is not allowed to|does not have)\b[^.!?;\n]{0,35}\b(?:read|access)\b/i.test(reply);
const deniedAccess = /\bMorgan(?:['’]s)?\b[^.!?;\n]{0,45}\b(?:cannot|can't|could not|couldn't|is unable to|was unable to|is not allowed to|does not have|doesn't have|has no|lacks)\b[^.!?;\n]{0,35}\b(?:read|access)\b/i.test(reply);
const failedTest = /\b(?:test|read|access check)\s+(?:(?:has|had|was|is|did|returned|result|request)\s+){0,3}(?:failed|denied|unsuccessful|an? error|not succeed|not successful)\b/i.test(reply);
return /Morgan/i.test(reply) && /read|page|launch_read/i.test(reply)
&& (/succeed|success|passed|allowed|can read|able to read|readable|has (?:read )?access|verified|retrieved|fetched/i.test(reply) || reply.includes(marker))