mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 11:01:16 +01:00
test(evals): reject explicit missing target access reports
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
a74bbc949b
commit
6743055493
2 files changed
+3
-1
No files matched your search
@@ -37,6 +37,8 @@ it.each([
|
||||
"Morgan cannot read the page; the test returned an error.",
|
||||
"Morgan cannot read the sample page, although this test succeeded.",
|
||||
"Morgan's read test failed; access was not verified.",
|
||||
"Morgan has no read access to the page, although the test passed.",
|
||||
"Morgan doesn't have read access to the page, although the test passed.",
|
||||
])("rejects contradictory access/test outcome despite retaining all real successful read proof: %s", reply => {
|
||||
const o = authorizedTestObservation(); o.reply = reply; o.phaseReplies[0]!.body = reply;
|
||||
expect(o.state.decisions[0]!.invocation.status).toBe("succeeded");
|
||||
|
||||
@@ -19,7 +19,7 @@ function allowedScope(value: unknown, agentId: unknown, read: unknown, write: un
|
||||
}
|
||||
/** A test reports an observed access result; it does not qualify a future runtime. */
|
||||
export function appTestReport(reply: string, marker: string) {
|
||||
const deniedAccess = /\bMorgan(?:['’]s)?\b[^.!?;\n]{0,45}\b(?:cannot|can't|could not|couldn't|is unable to|was unable to|is not allowed to|does not have)\b[^.!?;\n]{0,35}\b(?:read|access)\b/i.test(reply);
|
||||
const deniedAccess = /\bMorgan(?:['’]s)?\b[^.!?;\n]{0,45}\b(?:cannot|can't|could not|couldn't|is unable to|was unable to|is not allowed to|does not have|doesn't have|has no|lacks)\b[^.!?;\n]{0,35}\b(?:read|access)\b/i.test(reply);
|
||||
const failedTest = /\b(?:test|read|access check)\s+(?:(?:has|had|was|is|did|returned|result|request)\s+){0,3}(?:failed|denied|unsuccessful|an? error|not succeed|not successful)\b/i.test(reply);
|
||||
return /Morgan/i.test(reply) && /read|page|launch_read/i.test(reply)
|
||||
&& (/succeed|success|passed|allowed|can read|able to read|readable|has (?:read )?access|verified|retrieved|fetched/i.test(reply) || reply.includes(marker))
|
||||
|
||||
Reference in new issue
Block a user