mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
fix(evals): admit ordinary secret menu directions without route examples
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
9ecaf0ad23
commit
6d9421a3f3
2 files changed
+8
-5
No files matched your search
@@ -10,6 +10,7 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("uses actua
|
||||
const scenarios=[
|
||||
{name:"ordinary menu",settings,secrets,accepted:true},
|
||||
{name:"company switcher menu",settings,secrets,accepted:true,menu:true},
|
||||
{name:"no route example",settings,secrets,accepted:true,body:"Open Settings then Secrets, New secret. Enter the deployment key in the secure form. After saving, I will bind its secret reference to Morgan's environment.",markup:"Settings then Secrets, New secret"},
|
||||
{name:"plain directions",settings,secrets,accepted:true,markup:"Company settings → Secrets (company-prefixed /company/settings/secrets) via New secret"},
|
||||
{name:"missing Settings",settings:"",secrets},
|
||||
{name:"hidden Settings",settings,secrets,hidden:true},
|
||||
@@ -31,8 +32,8 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("uses actua
|
||||
await route.fulfill({contentType:"text/html; charset=utf-8",body:content});
|
||||
});
|
||||
await page.goto(chat);
|
||||
if(scenario.throws)await expect(companySecretSetupURL(page,{body,commentId:"native"},origin,"RUN",1500),scenario.name).rejects.toThrow();
|
||||
else expect(await companySecretSetupURL(page,{body,commentId:"native"},origin,"RUN",1500),scenario.name).toBe(scenario.accepted?origin+secrets:undefined);
|
||||
if(scenario.throws)await expect(companySecretSetupURL(page,{body:scenario.body??body,commentId:"native"},origin,"RUN",1500),scenario.name).rejects.toThrow();
|
||||
else expect(await companySecretSetupURL(page,{body:scenario.body??body,commentId:"native"},origin,"RUN",1500),scenario.name).toBe(scenario.accepted?origin+secrets:undefined);
|
||||
expect(page.url(),scenario.name).toBe(chat);
|
||||
if(scenario.accepted)expect(clicked).toEqual([settings,secrets]);
|
||||
else expect(clicked.some(path=>path.startsWith("/OTHER/")),scenario.name).toBe(false);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type {Locator,Page} from "@playwright/test";
|
||||
export function companySecretMenuDirections(body:string){
|
||||
return /\bCompany settings\s*(?:→|>)\s*Secrets\b/i.test(body)&&/\bNew secret\b/i.test(body)&&/\bcompany-prefixed\s+[`"']?\/company\/settings\/secrets\b/i.test(body);
|
||||
return /\b(?:Company\s+)?Settings\s*(?:→|>|then)\s*Secrets\b/i.test(body)&&/\bNew secret\b/i.test(body);
|
||||
}
|
||||
export async function companySecretMenuURL(page:Page,reply:{body:string;commentId?:string},origin:string,prefix:string,timeout:number){
|
||||
if(!reply.commentId||!companySecretMenuDirections(reply.body))return;
|
||||
@@ -14,12 +14,14 @@ export async function companySecretMenuURL(page:Page,reply:{body:string;commentI
|
||||
const supported=(href:string|null,suffix:string)=>{try{const url=new URL(href??"",origin);return url.origin===origin&&url.pathname===`/${prefix}/company/settings${suffix}`&&!url.search&&!url.hash&&!url.username&&!url.password?url.href:undefined;}catch{return;}};
|
||||
await comment.waitFor({state:"visible",timeout:remaining()});
|
||||
const fragment=comment.getByText("/company/settings/secrets",{exact:false}).last();
|
||||
if(!await visible(comment.getByText(/Company settings\s*(?:→|>)\s*Secrets/i).last())||!await visible(comment.getByText("New secret",{exact:false}).last())||!await visible(fragment)||await fragment.evaluate(node=>!!node.closest("a[href]")))return;
|
||||
if(!await visible(comment.getByText(/(?:Company\s+)?Settings\s*(?:→|>|then)\s*Secrets/i).last())||!await visible(comment.getByText("New secret",{exact:false}).last()))return;
|
||||
const tokens:string[]=reply.body.match(/https?:\/\/[^\s`<>"')]+|(?<![\w/:])\/[^\s`<>"')]+/g)??[];
|
||||
if(tokens.includes("/company/settings/secrets")&&(!await visible(fragment)||await fragment.evaluate(node=>!!node.closest("a[href]"))))return;
|
||||
for(const anchor of await comment.locator("a[href]").all()){
|
||||
const href=await anchor.getAttribute("href"),label=await anchor.innerText();
|
||||
if(/settings|secrets?/i.test(`${href} ${label}`)&&!supported(href,"")&&!supported(href,"/secrets"))return;
|
||||
}
|
||||
for(const token of reply.body.match(/https?:\/\/[^\s`<>"')]+|(?<![\w/:])\/[^\s`<>"')]+/g)??[]){
|
||||
for(const token of tokens){
|
||||
if(token!=="/company/settings/secrets"&&/settings|secrets?/i.test(token)&&!supported(token,"")&&!supported(token,"/secrets"))return;
|
||||
}
|
||||
const originalURL=page.url();let navigated=false,openedMenu=false;
|
||||
|
||||
Reference in new issue
Block a user