fix(evals): admit ordinary secret menu directions without route examples

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-05 15:15:18 -06:00
1 parent 9ecaf0ad23
commit 6d9421a3f3
2 files changed
+8 -5

No files matched your search

@@ -10,6 +10,7 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("uses actua
const scenarios=[
{name:"ordinary menu",settings,secrets,accepted:true},
{name:"company switcher menu",settings,secrets,accepted:true,menu:true},
{name:"no route example",settings,secrets,accepted:true,body:"Open Settings then Secrets, New secret. Enter the deployment key in the secure form. After saving, I will bind its secret reference to Morgan's environment.",markup:"Settings then Secrets, New secret"},
{name:"plain directions",settings,secrets,accepted:true,markup:"Company settings → Secrets (company-prefixed /company/settings/secrets) via New secret"},
{name:"missing Settings",settings:"",secrets},
{name:"hidden Settings",settings,secrets,hidden:true},
@@ -31,8 +32,8 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("uses actua
await route.fulfill({contentType:"text/html; charset=utf-8",body:content});
});
await page.goto(chat);
if(scenario.throws)await expect(companySecretSetupURL(page,{body,commentId:"native"},origin,"RUN",1500),scenario.name).rejects.toThrow();
else expect(await companySecretSetupURL(page,{body,commentId:"native"},origin,"RUN",1500),scenario.name).toBe(scenario.accepted?origin+secrets:undefined);
if(scenario.throws)await expect(companySecretSetupURL(page,{body:scenario.body??body,commentId:"native"},origin,"RUN",1500),scenario.name).rejects.toThrow();
else expect(await companySecretSetupURL(page,{body:scenario.body??body,commentId:"native"},origin,"RUN",1500),scenario.name).toBe(scenario.accepted?origin+secrets:undefined);
expect(page.url(),scenario.name).toBe(chat);
if(scenario.accepted)expect(clicked).toEqual([settings,secrets]);
else expect(clicked.some(path=>path.startsWith("/OTHER/")),scenario.name).toBe(false);
@@ -1,6 +1,6 @@
import type {Locator,Page} from "@playwright/test";
export function companySecretMenuDirections(body:string){
return /\bCompany settings\s*(?:→|>)\s*Secrets\b/i.test(body)&&/\bNew secret\b/i.test(body)&&/\bcompany-prefixed\s+[`"']?\/company\/settings\/secrets\b/i.test(body);
return /\b(?:Company\s+)?Settings\s*(?:→|>|then)\s*Secrets\b/i.test(body)&&/\bNew secret\b/i.test(body);
}
export async function companySecretMenuURL(page:Page,reply:{body:string;commentId?:string},origin:string,prefix:string,timeout:number){
if(!reply.commentId||!companySecretMenuDirections(reply.body))return;
@@ -14,12 +14,14 @@ export async function companySecretMenuURL(page:Page,reply:{body:string;commentI
const supported=(href:string|null,suffix:string)=>{try{const url=new URL(href??"",origin);return url.origin===origin&&url.pathname===`/${prefix}/company/settings${suffix}`&&!url.search&&!url.hash&&!url.username&&!url.password?url.href:undefined;}catch{return;}};
await comment.waitFor({state:"visible",timeout:remaining()});
const fragment=comment.getByText("/company/settings/secrets",{exact:false}).last();
if(!await visible(comment.getByText(/Company settings\s*(?:→|>)\s*Secrets/i).last())||!await visible(comment.getByText("New secret",{exact:false}).last())||!await visible(fragment)||await fragment.evaluate(node=>!!node.closest("a[href]")))return;
if(!await visible(comment.getByText(/(?:Company\s+)?Settings\s*(?:→|>|then)\s*Secrets/i).last())||!await visible(comment.getByText("New secret",{exact:false}).last()))return;
const tokens:string[]=reply.body.match(/https?:\/\/[^\s`<>"')]+|(?<![\w/:])\/[^\s`<>"')]+/g)??[];
if(tokens.includes("/company/settings/secrets")&&(!await visible(fragment)||await fragment.evaluate(node=>!!node.closest("a[href]"))))return;
for(const anchor of await comment.locator("a[href]").all()){
const href=await anchor.getAttribute("href"),label=await anchor.innerText();
if(/settings|secrets?/i.test(`${href} ${label}`)&&!supported(href,"")&&!supported(href,"/secrets"))return;
}
for(const token of reply.body.match(/https?:\/\/[^\s`<>"')]+|(?<![\w/:])\/[^\s`<>"')]+/g)??[]){
for(const token of tokens){
if(token!=="/company/settings/secrets"&&/settings|secrets?/i.test(token)&&!supported(token,"")&&!supported(token,"/secrets"))return;
}
const originalURL=page.url();let navigated=false,openedMenu=false;