fix(evals): read secret menu directions through Markdown emphasis

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-05 15:16:05 -06:00
1 parent 6d9421a3f3
commit b269f98160
2 files changed
+2

No files matched your search

@@ -10,6 +10,7 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("uses actua
const scenarios=[
{name:"ordinary menu",settings,secrets,accepted:true},
{name:"company switcher menu",settings,secrets,accepted:true,menu:true},
{name:"separate Markdown emphasis",settings,secrets,accepted:true,body:"Open **Settings** then **Secrets**, **New secret**. Enter the deployment key in the secure form. After saving, I will bind its secret reference to Morgan's environment.",markup:"<strong>Settings</strong> then <strong>Secrets</strong>, <strong>New secret</strong>"},
{name:"no route example",settings,secrets,accepted:true,body:"Open Settings then Secrets, New secret. Enter the deployment key in the secure form. After saving, I will bind its secret reference to Morgan's environment.",markup:"Settings then Secrets, New secret"},
{name:"plain directions",settings,secrets,accepted:true,markup:"Company settings → Secrets (company-prefixed /company/settings/secrets) via New secret"},
{name:"missing Settings",settings:"",secrets},
@@ -1,5 +1,6 @@
import type {Locator,Page} from "@playwright/test";
export function companySecretMenuDirections(body:string){
body=body.replace(/[*_`]/g,"");
return /\b(?:Company\s+)?Settings\s*(?:→|>|then)\s*Secrets\b/i.test(body)&&/\bNew secret\b/i.test(body);
}
export async function companySecretMenuURL(page:Page,reply:{body:string;commentId?:string},origin:string,prefix:string,timeout:number){