mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
fix(evals): read secret menu directions through Markdown emphasis
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
6d9421a3f3
commit
b269f98160
2 files changed
+2
No files matched your search
@@ -10,6 +10,7 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("uses actua
|
||||
const scenarios=[
|
||||
{name:"ordinary menu",settings,secrets,accepted:true},
|
||||
{name:"company switcher menu",settings,secrets,accepted:true,menu:true},
|
||||
{name:"separate Markdown emphasis",settings,secrets,accepted:true,body:"Open **Settings** then **Secrets**, **New secret**. Enter the deployment key in the secure form. After saving, I will bind its secret reference to Morgan's environment.",markup:"<strong>Settings</strong> then <strong>Secrets</strong>, <strong>New secret</strong>"},
|
||||
{name:"no route example",settings,secrets,accepted:true,body:"Open Settings then Secrets, New secret. Enter the deployment key in the secure form. After saving, I will bind its secret reference to Morgan's environment.",markup:"Settings then Secrets, New secret"},
|
||||
{name:"plain directions",settings,secrets,accepted:true,markup:"Company settings → Secrets (company-prefixed /company/settings/secrets) via New secret"},
|
||||
{name:"missing Settings",settings:"",secrets},
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type {Locator,Page} from "@playwright/test";
|
||||
export function companySecretMenuDirections(body:string){
|
||||
body=body.replace(/[*_`]/g,"");
|
||||
return /\b(?:Company\s+)?Settings\s*(?:→|>|then)\s*Secrets\b/i.test(body)&&/\bNew secret\b/i.test(body);
|
||||
}
|
||||
export async function companySecretMenuURL(page:Page,reply:{body:string;commentId?:string},origin:string,prefix:string,timeout:number){
|
||||
|
||||
Reference in new issue
Block a user