diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index b99922ef6ba..138c9d8ae68 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -474,3 +474,13 @@ UI answer, model and 300000ms attempt unchanged. Result 5172 remains its origina PASSED result for the earlier persistence/continuation scope, with the recorded confusing duplicate-form and inaccurate walkthrough metadata limitations. It is not retroactively regraded, and no new native result is claimed. + +Personal secret v14 admits the measured result 5169's safe My secrets/definition +menu directions with a noncanonical inline code hint. The hint is retained as an +advertised hint, never rewritten into a trusted URL. Actual visible Account → +Settings → Secrets hrefs must independently match the current company, and the +My secrets control must expose the required own-value entry. Foreign links, +other paths, hidden navigation and mismatched definitions remain inadmissible. +The private worker still independently verifies the responsible person and exact +definition before entering the value. Result 5169 remains FAILED; this local +observer correction does not qualify the unexercised save, consumer or final reply. diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 92e5d5982d2..d29741a87b0 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4978,6 +4978,8 @@ "sourceFiles": [ "../../packages/paperclip-runner/src/protocol-actions/request-human-input.ts", "chat-catalog-domains/external-personal-secret.ts", + "chat-catalog-domains/external-personal-secret-menu.ts", + "../../ui/src/pages/secrets/MyUserSecretsTab.tsx", "chat-catalog-domains/external-personal-secret-confirmation.ts", "chat-catalog-contract.ts", "chat-catalog-flow.ts", @@ -5006,7 +5008,7 @@ "../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.personal-secret.v13", + "oracleVersion": "paperclip.chat-catalog.personal-secret.v14", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "minimumRunCount": 3, diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-menu.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-menu.test.ts new file mode 100644 index 00000000000..33572cbd85a --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-menu.test.ts @@ -0,0 +1,34 @@ +import {expect,it} from "vitest"; +import {chromium} from "@playwright/test"; +import {personalSecretSetupURL,personalSecretGuidance} from "./external-personal-secret.js"; +const origin="https://paperclip.example",chat=origin+"/RUN/chat",settings="/RUN/company/settings",secrets=settings+"/secrets",definition="6263be21-6958-4e2b-a76a-f0e20331245c"; +const body=`I can't set a personal credential for you; never share it in chat. Open Settings → Secrets at \`/company/settings/secrets\`. Under My secrets, find PERSONAL_PUBLISHING_KEY (definition \`${definition}\`) and choose Set my value. Enter the publishing credential there and save.`; +const markup=`

Open Settings → Secrets at /company/settings/secrets.

Under My secrets, find PERSONAL_PUBLISHING_KEY (definition ${definition}) and choose Set my value.

`; +it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("discovers the current Personal UI route independently of the5169 inline hint and rejects unbound alternatives",async()=>{ + expect(personalSecretGuidance(body)).toBe(true); + const browser=await chromium.launch({headless:true,...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?{executablePath:process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE}:{}}),page=await browser.newPage(); + const scenarios=[ + {name:"actual-style Account navigation",accepted:true}, + {name:"missing menu",missing:true}, + {name:"hidden menu",hidden:true}, + {name:"wrong company Settings",settings:"/OTHER/company/settings"}, + {name:"foreign Secrets",secrets:"https://foreign.example/company/settings/secrets"}, + {name:"wrong definition",body:body.replace(definition,"other-definition"),markup:markup.replace(definition,"other-definition")}, + {name:"foreign anchor",extra:'Setup'}, + {name:"anchored hint",markup:markup.replace('/company/settings/secrets','/company/settings/secrets')}, + {name:"other bare path",body:body.replace('/company/settings/secrets','/OTHER/company/settings/secrets'),markup:markup.replace('/company/settings/secrets','/OTHER/company/settings/secrets')}, + {name:"unsafe guidance",body:body+" Paste the key here in chat."}, + {name:"missing own entry",noEntry:true}, + ]; + try{for(const scenario of scenarios){ + const paths:string[]=[];await page.unroute("**/*");await page.route("**/*",async route=>{ + const path=new URL(route.request().url()).pathname;paths.push(path); + const content=path===settings?``:path===secrets?`${scenario.noEntry?"":''}`:`
${scenario.markup??markup}${scenario.extra??""}
${scenario.missing?"":``}`; + await route.fulfill({contentType:"text/html; charset=utf-8",body:content}); + });await page.goto(chat); + let proof:unknown; + expect(await personalSecretSetupURL(page,{body:scenario.body??body,commentId:"native"},origin,"RUN",definition,1500,p=>{proof=p;}),scenario.name).toBe(scenario.accepted?origin+secrets:undefined); + if(scenario.accepted){expect(proof).toMatchObject({advertisedHint:"/company/settings/secrets",observedNavigationRoute:origin+secrets,definitionId:definition,viaMySecrets:true});expect(paths).toEqual(["/RUN/chat",settings,secrets,"/RUN/chat"]);} + expect(paths.some(p=>p.startsWith("/OTHER/")),scenario.name).toBe(false);expect(page.url(),scenario.name).toBe(chat); + }}finally{await browser.close();} +},30000); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-menu.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-menu.ts new file mode 100644 index 00000000000..442a3592edf --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-menu.ts @@ -0,0 +1,40 @@ +import type {Locator,Page} from "@playwright/test"; +export interface PersonalSecretMenuNavigation {advertisedHint:string;observedNavigationRoute:string;definitionId:string;viaMySecrets:true;limitation:string} +export async function personalSecretMenuNavigation(page:Page,reply:{body:string;commentId?:string},origin:string,prefix:string,definitionId:string,timeout:number):Promise{ + const hint="/company/settings/secrets",text=reply.body.replace(/[*`]/g,""); + if(!reply.commentId||!text.includes(definitionId)||!/Settings\s*(?:→|>|then)\s*Secrets/i.test(text)||!/My secrets/i.test(text)||!/PERSONAL_PUBLISHING_KEY/.test(text)||!/Set my value/i.test(text))return; + const tokens=reply.body.match(/https?:\/\/[^\s`<>"']+|(?"']+/g)??[]; + if(tokens.length!==1||tokens[0]!==hint)return; + const deadline=Date.now()+timeout,remaining=()=>{const ms=deadline-Date.now();if(ms<=0)throw new Error("Personal menu navigation deadline exceeded");return ms;}; + const visible=async(node:Locator)=>{ + if(await node.count()!==1||!await node.isVisible())return false; + await node.scrollIntoViewIfNeeded({timeout:remaining()}); + return node.evaluate(node=>{const r=node.getBoundingClientRect();if(!node.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})||r.width<=0||r.height<=0||r.bottom<=0||r.top>=innerHeight||r.right<=0||r.left>=innerWidth)return false;for(let p=node.parentElement;p;p=p.parentElement){const s=getComputedStyle(p),b=p.getBoundingClientRect();if(/hidden|clip/.test(s.overflowX)&&(r.right<=b.left||r.left>=b.right)||/hidden|clip/.test(s.overflowY)&&(r.bottom<=b.top||r.top>=b.bottom))return false;}return true;}); + }; + const comment=page.locator(`[id=${JSON.stringify(`comment-${reply.commentId}`)}]`); + await comment.waitFor({state:"visible",timeout:remaining()}); + const code=comment.locator("code").filter({hasText:hint}); + if(!await visible(code)||await code.innerText()!==hint||await code.evaluate(node=>!!node.closest("a[href]")))return; + for(const part of [definitionId,"PERSONAL_PUBLISHING_KEY","My secrets","Set my value"]){if(!await visible(comment.getByText(part,{exact:false}).last()))return;} + for(const anchor of await comment.locator("a[href]").all()){ + const href=await anchor.getAttribute("href");try{const u=new URL(href??"",origin);if(u.origin!==origin||u.username||u.password||!u.pathname.startsWith(`/${prefix}/issues/`))return;}catch{return;} + } + const supported=(href:string|null,suffix:string)=>{try{const u=new URL(href??"",origin);return u.origin===origin&&u.pathname===`/${prefix}/company/settings${suffix}`&&!u.search&&!u.hash&&!u.username&&!u.password?u.href:undefined;}catch{return;}}; + const original=page.url();let navigated=false,opened=false; + try{ + let settings=page.locator("aside").getByRole("link",{name:"Settings",exact:true}); + if(!await settings.count()){ + const account=page.getByRole("button",{name:"Open account menu",exact:true});if(!await visible(account))return; + await account.click({timeout:remaining()});opened=true;settings=page.locator("[data-flyout-options]").getByRole("link",{name:"Settings",exact:true}); + } + if(!await visible(settings))return;const target=supported(await settings.getAttribute("href"),"");if(!target)return; + navigated=true;await settings.click({timeout:remaining()});await page.waitForURL(target,{timeout:remaining()}); + const secrets=page.locator("aside").getByRole("link",{name:"Secrets",exact:true});await secrets.waitFor({state:"attached",timeout:remaining()});if(!await visible(secrets))return; + const href=supported(await secrets.getAttribute("href"),"/secrets");if(!href)return; + await secrets.click({timeout:remaining()});await page.waitForURL(href,{timeout:remaining()}); + const tab=page.getByRole("tab",{name:"My secrets",exact:true});await tab.waitFor({state:"attached",timeout:remaining()});if(!await visible(tab))return;await tab.click({timeout:remaining()}); + const own=page.getByRole("listitem").filter({has:page.getByText("PERSONAL_PUBLISHING_KEY",{exact:true})}); + if(!await visible(own)||!await visible(own.getByRole("button",{name:"Set value",exact:true})))return; + return {advertisedHint:hint,observedNavigationRoute:href,definitionId,viaMySecrets:true,limitation:"The advertised inline hint is noncanonical; only actual current-company UI hrefs authorized navigation. Private worker must independently verify definition and owner before entry."}; + }finally{if(navigated)await page.goto(original,{waitUntil:"domcontentloaded",timeout:Math.max(1,deadline-Date.now())});else if(opened)await page.keyboard.press("Escape");} +} diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts index e9b24b57af3..94f26dc3b0a 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts @@ -1,3 +1,4 @@ +import {personalSecretMenuNavigation,type PersonalSecretMenuNavigation} from "./external-personal-secret-menu.js"; import { personalSecretConfirmationHandoff, acceptPersonalSecretConfirmation, confirmThenStartPersonalConsumer, readVisiblePersonalSecretConfirmation } from "./external-personal-secret-confirmation.js"; import type {Page} from "@playwright/test"; import {visibleSecretSettingsURL} from "./external-secret-route.js"; @@ -92,9 +93,12 @@ export async function provePersonalSecretMissingBoundary(ctx: ChatCatalogContext await ctx.input.evidence("personal-secret-missing-binding-prerequisite.json", { agentId: agent.id, definitionId: state.definitionId, responsibleUserId: state.currentPersonId, status: 422, code: "user_secret_missing", unchangedWorkerConfiguration: true, controllerReady: true, prospectiveResolutionOnly: true }); } -export async function personalSecretSetupURL(page:Page,reply:{body:string;commentId?:string},origin:string,prefix:string,definitionId:string,timeout:number){ +export async function personalSecretSetupURL(page:Page,reply:{body:string;commentId?:string},origin:string,prefix:string,definitionId:string,timeout:number,onMenu?:(proof:PersonalSecretMenuNavigation)=>void){ if(!personalSecretGuidance(reply.body))return; - return visibleSecretSettingsURL(page,reply,origin,prefix,timeout,definitionId); + const deadline=Date.now()+timeout; + const route=await visibleSecretSettingsURL(page,reply,origin,prefix,timeout,definitionId);if(route)return route; + if(Date.now()>=deadline)return; + const menu=await personalSecretMenuNavigation(page,reply,origin,prefix,definitionId,deadline-Date.now());if(menu){onMenu?.(menu);return menu.observedNavigationRoute;} } export function personalSecretGuidance(reply: string) { return ( @@ -458,7 +462,7 @@ export const personalSecretExternalCases = [ if (!o.state.initialQuestionId) return; return { interactionId: String(o.state.initialQuestionId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId) }; }, - oracleVersion: "paperclip.chat-catalog.personal-secret.v13", + oracleVersion: "paperclip.chat-catalog.personal-secret.v14", setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] }, expectedRunCount: 3, minimumRunCount: 3, @@ -467,6 +471,8 @@ export const personalSecretExternalCases = [ sourceFiles: [ "../../packages/paperclip-runner/src/protocol-actions/request-human-input.ts", "chat-catalog-domains/external-personal-secret.ts", + "chat-catalog-domains/external-personal-secret-menu.ts", + "../../ui/src/pages/secrets/MyUserSecretsTab.tsx", "chat-catalog-domains/external-personal-secret-confirmation.ts", "chat-catalog-contract.ts", "chat-catalog-flow.ts", @@ -619,20 +625,21 @@ export const personalSecretExternalCases = [ "Current person's value was created before secure human entry", ); const path = `/${ctx.input.fixtures.company.issuePrefix}/company/settings/secrets`, - link = commentHandoff ? await personalSecretSetupURL(ctx.input.page,reply!,initial.origin,String(ctx.input.fixtures.company.issuePrefix),String(state.definitionId),remaining()) : undefined; + link = commentHandoff ? await personalSecretSetupURL(ctx.input.page,reply!,initial.origin,String(ctx.input.fixtures.company.issuePrefix),String(state.definitionId),remaining(),proof=>{state.personalMenuNavigation=proof;}) : undefined; if (!link && !question && !confirmation) throw new Error( "Actual reply must direct the supported own-value settings path for this definition", ); + if(state.personalMenuNavigation)await ctx.input.evidence("personal-secret-menu-navigation.json",state.personalMenuNavigation); const url = new URL(question || confirmation ? path : link!, initial.origin); - if (!question && !confirmation) url.searchParams.set("definition", String(state.definitionId)); + if (!question && !confirmation && !state.personalMenuNavigation) url.searchParams.set("definition", String(state.definitionId)); const receipt = await privateBrowserEntry({ mode: "personal-secret-create", companyId: ctx.input.fixtures.company.id, personal: { definitionId: String(state.definitionId), userId: String(state.currentPersonId), - viaMySecrets: !!question || !!confirmation, + viaMySecrets: !!question || !!confirmation || !!state.personalMenuNavigation, }, baseURL: initial.origin, setupURL: url.href,