fix(evals): discover personal secure entry through visible menus

tests/runner-e2e/CHAT-FEATURES.md

@@ -474,3 +474,13 @@ UI answer, model and 300000ms attempt unchanged. Result 5172 remains its origina
 PASSED result for the earlier persistence/continuation scope, with the recorded
 confusing duplicate-form and inaccurate walkthrough metadata limitations. It is
 not retroactively regraded, and no new native result is claimed.
+
+Personal secret v14 admits the measured result 5169's safe My secrets/definition
+menu directions with a noncanonical inline code hint. The hint is retained as an
+advertised hint, never rewritten into a trusted URL. Actual visible Account →
+Settings → Secrets hrefs must independently match the current company, and the
+My secrets control must expose the required own-value entry. Foreign links,
+other paths, hidden navigation and mismatched definitions remain inadmissible.
+The private worker still independently verifies the responsible person and exact
+definition before entering the value. Result 5169 remains FAILED; this local
+observer correction does not qualify the unexercised save, consumer or final reply.

tests/runner-e2e/chat-catalog-descriptors.json

@@ -4978,6 +4978,8 @@
     "sourceFiles": [
       "../../packages/paperclip-runner/src/protocol-actions/request-human-input.ts",
       "chat-catalog-domains/external-personal-secret.ts",
+      "chat-catalog-domains/external-personal-secret-menu.ts",
+      "../../ui/src/pages/secrets/MyUserSecretsTab.tsx",
       "chat-catalog-domains/external-personal-secret-confirmation.ts",
       "chat-catalog-contract.ts",
       "chat-catalog-flow.ts",
@@ -5006,7 +5008,7 @@
       "../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx",
       "chat-scenarios/secrets.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.personal-secret.v13",
+    "oracleVersion": "paperclip.chat-catalog.personal-secret.v14",
     "expectedRunCount": 3,
     "expectedHumanMessageCount": 2,
     "minimumRunCount": 3,

tests/runner-e2e/chat-catalog-domains/external-personal-secret-menu.test.ts

@@ -0,0 +1,34 @@
+import {expect,it} from "vitest";
+import {chromium} from "@playwright/test";
+import {personalSecretSetupURL,personalSecretGuidance} from "./external-personal-secret.js";
+const origin="https://paperclip.example",chat=origin+"/RUN/chat",settings="/RUN/company/settings",secrets=settings+"/secrets",definition="6263be21-6958-4e2b-a76a-f0e20331245c";
+const body=`I can't set a personal credential for you; never share it in chat. Open Settings → Secrets at \`/company/settings/secrets\`. Under My secrets, find PERSONAL_PUBLISHING_KEY (definition \`${definition}\`) and choose Set my value. Enter the publishing credential there and save.`;
+const markup=`<p>Open Settings → Secrets at <code>/company/settings/secrets</code>.</p><p>Under My secrets, find PERSONAL_PUBLISHING_KEY (definition <code>${definition}</code>) and choose Set my value.</p>`;
+it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("discovers the current Personal UI route independently of the5169 inline hint and rejects unbound alternatives",async()=>{
+ expect(personalSecretGuidance(body)).toBe(true);
+ const browser=await chromium.launch({headless:true,...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?{executablePath:process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE}:{}}),page=await browser.newPage();
+ const scenarios=[
+  {name:"actual-style Account navigation",accepted:true},
+  {name:"missing menu",missing:true},
+  {name:"hidden menu",hidden:true},
+  {name:"wrong company Settings",settings:"/OTHER/company/settings"},
+  {name:"foreign Secrets",secrets:"https://foreign.example/company/settings/secrets"},
+  {name:"wrong definition",body:body.replace(definition,"other-definition"),markup:markup.replace(definition,"other-definition")},
+  {name:"foreign anchor",extra:'<a href="https://foreign.example">Setup</a>'},
+  {name:"anchored hint",markup:markup.replace('<code>/company/settings/secrets</code>','<a href="/company/settings/secrets"><code>/company/settings/secrets</code></a>')},
+  {name:"other bare path",body:body.replace('/company/settings/secrets','/OTHER/company/settings/secrets'),markup:markup.replace('/company/settings/secrets','/OTHER/company/settings/secrets')},
+  {name:"unsafe guidance",body:body+" Paste the key here in chat."},
+  {name:"missing own entry",noEntry:true},
+ ];
+ try{for(const scenario of scenarios){
+  const paths:string[]=[];await page.unroute("**/*");await page.route("**/*",async route=>{
+   const path=new URL(route.request().url()).pathname;paths.push(path);
+   const content=path===settings?`<aside><a href="${scenario.secrets??secrets}">Secrets</a></aside>`:path===secrets?`<button role="tab">My secrets</button>${scenario.noEntry?"":'<ul><li><code>PERSONAL_PUBLISHING_KEY</code><button>Set value</button></li></ul>'}`:`<article id="comment-native">${scenario.markup??markup}${scenario.extra??""}</article>${scenario.missing?"":`<aside><button ${scenario.hidden?"hidden":""} aria-label="Open account menu" onclick="document.querySelector('[data-flyout-options]').hidden=false">Person</button></aside><div data-flyout-options hidden><a href="${scenario.settings??settings}">Settings</a></div>`}`;
+   await route.fulfill({contentType:"text/html; charset=utf-8",body:content});
+  });await page.goto(chat);
+  let proof:unknown;
+  expect(await personalSecretSetupURL(page,{body:scenario.body??body,commentId:"native"},origin,"RUN",definition,1500,p=>{proof=p;}),scenario.name).toBe(scenario.accepted?origin+secrets:undefined);
+  if(scenario.accepted){expect(proof).toMatchObject({advertisedHint:"/company/settings/secrets",observedNavigationRoute:origin+secrets,definitionId:definition,viaMySecrets:true});expect(paths).toEqual(["/RUN/chat",settings,secrets,"/RUN/chat"]);}
+  expect(paths.some(p=>p.startsWith("/OTHER/")),scenario.name).toBe(false);expect(page.url(),scenario.name).toBe(chat);
+ }}finally{await browser.close();}
+},30000);

tests/runner-e2e/chat-catalog-domains/external-personal-secret-menu.ts

@@ -0,0 +1,40 @@
+import type {Locator,Page} from "@playwright/test";
+export interface PersonalSecretMenuNavigation {advertisedHint:string;observedNavigationRoute:string;definitionId:string;viaMySecrets:true;limitation:string}
+export async function personalSecretMenuNavigation(page:Page,reply:{body:string;commentId?:string},origin:string,prefix:string,definitionId:string,timeout:number):Promise<PersonalSecretMenuNavigation|undefined>{
+ const hint="/company/settings/secrets",text=reply.body.replace(/[*`]/g,"");
+ if(!reply.commentId||!text.includes(definitionId)||!/Settings\s*(?:→|>|then)\s*Secrets/i.test(text)||!/My secrets/i.test(text)||!/PERSONAL_PUBLISHING_KEY/.test(text)||!/Set my value/i.test(text))return;
+ const tokens=reply.body.match(/https?:\/\/[^\s`<>"']+|(?<![\w/:])\/[^\s`<>"']+/g)??[];
+ if(tokens.length!==1||tokens[0]!==hint)return;
+ const deadline=Date.now()+timeout,remaining=()=>{const ms=deadline-Date.now();if(ms<=0)throw new Error("Personal menu navigation deadline exceeded");return ms;};
+ const visible=async(node:Locator)=>{
+  if(await node.count()!==1||!await node.isVisible())return false;
+  await node.scrollIntoViewIfNeeded({timeout:remaining()});
+  return node.evaluate(node=>{const r=node.getBoundingClientRect();if(!node.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})||r.width<=0||r.height<=0||r.bottom<=0||r.top>=innerHeight||r.right<=0||r.left>=innerWidth)return false;for(let p=node.parentElement;p;p=p.parentElement){const s=getComputedStyle(p),b=p.getBoundingClientRect();if(/hidden|clip/.test(s.overflowX)&&(r.right<=b.left||r.left>=b.right)||/hidden|clip/.test(s.overflowY)&&(r.bottom<=b.top||r.top>=b.bottom))return false;}return true;});
+ };
+ const comment=page.locator(`[id=${JSON.stringify(`comment-${reply.commentId}`)}]`);
+ await comment.waitFor({state:"visible",timeout:remaining()});
+ const code=comment.locator("code").filter({hasText:hint});
+ if(!await visible(code)||await code.innerText()!==hint||await code.evaluate(node=>!!node.closest("a[href]")))return;
+ for(const part of [definitionId,"PERSONAL_PUBLISHING_KEY","My secrets","Set my value"]){if(!await visible(comment.getByText(part,{exact:false}).last()))return;}
+ for(const anchor of await comment.locator("a[href]").all()){
+  const href=await anchor.getAttribute("href");try{const u=new URL(href??"",origin);if(u.origin!==origin||u.username||u.password||!u.pathname.startsWith(`/${prefix}/issues/`))return;}catch{return;}
+ }
+ const supported=(href:string|null,suffix:string)=>{try{const u=new URL(href??"",origin);return u.origin===origin&&u.pathname===`/${prefix}/company/settings${suffix}`&&!u.search&&!u.hash&&!u.username&&!u.password?u.href:undefined;}catch{return;}};
+ const original=page.url();let navigated=false,opened=false;
+ try{
+  let settings=page.locator("aside").getByRole("link",{name:"Settings",exact:true});
+  if(!await settings.count()){
+   const account=page.getByRole("button",{name:"Open account menu",exact:true});if(!await visible(account))return;
+   await account.click({timeout:remaining()});opened=true;settings=page.locator("[data-flyout-options]").getByRole("link",{name:"Settings",exact:true});
+  }
+  if(!await visible(settings))return;const target=supported(await settings.getAttribute("href"),"");if(!target)return;
+  navigated=true;await settings.click({timeout:remaining()});await page.waitForURL(target,{timeout:remaining()});
+  const secrets=page.locator("aside").getByRole("link",{name:"Secrets",exact:true});await secrets.waitFor({state:"attached",timeout:remaining()});if(!await visible(secrets))return;
+  const href=supported(await secrets.getAttribute("href"),"/secrets");if(!href)return;
+  await secrets.click({timeout:remaining()});await page.waitForURL(href,{timeout:remaining()});
+  const tab=page.getByRole("tab",{name:"My secrets",exact:true});await tab.waitFor({state:"attached",timeout:remaining()});if(!await visible(tab))return;await tab.click({timeout:remaining()});
+  const own=page.getByRole("listitem").filter({has:page.getByText("PERSONAL_PUBLISHING_KEY",{exact:true})});
+  if(!await visible(own)||!await visible(own.getByRole("button",{name:"Set value",exact:true})))return;
+  return {advertisedHint:hint,observedNavigationRoute:href,definitionId,viaMySecrets:true,limitation:"The advertised inline hint is noncanonical; only actual current-company UI hrefs authorized navigation. Private worker must independently verify definition and owner before entry."};
+ }finally{if(navigated)await page.goto(original,{waitUntil:"domcontentloaded",timeout:Math.max(1,deadline-Date.now())});else if(opened)await page.keyboard.press("Escape");}
+}

tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts

@@ -1,3 +1,4 @@
+import {personalSecretMenuNavigation,type PersonalSecretMenuNavigation} from "./external-personal-secret-menu.js";
 import { personalSecretConfirmationHandoff, acceptPersonalSecretConfirmation, confirmThenStartPersonalConsumer, readVisiblePersonalSecretConfirmation } from "./external-personal-secret-confirmation.js";
 import type {Page} from "@playwright/test";
 import {visibleSecretSettingsURL} from "./external-secret-route.js";
@@ -92,9 +93,12 @@ export async function provePersonalSecretMissingBoundary(ctx: ChatCatalogContext
   await ctx.input.evidence("personal-secret-missing-binding-prerequisite.json", { agentId: agent.id, definitionId: state.definitionId, responsibleUserId: state.currentPersonId, status: 422, code: "user_secret_missing", unchangedWorkerConfiguration: true, controllerReady: true, prospectiveResolutionOnly: true });
 }
 
-export async function personalSecretSetupURL(page:Page,reply:{body:string;commentId?:string},origin:string,prefix:string,definitionId:string,timeout:number){
+export async function personalSecretSetupURL(page:Page,reply:{body:string;commentId?:string},origin:string,prefix:string,definitionId:string,timeout:number,onMenu?:(proof:PersonalSecretMenuNavigation)=>void){
  if(!personalSecretGuidance(reply.body))return;
- return visibleSecretSettingsURL(page,reply,origin,prefix,timeout,definitionId);
+ const deadline=Date.now()+timeout;
+ const route=await visibleSecretSettingsURL(page,reply,origin,prefix,timeout,definitionId);if(route)return route;
+ if(Date.now()>=deadline)return;
+ const menu=await personalSecretMenuNavigation(page,reply,origin,prefix,definitionId,deadline-Date.now());if(menu){onMenu?.(menu);return menu.observedNavigationRoute;}
 }
 export function personalSecretGuidance(reply: string) {
   return (
@@ -458,7 +462,7 @@ export const personalSecretExternalCases = [
         if (!o.state.initialQuestionId) return;
         return { interactionId: String(o.state.initialQuestionId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId) };
       },
-      oracleVersion: "paperclip.chat-catalog.personal-secret.v13",
+      oracleVersion: "paperclip.chat-catalog.personal-secret.v14",
       setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] },
       expectedRunCount: 3,
       minimumRunCount: 3,
@@ -467,6 +471,8 @@ export const personalSecretExternalCases = [
       sourceFiles: [
         "../../packages/paperclip-runner/src/protocol-actions/request-human-input.ts",
         "chat-catalog-domains/external-personal-secret.ts",
+        "chat-catalog-domains/external-personal-secret-menu.ts",
+        "../../ui/src/pages/secrets/MyUserSecretsTab.tsx",
         "chat-catalog-domains/external-personal-secret-confirmation.ts",
         "chat-catalog-contract.ts",
         "chat-catalog-flow.ts",
@@ -619,20 +625,21 @@ export const personalSecretExternalCases = [
             "Current person's value was created before secure human entry",
           );
         const path = `/${ctx.input.fixtures.company.issuePrefix}/company/settings/secrets`,
-          link = commentHandoff ? await personalSecretSetupURL(ctx.input.page,reply!,initial.origin,String(ctx.input.fixtures.company.issuePrefix),String(state.definitionId),remaining()) : undefined;
+          link = commentHandoff ? await personalSecretSetupURL(ctx.input.page,reply!,initial.origin,String(ctx.input.fixtures.company.issuePrefix),String(state.definitionId),remaining(),proof=>{state.personalMenuNavigation=proof;}) : undefined;
         if (!link && !question && !confirmation)
           throw new Error(
             "Actual reply must direct the supported own-value settings path for this definition",
           );
+        if(state.personalMenuNavigation)await ctx.input.evidence("personal-secret-menu-navigation.json",state.personalMenuNavigation);
         const url = new URL(question || confirmation ? path : link!, initial.origin);
-        if (!question && !confirmation) url.searchParams.set("definition", String(state.definitionId));
+        if (!question && !confirmation && !state.personalMenuNavigation) url.searchParams.set("definition", String(state.definitionId));
         const receipt = await privateBrowserEntry({
           mode: "personal-secret-create",
           companyId: ctx.input.fixtures.company.id,
           personal: {
             definitionId: String(state.definitionId),
             userId: String(state.currentPersonId),
-            viaMySecrets: !!question || !!confirmation,
+            viaMySecrets: !!question || !!confirmation || !!state.personalMenuNavigation,
           },
           baseURL: initial.origin,
           setupURL: url.href,