fix(evals): observe secure handoffs before private value entry

Co-Authored-By: Paperclip <paperclip@paperclip.ing>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -4908,7 +4908,7 @@
       "chat-flow.ts",
       "chat-scenarios/secrets.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v8",
+    "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v9",
     "expectedRunCount": 2,
     "expectedHumanMessageCount": 2,
     "turnTimeoutMs": 120000,
@@ -4988,7 +4988,7 @@
       "../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx",
       "chat-scenarios/secrets.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.personal-secret.v10",
+    "oracleVersion": "paperclip.chat-catalog.personal-secret.v11",
     "expectedRunCount": 3,
     "expectedHumanMessageCount": 2,
     "minimumRunCount": 3,

tests/runner-e2e/chat-catalog-domains/external-company-secret-route.test.ts

@@ -7,6 +7,10 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("admits the
  const browser=await chromium.launch({headless:true,...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?{executablePath:process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE}:{}}),page=await browser.newPage();
  const render=async(text:string,html?:string,id="native")=>{await page.setContent(`<article id="comment-${id}"><p>Use secure Settings Secrets for Morgan's deployment credential environment binding.</p>${html??`<code>${text}</code>`}</article>`);};
  try{
+  await page.setContent(`<div style="height:100px;overflow-x:hidden;overflow-y:auto"><article id="comment-native"><code>${route}</code><div style="height:1500px">Long reply</div></article></div>`);
+  await page.locator("body > div").evaluate(node=>node.scrollTop=node.scrollHeight);
+  expect(await companySecretSetupURL(page,{body,commentId:"native"},origin,"RUN",1000)).toBe(origin+route);
+
   await render(route);expect(await companySecretSetupURL(page,{body,commentId:"native"},origin,"RUN",1000)).toBe(origin+route);
   await render(route,`<p style="height:1500px">Ordinary reply context</p><code>${route}</code>`);expect(await companySecretSetupURL(page,{body,commentId:"native"},origin,"RUN",1000)).toBe(origin+route);
   for(const html of [`<code style="position:absolute;left:-9999px">${route}</code>`,`<details><summary>More</summary><code>${route}</code></details>`,`<code style="font-size:0">${route}</code>`,`<code style="color:transparent">${route}</code>`,`<code style="opacity:0.001">${route}</code>`,`<div style="overflow:hidden;height:1px"><p style="margin-top:200px"><code>${route}</code></p></div>`]){

tests/runner-e2e/chat-catalog-domains/external-company-secret.ts

@@ -162,7 +162,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s
   return {interactionId:String(o.state.initialQuestionId),sourceRunId:String(o.state.initialRunId),userId:String(o.state.currentPersonId),
    ...(o.state.initialQuestionSavedSecretName===true?{companySavedSecretNameAnswer:{questionId:String(o.state.initialQuestionFieldId),payloadHash:String(o.state.initialQuestionPayloadHash),value:String(o.secrets.find(row=>row.id===o.state.secretId)?.key??""),secretId:String(o.state.secretId)}}:{}),
    ...(o.state.initialQuestionEnvironmentName==="DEPLOYMENT_API_KEY"?{companyEnvironmentNameAnswer:{questionId:String(o.state.initialQuestionFieldId),payloadHash:String(o.state.initialQuestionPayloadHash),value:"DEPLOYMENT_API_KEY" as const}}:{})};
- },oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v8",expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000,
+ },oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v9",expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000,
  sourceFiles:["chat-catalog-domains/external-company-secret.ts","chat-catalog-domains/external-secret-question.ts","chat-catalog-domains/external-secret-route.ts","chat-catalog-contract.ts","chat-catalog-flow.ts","chat-catalog-domains/external-private-browser.ts","chat-catalog-domains/external-private-browser-worker.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","live-fixtures.ts","../../server/src/services/secrets.ts","run-observations.ts","chat-flow.ts","chat-scenarios/secrets.yaml"],
  async act(ctx,state){
   const deadline=chatCatalogOriginalPhaseDeadline(ctx),remaining=()=>remainingIdentityObservationBudget(ctx);

tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation-visibility.test.ts

@@ -0,0 +1,32 @@
+import { expect, it } from "vitest";
+import { chromium } from "@playwright/test";
+import { readVisiblePersonalSecretConfirmation, acceptPersonalSecretConfirmation } from "./external-personal-secret-confirmation.js";
+import { personalSecretGuidance } from "./external-personal-secret.js";
+const prompt = "Set your personal publishing credential in secure Settings Secrets, My secrets, Set my value for PERSONAL_PUBLISHING_KEY. Never paste credentials in chat. When saved, confirm this card.";
+it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY !== "1")("reads the production-style collapsed Details before entry and never confirms unseen targets", async () => {
+  const browser = await chromium.launch({ headless: true, ...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ? { executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE } : {} }), page = await browser.newPage();
+  const target = { origin: "https://paperclip.example", prefix: "RUN", definitionId: "current-definition" };
+  const render = async (details: string) => page.setContent(`<div data-testid="task-chat-composer-takeover"><p>${prompt}</p><details><summary>Details</summary><div>${details}</div></details><button>Confirm</button></div>`);
+  try {
+    await render("Definition: current-definition");
+    expect(await page.locator("details").getAttribute("open")).toBeNull();
+    await expect(readVisiblePersonalSecretConfirmation(page, { prompt }, target, personalSecretGuidance, () => 500)).resolves.toMatchObject({ definitionVisible: true });
+    expect(await page.locator("details").getAttribute("open")).not.toBeNull();
+    const shortPrompt = "Save confirmation for current-definition";
+    await page.setContent(`<div data-testid="task-chat-composer-takeover"><p>${shortPrompt}</p><details><summary>Details</summary><div>${prompt}</div></details><button>Confirm</button></div>`);
+    await expect(readVisiblePersonalSecretConfirmation(page, { prompt: shortPrompt }, target, personalSecretGuidance, () => 500)).resolves.toMatchObject({ definitionVisible: true });
+    await render('<a href="https://paperclip.example/RUN/company/settings/secrets?definition=current-definition">Open My secrets</a>');
+    await expect(readVisiblePersonalSecretConfirmation(page, { prompt }, target, personalSecretGuidance, () => 500)).resolves.toMatchObject({ definitionVisible: true });
+    for (const details of ['<span hidden>current-definition</span>', '<span style="opacity:0">current-definition</span>', '<span style="position:absolute;left:-9999px">current-definition</span>', 'Definition: foreign-definition']) {
+      await render(details);
+      await expect(readVisiblePersonalSecretConfirmation(page, { prompt }, target, personalSecretGuidance, () => 300)).rejects.toThrow();
+    }
+    await render("Definition: current-definition");
+    let accepted = 0;
+    await page.route("**/api/issues/source/interactions/card/accept", async route => { accepted++; await route.fulfill({ status: 200, contentType: "application/json", body: '{"status":"accepted"}' }); });
+    await page.locator("button").evaluate(button => button.addEventListener("click", () => { void fetch("https://paperclip.example/api/issues/source/interactions/card/accept", { method: "POST" }); }));
+    await readVisiblePersonalSecretConfirmation(page, { prompt }, target, personalSecretGuidance, () => 500);
+    await acceptPersonalSecretConfirmation(page, "source", "card", "Confirm", prompt, () => 1000);
+    expect(accepted).toBe(1);
+  } finally { await browser.close(); }
+}, 15000);

tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation.ts

@@ -35,3 +35,42 @@ export async function confirmThenStartPersonalConsumer(confirm: () => Promise<vo
   await confirm();
   await start();
 }
+
+export async function readVisiblePersonalSecretConfirmation(page: Page, handoff: { prompt: string }, target: { origin: string; prefix: string; definitionId: string }, guidance: (text: string) => boolean, remaining: () => number) {
+  const card = page.getByTestId("task-chat-composer-takeover");
+  await expect(card.getByText(handoff.prompt, { exact: true })).toBeVisible({ timeout: remaining() });
+  const read = () => card.evaluate((root, target) => {
+    const visibility = { shown(node: Element, rect: DOMRect) {
+      let opacity = 1;
+      for (let element: Element | null = node; element; element = element.parentElement) {
+        const style = getComputedStyle(element), box = element.getBoundingClientRect(); opacity *= Number(style.opacity);
+        if (style.display === "none" || style.visibility !== "visible" || opacity < 0.05 || parseFloat(style.fontSize) <= 0) return false;
+        if (element instanceof HTMLDetailsElement && !element.open && !element.querySelector(":scope > summary")?.contains(node)) return false;
+        if (/hidden|clip/.test(style.overflowX) && (rect.right <= box.left || rect.left >= box.right)) return false;
+        if (/hidden|clip/.test(style.overflowY) && (rect.bottom <= box.top || rect.top >= box.bottom)) return false;
+      }
+      return rect.width > 0 && rect.height > 0 && rect.right > 0 && rect.left < innerWidth && rect.bottom > 0 && rect.top < innerHeight;
+    } };
+    const parts: string[] = [], walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT);
+    for (let node = walker.nextNode(); node; node = walker.nextNode()) if (node.parentElement) for (const match of (node.textContent ?? "").matchAll(/\S+/g)) {
+      const range = document.createRange(); range.setStart(node, match.index!); range.setEnd(node, match.index! + match[0].length);
+      if (Array.from(range.getClientRects()).some(rect => visibility.shown(node.parentElement!, rect))) parts.push(match[0]);
+    }
+    const links = Array.from(root.querySelectorAll<HTMLAnchorElement>("a[href]")).filter(node => visibility.shown(node, node.getBoundingClientRect())).map(node => node.href);
+    const linked = links.some(href => { try { const url = new URL(href, target.origin); return url.origin === target.origin && url.pathname === `/${target.prefix}/company/settings/secrets` && !url.username && !url.password && !url.hash && url.searchParams.getAll("definition").length === 1 && url.searchParams.get("definition") === target.definitionId && [...url.searchParams.keys()].every(key => key === "definition"); } catch { return false; } });
+    return { text: parts.join(" "), linked };
+  }, target);
+  let visible = await read(); const before = visible.text;
+  const seenDefinition = chatCatalogHasExactDefinitionId(visible.text, target.definitionId) || visible.linked;
+  if (!seenDefinition || !guidance(visible.text)) {
+    const details = card.locator("details");
+    if (await details.count() !== 1) throw new Error("Personal confirmation definition is not visibly bound");
+    if (await details.getAttribute("open") === null) await details.locator("summary").click({ timeout: remaining() });
+    const text = details.getByText(target.definitionId, { exact: false }).last();
+    if (await text.count()) await text.scrollIntoViewIfNeeded({ timeout: remaining() });
+    else { const link = details.locator("a[href]").last(); if (await link.count()) await link.scrollIntoViewIfNeeded({ timeout: remaining() }); }
+    visible = await read();
+  }
+  if ((!seenDefinition && !chatCatalogHasExactDefinitionId(visible.text, target.definitionId) && !visible.linked) || !guidance(before + "\n" + visible.text)) throw new Error("Personal confirmation secure handoff is not actually visible");
+  return { definitionVisible: true, visibleTextHash: createHash("sha256").update(before + "\n" + visible.text).digest("hex") };
+}

tests/runner-e2e/chat-catalog-domains/external-personal-secret-question.test.ts

@@ -2,6 +2,7 @@ import { afterEach, expect, it, vi } from "vitest";
 import type { ChatCatalogContext } from "../chat-catalog-contract.js";
 import { questionSetToAskUserQuestionsPayload } from "../../../packages/shared/src/question-set.js";
 const port = vi.hoisted(() => ({ observation: {} as Record<string, unknown> }));
+vi.mock("./external-personal-secret-confirmation.js", async original => ({ ...await original<typeof import("./external-personal-secret-confirmation.js")>(), readVisiblePersonalSecretConfirmation: async () => ({ definitionVisible: true, visibleTextHash: "reader-tested-separately" }) }));
 vi.mock("./external-support.js", async original => ({ ...await original<typeof import("./external-support.js")>(), observeExternal: async () => port.observation }));
 import { personalSecretExternalCases, personalSecretQuestionHandoff } from "./external-personal-secret.js";
 afterEach(() => { vi.useRealTimers(); });

tests/runner-e2e/chat-catalog-domains/external-personal-secret-route.test.ts

@@ -7,6 +7,10 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("admits the
  const browser=await chromium.launch({headless:true,...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?{executablePath:process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE}:{}}),page=await browser.newPage();
  const render=async(text:string,html?:string)=>page.setContent(`<article id="comment-native"><p>Personal credential setup</p>${html??`<code>${text}</code>`}</article>`);
  try{
+  await page.setContent(`<div style="height:100px;overflow-x:hidden;overflow-y:auto"><article id="comment-native"><code>${path+"?definition="+definition}</code><div style="height:1500px">Long reply</div></article></div>`);
+  await page.locator("body > div").evaluate(node=>node.scrollTop=node.scrollHeight);
+  expect(await personalSecretSetupURL(page,{body,commentId:"native"},origin,"RUN",definition,1000)).toBe(origin+path+"?definition="+definition);
+
   await render(path+"?definition="+definition);
   expect(await personalSecretSetupURL(page,{body,commentId:"native"},origin,"RUN",definition,1000)).toBe(origin+path+"?definition="+definition);
   await render("",`<a href="${path}?definition=${definition}">My secrets</a>`);

tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts

@@ -288,7 +288,7 @@ it("accepts a saved-value confirmation without a fabricated initial reply and re
   const o = valid();
   const payload = { version: 1, prompt: "Open Settings Secrets, My secrets, Set my value for your personal publishing credential PERSONAL_PUBLISHING_KEY definition definition. Never paste the value in chat. When you have saved it, confirm this card.", acceptLabel: "Confirm" };
   const row = { id: "confirmation", companyId: "company", issueId: "source", sourceRunId: "initial", createdByAgentId: "ceo", originCommentIds: ["human"], kind: "request_confirmation", status: "accepted", resolverPolicy: "anyone", continuationPolicy: "wake_assignee", resolvedByUserId: "person", resolvedAt: "2026-11-03T23:59:59Z", result: { version: 1, outcome: "accepted" }, payload };
-  Object.assign(o.state, { sourceIssueId: "source", initialConfirmationId: row.id, canonicalHumanCommentId: "human", initialConfirmationPayloadHash: createHash("sha256").update(JSON.stringify(payload)).digest("hex") });
+  Object.assign(o.state, { sourceIssueId: "source", confirmationVisibleBeforeEntry: true, initialConfirmationId: row.id, canonicalHumanCommentId: "human", initialConfirmationPayloadHash: createHash("sha256").update(JSON.stringify(payload)).digest("hex") });
   Object.assign(o.state.currentSecret, { createdAt: "2026-11-03T23:59:58Z" });
   Object.assign(o.runs[2]!, { agentId: "ceo", contextSnapshot: { issueId: "source", interactionId: row.id } });
   o.phaseReplies.shift();

tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts

@@ -1,4 +1,4 @@
-import { personalSecretConfirmationHandoff, acceptPersonalSecretConfirmation, confirmThenStartPersonalConsumer } from "./external-personal-secret-confirmation.js";
+import { personalSecretConfirmationHandoff, acceptPersonalSecretConfirmation, confirmThenStartPersonalConsumer, readVisiblePersonalSecretConfirmation } from "./external-personal-secret-confirmation.js";
 import type {Page} from "@playwright/test";
 import {visibleSecretSettingsURL} from "./external-secret-route.js";
 import { createHash, randomBytes } from "node:crypto";
@@ -231,7 +231,7 @@ export function gradePersonalSecret(value: unknown) {
       .at(-1);
   const confirmationRow = o.interactions.find(row => row.id === o.state.initialConfirmationId);
   const confirmation = personalSecretConfirmationHandoff(confirmationRow, { companyId: o.companyId, issueId: String(o.state.sourceIssueId), agentId: o.leadId, userId: String(o.state.currentPersonId), runId: String(o.state.initialRunId), definitionId: String(o.state.definitionId), canonicalCommentId: String(o.state.canonicalHumanCommentId) }, personalSecretGuidance, "accepted");
-  const confirmationAccepted = !!confirmation && confirmation.payloadHash === o.state.initialConfirmationPayloadHash
+  const confirmationAccepted = o.state.confirmationVisibleBeforeEntry === true && !!confirmation && confirmation.payloadHash === o.state.initialConfirmationPayloadHash
     && o.activity.some(event => event.action === "issue.thread_interaction_accepted" && event.companyId === o.companyId && event.entityId === o.state.sourceIssueId && event.actorType === "user" && event.actorId === o.state.currentPersonId && event.details?.interactionId === confirmation.interactionId && event.details.interactionStatus === "accepted");
   const secret = o.state.currentSecret as Partial<CompanySecret> | undefined;
   c.add(
@@ -457,7 +457,7 @@ export const personalSecretExternalCases = [
         if (!o.state.initialQuestionId) return;
         return { interactionId: String(o.state.initialQuestionId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId) };
       },
-      oracleVersion: "paperclip.chat-catalog.personal-secret.v10",
+      oracleVersion: "paperclip.chat-catalog.personal-secret.v11",
       setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] },
       expectedRunCount: 3,
       minimumRunCount: 3,
@@ -600,6 +600,11 @@ export const personalSecretExternalCases = [
           throw new Error("Personal setup must direct the current person's secure publishing value before human action");
         state.initialRunId = run.id;
         if (confirmation) Object.assign(state, { initialConfirmationId: confirmation.interactionId, initialConfirmationPayloadHash: confirmation.payloadHash, canonicalHumanCommentId: canonical[0]!.id });
+        if (confirmation) {
+          const visible = await readVisiblePersonalSecretConfirmation(ctx.input.page, confirmation, { origin: initial.origin, prefix: String(ctx.input.fixtures.company.issuePrefix), definitionId: String(state.definitionId) }, personalSecretGuidance, remaining);
+          state.confirmationVisibleBeforeEntry = visible.definitionVisible;
+          state.confirmationVisibleTextHash = visible.visibleTextHash;
+        }
         if (question) Object.assign(state, { initialQuestionId: question.interactionId,
           initialQuestionOptionId: question.optionId, initialQuestionPayloadHash: question.payloadHash });
         if (

tests/runner-e2e/chat-catalog-domains/external-secret-route.ts

@@ -10,8 +10,10 @@ export async function visibleSecretSettingsURL(page:Page,reply:{body:string;comm
   const reachable=await node.evaluate(node=>{
    const rect=node.getBoundingClientRect(),style=getComputedStyle(node);
    if(/absolute|fixed/.test(style.position)&&(rect.right<=0||rect.left>=innerWidth||rect.bottom<=0||rect.top>=innerHeight))return false;
-   for(let parent=node.parentElement;parent;parent=parent.parentElement)if(/hidden|clip/.test(getComputedStyle(parent).overflowX+getComputedStyle(parent).overflowY)){
-    const box=parent.getBoundingClientRect();if(rect.right<=box.left||rect.left>=box.right||rect.bottom<=box.top||rect.top>=box.bottom)return false;
+   for(let parent=node.parentElement;parent;parent=parent.parentElement){
+    const style=getComputedStyle(parent),box=parent.getBoundingClientRect();
+    if(/hidden|clip/.test(style.overflowX)&&(rect.right<=box.left||rect.left>=box.right))return false;
+    if(/hidden|clip/.test(style.overflowY)&&(rect.bottom<=box.top||rect.top>=box.bottom))return false;
    }
    return true;
   });
@@ -31,9 +33,9 @@ export async function visibleSecretSettingsURL(page:Page,reply:{body:string;comm
     const style=getComputedStyle(element);opacity*=Number(style.opacity);
     if(style.display==="none"||style.visibility==="hidden"||style.visibility==="collapse"||opacity<0.05)return false;
     if(element instanceof HTMLDetailsElement&&!element.open&&!element.querySelector(":scope > summary")?.contains(node))return false;
-    if(/hidden|clip/.test(style.overflowX+style.overflowY)){
-     const box=element.getBoundingClientRect();if(rect.right<=box.left||rect.left>=box.right||rect.bottom<=box.top||rect.top>=box.bottom)return false;
-    }
+    const box=element.getBoundingClientRect();
+    if(/hidden|clip/.test(style.overflowX)&&(rect.right<=box.left||rect.left>=box.right))return false;
+    if(/hidden|clip/.test(style.overflowY)&&(rect.bottom<=box.top||rect.top>=box.bottom))return false;
    }
    return rect.width>0&&rect.height>0&&rect.right>0&&rect.left<innerWidth&&rect.bottom>0&&rect.top<innerHeight;
   };