From 3514765f297e037daba8540a519f5d5cbcf5f0b8 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 14:55:30 -0600 Subject: [PATCH] fix(evals): observe secure handoffs before private value entry Co-Authored-By: Paperclip --- .../runner-e2e/chat-catalog-descriptors.json | 4 +- .../external-company-secret-route.test.ts | 4 ++ .../external-company-secret.ts | 2 +- ...nal-secret-confirmation-visibility.test.ts | 32 +++++++++++++++ .../external-personal-secret-confirmation.ts | 39 +++++++++++++++++++ .../external-personal-secret-question.test.ts | 1 + .../external-personal-secret-route.test.ts | 4 ++ .../external-personal-secret.test.ts | 2 +- .../external-personal-secret.ts | 11 ++++-- .../external-secret-route.ts | 12 +++--- 10 files changed, 99 insertions(+), 12 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation-visibility.test.ts diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 8fc8d0634bf..cf3a9b2e678 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4908,7 +4908,7 @@ "chat-flow.ts", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v8", + "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v9", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, @@ -4988,7 +4988,7 @@ "../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.personal-secret.v10", + "oracleVersion": "paperclip.chat-catalog.personal-secret.v11", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "minimumRunCount": 3, diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-route.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-route.test.ts index 4449e5dd605..e3fef9ee3a6 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-route.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-route.test.ts @@ -7,6 +7,10 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("admits the const browser=await chromium.launch({headless:true,...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?{executablePath:process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE}:{}}),page=await browser.newPage(); const render=async(text:string,html?:string,id="native")=>{await page.setContent(`

Use secure Settings Secrets for Morgan's deployment credential environment binding.

${html??`${text}`}
`);}; try{ + await page.setContent(`
${route}
Long reply
`); + await page.locator("body > div").evaluate(node=>node.scrollTop=node.scrollHeight); + expect(await companySecretSetupURL(page,{body,commentId:"native"},origin,"RUN",1000)).toBe(origin+route); + await render(route);expect(await companySecretSetupURL(page,{body,commentId:"native"},origin,"RUN",1000)).toBe(origin+route); await render(route,`

Ordinary reply context

${route}`);expect(await companySecretSetupURL(page,{body,commentId:"native"},origin,"RUN",1000)).toBe(origin+route); for(const html of [`${route}`,`
More${route}
`,`${route}`,`${route}`,`${route}`,`

${route}

`]){ diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts index b7c272d9a55..6dbac435064 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts @@ -162,7 +162,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s return {interactionId:String(o.state.initialQuestionId),sourceRunId:String(o.state.initialRunId),userId:String(o.state.currentPersonId), ...(o.state.initialQuestionSavedSecretName===true?{companySavedSecretNameAnswer:{questionId:String(o.state.initialQuestionFieldId),payloadHash:String(o.state.initialQuestionPayloadHash),value:String(o.secrets.find(row=>row.id===o.state.secretId)?.key??""),secretId:String(o.state.secretId)}}:{}), ...(o.state.initialQuestionEnvironmentName==="DEPLOYMENT_API_KEY"?{companyEnvironmentNameAnswer:{questionId:String(o.state.initialQuestionFieldId),payloadHash:String(o.state.initialQuestionPayloadHash),value:"DEPLOYMENT_API_KEY" as const}}:{})}; - },oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v8",expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, + },oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v9",expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, sourceFiles:["chat-catalog-domains/external-company-secret.ts","chat-catalog-domains/external-secret-question.ts","chat-catalog-domains/external-secret-route.ts","chat-catalog-contract.ts","chat-catalog-flow.ts","chat-catalog-domains/external-private-browser.ts","chat-catalog-domains/external-private-browser-worker.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","live-fixtures.ts","../../server/src/services/secrets.ts","run-observations.ts","chat-flow.ts","chat-scenarios/secrets.yaml"], async act(ctx,state){ const deadline=chatCatalogOriginalPhaseDeadline(ctx),remaining=()=>remainingIdentityObservationBudget(ctx); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation-visibility.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation-visibility.test.ts new file mode 100644 index 00000000000..135d2e6643f --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation-visibility.test.ts @@ -0,0 +1,32 @@ +import { expect, it } from "vitest"; +import { chromium } from "@playwright/test"; +import { readVisiblePersonalSecretConfirmation, acceptPersonalSecretConfirmation } from "./external-personal-secret-confirmation.js"; +import { personalSecretGuidance } from "./external-personal-secret.js"; +const prompt = "Set your personal publishing credential in secure Settings Secrets, My secrets, Set my value for PERSONAL_PUBLISHING_KEY. Never paste credentials in chat. When saved, confirm this card."; +it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY !== "1")("reads the production-style collapsed Details before entry and never confirms unseen targets", async () => { + const browser = await chromium.launch({ headless: true, ...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ? { executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE } : {} }), page = await browser.newPage(); + const target = { origin: "https://paperclip.example", prefix: "RUN", definitionId: "current-definition" }; + const render = async (details: string) => page.setContent(`

${prompt}

Details
${details}
`); + try { + await render("Definition: current-definition"); + expect(await page.locator("details").getAttribute("open")).toBeNull(); + await expect(readVisiblePersonalSecretConfirmation(page, { prompt }, target, personalSecretGuidance, () => 500)).resolves.toMatchObject({ definitionVisible: true }); + expect(await page.locator("details").getAttribute("open")).not.toBeNull(); + const shortPrompt = "Save confirmation for current-definition"; + await page.setContent(`

${shortPrompt}

Details
${prompt}
`); + await expect(readVisiblePersonalSecretConfirmation(page, { prompt: shortPrompt }, target, personalSecretGuidance, () => 500)).resolves.toMatchObject({ definitionVisible: true }); + await render('Open My secrets'); + await expect(readVisiblePersonalSecretConfirmation(page, { prompt }, target, personalSecretGuidance, () => 500)).resolves.toMatchObject({ definitionVisible: true }); + for (const details of ['', 'current-definition', 'current-definition', 'Definition: foreign-definition']) { + await render(details); + await expect(readVisiblePersonalSecretConfirmation(page, { prompt }, target, personalSecretGuidance, () => 300)).rejects.toThrow(); + } + await render("Definition: current-definition"); + let accepted = 0; + await page.route("**/api/issues/source/interactions/card/accept", async route => { accepted++; await route.fulfill({ status: 200, contentType: "application/json", body: '{"status":"accepted"}' }); }); + await page.locator("button").evaluate(button => button.addEventListener("click", () => { void fetch("https://paperclip.example/api/issues/source/interactions/card/accept", { method: "POST" }); })); + await readVisiblePersonalSecretConfirmation(page, { prompt }, target, personalSecretGuidance, () => 500); + await acceptPersonalSecretConfirmation(page, "source", "card", "Confirm", prompt, () => 1000); + expect(accepted).toBe(1); + } finally { await browser.close(); } +}, 15000); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation.ts index 9eabfb5d123..ce8671d6f83 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation.ts @@ -35,3 +35,42 @@ export async function confirmThenStartPersonalConsumer(confirm: () => Promise boolean, remaining: () => number) { + const card = page.getByTestId("task-chat-composer-takeover"); + await expect(card.getByText(handoff.prompt, { exact: true })).toBeVisible({ timeout: remaining() }); + const read = () => card.evaluate((root, target) => { + const visibility = { shown(node: Element, rect: DOMRect) { + let opacity = 1; + for (let element: Element | null = node; element; element = element.parentElement) { + const style = getComputedStyle(element), box = element.getBoundingClientRect(); opacity *= Number(style.opacity); + if (style.display === "none" || style.visibility !== "visible" || opacity < 0.05 || parseFloat(style.fontSize) <= 0) return false; + if (element instanceof HTMLDetailsElement && !element.open && !element.querySelector(":scope > summary")?.contains(node)) return false; + if (/hidden|clip/.test(style.overflowX) && (rect.right <= box.left || rect.left >= box.right)) return false; + if (/hidden|clip/.test(style.overflowY) && (rect.bottom <= box.top || rect.top >= box.bottom)) return false; + } + return rect.width > 0 && rect.height > 0 && rect.right > 0 && rect.left < innerWidth && rect.bottom > 0 && rect.top < innerHeight; + } }; + const parts: string[] = [], walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT); + for (let node = walker.nextNode(); node; node = walker.nextNode()) if (node.parentElement) for (const match of (node.textContent ?? "").matchAll(/\S+/g)) { + const range = document.createRange(); range.setStart(node, match.index!); range.setEnd(node, match.index! + match[0].length); + if (Array.from(range.getClientRects()).some(rect => visibility.shown(node.parentElement!, rect))) parts.push(match[0]); + } + const links = Array.from(root.querySelectorAll("a[href]")).filter(node => visibility.shown(node, node.getBoundingClientRect())).map(node => node.href); + const linked = links.some(href => { try { const url = new URL(href, target.origin); return url.origin === target.origin && url.pathname === `/${target.prefix}/company/settings/secrets` && !url.username && !url.password && !url.hash && url.searchParams.getAll("definition").length === 1 && url.searchParams.get("definition") === target.definitionId && [...url.searchParams.keys()].every(key => key === "definition"); } catch { return false; } }); + return { text: parts.join(" "), linked }; + }, target); + let visible = await read(); const before = visible.text; + const seenDefinition = chatCatalogHasExactDefinitionId(visible.text, target.definitionId) || visible.linked; + if (!seenDefinition || !guidance(visible.text)) { + const details = card.locator("details"); + if (await details.count() !== 1) throw new Error("Personal confirmation definition is not visibly bound"); + if (await details.getAttribute("open") === null) await details.locator("summary").click({ timeout: remaining() }); + const text = details.getByText(target.definitionId, { exact: false }).last(); + if (await text.count()) await text.scrollIntoViewIfNeeded({ timeout: remaining() }); + else { const link = details.locator("a[href]").last(); if (await link.count()) await link.scrollIntoViewIfNeeded({ timeout: remaining() }); } + visible = await read(); + } + if ((!seenDefinition && !chatCatalogHasExactDefinitionId(visible.text, target.definitionId) && !visible.linked) || !guidance(before + "\n" + visible.text)) throw new Error("Personal confirmation secure handoff is not actually visible"); + return { definitionVisible: true, visibleTextHash: createHash("sha256").update(before + "\n" + visible.text).digest("hex") }; +} diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-question.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-question.test.ts index d4698b812f9..42f24893513 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-question.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-question.test.ts @@ -2,6 +2,7 @@ import { afterEach, expect, it, vi } from "vitest"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { questionSetToAskUserQuestionsPayload } from "../../../packages/shared/src/question-set.js"; const port = vi.hoisted(() => ({ observation: {} as Record })); +vi.mock("./external-personal-secret-confirmation.js", async original => ({ ...await original(), readVisiblePersonalSecretConfirmation: async () => ({ definitionVisible: true, visibleTextHash: "reader-tested-separately" }) })); vi.mock("./external-support.js", async original => ({ ...await original(), observeExternal: async () => port.observation })); import { personalSecretExternalCases, personalSecretQuestionHandoff } from "./external-personal-secret.js"; afterEach(() => { vi.useRealTimers(); }); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-route.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-route.test.ts index dd376effbc4..277366afe06 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-route.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-route.test.ts @@ -7,6 +7,10 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("admits the const browser=await chromium.launch({headless:true,...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?{executablePath:process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE}:{}}),page=await browser.newPage(); const render=async(text:string,html?:string)=>page.setContent(`

Personal credential setup

${html??`${text}`}
`); try{ + await page.setContent(`
${path+"?definition="+definition}
Long reply
`); + await page.locator("body > div").evaluate(node=>node.scrollTop=node.scrollHeight); + expect(await personalSecretSetupURL(page,{body,commentId:"native"},origin,"RUN",definition,1000)).toBe(origin+path+"?definition="+definition); + await render(path+"?definition="+definition); expect(await personalSecretSetupURL(page,{body,commentId:"native"},origin,"RUN",definition,1000)).toBe(origin+path+"?definition="+definition); await render("",`My secrets`); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts index 6b3f4313988..ad2ab4d5ee6 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts @@ -288,7 +288,7 @@ it("accepts a saved-value confirmation without a fabricated initial reply and re const o = valid(); const payload = { version: 1, prompt: "Open Settings Secrets, My secrets, Set my value for your personal publishing credential PERSONAL_PUBLISHING_KEY definition definition. Never paste the value in chat. When you have saved it, confirm this card.", acceptLabel: "Confirm" }; const row = { id: "confirmation", companyId: "company", issueId: "source", sourceRunId: "initial", createdByAgentId: "ceo", originCommentIds: ["human"], kind: "request_confirmation", status: "accepted", resolverPolicy: "anyone", continuationPolicy: "wake_assignee", resolvedByUserId: "person", resolvedAt: "2026-11-03T23:59:59Z", result: { version: 1, outcome: "accepted" }, payload }; - Object.assign(o.state, { sourceIssueId: "source", initialConfirmationId: row.id, canonicalHumanCommentId: "human", initialConfirmationPayloadHash: createHash("sha256").update(JSON.stringify(payload)).digest("hex") }); + Object.assign(o.state, { sourceIssueId: "source", confirmationVisibleBeforeEntry: true, initialConfirmationId: row.id, canonicalHumanCommentId: "human", initialConfirmationPayloadHash: createHash("sha256").update(JSON.stringify(payload)).digest("hex") }); Object.assign(o.state.currentSecret, { createdAt: "2026-11-03T23:59:58Z" }); Object.assign(o.runs[2]!, { agentId: "ceo", contextSnapshot: { issueId: "source", interactionId: row.id } }); o.phaseReplies.shift(); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts index fa576e322e2..9cfb98b6ae6 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts @@ -1,4 +1,4 @@ -import { personalSecretConfirmationHandoff, acceptPersonalSecretConfirmation, confirmThenStartPersonalConsumer } from "./external-personal-secret-confirmation.js"; +import { personalSecretConfirmationHandoff, acceptPersonalSecretConfirmation, confirmThenStartPersonalConsumer, readVisiblePersonalSecretConfirmation } from "./external-personal-secret-confirmation.js"; import type {Page} from "@playwright/test"; import {visibleSecretSettingsURL} from "./external-secret-route.js"; import { createHash, randomBytes } from "node:crypto"; @@ -231,7 +231,7 @@ export function gradePersonalSecret(value: unknown) { .at(-1); const confirmationRow = o.interactions.find(row => row.id === o.state.initialConfirmationId); const confirmation = personalSecretConfirmationHandoff(confirmationRow, { companyId: o.companyId, issueId: String(o.state.sourceIssueId), agentId: o.leadId, userId: String(o.state.currentPersonId), runId: String(o.state.initialRunId), definitionId: String(o.state.definitionId), canonicalCommentId: String(o.state.canonicalHumanCommentId) }, personalSecretGuidance, "accepted"); - const confirmationAccepted = !!confirmation && confirmation.payloadHash === o.state.initialConfirmationPayloadHash + const confirmationAccepted = o.state.confirmationVisibleBeforeEntry === true && !!confirmation && confirmation.payloadHash === o.state.initialConfirmationPayloadHash && o.activity.some(event => event.action === "issue.thread_interaction_accepted" && event.companyId === o.companyId && event.entityId === o.state.sourceIssueId && event.actorType === "user" && event.actorId === o.state.currentPersonId && event.details?.interactionId === confirmation.interactionId && event.details.interactionStatus === "accepted"); const secret = o.state.currentSecret as Partial | undefined; c.add( @@ -457,7 +457,7 @@ export const personalSecretExternalCases = [ if (!o.state.initialQuestionId) return; return { interactionId: String(o.state.initialQuestionId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId) }; }, - oracleVersion: "paperclip.chat-catalog.personal-secret.v10", + oracleVersion: "paperclip.chat-catalog.personal-secret.v11", setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] }, expectedRunCount: 3, minimumRunCount: 3, @@ -600,6 +600,11 @@ export const personalSecretExternalCases = [ throw new Error("Personal setup must direct the current person's secure publishing value before human action"); state.initialRunId = run.id; if (confirmation) Object.assign(state, { initialConfirmationId: confirmation.interactionId, initialConfirmationPayloadHash: confirmation.payloadHash, canonicalHumanCommentId: canonical[0]!.id }); + if (confirmation) { + const visible = await readVisiblePersonalSecretConfirmation(ctx.input.page, confirmation, { origin: initial.origin, prefix: String(ctx.input.fixtures.company.issuePrefix), definitionId: String(state.definitionId) }, personalSecretGuidance, remaining); + state.confirmationVisibleBeforeEntry = visible.definitionVisible; + state.confirmationVisibleTextHash = visible.visibleTextHash; + } if (question) Object.assign(state, { initialQuestionId: question.interactionId, initialQuestionOptionId: question.optionId, initialQuestionPayloadHash: question.payloadHash }); if ( diff --git a/tests/runner-e2e/chat-catalog-domains/external-secret-route.ts b/tests/runner-e2e/chat-catalog-domains/external-secret-route.ts index 834eab17874..776b2eab45b 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-secret-route.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-secret-route.ts @@ -10,8 +10,10 @@ export async function visibleSecretSettingsURL(page:Page,reply:{body:string;comm const reachable=await node.evaluate(node=>{ const rect=node.getBoundingClientRect(),style=getComputedStyle(node); if(/absolute|fixed/.test(style.position)&&(rect.right<=0||rect.left>=innerWidth||rect.bottom<=0||rect.top>=innerHeight))return false; - for(let parent=node.parentElement;parent;parent=parent.parentElement)if(/hidden|clip/.test(getComputedStyle(parent).overflowX+getComputedStyle(parent).overflowY)){ - const box=parent.getBoundingClientRect();if(rect.right<=box.left||rect.left>=box.right||rect.bottom<=box.top||rect.top>=box.bottom)return false; + for(let parent=node.parentElement;parent;parent=parent.parentElement){ + const style=getComputedStyle(parent),box=parent.getBoundingClientRect(); + if(/hidden|clip/.test(style.overflowX)&&(rect.right<=box.left||rect.left>=box.right))return false; + if(/hidden|clip/.test(style.overflowY)&&(rect.bottom<=box.top||rect.top>=box.bottom))return false; } return true; }); @@ -31,9 +33,9 @@ export async function visibleSecretSettingsURL(page:Page,reply:{body:string;comm const style=getComputedStyle(element);opacity*=Number(style.opacity); if(style.display==="none"||style.visibility==="hidden"||style.visibility==="collapse"||opacity<0.05)return false; if(element instanceof HTMLDetailsElement&&!element.open&&!element.querySelector(":scope > summary")?.contains(node))return false; - if(/hidden|clip/.test(style.overflowX+style.overflowY)){ - const box=element.getBoundingClientRect();if(rect.right<=box.left||rect.left>=box.right||rect.bottom<=box.top||rect.top>=box.bottom)return false; - } + const box=element.getBoundingClientRect(); + if(/hidden|clip/.test(style.overflowX)&&(rect.right<=box.left||rect.left>=box.right))return false; + if(/hidden|clip/.test(style.overflowY)&&(rect.bottom<=box.top||rect.top>=box.bottom))return false; } return rect.width>0&&rect.height>0&&rect.right>0&&rect.left0&&rect.top