From 2b68f4a902a080e2aec89c3b941d89d9dae045a4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 14:59:59 -0600 Subject: [PATCH] fix(evals): bind personal save wait to entry dialog --- .../runner-e2e/chat-catalog-descriptors.json | 1 + ...ernal-personal-secret-entry-dialog.test.ts | 31 +++++++++++++++++++ .../external-personal-secret-entry-dialog.ts | 4 +++ .../external-personal-secret.ts | 1 + .../external-private-browser-worker.ts | 3 +- 5 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.ts diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index cf3a9b2e678..3298e8fc4a7 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4969,6 +4969,7 @@ "chat-catalog-domains/external-secret-endpoint.ts", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", + "chat-catalog-domains/external-personal-secret-entry-dialog.ts", "chat-catalog-domains/external-credential-language.ts", "chat-catalog-domains/external-secret-scan.ts", "chat-catalog-domains/identity-auth.ts", diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.test.ts new file mode 100644 index 00000000000..accfbeb7555 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.test.ts @@ -0,0 +1,31 @@ +import { expect, it } from "vitest"; +import { chromium } from "@playwright/test"; +import { personalSecretEntryDialog } from "./external-personal-secret-entry-dialog.js"; +it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY !== "1")("observes the actual entry-dialog helper after save while the definition Sheet remains open", async () => { + const browser = await chromium.launch({ headless: true, ...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ? { executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE } : {} }), page = await browser.newPage(); + const render = () => page.setContent(''); + try { + await render(); + const entry = personalSecretEntryDialog(page); + await entry.getByRole("heading", { name: /^Set your value/ }).waitFor(); + await entry.getByRole("textbox").fill("synthetic-model-free-value"); + let persisted = false; + await page.route("**/api/private-fixture-save", async route => { persisted = true; await route.fulfill({ status: 201, contentType: "application/json", headers: { "access-control-allow-origin": "*" }, body: '{"id":"synthetic-saved-value"}' }); }); + await page.locator("#entry button").evaluate(button => button.addEventListener("click", async () => { const response = await fetch("https://paperclip.example/api/private-fixture-save", { method: "POST" }); if (response.status === 201) { document.querySelector("#entry")!.remove(); document.querySelector("#sheet")!.removeAttribute("aria-hidden"); } })); + await entry.getByRole("button", { name: "Save value", exact: true }).click(); + await entry.waitFor({ state: "hidden", timeout: 300 }); + expect(persisted).toBe(true); + expect(await page.locator("#entry").count()).toBe(0); + expect(await page.locator("#sheet").isVisible()).toBe(true); + await expect(page.getByRole("dialog").waitFor({ state: "hidden", timeout: 100 })).rejects.toThrow(); + await render(); + await page.unroute("**/api/private-fixture-save"); persisted = false; + await page.route("**/api/private-fixture-save", route => route.fulfill({ status: 500, headers: { "access-control-allow-origin": "*" }, body: "failed" })); + await page.locator("#entry button").evaluate(button => button.addEventListener("click", async () => { const response = await fetch("https://paperclip.example/api/private-fixture-save", { method: "POST" }); if (response.status === 201) document.querySelector("#entry")!.remove(); })); + await personalSecretEntryDialog(page).getByRole("textbox").fill("synthetic-model-free-value"); + await personalSecretEntryDialog(page).getByRole("button", { name: "Save value", exact: true }).click(); + await expect(personalSecretEntryDialog(page).waitFor({ state: "hidden", timeout: 100 })).rejects.toThrow(); + expect(await page.locator("#entry").isVisible()).toBe(true); + expect(persisted).toBe(false); + } finally { await browser.close(); } +}, 10000); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.ts new file mode 100644 index 00000000000..7047fda39ef --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.ts @@ -0,0 +1,4 @@ +import type { Page } from "@playwright/test"; +export function personalSecretEntryDialog(page: Page) { + return page.getByRole("dialog").filter({ has: page.getByRole("heading", { name: /^Set your value/ }) }); +} diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts index 9cfb98b6ae6..3359abcc26c 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts @@ -473,6 +473,7 @@ export const personalSecretExternalCases = [ "chat-catalog-domains/external-secret-endpoint.ts", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", + "chat-catalog-domains/external-personal-secret-entry-dialog.ts", "chat-catalog-domains/external-credential-language.ts", "chat-catalog-domains/external-secret-scan.ts", "chat-catalog-domains/identity-auth.ts", diff --git a/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts b/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts index 5972898c0cd..c79bcbfb740 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts @@ -1,3 +1,4 @@ +import { personalSecretEntryDialog } from "./external-personal-secret-entry-dialog.js"; import { validDevicePrivateInput, type DevicePrivateInput } from "./external-device-private-contract.js"; import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; @@ -219,7 +220,7 @@ async function main() { await page.getByRole('tab',{name:'My secrets',exact:true}).click(); await page.getByRole('listitem').filter({hasText:entry.definition.key}).getByRole('button',{name:'Set value',exact:true}).click(); } else await page.getByRole('button',{name:'Set my value',exact:true}).click(); - const dialog=page.getByRole('dialog');await dialog.getByRole('heading',{name:/^Set your value/}).waitFor(); + const dialog=personalSecretEntryDialog(page);await dialog.getByRole('heading',{name:/^Set your value/}).waitFor(); if(!await dialog.getByText('Personal publishing credential',{exact:true}).isVisible())throw new Error('Wrong personal credential form'); await dialog.getByRole('textbox').fill(input.credential); const created=page.waitForResponse(response=>new URL(response.url()).pathname===`/api/companies/${input.companyId}/me/user-secrets`&&response.request().method()==='POST'&&response.status()===201);