mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
fix(evals): preserve pending forms when sending ordinary chat
This commit is contained in:
1 parent
58449c4ecd
commit
78e791b0fd
2 files changed
+23
-6
No files matched your search
@@ -341,12 +341,7 @@ export function assertChatHandoff(
|
||||
export async function sendChatMessage(page: Page, message: string, remainingBudget?: () => number) {
|
||||
const options = () => remainingBudget ? { timeout: remainingBudget() } : {};
|
||||
const composer = page.getByTestId("task-chat-composer-input").last();
|
||||
const takeover = page.getByTestId("task-chat-composer-takeover").last();
|
||||
await expect(composer.or(takeover).first()).toBeVisible(options());
|
||||
if (await takeover.isVisible()) {
|
||||
// Close the card's composer view without answering, skipping, or approving it.
|
||||
await takeover.getByRole("button", { name: /^Dismiss / }).click(options());
|
||||
}
|
||||
await expect(composer).toBeVisible(options());
|
||||
await composer
|
||||
.locator('[contenteditable="true"], textarea')
|
||||
.first()
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { chromium } from "@playwright/test";
|
||||
import { sendChatMessage } from "./chat-flow.js";
|
||||
|
||||
it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_MESSAGE_CANARY !== "1")("sends ordinary chat without dismissing a current form on desktop and mobile", async () => {
|
||||
const browser = await chromium.launch({ headless: true, ...(process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ? { executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE } : {}) });
|
||||
try {
|
||||
for (const viewport of [{ width: 1280, height: 900 }, { width: 390, height: 844 }]) {
|
||||
const page = await browser.newPage({ viewport });
|
||||
await page.setContent(`<section data-testid="task-chat-composer-takeover"><p>Launch page details</p><button aria-label="Dismiss Question">Close</button></section><div data-testid="task-chat-composer-input"><textarea></textarea></div><button data-testid="task-chat-composer-send">Send</button><script>
|
||||
window.formState = { status: "pending", dismissedAt: null, messages: [] };
|
||||
document.querySelector('[aria-label="Dismiss Question"]').onclick = () => { window.formState.dismissedAt = "dismissed"; document.querySelector('section').remove(); };
|
||||
document.querySelector('[data-testid="task-chat-composer-send"]').onclick = () => { window.formState.messages.push(document.querySelector('textarea').value); };
|
||||
</script>`);
|
||||
const message = "Walk me through the existing form before building anything.";
|
||||
await sendChatMessage(page, message, () => 5000);
|
||||
expect(await page.evaluate(() => (window as unknown as { formState: unknown }).formState)).toEqual({ status: "pending", dismissedAt: null, messages: [message] });
|
||||
expect(await page.getByTestId("task-chat-composer-takeover").isVisible()).toBe(true);
|
||||
await page.close();
|
||||
}
|
||||
} finally { await browser.close(); }
|
||||
});
|
||||
Reference in new issue
Block a user