From 41743cb0ae6fb356f8f5b9fcc5985916e2559ddc Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 20:22:04 -0600 Subject: [PATCH] test(evals): observe required question validation on Next Replace the disabled-button assumption with real invalid-click validation, no page advance, no saved answer and no new work. Keep all subsequent exact answers, page retention and continuation checks. Calibrate the real QuestionForm with a pure React test; no product UI behavior changes. Co-Authored-By: Paperclip --- .../identity-questions-and-approvals.ts | 16 +++++++++---- .../chat-multipart-required-ui.test.ts | 23 ++++++++++--------- .../task-chat/QuestionForm.test.tsx | 13 +++++++++++ 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts index 03a198477a0..2b28baaf7a6 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts @@ -201,7 +201,7 @@ export const questionIdentityCases = [ }, }), identityDefinition("questions-and-approvals.multi-part-question", ctx => questionFixture(ctx, true), gradeQuestions, { - expectedRunCount: 2, observe: observeQuestion, + expectedRunCount: 2, observe: observeQuestion, oracleVersion: "paperclip.chat-catalog.required-question.v2", async act(ctx, state) { await waitForReply(ctx); const row = ctx.input.page.locator(`[id=${JSON.stringify(`interaction-${entity(state, "original").id}`)}]`); @@ -211,14 +211,22 @@ export const questionIdentityCases = [ const firstQuestion = original.payload.questionSet?.questions[0]; if (!firstQuestion?.required) throw new Error("Multipart first question must retain its saved required contract"); const before = (await cards(ctx)).find(card => card.id === original.id); + const beforeRuns = (await ctx.allRuns()).map(run => ({ id: run.id, status: run.status })); + const tasksPath = `/api/companies/${ctx.input.fixtures.company.id}/issues?includeAgentCreatedTasks=true`; + const beforeTasks = (await ctx.input.api.get(tasksPath)).filter(task => task.id !== ctx.issue()!.id); await expect(form.getByText(firstQuestion.prompt, { exact: true })).toBeVisible(); const next = form.getByRole("button", { name: "Next", exact: true }); - await expect(next).toBeDisabled(); + // The ordinary form validates on an explicit Next click; enabled is not consent. + await next.click(); + await expect(form.getByText("This question is required.", { exact: true })).toBeVisible(); const after = (await cards(ctx)).find(card => card.id === original.id); await expect(form.getByText(firstQuestion.prompt, { exact: true })).toBeVisible(); - state.validationObserved = await next.isDisabled() && before?.status === "pending" && after?.status === "pending" + const afterTasks = (await ctx.input.api.get(tasksPath)).filter(task => task.id !== ctx.issue()!.id); + const afterRuns = (await ctx.allRuns()).map(run => ({ id: run.id, status: run.status })); + state.validationObserved = before?.status === "pending" && after?.status === "pending" && !before.result && !after.result && JSON.stringify(before.payload) === JSON.stringify(original.payload) - && JSON.stringify(after.payload) === JSON.stringify(original.payload); + && JSON.stringify(after.payload) === JSON.stringify(original.payload) && beforeTasks.length === 0 && afterTasks.length === 0 + && JSON.stringify(afterRuns) === JSON.stringify(beforeRuns); if (!state.validationObserved) throw new Error("Required multipart question advanced or changed before an answer"); await form.getByRole("radio", { name: "Researchers", exact: true }).click(); await form.getByRole("button", { name: "Next", exact: true }).click(); diff --git a/tests/runner-e2e/chat-multipart-required-ui.test.ts b/tests/runner-e2e/chat-multipart-required-ui.test.ts index 26b1c6935a9..ed1d9a9ed72 100644 --- a/tests/runner-e2e/chat-multipart-required-ui.test.ts +++ b/tests/runner-e2e/chat-multipart-required-ui.test.ts @@ -5,18 +5,19 @@ vi.mock("@playwright/test", () => ({ expect: (target: { isDisabled?: () => boole }) })); import { questionIdentityCases } from "./chat-catalog-domains/identity-questions-and-approvals.js"; import type { ChatCatalogContext } from "./chat-catalog-contract.js"; -function fixture(options: { nextEnabled?: boolean; wrongPrompt?: boolean; changedPayload?: boolean; prematurelyResolved?: boolean } = {}) { - let page = 0, answered = false, reads = 0, submitted = false; +function fixture(options: { advancesMissing?: boolean; missingValidation?: boolean; createsWork?: boolean; resumesWork?: boolean; wrongPrompt?: boolean; changedPayload?: boolean; prematurelyResolved?: boolean } = {}) { + let page = 0, answered = false, reads = 0, submitted = false, attemptedMissing = false; const clicks: string[] = [], answers: Record = {}; const original = { id: "question", kind: "ask_user_questions", status: "pending", payload: { questionSet: { questions: [ { id: "audience", prompt: "Who is the audience?", required: true }, { id: "tone", prompt: "What tone should we use?", required: true }, { id: "headline", prompt: "What is your custom headline?", required: true }, ] } } }; - const form: any = { isVisible: () => true, getByText: (prompt: string) => ({ isVisible: () => !options.wrongPrompt && prompt === original.payload.questionSet.questions[page]?.prompt }), + const form: any = { isVisible: () => true, getByText: (prompt: string) => ({ isVisible: () => prompt === "This question is required." + ? attemptedMissing && !options.missingValidation : !options.wrongPrompt && prompt === original.payload.questionSet.questions[page]?.prompt }), getByRole: (_role: string, { name }: { name: string | RegExp }) => ({ - isDisabled: () => !options.nextEnabled && !answered, + isDisabled: () => false, isChecked: () => answers.audience === "Researchers", click: async () => { const label = String(name); clicks.push(label); - if (label === "Next") { if (!answered) throw new Error("Disabled Next cannot be clicked"); page++; answered = !!answers[original.payload.questionSet.questions[page]?.id ?? ""]; } + if (label === "Next") { if (!answered) { attemptedMissing = true; if (options.advancesMissing) page++; return; } page++; answered = !!answers[original.payload.questionSet.questions[page]?.id ?? ""]; } else if (label === "Back") { page--; answered = !!answers[original.payload.questionSet.questions[page]!.id]; } else if (label === "Researchers" || label === "Clear") { answers[original.payload.questionSet.questions[page]!.id] = label; answered = true; } else { if (page !== 2 || !answers.headline) throw new Error("Cannot submit incomplete questions"); submitted = true; } @@ -24,22 +25,22 @@ function fixture(options: { nextEnabled?: boolean; wrongPrompt?: boolean; change }), getByTestId: () => ({ isVisible: async () => page === 2, locator: () => ({ first: () => ({ fill: async (value: string) => { answers.headline = value; } }) }) }), }; - const ctx = { issue: () => ({ id: "source", conversationState: "waiting" }), refreshIssue: async () => {}, allRuns: async () => [{ status: "succeeded" }], + const ctx = { issue: () => ({ id: "source", conversationState: "waiting" }), refreshIssue: async () => {}, allRuns: async () => [{ id: "native", status: "succeeded" }, ...(options.resumesWork && attemptedMissing ? [{ id: "forbidden-continuation", status: "running" }] : [])], input: { fixtures: { company: { id: "company" } }, page: { locator: () => ({ click: async () => {} }), getByTestId: () => form }, api: { get: async (url: string) => { if (url.endsWith("interactions")) { reads++; return [{ ...original, status: options.prematurelyResolved ? "resolved" : "pending", payload: options.changedPayload && reads >= 2 ? { replaced: true } : original.payload }]; } - return []; + return options.createsWork && attemptedMissing && url.includes("/issues?") ? [{ id: "forbidden-work" }] : []; } } }, } as unknown as ChatCatalogContext; return { ctx, state: { original, answer: "Acorn unchanged human answer" } as Record, clicks, answers, submitted: () => submitted }; } -it("actual multipart act observes required blocking without clicking disabled Next, retains earlier choice and submits the exact supplied answer", async () => { +it("actual multipart act clicks enabled Next to observe required blocking, retains earlier choice and submits the exact supplied answer", async () => { const f = fixture(), c = questionIdentityCases.find(row => row.scenarioId === "questions-and-approvals.multi-part-question")!; await c.act!(f.ctx, f.state); expect(f.state.validationObserved).toBe(true); expect(f.state.retainedAudience).toBe(true); expect(f.state.unsubmittedPending).toBe(true); - expect(f.clicks).toEqual(["Researchers", "Next", "Back", "Next", "Clear", "Next", "/^(Submit|Send) answers$/"]); + expect(f.clicks).toEqual(["Next", "Researchers", "Next", "Back", "Next", "Clear", "Next", "/^(Submit|Send) answers$/"]); expect(f.answers).toEqual({ audience: "Researchers", tone: "Clear", headline: "Acorn unchanged human answer" }); expect(f.submitted()).toBe(true); }); -it.each([{ nextEnabled: true }, { wrongPrompt: true }, { changedPayload: true }, { prematurelyResolved: true }])("rejects invalid current required-question proof %j before any answer", async options => { +it.each([{ advancesMissing: true }, { missingValidation: true }, { createsWork: true }, { resumesWork: true }, { wrongPrompt: true }, { changedPayload: true }, { prematurelyResolved: true }])("rejects invalid current required-question proof %j before any answer", async options => { const f = fixture(options), c = questionIdentityCases.find(row => row.scenarioId === "questions-and-approvals.multi-part-question")!; - await expect(c.act!(f.ctx, f.state)).rejects.toThrow(); expect(f.clicks).toEqual([]); expect(f.submitted()).toBe(false); + await expect(c.act!(f.ctx, f.state)).rejects.toThrow(); expect(f.clicks.every(click => click === "Next")).toBe(true); expect(f.answers).toEqual({}); expect(f.submitted()).toBe(false); }); diff --git a/ui/src/components/task-chat/QuestionForm.test.tsx b/ui/src/components/task-chat/QuestionForm.test.tsx index b132e6381db..7d3e1b1893a 100644 --- a/ui/src/components/task-chat/QuestionForm.test.tsx +++ b/ui/src/components/task-chat/QuestionForm.test.tsx @@ -46,6 +46,19 @@ describe("QuestionForm initial text", () => { await remount(refreshed); expect(input().value).toBe("Operator edit\n"); expect(submit).not.toHaveBeenCalled(); }); + it("shows required validation without advancing or submitting after enabled Next", async () => { + await render({ schema: "paperclip.question_set.v1", questions: [ + { id: "audience", prompt: "Who is the audience?", required: true, answerMode: "text" }, + { id: "tone", prompt: "What tone should we use?", required: true, answerMode: "text" }, + ] }); + const next = Array.from(container.querySelectorAll("button")).find(button => button.textContent === "Next")!; + expect(next.disabled).toBe(false); + await act(async () => next.click()); + expect(container.textContent).toContain("This question is required."); + expect(container.textContent).toContain("Who is the audience?"); + expect(container.textContent).not.toContain("What tone should we use?"); + expect(submit).not.toHaveBeenCalled(); + }); it("retains an explicitly cleared draft and required validation across reload", async () => { await render(); await edit(""); await remount(); expect(input().value).toBe("");