test(evals): observe required question validation on Next

Replace the disabled-button assumption with real invalid-click validation, no page advance, no saved answer and no new work. Keep all subsequent exact answers, page retention and continuation checks. Calibrate the real QuestionForm with a pure React test; no product UI behavior changes.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts

@@ -201,7 +201,7 @@ export const questionIdentityCases = [
     },
   }),
   identityDefinition("questions-and-approvals.multi-part-question", ctx => questionFixture(ctx, true), gradeQuestions, {
-    expectedRunCount: 2, observe: observeQuestion,
+    expectedRunCount: 2, observe: observeQuestion, oracleVersion: "paperclip.chat-catalog.required-question.v2",
     async act(ctx, state) {
       await waitForReply(ctx);
       const row = ctx.input.page.locator(`[id=${JSON.stringify(`interaction-${entity(state, "original").id}`)}]`);
@@ -211,14 +211,22 @@ export const questionIdentityCases = [
       const firstQuestion = original.payload.questionSet?.questions[0];
       if (!firstQuestion?.required) throw new Error("Multipart first question must retain its saved required contract");
       const before = (await cards(ctx)).find(card => card.id === original.id);
+      const beforeRuns = (await ctx.allRuns()).map(run => ({ id: run.id, status: run.status }));
+      const tasksPath = `/api/companies/${ctx.input.fixtures.company.id}/issues?includeAgentCreatedTasks=true`;
+      const beforeTasks = (await ctx.input.api.get<Entity[]>(tasksPath)).filter(task => task.id !== ctx.issue()!.id);
       await expect(form.getByText(firstQuestion.prompt, { exact: true })).toBeVisible();
       const next = form.getByRole("button", { name: "Next", exact: true });
-      await expect(next).toBeDisabled();
+      // The ordinary form validates on an explicit Next click; enabled is not consent.
+      await next.click();
+      await expect(form.getByText("This question is required.", { exact: true })).toBeVisible();
       const after = (await cards(ctx)).find(card => card.id === original.id);
       await expect(form.getByText(firstQuestion.prompt, { exact: true })).toBeVisible();
-      state.validationObserved = await next.isDisabled() && before?.status === "pending" && after?.status === "pending"
+      const afterTasks = (await ctx.input.api.get<Entity[]>(tasksPath)).filter(task => task.id !== ctx.issue()!.id);
+      const afterRuns = (await ctx.allRuns()).map(run => ({ id: run.id, status: run.status }));
+      state.validationObserved = before?.status === "pending" && after?.status === "pending"
         && !before.result && !after.result && JSON.stringify(before.payload) === JSON.stringify(original.payload)
-        && JSON.stringify(after.payload) === JSON.stringify(original.payload);
+        && JSON.stringify(after.payload) === JSON.stringify(original.payload) && beforeTasks.length === 0 && afterTasks.length === 0
+        && JSON.stringify(afterRuns) === JSON.stringify(beforeRuns);
       if (!state.validationObserved) throw new Error("Required multipart question advanced or changed before an answer");
       await form.getByRole("radio", { name: "Researchers", exact: true }).click();
       await form.getByRole("button", { name: "Next", exact: true }).click();

tests/runner-e2e/chat-multipart-required-ui.test.ts

@@ -5,18 +5,19 @@ vi.mock("@playwright/test", () => ({ expect: (target: { isDisabled?: () => boole
 }) }));
 import { questionIdentityCases } from "./chat-catalog-domains/identity-questions-and-approvals.js";
 import type { ChatCatalogContext } from "./chat-catalog-contract.js";
-function fixture(options: { nextEnabled?: boolean; wrongPrompt?: boolean; changedPayload?: boolean; prematurelyResolved?: boolean } = {}) {
-  let page = 0, answered = false, reads = 0, submitted = false;
+function fixture(options: { advancesMissing?: boolean; missingValidation?: boolean; createsWork?: boolean; resumesWork?: boolean; wrongPrompt?: boolean; changedPayload?: boolean; prematurelyResolved?: boolean } = {}) {
+  let page = 0, answered = false, reads = 0, submitted = false, attemptedMissing = false;
   const clicks: string[] = [], answers: Record<string, string> = {};
   const original = { id: "question", kind: "ask_user_questions", status: "pending", payload: { questionSet: { questions: [
     { id: "audience", prompt: "Who is the audience?", required: true }, { id: "tone", prompt: "What tone should we use?", required: true }, { id: "headline", prompt: "What is your custom headline?", required: true },
   ] } } };
-  const form: any = { isVisible: () => true, getByText: (prompt: string) => ({ isVisible: () => !options.wrongPrompt && prompt === original.payload.questionSet.questions[page]?.prompt }),
+  const form: any = { isVisible: () => true, getByText: (prompt: string) => ({ isVisible: () => prompt === "This question is required."
+    ? attemptedMissing && !options.missingValidation : !options.wrongPrompt && prompt === original.payload.questionSet.questions[page]?.prompt }),
     getByRole: (_role: string, { name }: { name: string | RegExp }) => ({
-      isDisabled: () => !options.nextEnabled && !answered,
+      isDisabled: () => false,
       isChecked: () => answers.audience === "Researchers",
       click: async () => { const label = String(name); clicks.push(label);
-        if (label === "Next") { if (!answered) throw new Error("Disabled Next cannot be clicked"); page++; answered = !!answers[original.payload.questionSet.questions[page]?.id ?? ""]; }
+        if (label === "Next") { if (!answered) { attemptedMissing = true; if (options.advancesMissing) page++; return; } page++; answered = !!answers[original.payload.questionSet.questions[page]?.id ?? ""]; }
         else if (label === "Back") { page--; answered = !!answers[original.payload.questionSet.questions[page]!.id]; }
         else if (label === "Researchers" || label === "Clear") { answers[original.payload.questionSet.questions[page]!.id] = label; answered = true; }
         else { if (page !== 2 || !answers.headline) throw new Error("Cannot submit incomplete questions"); submitted = true; }
@@ -24,22 +25,22 @@ function fixture(options: { nextEnabled?: boolean; wrongPrompt?: boolean; change
     }),
     getByTestId: () => ({ isVisible: async () => page === 2, locator: () => ({ first: () => ({ fill: async (value: string) => { answers.headline = value; } }) }) }),
   };
-  const ctx = { issue: () => ({ id: "source", conversationState: "waiting" }), refreshIssue: async () => {}, allRuns: async () => [{ status: "succeeded" }],
+  const ctx = { issue: () => ({ id: "source", conversationState: "waiting" }), refreshIssue: async () => {}, allRuns: async () => [{ id: "native", status: "succeeded" }, ...(options.resumesWork && attemptedMissing ? [{ id: "forbidden-continuation", status: "running" }] : [])],
     input: { fixtures: { company: { id: "company" } }, page: { locator: () => ({ click: async () => {} }), getByTestId: () => form }, api: { get: async (url: string) => {
       if (url.endsWith("interactions")) { reads++; return [{ ...original, status: options.prematurelyResolved ? "resolved" : "pending", payload: options.changedPayload && reads >= 2 ? { replaced: true } : original.payload }]; }
-      return [];
+      return options.createsWork && attemptedMissing && url.includes("/issues?") ? [{ id: "forbidden-work" }] : [];
     } } },
   } as unknown as ChatCatalogContext;
   return { ctx, state: { original, answer: "Acorn unchanged human answer" } as Record<string, unknown>, clicks, answers, submitted: () => submitted };
 }
-it("actual multipart act observes required blocking without clicking disabled Next, retains earlier choice and submits the exact supplied answer", async () => {
+it("actual multipart act clicks enabled Next to observe required blocking, retains earlier choice and submits the exact supplied answer", async () => {
   const f = fixture(), c = questionIdentityCases.find(row => row.scenarioId === "questions-and-approvals.multi-part-question")!;
   await c.act!(f.ctx, f.state);
   expect(f.state.validationObserved).toBe(true); expect(f.state.retainedAudience).toBe(true); expect(f.state.unsubmittedPending).toBe(true);
-  expect(f.clicks).toEqual(["Researchers", "Next", "Back", "Next", "Clear", "Next", "/^(Submit|Send) answers$/"]);
+  expect(f.clicks).toEqual(["Next", "Researchers", "Next", "Back", "Next", "Clear", "Next", "/^(Submit|Send) answers$/"]);
   expect(f.answers).toEqual({ audience: "Researchers", tone: "Clear", headline: "Acorn unchanged human answer" }); expect(f.submitted()).toBe(true);
 });
-it.each([{ nextEnabled: true }, { wrongPrompt: true }, { changedPayload: true }, { prematurelyResolved: true }])("rejects invalid current required-question proof %j before any answer", async options => {
+it.each([{ advancesMissing: true }, { missingValidation: true }, { createsWork: true }, { resumesWork: true }, { wrongPrompt: true }, { changedPayload: true }, { prematurelyResolved: true }])("rejects invalid current required-question proof %j before any answer", async options => {
   const f = fixture(options), c = questionIdentityCases.find(row => row.scenarioId === "questions-and-approvals.multi-part-question")!;
-  await expect(c.act!(f.ctx, f.state)).rejects.toThrow(); expect(f.clicks).toEqual([]); expect(f.submitted()).toBe(false);
+  await expect(c.act!(f.ctx, f.state)).rejects.toThrow(); expect(f.clicks.every(click => click === "Next")).toBe(true); expect(f.answers).toEqual({}); expect(f.submitted()).toBe(false);
 });

ui/src/components/task-chat/QuestionForm.test.tsx

@@ -46,6 +46,19 @@ describe("QuestionForm initial text", () => {
     await remount(refreshed); expect(input().value).toBe("Operator edit\n");
     expect(submit).not.toHaveBeenCalled();
   });
+  it("shows required validation without advancing or submitting after enabled Next", async () => {
+    await render({ schema: "paperclip.question_set.v1", questions: [
+      { id: "audience", prompt: "Who is the audience?", required: true, answerMode: "text" },
+      { id: "tone", prompt: "What tone should we use?", required: true, answerMode: "text" },
+    ] });
+    const next = Array.from(container.querySelectorAll("button")).find(button => button.textContent === "Next")!;
+    expect(next.disabled).toBe(false);
+    await act(async () => next.click());
+    expect(container.textContent).toContain("This question is required.");
+    expect(container.textContent).toContain("Who is the audience?");
+    expect(container.textContent).not.toContain("What tone should we use?");
+    expect(submit).not.toHaveBeenCalled();
+  });
   it("retains an explicitly cleared draft and required validation across reload", async () => {
     await render(); await edit(""); await remount();
     expect(input().value).toBe("");