Evaluate question tool documentation and wait for rendered screenshots

Co-Authored-By: Paperclip <noreply@paperclip.ing>

package.json

@@ -71,6 +71,7 @@
     "test:e2e:runner:models:update": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/openrouter-models-update.ts",
     "test:e2e:runner:history:publish": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/history-publish.ts",
     "test:runner-recovery": "vitest run server/src/services/native-runtime/native-replacement-evidence.test.ts server/src/services/native-runtime/stopped-codex-turn.test.ts server/src/services/native-runtime/native-safe-replacement.test.ts",
+    "test:e2e:runner:browser-support": "playwright test --config tests/runner-e2e/playwright-support.config.ts",
     "test:e2e:runner:unit": "vitest run --config tests/runner-e2e/vitest.config.ts",
     "test:e2e:runner:typecheck": "tsc -p tests/runner-e2e/tsconfig.json",
     "test:e2e:runner:report": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/report.ts",

server/src/services/native-runtime/native-execution-input.test.ts

@@ -444,7 +444,12 @@ describe("native execution input external-chat framing", () => {
       }
     },
   );
-  it("keeps question tool documentation out of the per-turn Board prompt", () => {
+  it.each([
+    { provider: "codex", resumedSession: false },
+    { provider: "codex", resumedSession: true },
+    { provider: "acpx", resumedSession: false },
+    { provider: "acpx", resumedSession: true },
+  ] as const)("keeps question documentation in the tool on $provider (resumed: $resumedSession)", ({ provider, resumedSession }) => {
     const input = buildNativeExecutionInput({
       companyId: "10000000-0000-4000-8000-000000000001",
       runId: "50000000-0000-4000-8000-000000000005",
@@ -452,8 +457,10 @@ describe("native execution input external-chat framing", () => {
       issue: { id: "20000000-0000-4000-8000-000000000002", identifier: "QA-1", title: "Welcome", description: null, workMode: "standard" },
       taskPrompt: "Ask whether the welcome should sound warm or formal before writing it.",
       workspace: { id: "40000000-0000-4000-8000-000000000004", cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
-      normalizedSessionId: null,
-      provider: "codex",
+      normalizedSessionId: resumedSession ? "60000000-0000-4000-8000-000000000006" : null,
+      provider, resumedSession,
+      acpxAgent: "claude",
+      model: provider === "acpx" ? "claude-sonnet-5" : "gpt-5.6-sol",
       completionContract: {
         id: "70000000-0000-4000-8000-000000000007", sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1",
         contract: { revision: "1", objective: "Write a welcome after the user's answer", criteria: [{ id: "objective", requirement: "Use the selected tone" }] },

tests/runner-e2e/README.md

@@ -161,7 +161,7 @@ Both suites save and restore experimental settings. Browser E2E always starts a
 throwaway instance; never point the authenticated suite at the running demo.
 Missing provider credentials fail paid preflight and are not passing coverage.
 
-The default `--all` selection is 160 cells (137 local and 23 Daytona) and 344
+The default `--all` selection is 162 cells (139 local and 23 Daytona) and 350
 expected paid agent turns. The explicit-only everyday suite adds 30 catalog cells
 and is excluded from `--all`. Follow-up steps remain ordered within their cell; all other
 cells are independent. Narrow selectors are strongly recommended while
@@ -448,7 +448,7 @@ Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped
 ephemeral AWS RunsOn fleet selected by
 `runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value uses the
 proven GitHub-hosted `ubuntu-latest` target. Set `RUNNER_E2E_MAX_PARALLEL` to an
-integer from 1–100 on AWS (default 100). The 160-cell default selection takes more than
+integer from 1–100 on AWS (default 100). The 162-cell default selection takes more than
 one wave at that limit; use suite selectors for smaller campaigns. The fallback runner retains its 1–57 limit and
 default of 32. Multi-turn steps are sequential inside their cell while
 independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports
@@ -705,3 +705,24 @@ The untrusted-evidence case reads a synthetic previous-assistant handoff file;
 server tests separately exercise actual tool-result, agent-summary, and mixed
 resolver projections. This is a regression sample, not an exhaustive injection
 or authorization evaluation.
+
+The native-only `question-tool-documentation` case adds two cells (Runner Codex
+and Runner ACPX Claude), for 22 continuation cells total. It asks for a clickable
+Morning/Afternoon question, followed by an open text question, then a saved note
+using both real answers. The user prompt contains no tool names or payload recipes.
+Checks inspect actual forms, ordered UI answers, the saved document, and every
+recorded native task prompt: the short routing sentence must remain, while the old
+question section and detailed tool-format instructions must be absent. Server
+contract tests separately verify that the advertised tool carries the documentation
+for fresh and resumed native executions. This tests the current documentation
+placement; it is not a statistical comparison with the former prompt arrangement.
+
+Continuation screenshots wait for the correct task heading and fully revealed
+conversation before capture. A loading screen or wrong task fails capture.
+Browser-only regressions exercise delayed rendering without provider calls:
+
+```sh
+pnpm test:e2e:runner:browser-support
+# To use an installed Chrome instead of Playwright's Chromium:
+PAPERCLIP_PLAYWRIGHT_CHANNEL=chrome pnpm test:e2e:runner:browser-support
+```

tests/runner-e2e/catalog.test.ts

@@ -41,10 +41,10 @@ describe("runner E2E catalog", () => {
     expect(localIntegrityTasks).toHaveLength(2);
     expect(openRouterBreadthTasks).toHaveLength(3);
     expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
-      20, 30, 48, 24, 42, 14, 10, 2,
+      22, 30, 48, 24, 42, 14, 10, 2,
     ]);
-    expect(validateRunnerCatalog()).toHaveLength(190);
-    expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(190);
+    expect(validateRunnerCatalog()).toHaveLength(192);
+    expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(192);
     expect(
       runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
     ).toHaveLength(42);
@@ -68,7 +68,7 @@ describe("runner E2E catalog", () => {
         (total, execution) => total + execution.task.expectedRunCount,
         0,
       ),
-    ).toBe(344);
+    ).toBe(350);
     expect(
       runnerTasks.find((task) => task.id === "plan-revise-accept")
         ?.attemptTimeoutMs,
@@ -561,10 +561,10 @@ describe("runner E2E selectors", () => {
     const jobs = buildMatrixJobs(
       selectRunnerExecutions(parseRunnerSelectors(["--all"])),
     );
-    expect(jobs).toHaveLength(160);
+    expect(jobs).toHaveLength(162);
     expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(23);
-    expect(jobs.filter((job) => !job.needsDaytona)).toHaveLength(137);
-    expect(new Set(jobs.map((job) => job.executionId)).size).toBe(160);
+    expect(jobs.filter((job) => !job.needsDaytona)).toHaveLength(139);
+    expect(new Set(jobs.map((job) => job.executionId)).size).toBe(162);
     expect(
       jobs.find(
         (job) =>

tests/runner-e2e/catalog.ts

@@ -898,8 +898,9 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
     description: "Human direction, approval boundaries, untrusted evidence, and completed actions across turns.",
     groups: ["local"], environments: [localEnvironment],
     profiles: runnerProfiles.filter(profile => ["legacy-codex", "legacy-claude", "runner-codex", "runner-acpx-claude"].includes(profile.id)).map(productionStoryProfile),
-    tasks: continuationTasks, expectedMatrixSize: 20,
-    definitionMetadata: { version: 2, grading: "durable-state-and-approval-boundaries", instructions: "production" },
+    tasks: continuationTasks, expectedMatrixSize: 22,
+    excludedExecutionIds: ["legacy-codex", "legacy-claude"].map(profile => `continuation.${profile}.local.question-tool-documentation`),
+    definitionMetadata: { version: 3, grading: "durable-state-and-approval-boundaries", instructions: "production" },
   },
   {
     id: "everyday-workflows", label: "Everyday Paperclip Work", manualOnly: true,

tests/runner-e2e/continuation-cases.ts

@@ -7,6 +7,7 @@ export const CONTINUATION_CASES = [
   "revision-preserves-approval",
   "untrusted-evidence",
   "completed-action-resume",
+  "question-tool-documentation",
 ] as const;
 export type ContinuationCase = (typeof CONTINUATION_CASES)[number];
 export function continuationScenario(id: string, nonce: string) {
@@ -20,6 +21,7 @@ export function continuationScenario(id: string, nonce: string) {
   const document =
     "Save the final note as a task document. No other deliverables or child tasks are needed.";
   const prompts: Record<ContinuationCase, string> = {
+    "question-tool-documentation": `Help me write a one-sentence welcome note for a club meetup. First let me choose Morning or Afternoon using clickable choices. After I choose, ask me for a reference to include using an open text field. Ask only one question at a time and wait for my answers. Then save the note as a task document, including the selected time and my reference exactly as supplied, and finish. Do not create any other tasks or deliverables.`,
     "answer-updates-scope": `I need a one-sentence welcome note containing ${old}. Before writing it, ask me one open-ended structured question about any changes I want. Then apply my answer and finish. ${document}`,
     "clarification-not-approval": `I need a one-sentence welcome note. First ask me one open-ended structured question for the word to include. After my answer, propose your approach and wait for my explicit approval before writing the note. ${document}`,
     "revision-preserves-approval": `Propose an approach for a one-sentence welcome note containing ${old}, and save that approach as the plan document. Wait for my explicit approval before writing the note. ${document}`,
@@ -61,6 +63,7 @@ export const continuationTasks: readonly RunnerTaskFixture[] =
         : [
               "clarification-not-approval",
               "revision-preserves-approval",
+              "question-tool-documentation",
             ].includes(id)
           ? 3
           : 2,

tests/runner-e2e/continuation-flow.ts

@@ -1,5 +1,6 @@
 import { expect, type Page } from "@playwright/test";
 import path from "node:path";
+import { captureLoadedContinuation } from "./continuation-screenshot.js";
 import { seedContinuationContext } from "./continuation-workspace.js";
 import { pollUntil, type RunnerApi } from "./api.js";
 import {
@@ -136,13 +137,13 @@ export async function runContinuationFlow(input: {
     });
     await input.evidence("api-state.json", checkpoints.at(-1));
     await open();
-    await input.capture(
+    await captureLoadedContinuation(page, String(issue!.title), () => input.capture(
       phase,
       `Continuation: ${phase}`,
       continuationScreenshotFile(phase),
-    );
+    ));
   }
-  async function answer() {
+  async function answer(choice?: string) {
     const interactions = await api.get<Row[]>(
       `/api/issues/${issue!.id}/interactions`,
     );
@@ -154,17 +155,18 @@ export async function runContinuationFlow(input: {
     expect(set.questions, "ask only the requested next question").toHaveLength(
       1,
     );
-    expect(
-      set.questions[0].answerMode,
-      "open-ended questions must render a text field, not a lone choice",
-    ).toBe("text");
     const before = new Set(runs.map((r) => r.id));
-    await page
-      .getByTestId("question-text-answer-composer")
-      .last()
-      .locator('[contenteditable="true"],textarea')
-      .first()
-      .fill(scenario.answer);
+    if (choice) {
+      expect(set.questions[0].answerMode, "choices must use radio controls").toBe("single_select");
+      const options = questions[0].payload.questionSet?.questions[0]?.options
+        ?? questions[0].payload.questions[0]?.options ?? [];
+      expect(new Set(options.map((o: Row) => String(o.label).trim().toLowerCase())).size).toBeGreaterThanOrEqual(2);
+      await page.getByRole("radio", { name: new RegExp(`^${choice}\\b`, "i") }).last().click();
+    } else {
+      expect(set.questions[0].answerMode, "open answers must render a text field, not a lone choice").toBe("text");
+      await page.getByTestId("question-text-answer-composer").last()
+        .locator('[contenteditable="true"],textarea').first().fill(scenario.answer);
+    }
     await page
       .getByRole("button", {
         name: set.submitLabel ?? "Submit answers",
@@ -225,7 +227,12 @@ export async function runContinuationFlow(input: {
       await input.restart();
       await open();
     }
-    if (scenario.id === "revision-preserves-approval")
+    if (scenario.id === "question-tool-documentation") {
+      await answer("Afternoon");
+      await snapshot("answered");
+      assertWaiting();
+      await answer();
+    } else if (scenario.id === "revision-preserves-approval")
       await reply(scenario.revision);
     else await answer();
     if (scenario.gate) {

tests/runner-e2e/continuation-scoring.ts

@@ -1,3 +1,4 @@
+import { gradeQuestionDocumentation } from "./question-documentation-scoring.js";
 import type { ContinuationCase } from "./continuation-cases.js";
 export interface ContinuationCheckpoint {
   phase: "initial" | "answered" | "revised" | "final";
@@ -103,5 +104,6 @@ export function gradeContinuation(input: {
       input.checkpoints.every((c) => c.children.length === 0),
       "No checkpoint may contain an unrequested child task.",
     );
+  if (input.id === "question-tool-documentation") checks.push(...gradeQuestionDocumentation(input.checkpoints, input.marker));
   return checks;
 }

tests/runner-e2e/continuation-screenshot.ts

@@ -0,0 +1,17 @@
+import { expect, type Page } from "@playwright/test";
+
+/** Capture only after the requested task's conversation is actually rendered. */
+export async function captureLoadedContinuation(
+  page: Page,
+  title: string,
+  capture: () => Promise<void>,
+  timeout = 30_000,
+) {
+  const thread = page.getByTestId("task-chat-thread");
+  await expect(thread).toBeVisible({ timeout });
+  await expect(thread.locator(':scope > [aria-busy="false"]')).toBeVisible({ timeout });
+  await expect(thread.getByTestId("task-chat-history-loading")).toHaveCount(0, { timeout });
+  await expect(thread.getByRole("heading", { name: title, exact: true })).toBeVisible({ timeout });
+  await page.evaluate(() => document.fonts.ready.then(() => undefined));
+  await capture();
+}

tests/runner-e2e/continuation.test.ts

@@ -67,7 +67,7 @@ const failures = (r: ReturnType<typeof recording>) =>
 describe("continuation behavioral evaluation", () => {
   it("registers all five cases for both runtime generations and providers", () => {
     const matrix = runnerMatrix.filter((c) => c.suite.id === "continuation");
-    expect(matrix).toHaveLength(20);
+    expect(matrix).toHaveLength(22);
     expect(new Set(matrix.map((c) => c.profile.id))).toEqual(
       new Set([
         "legacy-codex",
@@ -78,7 +78,7 @@ describe("continuation behavioral evaluation", () => {
     );
     expect(matrix.every((c) => !c.suite.manualOnly)).toBe(true);
   });
-  it.each(CONTINUATION_CASES)("accepts a complete %s recording", (id) =>
+  it.each(CONTINUATION_CASES.filter(id => id !== "question-tool-documentation"))("accepts a complete %s recording", (id) =>
     expect(failures(recording(id))).toEqual([]),
   );
   it("fails premature output even when the final result is correct", () => {

tests/runner-e2e/playwright-support.config.ts

@@ -0,0 +1,18 @@
+import { defineConfig } from "@playwright/test";
+
+/** Browser-only harness regressions: no Paperclip instance or provider credentials. */
+export default defineConfig({
+  testDir: ".",
+  testMatch: "screenshot-readiness.spec.ts",
+  workers: 1,
+  retries: 0,
+  timeout: 10_000,
+  use: {
+    headless: true,
+    ...(process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL
+      ? { channel: process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL }
+      : {}),
+  },
+  outputDir: "./results/browser-support",
+  reporter: "list",
+});

tests/runner-e2e/question-documentation-scoring.ts

@@ -0,0 +1,69 @@
+import type { ContinuationCheckpoint } from "./continuation-scoring.js";
+
+type Row = Record<string, any>;
+const questions = (checkpoint?: ContinuationCheckpoint): Row[] =>
+  (checkpoint?.interactions ?? []).filter((value: any) => value.kind === "ask_user_questions") as Row[];
+function form(row?: Row) {
+  const payload = row?.payload;
+  const canonical = payload?.questionSet?.questions;
+  if (canonical) return canonical;
+  return (payload?.questions ?? []).map((q: Row) => ({
+    ...q, answerMode: q.selectionMode === "multi" ? "multi_select" : "single_select",
+  }));
+}
+
+/** Checks real recorded forms, responses, and native task inputs; no model judge. */
+export function gradeQuestionDocumentation(checkpoints: ContinuationCheckpoint[], marker: string) {
+  const initial = checkpoints.find(c => c.phase === "initial");
+  const middle = checkpoints.find(c => c.phase === "answered");
+  const final = checkpoints.find(c => c.phase === "final");
+  const first = questions(initial)[0];
+  const next = questions(middle).filter(q => q.status === "pending");
+  const choice = form(first)[0];
+  const text = form(next[0])[0];
+  const options: Row[] = choice?.options ?? [];
+  const afternoon = options.find(o => /^afternoon\b/i.test(String(o.label).trim()));
+  const firstAnswer = questions(final).find(q => q.id === first?.id);
+  const secondAnswer = questions(final).find(q => q.id === next[0]?.id);
+  const inputs = (final?.runs ?? []).map((run: any) => run.runnerProfileJson?.nativeExecutionInput?.task?.prompt);
+  const checks = [
+    {
+      id: "meaningful-choice-form",
+      passed: questions(initial).length === 1 && first?.status === "pending" && form(first).length === 1
+        && choice?.answerMode === "single_select" && options.length >= 2
+        && new Set(options.map(o => o.id)).size === options.length
+        && options.some(o => /^morning\b/i.test(String(o.label).trim())) && Boolean(afternoon),
+      detail: "The first card must show one single-choice question with distinct Morning and Afternoon options.",
+    },
+    {
+      id: "text-form-after-choice",
+      passed: questions(middle).length === 2 && next.length === 1 && form(next[0]).length === 1
+        && next[0]?.id !== first?.id && text?.answerMode === "text"
+        && !text?.options?.length && !text?.customAnswer
+        && questions(middle).some(q => q.id === first?.id && q.status === "answered"),
+      detail: "Only after the choice is answered may the second, text-only question appear.",
+    },
+    {
+      id: "actual-question-answers",
+      passed: questions(final).length === 2 && firstAnswer?.status === "answered" && secondAnswer?.status === "answered"
+        && firstAnswer?.result?.answers?.some((a: Row) => a.questionId === choice?.id && a.optionIds?.includes(afternoon?.id))
+        && secondAnswer?.result?.answers?.some((a: Row) => a.questionId === text?.id && a.otherText?.includes(marker)),
+      detail: "Both real UI submissions must be persisted against their original question IDs, with no duplicate question cards.",
+    },
+    {
+      id: "question-guidance-not-in-wake",
+      passed: inputs.length >= 3 && inputs.every(prompt => typeof prompt === "string"
+        && prompt.includes("Use Paperclip's request_human_input for durable task questions.")
+        && !prompt.includes("## Questions that need a user response")
+        && !prompt.includes("payload.questionSet")
+        && !prompt.includes("Create a durable human question or approval card on the current Paperclip task bound to this run")),
+      detail: "Every recorded native turn must retain the short routing hint without the old question block or copied tool-format instructions.",
+    },
+    {
+      id: "both-answers-in-output",
+      passed: Boolean(final?.documents.some(d => d.key !== "plan" && d.body.includes(marker) && /\bafternoon\b/i.test(d.body))),
+      detail: "The saved note must use both the selected time and the literal reference supplied in the text answer.",
+    },
+  ];
+  return checks.map(check => ({ ...check, passed: Boolean(check.passed) }));
+}

tests/runner-e2e/question-documentation.test.ts

@@ -0,0 +1,75 @@
+import { describe, expect, it } from "vitest";
+import { gradeQuestionDocumentation } from "./question-documentation-scoring.js";
+import type { ContinuationCheckpoint } from "./continuation-scoring.js";
+import { runnerMatrix } from "./catalog.js";
+
+function recording() {
+  const choice = { id: "time", prompt: "Which time?", answerMode: "single_select", options: [
+    { id: "am", label: "Morning" }, { id: "pm", label: "Afternoon" },
+  ] };
+  const text = { id: "reference", prompt: "What reference?", answerMode: "text" };
+  const first: any = { id: "first", kind: "ask_user_questions", status: "pending", payload: { questionSet: { questions: [choice] } } };
+  const second: any = { id: "second", kind: "ask_user_questions", status: "pending", payload: { questionSet: { questions: [text] } } };
+  const answeredFirst = { ...structuredClone(first), status: "answered", result: { answers: [{ questionId: "time", optionIds: ["pm"] }] } };
+  const answeredSecond = { ...structuredClone(second), status: "answered", result: { answers: [{ questionId: "reference", optionIds: [], otherText: "Use AMBER123." }] } };
+  const runs = [1, 2, 3].map(id => ({ id: String(id), status: "succeeded", runtimeMode: "native", runnerProfileJson: { nativeExecutionInput: { task: { prompt: "Use Paperclip's request_human_input for durable task questions." } } } }));
+  const checkpoint = (phase: ContinuationCheckpoint["phase"], interactions: any[], count: number): ContinuationCheckpoint => ({
+    phase, issue: { id: "task", status: phase === "final" ? "done" : "in_review" }, children: [], attachments: [], comments: [],
+    interactions, runs: runs.slice(0, count),
+    documents: phase === "final" ? [{ key: "note", latestRevisionId: "saved", body: "Welcome to the afternoon meetup, AMBER123." }] : [],
+  });
+  return [checkpoint("initial", [first], 1), checkpoint("answered", [answeredFirst, second], 2), checkpoint("final", [structuredClone(answeredFirst), answeredSecond], 3)];
+}
+const failed = (checkpoints: ContinuationCheckpoint[]) => gradeQuestionDocumentation(checkpoints, "AMBER123").filter(c => !c.passed).map(c => c.id);
+
+describe("question tool documentation regression", () => {
+  it("runs against both native providers with production instructions", () => {
+    const cells = runnerMatrix.filter(c => c.task.id === "question-tool-documentation");
+    expect(cells.map(c => c.profile.id).sort()).toEqual(["runner-acpx-claude", "runner-codex"]);
+    for (const c of cells) {
+      const prompt = c.task.buildPrompt("nonce");
+      expect(prompt).not.toMatch(/request_human_input|questionSet|selectionMode|payload\./);
+      expect(c.suite.manualOnly).not.toBe(true);
+    }
+  });
+  it("passes real choices followed by a text field and both recorded answers", () => {
+    expect(failed(recording())).toEqual([]);
+  });
+  it("rejects a lone choice", () => {
+    const r = recording();
+    (r[0].interactions[0] as any).payload.questionSet.questions[0].options.shift();
+    expect(failed(r)).toContain("meaningful-choice-form");
+  });
+  it("rejects a lone Other choice instead of a text field", () => {
+    const r = recording();
+    Object.assign((r[1].interactions[1] as any).payload.questionSet.questions[0], { answerMode: "single_select", options: [{ id: "other", label: "Other" }] });
+    expect(failed(r)).toContain("text-form-after-choice");
+  });
+  it("rejects asking both questions before the first answer", () => {
+    const r = recording();
+    r[0].interactions.push(structuredClone(r[1].interactions[1]));
+    expect(failed(r)).toContain("meaningful-choice-form");
+  });
+  it("rejects fabricated or duplicated responses", () => {
+    const r = recording();
+    (r[2].interactions[0] as any).result.answers[0].optionIds = ["am"];
+    expect(failed(r)).toContain("actual-question-answers");
+    r[2].interactions.push(structuredClone(r[2].interactions[1]));
+    expect(failed(r)).toContain("actual-question-answers");
+  });
+  it("rejects the old instruction block copied into a later turn", () => {
+    const r = recording();
+    (r[2].runs[2] as any).runnerProfileJson.nativeExecutionInput.task.prompt += "\n## Questions that need a user response\npayload.questionSet";
+    expect(failed(r)).toContain("question-guidance-not-in-wake");
+  });
+  it("requires recorded inputs rather than assuming docs-only delivery", () => {
+    const r = recording();
+    delete (r[2].runs[2] as any).runnerProfileJson;
+    expect(failed(r)).toContain("question-guidance-not-in-wake");
+  });
+  it("requires both real answers in the saved result", () => {
+    const r = recording();
+    r[2].documents[0].body = "Welcome AMBER123.";
+    expect(failed(r)).toContain("both-answers-in-output");
+  });
+});

tests/runner-e2e/screenshot-readiness.spec.ts

@@ -0,0 +1,33 @@
+import { expect, test } from "@playwright/test";
+import { captureLoadedContinuation } from "./continuation-screenshot.js";
+
+const shell = (busy: boolean, title = "Continuation example") => `
+  <main data-testid="task-chat-thread"><div aria-busy="${busy}">
+    <div data-testid="issue-detail-header"><h2>${title}</h2></div>
+    ${busy ? '<div data-testid="task-chat-history-loading">Loading conversation</div>' : '<p>Saved note is ready</p>'}
+  </div></main>`;
+
+test("capture waits through app loading and conversation loading", async ({ page }) => {
+  await page.setContent(`<p>App loading</p><script>
+    setTimeout(() => document.body.innerHTML = ${JSON.stringify(shell(true))}, 150);
+    setTimeout(() => document.body.innerHTML = ${JSON.stringify(shell(false))}, 450);
+  </script>`);
+  let captured = "";
+  await captureLoadedContinuation(page, "Continuation example", async () => {
+    captured = await page.locator("body").innerText();
+    await page.screenshot();
+  }, 3000);
+  expect(captured).toContain("Saved note is ready");
+  expect(captured).not.toContain("Loading");
+});
+
+for (const state of ["app-loader", "history-loader", "wrong-task"] as const) {
+  test(`does not capture ${state} as a successful checkpoint`, async ({ page }) => {
+    await page.setContent(state === "app-loader" ? "<p>App loading</p>" : shell(state === "history-loader", state === "wrong-task" ? "Unrelated task" : undefined));
+    let captures = 0;
+    await expect(captureLoadedContinuation(page, "Continuation example", async () => {
+      captures++;
+    }, 250)).rejects.toThrow();
+    expect(captures).toBe(0);
+  });
+}