diff --git a/package.json b/package.json index da311d64d92..cb34da76f19 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "test:e2e:runner:models:update": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/openrouter-models-update.ts", "test:e2e:runner:history:publish": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/history-publish.ts", "test:runner-recovery": "vitest run server/src/services/native-runtime/native-replacement-evidence.test.ts server/src/services/native-runtime/stopped-codex-turn.test.ts server/src/services/native-runtime/native-safe-replacement.test.ts", + "test:e2e:runner:browser-support": "playwright test --config tests/runner-e2e/playwright-support.config.ts", "test:e2e:runner:unit": "vitest run --config tests/runner-e2e/vitest.config.ts", "test:e2e:runner:typecheck": "tsc -p tests/runner-e2e/tsconfig.json", "test:e2e:runner:report": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/report.ts", diff --git a/server/src/services/native-runtime/native-execution-input.test.ts b/server/src/services/native-runtime/native-execution-input.test.ts index bfe598f8fce..2afc93b12e9 100644 --- a/server/src/services/native-runtime/native-execution-input.test.ts +++ b/server/src/services/native-runtime/native-execution-input.test.ts @@ -444,7 +444,12 @@ describe("native execution input external-chat framing", () => { } }, ); - it("keeps question tool documentation out of the per-turn Board prompt", () => { + it.each([ + { provider: "codex", resumedSession: false }, + { provider: "codex", resumedSession: true }, + { provider: "acpx", resumedSession: false }, + { provider: "acpx", resumedSession: true }, + ] as const)("keeps question documentation in the tool on $provider (resumed: $resumedSession)", ({ provider, resumedSession }) => { const input = buildNativeExecutionInput({ companyId: "10000000-0000-4000-8000-000000000001", runId: "50000000-0000-4000-8000-000000000005", @@ -452,8 +457,10 @@ describe("native execution input external-chat framing", () => { issue: { id: "20000000-0000-4000-8000-000000000002", identifier: "QA-1", title: "Welcome", description: null, workMode: "standard" }, taskPrompt: "Ask whether the welcome should sound warm or formal before writing it.", workspace: { id: "40000000-0000-4000-8000-000000000004", cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, - normalizedSessionId: null, - provider: "codex", + normalizedSessionId: resumedSession ? "60000000-0000-4000-8000-000000000006" : null, + provider, resumedSession, + acpxAgent: "claude", + model: provider === "acpx" ? "claude-sonnet-5" : "gpt-5.6-sol", completionContract: { id: "70000000-0000-4000-8000-000000000007", sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1", contract: { revision: "1", objective: "Write a welcome after the user's answer", criteria: [{ id: "objective", requirement: "Use the selected tone" }] }, diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index f8de10418f4..5697b1e88fa 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -161,7 +161,7 @@ Both suites save and restore experimental settings. Browser E2E always starts a throwaway instance; never point the authenticated suite at the running demo. Missing provider credentials fail paid preflight and are not passing coverage. -The default `--all` selection is 160 cells (137 local and 23 Daytona) and 344 +The default `--all` selection is 162 cells (139 local and 23 Daytona) and 350 expected paid agent turns. The explicit-only everyday suite adds 30 catalog cells and is excluded from `--all`. Follow-up steps remain ordered within their cell; all other cells are independent. Narrow selectors are strongly recommended while @@ -448,7 +448,7 @@ Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped ephemeral AWS RunsOn fleet selected by `runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value uses the proven GitHub-hosted `ubuntu-latest` target. Set `RUNNER_E2E_MAX_PARALLEL` to an -integer from 1–100 on AWS (default 100). The 160-cell default selection takes more than +integer from 1–100 on AWS (default 100). The 162-cell default selection takes more than one wave at that limit; use suite selectors for smaller campaigns. The fallback runner retains its 1–57 limit and default of 32. Multi-turn steps are sequential inside their cell while independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports @@ -705,3 +705,24 @@ The untrusted-evidence case reads a synthetic previous-assistant handoff file; server tests separately exercise actual tool-result, agent-summary, and mixed resolver projections. This is a regression sample, not an exhaustive injection or authorization evaluation. + +The native-only `question-tool-documentation` case adds two cells (Runner Codex +and Runner ACPX Claude), for 22 continuation cells total. It asks for a clickable +Morning/Afternoon question, followed by an open text question, then a saved note +using both real answers. The user prompt contains no tool names or payload recipes. +Checks inspect actual forms, ordered UI answers, the saved document, and every +recorded native task prompt: the short routing sentence must remain, while the old +question section and detailed tool-format instructions must be absent. Server +contract tests separately verify that the advertised tool carries the documentation +for fresh and resumed native executions. This tests the current documentation +placement; it is not a statistical comparison with the former prompt arrangement. + +Continuation screenshots wait for the correct task heading and fully revealed +conversation before capture. A loading screen or wrong task fails capture. +Browser-only regressions exercise delayed rendering without provider calls: + +```sh +pnpm test:e2e:runner:browser-support +# To use an installed Chrome instead of Playwright's Chromium: +PAPERCLIP_PLAYWRIGHT_CHANNEL=chrome pnpm test:e2e:runner:browser-support +``` diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index adcc9e8c6d9..b8beaf36298 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -41,10 +41,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 20, 30, 48, 24, 42, 14, 10, 2, + 22, 30, 48, 24, 42, 14, 10, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(190); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(190); + expect(validateRunnerCatalog()).toHaveLength(192); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(192); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(42); @@ -68,7 +68,7 @@ describe("runner E2E catalog", () => { (total, execution) => total + execution.task.expectedRunCount, 0, ), - ).toBe(344); + ).toBe(350); expect( runnerTasks.find((task) => task.id === "plan-revise-accept") ?.attemptTimeoutMs, @@ -561,10 +561,10 @@ describe("runner E2E selectors", () => { const jobs = buildMatrixJobs( selectRunnerExecutions(parseRunnerSelectors(["--all"])), ); - expect(jobs).toHaveLength(160); + expect(jobs).toHaveLength(162); expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(23); - expect(jobs.filter((job) => !job.needsDaytona)).toHaveLength(137); - expect(new Set(jobs.map((job) => job.executionId)).size).toBe(160); + expect(jobs.filter((job) => !job.needsDaytona)).toHaveLength(139); + expect(new Set(jobs.map((job) => job.executionId)).size).toBe(162); expect( jobs.find( (job) => diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index c033efbb6ed..8bb4a538c34 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -898,8 +898,9 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Human direction, approval boundaries, untrusted evidence, and completed actions across turns.", groups: ["local"], environments: [localEnvironment], profiles: runnerProfiles.filter(profile => ["legacy-codex", "legacy-claude", "runner-codex", "runner-acpx-claude"].includes(profile.id)).map(productionStoryProfile), - tasks: continuationTasks, expectedMatrixSize: 20, - definitionMetadata: { version: 2, grading: "durable-state-and-approval-boundaries", instructions: "production" }, + tasks: continuationTasks, expectedMatrixSize: 22, + excludedExecutionIds: ["legacy-codex", "legacy-claude"].map(profile => `continuation.${profile}.local.question-tool-documentation`), + definitionMetadata: { version: 3, grading: "durable-state-and-approval-boundaries", instructions: "production" }, }, { id: "everyday-workflows", label: "Everyday Paperclip Work", manualOnly: true, diff --git a/tests/runner-e2e/continuation-cases.ts b/tests/runner-e2e/continuation-cases.ts index 495a047a6bc..040a098d1ef 100644 --- a/tests/runner-e2e/continuation-cases.ts +++ b/tests/runner-e2e/continuation-cases.ts @@ -7,6 +7,7 @@ export const CONTINUATION_CASES = [ "revision-preserves-approval", "untrusted-evidence", "completed-action-resume", + "question-tool-documentation", ] as const; export type ContinuationCase = (typeof CONTINUATION_CASES)[number]; export function continuationScenario(id: string, nonce: string) { @@ -20,6 +21,7 @@ export function continuationScenario(id: string, nonce: string) { const document = "Save the final note as a task document. No other deliverables or child tasks are needed."; const prompts: Record = { + "question-tool-documentation": `Help me write a one-sentence welcome note for a club meetup. First let me choose Morning or Afternoon using clickable choices. After I choose, ask me for a reference to include using an open text field. Ask only one question at a time and wait for my answers. Then save the note as a task document, including the selected time and my reference exactly as supplied, and finish. Do not create any other tasks or deliverables.`, "answer-updates-scope": `I need a one-sentence welcome note containing ${old}. Before writing it, ask me one open-ended structured question about any changes I want. Then apply my answer and finish. ${document}`, "clarification-not-approval": `I need a one-sentence welcome note. First ask me one open-ended structured question for the word to include. After my answer, propose your approach and wait for my explicit approval before writing the note. ${document}`, "revision-preserves-approval": `Propose an approach for a one-sentence welcome note containing ${old}, and save that approach as the plan document. Wait for my explicit approval before writing the note. ${document}`, @@ -61,6 +63,7 @@ export const continuationTasks: readonly RunnerTaskFixture[] = : [ "clarification-not-approval", "revision-preserves-approval", + "question-tool-documentation", ].includes(id) ? 3 : 2, diff --git a/tests/runner-e2e/continuation-flow.ts b/tests/runner-e2e/continuation-flow.ts index 7c26f0aa228..60894089961 100644 --- a/tests/runner-e2e/continuation-flow.ts +++ b/tests/runner-e2e/continuation-flow.ts @@ -1,5 +1,6 @@ import { expect, type Page } from "@playwright/test"; import path from "node:path"; +import { captureLoadedContinuation } from "./continuation-screenshot.js"; import { seedContinuationContext } from "./continuation-workspace.js"; import { pollUntil, type RunnerApi } from "./api.js"; import { @@ -136,13 +137,13 @@ export async function runContinuationFlow(input: { }); await input.evidence("api-state.json", checkpoints.at(-1)); await open(); - await input.capture( + await captureLoadedContinuation(page, String(issue!.title), () => input.capture( phase, `Continuation: ${phase}`, continuationScreenshotFile(phase), - ); + )); } - async function answer() { + async function answer(choice?: string) { const interactions = await api.get( `/api/issues/${issue!.id}/interactions`, ); @@ -154,17 +155,18 @@ export async function runContinuationFlow(input: { expect(set.questions, "ask only the requested next question").toHaveLength( 1, ); - expect( - set.questions[0].answerMode, - "open-ended questions must render a text field, not a lone choice", - ).toBe("text"); const before = new Set(runs.map((r) => r.id)); - await page - .getByTestId("question-text-answer-composer") - .last() - .locator('[contenteditable="true"],textarea') - .first() - .fill(scenario.answer); + if (choice) { + expect(set.questions[0].answerMode, "choices must use radio controls").toBe("single_select"); + const options = questions[0].payload.questionSet?.questions[0]?.options + ?? questions[0].payload.questions[0]?.options ?? []; + expect(new Set(options.map((o: Row) => String(o.label).trim().toLowerCase())).size).toBeGreaterThanOrEqual(2); + await page.getByRole("radio", { name: new RegExp(`^${choice}\\b`, "i") }).last().click(); + } else { + expect(set.questions[0].answerMode, "open answers must render a text field, not a lone choice").toBe("text"); + await page.getByTestId("question-text-answer-composer").last() + .locator('[contenteditable="true"],textarea').first().fill(scenario.answer); + } await page .getByRole("button", { name: set.submitLabel ?? "Submit answers", @@ -225,7 +227,12 @@ export async function runContinuationFlow(input: { await input.restart(); await open(); } - if (scenario.id === "revision-preserves-approval") + if (scenario.id === "question-tool-documentation") { + await answer("Afternoon"); + await snapshot("answered"); + assertWaiting(); + await answer(); + } else if (scenario.id === "revision-preserves-approval") await reply(scenario.revision); else await answer(); if (scenario.gate) { diff --git a/tests/runner-e2e/continuation-scoring.ts b/tests/runner-e2e/continuation-scoring.ts index d04a45d2bb0..1f0035dc40a 100644 --- a/tests/runner-e2e/continuation-scoring.ts +++ b/tests/runner-e2e/continuation-scoring.ts @@ -1,3 +1,4 @@ +import { gradeQuestionDocumentation } from "./question-documentation-scoring.js"; import type { ContinuationCase } from "./continuation-cases.js"; export interface ContinuationCheckpoint { phase: "initial" | "answered" | "revised" | "final"; @@ -103,5 +104,6 @@ export function gradeContinuation(input: { input.checkpoints.every((c) => c.children.length === 0), "No checkpoint may contain an unrequested child task.", ); + if (input.id === "question-tool-documentation") checks.push(...gradeQuestionDocumentation(input.checkpoints, input.marker)); return checks; } diff --git a/tests/runner-e2e/continuation-screenshot.ts b/tests/runner-e2e/continuation-screenshot.ts new file mode 100644 index 00000000000..85506bcacbc --- /dev/null +++ b/tests/runner-e2e/continuation-screenshot.ts @@ -0,0 +1,17 @@ +import { expect, type Page } from "@playwright/test"; + +/** Capture only after the requested task's conversation is actually rendered. */ +export async function captureLoadedContinuation( + page: Page, + title: string, + capture: () => Promise, + timeout = 30_000, +) { + const thread = page.getByTestId("task-chat-thread"); + await expect(thread).toBeVisible({ timeout }); + await expect(thread.locator(':scope > [aria-busy="false"]')).toBeVisible({ timeout }); + await expect(thread.getByTestId("task-chat-history-loading")).toHaveCount(0, { timeout }); + await expect(thread.getByRole("heading", { name: title, exact: true })).toBeVisible({ timeout }); + await page.evaluate(() => document.fonts.ready.then(() => undefined)); + await capture(); +} diff --git a/tests/runner-e2e/continuation.test.ts b/tests/runner-e2e/continuation.test.ts index 3b1fa54e676..0f2c98d6f86 100644 --- a/tests/runner-e2e/continuation.test.ts +++ b/tests/runner-e2e/continuation.test.ts @@ -67,7 +67,7 @@ const failures = (r: ReturnType) => describe("continuation behavioral evaluation", () => { it("registers all five cases for both runtime generations and providers", () => { const matrix = runnerMatrix.filter((c) => c.suite.id === "continuation"); - expect(matrix).toHaveLength(20); + expect(matrix).toHaveLength(22); expect(new Set(matrix.map((c) => c.profile.id))).toEqual( new Set([ "legacy-codex", @@ -78,7 +78,7 @@ describe("continuation behavioral evaluation", () => { ); expect(matrix.every((c) => !c.suite.manualOnly)).toBe(true); }); - it.each(CONTINUATION_CASES)("accepts a complete %s recording", (id) => + it.each(CONTINUATION_CASES.filter(id => id !== "question-tool-documentation"))("accepts a complete %s recording", (id) => expect(failures(recording(id))).toEqual([]), ); it("fails premature output even when the final result is correct", () => { diff --git a/tests/runner-e2e/playwright-support.config.ts b/tests/runner-e2e/playwright-support.config.ts new file mode 100644 index 00000000000..c9905f3ecd2 --- /dev/null +++ b/tests/runner-e2e/playwright-support.config.ts @@ -0,0 +1,18 @@ +import { defineConfig } from "@playwright/test"; + +/** Browser-only harness regressions: no Paperclip instance or provider credentials. */ +export default defineConfig({ + testDir: ".", + testMatch: "screenshot-readiness.spec.ts", + workers: 1, + retries: 0, + timeout: 10_000, + use: { + headless: true, + ...(process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL + ? { channel: process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL } + : {}), + }, + outputDir: "./results/browser-support", + reporter: "list", +}); diff --git a/tests/runner-e2e/question-documentation-scoring.ts b/tests/runner-e2e/question-documentation-scoring.ts new file mode 100644 index 00000000000..a10ddc9390d --- /dev/null +++ b/tests/runner-e2e/question-documentation-scoring.ts @@ -0,0 +1,69 @@ +import type { ContinuationCheckpoint } from "./continuation-scoring.js"; + +type Row = Record; +const questions = (checkpoint?: ContinuationCheckpoint): Row[] => + (checkpoint?.interactions ?? []).filter((value: any) => value.kind === "ask_user_questions") as Row[]; +function form(row?: Row) { + const payload = row?.payload; + const canonical = payload?.questionSet?.questions; + if (canonical) return canonical; + return (payload?.questions ?? []).map((q: Row) => ({ + ...q, answerMode: q.selectionMode === "multi" ? "multi_select" : "single_select", + })); +} + +/** Checks real recorded forms, responses, and native task inputs; no model judge. */ +export function gradeQuestionDocumentation(checkpoints: ContinuationCheckpoint[], marker: string) { + const initial = checkpoints.find(c => c.phase === "initial"); + const middle = checkpoints.find(c => c.phase === "answered"); + const final = checkpoints.find(c => c.phase === "final"); + const first = questions(initial)[0]; + const next = questions(middle).filter(q => q.status === "pending"); + const choice = form(first)[0]; + const text = form(next[0])[0]; + const options: Row[] = choice?.options ?? []; + const afternoon = options.find(o => /^afternoon\b/i.test(String(o.label).trim())); + const firstAnswer = questions(final).find(q => q.id === first?.id); + const secondAnswer = questions(final).find(q => q.id === next[0]?.id); + const inputs = (final?.runs ?? []).map((run: any) => run.runnerProfileJson?.nativeExecutionInput?.task?.prompt); + const checks = [ + { + id: "meaningful-choice-form", + passed: questions(initial).length === 1 && first?.status === "pending" && form(first).length === 1 + && choice?.answerMode === "single_select" && options.length >= 2 + && new Set(options.map(o => o.id)).size === options.length + && options.some(o => /^morning\b/i.test(String(o.label).trim())) && Boolean(afternoon), + detail: "The first card must show one single-choice question with distinct Morning and Afternoon options.", + }, + { + id: "text-form-after-choice", + passed: questions(middle).length === 2 && next.length === 1 && form(next[0]).length === 1 + && next[0]?.id !== first?.id && text?.answerMode === "text" + && !text?.options?.length && !text?.customAnswer + && questions(middle).some(q => q.id === first?.id && q.status === "answered"), + detail: "Only after the choice is answered may the second, text-only question appear.", + }, + { + id: "actual-question-answers", + passed: questions(final).length === 2 && firstAnswer?.status === "answered" && secondAnswer?.status === "answered" + && firstAnswer?.result?.answers?.some((a: Row) => a.questionId === choice?.id && a.optionIds?.includes(afternoon?.id)) + && secondAnswer?.result?.answers?.some((a: Row) => a.questionId === text?.id && a.otherText?.includes(marker)), + detail: "Both real UI submissions must be persisted against their original question IDs, with no duplicate question cards.", + }, + { + id: "question-guidance-not-in-wake", + passed: inputs.length >= 3 && inputs.every(prompt => typeof prompt === "string" + && prompt.includes("Use Paperclip's request_human_input for durable task questions.") + && !prompt.includes("## Questions that need a user response") + && !prompt.includes("payload.questionSet") + && !prompt.includes("Create a durable human question or approval card on the current Paperclip task bound to this run")), + detail: "Every recorded native turn must retain the short routing hint without the old question block or copied tool-format instructions.", + }, + { + id: "both-answers-in-output", + passed: Boolean(final?.documents.some(d => d.key !== "plan" && d.body.includes(marker) && /\bafternoon\b/i.test(d.body))), + detail: "The saved note must use both the selected time and the literal reference supplied in the text answer.", + }, + ]; + return checks.map(check => ({ ...check, passed: Boolean(check.passed) })); +} diff --git a/tests/runner-e2e/question-documentation.test.ts b/tests/runner-e2e/question-documentation.test.ts new file mode 100644 index 00000000000..516f2026f46 --- /dev/null +++ b/tests/runner-e2e/question-documentation.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { gradeQuestionDocumentation } from "./question-documentation-scoring.js"; +import type { ContinuationCheckpoint } from "./continuation-scoring.js"; +import { runnerMatrix } from "./catalog.js"; + +function recording() { + const choice = { id: "time", prompt: "Which time?", answerMode: "single_select", options: [ + { id: "am", label: "Morning" }, { id: "pm", label: "Afternoon" }, + ] }; + const text = { id: "reference", prompt: "What reference?", answerMode: "text" }; + const first: any = { id: "first", kind: "ask_user_questions", status: "pending", payload: { questionSet: { questions: [choice] } } }; + const second: any = { id: "second", kind: "ask_user_questions", status: "pending", payload: { questionSet: { questions: [text] } } }; + const answeredFirst = { ...structuredClone(first), status: "answered", result: { answers: [{ questionId: "time", optionIds: ["pm"] }] } }; + const answeredSecond = { ...structuredClone(second), status: "answered", result: { answers: [{ questionId: "reference", optionIds: [], otherText: "Use AMBER123." }] } }; + const runs = [1, 2, 3].map(id => ({ id: String(id), status: "succeeded", runtimeMode: "native", runnerProfileJson: { nativeExecutionInput: { task: { prompt: "Use Paperclip's request_human_input for durable task questions." } } } })); + const checkpoint = (phase: ContinuationCheckpoint["phase"], interactions: any[], count: number): ContinuationCheckpoint => ({ + phase, issue: { id: "task", status: phase === "final" ? "done" : "in_review" }, children: [], attachments: [], comments: [], + interactions, runs: runs.slice(0, count), + documents: phase === "final" ? [{ key: "note", latestRevisionId: "saved", body: "Welcome to the afternoon meetup, AMBER123." }] : [], + }); + return [checkpoint("initial", [first], 1), checkpoint("answered", [answeredFirst, second], 2), checkpoint("final", [structuredClone(answeredFirst), answeredSecond], 3)]; +} +const failed = (checkpoints: ContinuationCheckpoint[]) => gradeQuestionDocumentation(checkpoints, "AMBER123").filter(c => !c.passed).map(c => c.id); + +describe("question tool documentation regression", () => { + it("runs against both native providers with production instructions", () => { + const cells = runnerMatrix.filter(c => c.task.id === "question-tool-documentation"); + expect(cells.map(c => c.profile.id).sort()).toEqual(["runner-acpx-claude", "runner-codex"]); + for (const c of cells) { + const prompt = c.task.buildPrompt("nonce"); + expect(prompt).not.toMatch(/request_human_input|questionSet|selectionMode|payload\./); + expect(c.suite.manualOnly).not.toBe(true); + } + }); + it("passes real choices followed by a text field and both recorded answers", () => { + expect(failed(recording())).toEqual([]); + }); + it("rejects a lone choice", () => { + const r = recording(); + (r[0].interactions[0] as any).payload.questionSet.questions[0].options.shift(); + expect(failed(r)).toContain("meaningful-choice-form"); + }); + it("rejects a lone Other choice instead of a text field", () => { + const r = recording(); + Object.assign((r[1].interactions[1] as any).payload.questionSet.questions[0], { answerMode: "single_select", options: [{ id: "other", label: "Other" }] }); + expect(failed(r)).toContain("text-form-after-choice"); + }); + it("rejects asking both questions before the first answer", () => { + const r = recording(); + r[0].interactions.push(structuredClone(r[1].interactions[1])); + expect(failed(r)).toContain("meaningful-choice-form"); + }); + it("rejects fabricated or duplicated responses", () => { + const r = recording(); + (r[2].interactions[0] as any).result.answers[0].optionIds = ["am"]; + expect(failed(r)).toContain("actual-question-answers"); + r[2].interactions.push(structuredClone(r[2].interactions[1])); + expect(failed(r)).toContain("actual-question-answers"); + }); + it("rejects the old instruction block copied into a later turn", () => { + const r = recording(); + (r[2].runs[2] as any).runnerProfileJson.nativeExecutionInput.task.prompt += "\n## Questions that need a user response\npayload.questionSet"; + expect(failed(r)).toContain("question-guidance-not-in-wake"); + }); + it("requires recorded inputs rather than assuming docs-only delivery", () => { + const r = recording(); + delete (r[2].runs[2] as any).runnerProfileJson; + expect(failed(r)).toContain("question-guidance-not-in-wake"); + }); + it("requires both real answers in the saved result", () => { + const r = recording(); + r[2].documents[0].body = "Welcome AMBER123."; + expect(failed(r)).toContain("both-answers-in-output"); + }); +}); diff --git a/tests/runner-e2e/screenshot-readiness.spec.ts b/tests/runner-e2e/screenshot-readiness.spec.ts new file mode 100644 index 00000000000..b4e5ed9e3ff --- /dev/null +++ b/tests/runner-e2e/screenshot-readiness.spec.ts @@ -0,0 +1,33 @@ +import { expect, test } from "@playwright/test"; +import { captureLoadedContinuation } from "./continuation-screenshot.js"; + +const shell = (busy: boolean, title = "Continuation example") => ` +
+

${title}

+ ${busy ? '
Loading conversation
' : '

Saved note is ready

'} +
`; + +test("capture waits through app loading and conversation loading", async ({ page }) => { + await page.setContent(`

App loading

`); + let captured = ""; + await captureLoadedContinuation(page, "Continuation example", async () => { + captured = await page.locator("body").innerText(); + await page.screenshot(); + }, 3000); + expect(captured).toContain("Saved note is ready"); + expect(captured).not.toContain("Loading"); +}); + +for (const state of ["app-loader", "history-loader", "wrong-task"] as const) { + test(`does not capture ${state} as a successful checkpoint`, async ({ page }) => { + await page.setContent(state === "app-loader" ? "

App loading

" : shell(state === "history-loader", state === "wrong-task" ? "Unrelated task" : undefined)); + let captures = 0; + await expect(captureLoadedContinuation(page, "Continuation example", async () => { + captures++; + }, 250)).rejects.toThrow(); + expect(captures).toBe(0); + }); +}