diff --git a/doc/evals.md b/doc/evals.md index e396d676907..517eb0b4ecd 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -71,7 +71,9 @@ tests do not establish a completed live pass or unattended CI qualification. The deployment-targeted `question-answer-resume` journey reuses the classic two-question scenario and native continuation oracle on an attached target. It checks the actual board answers, one to three bound work runs, an agent-authored -task document and browser readback after reload. It accepts provider questions and +task document and browser readback after reload. A typed reference may use a text +field or the native card's explicit written-answer control; a predefined option +cannot replace the exact typed answer. It accepts provider questions and durable questions proven by an applied semantic receipt or an immutable completed native API mutation receipt. It does not qualify named-tool selection. Its provider owns target setup, usage and cleanup. Registration and local calibration do not establish a fresh diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 34cdd31a158..f0ace922eeb 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -40,7 +40,12 @@ never start a server or depend on a tenant filesystem. The question-resumption journey uses the same natural-language scenario, browser answer helper and behavioral grader as the classic `question-resume` suite. Native -provider questions can stay in one run; durable native questions create one exactly bound +reference cards may expose a text field or an explicit canonical written-answer +control on one single-select question. The browser must type the complete reference +and submit no predefined option. An absent or disabled custom-answer capability is +not an equivalent form. The user still answers exactly two distinct native questions; +the document must retain the actual choice and typed reference after reload. +Provider questions can stay in one run; durable native questions create one exactly bound continuation each. Creation must have an applied semantic receipt or a completed native API mutation receipt with the original card, exact scope, form and idempotency key. An API receipt must retain its creation identity unchanged after the answer. A form or diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index fbfbe27169c..33231603802 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1771,7 +1771,13 @@ remain unchanged after the answer. Both durable paths require one new response wake bound to that interaction and source run. Missing, pending, foreign or mismatched receipts fail; form shape and run counts cannot replace them. Only a verified provider choice may carry the adapter's optional Other field. A second substantive question -is still rejected; the subsequent reference question must be text-only. Both real +is still rejected. The subsequent reference must use a text field or one native +provider single-select question with an explicit canonical `customAnswer.enabled=true`. +The browser selects its written-answer control and submits the exact typed reference +with no predefined option. Missing or disabled custom answers, inferred legacy +capabilities, extra questions and substituted choices fail. This behavioral form +equivalence does not change the text-only necessary-question or named-tool +documentation qualifications. Both real board answers must remain bound to unchanged forms, and one saved document must include both answers. Existing lifecycle, ownership, premature-output, terminal state and no-extra-task checks apply. Three checkpoints are required even when @@ -1785,6 +1791,12 @@ Other field stopped the driver before any answer remains a FAIL. A new behavior pass does not establish semantic-tool selection, documentation placement, default hiring behavior or general reliability. +The original scheduled attempt that stopped at a closed reference card remains a +FAIL, including its separate cleanup failure. Supporting a server-bound written +answer does not regrade that card, infer a missing provider flag, or establish a +fresh Cloud pass. The general OpenCode omitted-default normalization fix requires +its own reviewed runtime artifact; browser/grader calibration cannot prove it served. + Continuation screenshots wait for the correct task heading and fully revealed conversation before capture. A loading screen or wrong task fails capture. Browser-only regressions exercise delayed rendering without provider calls: diff --git a/tests/runner-e2e/continuation-flow.ts b/tests/runner-e2e/continuation-flow.ts index 89072dc005e..3da0668ebae 100644 --- a/tests/runner-e2e/continuation-flow.ts +++ b/tests/runner-e2e/continuation-flow.ts @@ -1,6 +1,6 @@ import { gradeNecessaryQuestion } from "./necessary-question.js"; import { necessaryQuestionContract } from "./continuation-cases.js"; -import { validResumeQuestionForm } from "./question-resume-scoring.js"; +import { validResumeQuestionForm, validResumeReferenceForm } from "./question-resume-scoring.js"; import { answerResumeQuestionThroughUi } from "./question-resume-ui.js"; import { gradeLifecycleBaseline, type LifecycleCheckpoint } from "./lifecycle-baseline.js"; import { lifecycleLiveCase, lifecycleLiveContinuation, gradeLifecycleNarrative } from "./lifecycle-live-cases.js"; @@ -220,12 +220,15 @@ export async function runContinuationFlow(input: { if (scenario.id === "necessary-free-text") { expect(validResumeQuestionForm(checkpoints[0]!, questions[0], false), "a bound text-only question").toBe(true); } else if (scenario.id === "question-answer-resume") { - expect(validResumeQuestionForm(checkpoints.at(-1)!, questions[0], !!choice), "a verified native question path and the requested form").toBe(true); + expect(choice ? validResumeQuestionForm(checkpoints.at(-1)!, questions[0], true) + : validResumeReferenceForm(checkpoints.at(-1)!, questions[0]), "a verified native question path and the requested form").toBe(true); } else if (scenario.id === "provider-question-bridge") { expect(isSingleClaudeQuestion(set.questions), "one choice question with only the optional provider Other field").toBe(true); } else expect(set.questions, "ask only the requested next question").toHaveLength(1); const before = new Set(runs.map((r) => r.id)); - await answerResumeQuestionThroughUi(page, questions[0], choice ? { choice } : { text: scenario.answer }); + await answerResumeQuestionThroughUi(page, questions[0], choice ? { choice } : { + text: scenario.answer, allowProviderCustomAnswer: scenario.id === "question-answer-resume", + }); await settle(before, false, questions[0].id); } async function reply(body: string) { diff --git a/tests/runner-e2e/playwright-support.config.ts b/tests/runner-e2e/playwright-support.config.ts index 000c7d502c6..72cc2bfac65 100644 --- a/tests/runner-e2e/playwright-support.config.ts +++ b/tests/runner-e2e/playwright-support.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from "@playwright/test"; /** Browser-only harness regressions: no Paperclip instance or provider credentials. */ export default defineConfig({ testDir: ".", - testMatch: ["clarification-readback.spec.ts", "project-context-readback.spec.ts", "skill-studio-flow.spec.ts", "native-task-reply.spec.ts", "screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts", "native-blocker-visible.spec.ts", "copilot-permission-card.spec.ts", "kimi-permission-card.spec.ts", "github-review-check.spec.ts", "antigravity-journey-browser.spec.ts", "task-search-filter.spec.ts"], + testMatch: ["question-resume-ui.spec.ts", "clarification-readback.spec.ts", "project-context-readback.spec.ts", "skill-studio-flow.spec.ts", "native-task-reply.spec.ts", "screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts", "native-blocker-visible.spec.ts", "copilot-permission-card.spec.ts", "kimi-permission-card.spec.ts", "github-review-check.spec.ts", "antigravity-journey-browser.spec.ts", "task-search-filter.spec.ts"], workers: 1, retries: 0, timeout: 10_000, diff --git a/tests/runner-e2e/product-journeys/clarification-evidence.test.ts b/tests/runner-e2e/product-journeys/clarification-evidence.test.ts index 88c0ccb47cf..2a0287cc8d3 100644 --- a/tests/runner-e2e/product-journeys/clarification-evidence.test.ts +++ b/tests/runner-e2e/product-journeys/clarification-evidence.test.ts @@ -28,7 +28,7 @@ import type { JourneyContext } from "./types.js"; type Row = Record; const context = { company: { id: "company" }, user: { id: "board" }, environmentId: "daytona", agent: { id: "agent", provider: "opencode", model: "model", runtimeMode: "native" } } as JourneyContext; -function recording(paths: Array<"provider" | "semantic" | "api">) { +function recording(paths: Array<"provider" | "semantic" | "api">, writtenReference = false) { const runs: Row[] = [], cards: Row[] = [], checkpoints: any[] = []; const createRun = (previous?: Row) => ({ id: `run-${runs.length}`, status: "running", runtimeMode: "native", companyId: "company", issueId: "task", agentId: "agent", contextSnapshot: { paperclipEnvironment: { id: "daytona" }, ...(previous ? { issueId: "task", sourceRunId: previous.sourceRunId, @@ -41,7 +41,9 @@ function recording(paths: Array<"provider" | "semantic" | "api">) { runs.push(createRun()); for (const [index, path] of paths.entries()) { const run = runs.at(-1)!; - const field = index === 0 ? { id: "choice", answerMode: "single_select", options: [{ id: "am", label: "Morning" }, { id: "pm", label: "Afternoon" }] } : { id: "reference", answerMode: "text" }; + const field = index === 0 ? { id: "choice", answerMode: "single_select", options: [{ id: "am", label: "Morning" }, { id: "pm", label: "Afternoon" }] } + : writtenReference ? { id: "reference", answerMode: "single_select", options: [{ id: "none", label: "No reference" }], customAnswer: { enabled: true } } + : { id: "reference", answerMode: "text" }; const request = `request-${index}`, semantic = path !== "provider"; const card: Row = { id: `question-${index}`, kind: "ask_user_questions", status: "pending", companyId: "company", issueId: "task", createdByAgentId: "agent", sourceRunId: run.id, idempotencyKey: semantic ? `semantic-${index}` : `paperclip-runner-question:${run.id}:${request}`, continuationPolicy: semantic ? "wake_assignee" : "none", createdByUserId: null, @@ -65,6 +67,11 @@ function recording(paths: Array<"provider" | "semantic" | "api">) { const failures = (f: ReturnType) => gradeClarificationEvidence(context, "task", "AMBERnonce", f.checkpoints, f.readback).filter(c => !c.passed); describe("attached clarification evidence", () => { + it("accepts an explicit provider written reference with both exact board answers and saved native work", () => { + const f = recording(["provider", "provider"], true), point = f.checkpoints[1]; + expect(clarificationQuestionFailure(point, point.interactions[1], false)).toBeUndefined(); + expect(failures(f)).toEqual([]); + }); it.each([true, false])("rejects an incomplete API receipt instead of inferring native question evidence (choice=%s)", choice => { const f = recording(["semantic", "semantic"]), point = f.checkpoints[choice ? 0 : 1]; const question = point.interactions.find((i: Row) => i.status === "pending"), source = point.runs.at(-1); @@ -87,8 +94,8 @@ describe("attached clarification evidence", () => { expect(failures(recording([...paths]))).toEqual([]); }); it.each(["foreign-user", "foreign-agent-document", "foreign-task-document", "lost-revision", "wrong-visible-answer", "changed-readback-body", "premature-document", "extra-child", "foreign-model", "foreign-environment", "missing-native-input", "duplicate-question", "wrong-answer"])("rejects a plausible false pass: %s", kind => { - for (const path of ["semantic", "api"] as const) { - const f = recording([path, path]), first = f.checkpoints[0], final = f.checkpoints[2]; + for (const path of ["semantic", "api", "provider"] as const) { + const f = recording([path, path], path === "provider"), first = f.checkpoints[0], final = f.checkpoints[2]; if (kind === "foreign-user") final.interactions[1].resolvedByUserId = "other-board"; if (kind === "foreign-agent-document") final.documents[0].updatedByAgentId = "other-agent"; if (kind === "foreign-task-document") final.documents[0].issueId = "other-task"; diff --git a/tests/runner-e2e/product-journeys/clarification-evidence.ts b/tests/runner-e2e/product-journeys/clarification-evidence.ts index 69e39b7fa62..4d3ab02e70a 100644 --- a/tests/runner-e2e/product-journeys/clarification-evidence.ts +++ b/tests/runner-e2e/product-journeys/clarification-evidence.ts @@ -1,4 +1,4 @@ -import { gradeQuestionResume, questionPath, validResumeQuestionForm } from "../question-resume-scoring.js"; +import { gradeQuestionResume, questionPath, validResumeQuestionForm, validResumeReferenceForm } from "../question-resume-scoring.js"; import type { ContinuationCheckpoint } from "../continuation-scoring.js"; import { validateAgentTaskState } from "./agent-flows.js"; import type { JourneyContext } from "./types.js"; @@ -19,7 +19,8 @@ export function clarificationQuestionFailure(point: ContinuationCheckpoint, ques }); return apiCreated ? `clarification_${stage}_api_fallback` : `clarification_${stage}_native_path_invalid`; } - return validResumeQuestionForm(point, question, choice) ? undefined : `clarification_${stage}_form_invalid`; + return (choice ? validResumeQuestionForm(point, question, true) : validResumeReferenceForm(point, question)) + ? undefined : `clarification_${stage}_form_invalid`; } export function gradeClarificationEvidence(context: JourneyContext, issueId: string, marker: string, diff --git a/tests/runner-e2e/product-journeys/clarification-flow.ts b/tests/runner-e2e/product-journeys/clarification-flow.ts index 800a747284d..8edb160c5e0 100644 --- a/tests/runner-e2e/product-journeys/clarification-flow.ts +++ b/tests/runner-e2e/product-journeys/clarification-flow.ts @@ -91,7 +91,7 @@ export async function runClarificationJourney(context: JourneyContext, session: await session.own({ kind: "issue", name: title, id: created.issueId, journeyId: "question-answer-resume" }); issue = await context.api.get(`/api/issues/${created.issueId}`); let state = await settle(); - for (const [index, input] of [{ choice: "Afternoon" }, { text: scenario.answer }].entries()) { + for (const [index, input] of [{ choice: "Afternoon" }, { text: scenario.answer, allowProviderCustomAnswer: true }].entries()) { const point = await snapshot(index === 0 ? "initial" : "answered", state); const question = state.interactions.find(i => i.kind === "ask_user_questions" && i.status === "pending")!; const questionFailure = clarificationQuestionFailure(point, question, index === 0); diff --git a/tests/runner-e2e/question-resume-scoring.ts b/tests/runner-e2e/question-resume-scoring.ts index 85f8902d95e..30f1e46c350 100644 --- a/tests/runner-e2e/question-resume-scoring.ts +++ b/tests/runner-e2e/question-resume-scoring.ts @@ -67,6 +67,15 @@ export function validResumeQuestionForm(checkpoint: ContinuationCheckpoint, i: R (fields.length === 1 || (path === "provider" && isSingleClaudeQuestion(fields))); } +export function validResumeReferenceForm(checkpoint: ContinuationCheckpoint, i: Row): boolean { + if (validResumeQuestionForm(checkpoint, i, false)) return true; + const fields = resumeQuestionFields(i), field = fields[0], options: Row[] = Array.isArray(field?.options) ? field.options : []; + return questionPath(checkpoint, i) === "provider" && fields.length === 1 && nonempty(field.id) + && field.answerMode === "single_select" && field.customAnswer?.enabled === true + && options.length > 0 && options.every(option => nonempty(option.id) && nonempty(option.label)) + && new Set(options.map(option => option.id)).size === options.length; +} + export function sameQuestion(a: Row, b?: Row) { return !!b && ["id", "kind", "sourceRunId", "issueId", "companyId", "createdByAgentId", "idempotencyKey", "continuationPolicy"] .every(k => a[k] === b[k]) && isDeepStrictEqual(a.payload, b.payload); @@ -110,8 +119,8 @@ export function gradeQuestionResume(checkpoints: ContinuationCheckpoint[], marke options.every(o => nonempty(o.id)) && new Set(options.map(o => o.id)).size === options.length && options.some(o => /^morning\b/i.test(String(o.label).trim())) && !!afternoon && questions(middle).length === 2 && questions(middle).filter(i => i.status === "pending").length === 1 && - nonempty(second?.id) && second?.id !== first?.id && validResumeQuestionForm(middle, second, false), - detail: "One choice plus only a verified provider Other companion, then a distinct text-only question.", + nonempty(second?.id) && second?.id !== first?.id && validResumeReferenceForm(middle, second), + detail: "One choice plus only a verified provider Other companion, then a distinct typed reference with an explicit provider written-answer control or a text field.", }]; for (const [before, after, i, field, answer] of [ [initial, middle, first, choice, "choice"], [middle, final, second, text, "text"], diff --git a/tests/runner-e2e/question-resume-ui.spec.ts b/tests/runner-e2e/question-resume-ui.spec.ts new file mode 100644 index 00000000000..e1510b3aee7 --- /dev/null +++ b/tests/runner-e2e/question-resume-ui.spec.ts @@ -0,0 +1,83 @@ +import path from "node:path"; +import type { IncomingMessage, ServerResponse } from "node:http"; +import { expect, test } from "@playwright/test"; +import { createServer, type ViteDevServer } from "../../ui/node_modules/vite/dist/node/index.js"; +import { answerResumeQuestionThroughUi } from "./question-resume-ui.js"; +import { chatQuestionPresentation } from "./chat-flow.js"; + +let server: ViteDevServer; +let origin: string; +const uiRoot = path.resolve(import.meta.dirname, "../../ui"); +const probe = path.join(uiRoot, "src/__question_resume_probe.tsx"); + +test.beforeAll(async () => { + server = await createServer({ + root: uiRoot, configFile: false, + resolve: { alias: { "@": path.join(uiRoot, "src") } }, + oxc: { jsx: { runtime: "automatic" } }, + server: { host: "127.0.0.1", port: 0 }, + plugins: [{ + name: "question-resume-controls", + resolveId(id: string) { if (id === "/src/__question_resume_probe.tsx") return probe; }, + load(id: string) { + if (id !== probe) return; + return `import React from "react"; + import { createRoot } from "react-dom/client"; + import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + import { QuestionForm } from "/src/components/task-chat/QuestionForm.tsx"; + const root = createRoot(document.getElementById("root")); + const client = new QueryClient(); + window.showQuestion = questionSet => root.render( + { window.submitted = response; }} /> + );`; + }, + configureServer(vite: ViteDevServer) { + vite.middlewares.use((req: IncomingMessage, res: ServerResponse, next: () => void) => { + if (req.url !== "/") return next(); + res.setHeader("Content-Type", "text/html"); + res.end('
'); + }); + }, + }], + }); + await server.listen(); + const address = server.httpServer!.address(); + if (!address || typeof address === "string") throw new Error("Missing test server address"); + origin = `http://127.0.0.1:${address.port}`; +}); +test.afterAll(async () => { await server?.close(); }); + +for (const mode of ["text", "single_select"] as const) { + test(`submits an exact typed reference through the real ${mode} QuestionForm control`, async ({ page }) => { + const field = { id: "reference", prompt: "What reference should I include?", required: true, answerMode: mode, + ...(mode === "single_select" ? { options: [{ id: "none", label: "No reference" }], customAnswer: { enabled: true, label: "Write reference" } } : {}) }; + const card = { payload: { questionSet: { schema: "paperclip.question_set.v1", questions: [field] } } }; + await page.goto(origin); + await page.waitForFunction(() => typeof (window as any).showQuestion === "function"); + await page.evaluate(set => (window as any).showQuestion(set), card.payload.questionSet); + if (mode === "single_select") await page.getByRole("radio", { name: "No reference", exact: true }).click(); + await answerResumeQuestionThroughUi(page, card, { text: "Use AMBERwritten.", allowProviderCustomAnswer: true }); + expect(await page.evaluate(() => (window as any).submitted)).toEqual({ + schema: "paperclip.question_response.v1", answers: { reference: mode === "text" + ? { text: "Use AMBERwritten." } : { selectedOptionIds: [], customText: "Use AMBERwritten." } }, + }); + if (mode === "single_select") await expect(page.getByRole("radio", { name: "No reference", exact: true })).toHaveAttribute("aria-checked", "false"); + }); +} + +for (const invalid of ["missing", "disabled", "legacy", "strict"] as const) { + test(`does not infer a written-answer capability from ${invalid} metadata`, async ({ page }) => { + const field = { id: "reference", prompt: "Reference?", answerMode: "single_select", options: [{ id: "none", label: "No reference" }], + ...(invalid === "missing" ? {} : { customAnswer: { enabled: invalid !== "disabled" } }) }; + const card = { payload: invalid === "legacy" ? { questions: [{ ...field, options: [{ id: "none", label: "No reference", freeText: true }] }] } + : { questionSet: { schema: "paperclip.question_set.v1", questions: [field] } } }; + await page.goto(origin); + await page.waitForFunction(() => typeof (window as any).showQuestion === "function"); + const set = invalid === "legacy" ? chatQuestionPresentation(card.payload as any) : card.payload.questionSet; + await page.evaluate(set => (window as any).showQuestion(set), set); + await expect(page.getByRole("radio", { name: "No reference", exact: true })).toBeVisible(); + await expect(page.getByRole("radio", { name: "Other", exact: true })).toHaveCount(invalid === "strict" ? 1 : 0); + await expect(answerResumeQuestionThroughUi(page, card, { text: "Use AMBERwritten.", ...(invalid === "strict" ? {} : { allowProviderCustomAnswer: true }) })).rejects.toThrow(); + expect(await page.evaluate(() => (window as any).submitted)).toBeUndefined(); + }); +} diff --git a/tests/runner-e2e/question-resume-ui.ts b/tests/runner-e2e/question-resume-ui.ts index ac6fa3f9de8..9e325337e93 100644 --- a/tests/runner-e2e/question-resume-ui.ts +++ b/tests/runner-e2e/question-resume-ui.ts @@ -3,7 +3,7 @@ import { chatQuestionPresentation } from "./chat-flow.js"; type Row = Record; -export async function answerResumeQuestionThroughUi(page: Page, interaction: Row, input: { choice: string } | { text: string }) { +export async function answerResumeQuestionThroughUi(page: Page, interaction: Row, input: { choice: string } | { text: string; allowProviderCustomAnswer?: boolean }) { const set = chatQuestionPresentation(interaction.payload); if ("choice" in input) { expect(set.questions[0]?.answerMode, "choices must use radio controls").toBe("single_select"); @@ -12,8 +12,14 @@ export async function answerResumeQuestionThroughUi(page: Page, interaction: Row const choice = input.choice.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); await page.getByRole("radio", { name: new RegExp(`^${choice}\\b`, "i") }).last().click(); } else { - expect(set.questions[0]?.answerMode, "open answers must render a text field, not a choice question").toBe("text"); - await page.getByTestId("question-text-answer-composer").last() + const field = set.questions[0]; + const custom = field?.answerMode !== "text"; + if (custom) { + expect(input.allowProviderCustomAnswer && interaction.payload.questionSet && field?.answerMode === "single_select" + && field.customAnswer?.enabled === true, "the canonical provider form must explicitly permit a written reference").toBeTruthy(); + await page.getByRole("radio", { name: field!.customAnswer?.label ?? "Other", exact: true }).last().click(); + } + await page.getByTestId(custom ? "question-other-answer-composer" : "question-text-answer-composer").last() .locator('[contenteditable="true"],textarea').first().fill(input.text); } for (let index = 1; index < set.questions.length; index += 1) { diff --git a/tests/runner-e2e/question-resume.test.ts b/tests/runner-e2e/question-resume.test.ts index bded4d1c8d2..c312d4af580 100644 --- a/tests/runner-e2e/question-resume.test.ts +++ b/tests/runner-e2e/question-resume.test.ts @@ -60,8 +60,55 @@ function recording(paths: QuestionPath[] = ["provider", "semantic"]) { } const failed = (r: any[]) => gradeQuestionResume(r, "AMBERnonce").filter(c => !c.passed).map(c => c.id); const question = (r: any[], n = 0) => r[n].interactions[n]; +function writtenReference(r: any[]) { + for (const point of r.slice(1)) Object.assign(point.interactions[1].payload.questionSet.questions[0], { + answerMode: "single_select", options: [{ id: "none", label: "No reference" }], customAnswer: { enabled: true }, + }); + return r; +} describe("native question/resume path qualification", () => { + it("accepts an explicit provider written answer without selecting a predefined option", () => { + const r = writtenReference(recording(["provider", "provider"])); + expect(failed(r)).toEqual([]); + expect(validResumeQuestionForm(r[1], question(r, 1), false)).toBe(false); + }); + it("rejects an inferred legacy freeText reference without a canonical question set", () => { + const r = writtenReference(recording(["provider", "provider"])); + for (const point of r.slice(1)) { + const payload = point.interactions[1].payload; + payload.questions = [{ id: "reference", prompt: "What reference should I include?", selectionMode: "single", + options: [{ id: "write", label: "Write reference", freeText: true }] }]; + delete payload.questionSet; + } + expect(failed(r)).toContain("question-resume.forms"); + }); + it.each(["missing-custom", "disabled-custom", "missing-options", "duplicate-options", "missing-option-id", "multi-select", "extra-question", + "wrong-request", "foreign-source", "foreign-agent", "foreign-company", "semantic", "api"])("rejects an unverified written-reference form: %s", kind => { + const r = writtenReference(recording(["provider", kind === "semantic" || kind === "api" ? kind : "provider"])); + const card = question(r, 1), field = card.payload.questionSet.questions[0]; + if (kind === "missing-custom") delete field.customAnswer; + if (kind === "disabled-custom") field.customAnswer.enabled = false; + if (kind === "missing-options") field.options = []; + if (kind === "duplicate-options") field.options.push(structuredClone(field.options[0])); + if (kind === "missing-option-id") delete field.options[0].id; + if (kind === "multi-select") field.answerMode = "multi_select"; + if (kind === "extra-question") card.payload.questionSet.questions.push({ id: "extra", answerMode: "text" }); + if (kind === "wrong-request") card.payload.runtimeRequestId = "other"; + if (kind === "foreign-source") card.sourceRunId = "other"; + if (kind === "foreign-agent") card.createdByAgentId = "other"; + if (kind === "foreign-company") card.companyId = "other"; + expect(failed(r)).toContain("question-resume.forms"); + }); + it.each(["predefined-option", "wrong-text", "missing-text", "foreign-user", "changed-form"])("rejects a corrupt written-reference answer: %s", kind => { + const r = writtenReference(recording(["provider", "provider"])), card = r[2].interactions[1], answer = card.result.answers[0]; + if (kind === "predefined-option") answer.optionIds = ["none"]; + if (kind === "wrong-text") answer.otherText = "Use OLD."; + if (kind === "missing-text") delete answer.otherText; + if (kind === "foreign-user") { card.resolvedByUserId = null; card.resolvedByAgentId = "agent"; } + if (kind === "changed-form") card.payload.questionSet.questions[0].customAnswer.label = "Changed"; + expect(failed(r)).toContain("question-resume.text.answer"); + }); it("accepts a native API creation receipt with the retained server response shape", () => { const point = JSON.parse(readFileSync(new URL("./fixtures/native-api-question.json", import.meta.url), "utf8")); expect(questionPath(point, point.interactions[0])).toBe("api");