From d55eea3246bff75ee6ce091112c78ca5525352c7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 12:15:36 -0500 Subject: [PATCH] Add a portable agent question and answer resumption journey Reuse the classic two-question scenario and native continuation oracle. Verify board-owned answers, bound runs, agent-authored output and persisted document readback on attached targets. Co-Authored-By: Paperclip --- doc/evals.md | 7 ++ tests/runner-e2e/PRODUCT-JOURNEYS.md | 11 ++ tests/runner-e2e/continuation-flow.ts | 25 +--- .../product-journeys/agent-flows.ts | 2 +- tests/runner-e2e/product-journeys/catalog.ts | 1 + .../clarification-evidence.test.ts | 96 ++++++++++++++++ .../clarification-evidence.ts | 43 +++++++ .../product-journeys/clarification-flow.ts | 108 ++++++++++++++++++ tests/runner-e2e/product-journeys/launch.ts | 10 +- .../product-journeys/success-contract.ts | 12 +- tests/runner-e2e/product-journeys/types.ts | 2 +- tests/runner-e2e/question-resume-ui.ts | 23 ++++ tests/runner-e2e/result-validation.ts | 4 +- 13 files changed, 315 insertions(+), 29 deletions(-) create mode 100644 tests/runner-e2e/product-journeys/clarification-evidence.test.ts create mode 100644 tests/runner-e2e/product-journeys/clarification-evidence.ts create mode 100644 tests/runner-e2e/product-journeys/clarification-flow.ts create mode 100644 tests/runner-e2e/question-resume-ui.ts diff --git a/doc/evals.md b/doc/evals.md index 0e42d6a56df..77dca7c299f 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -58,6 +58,13 @@ self-hosted/Cloud targets and separately selected local/Daytona agents. Account and credential bindings stay in the private eval repo; registration and oracle tests do not establish a completed live pass or unattended CI qualification. +The deployment-targeted `question-answer-resume` journey reuses the classic +two-question scenario and native continuation oracle on an attached target. +It checks the actual board answers, one to three bound work runs, an agent-authored +task document and browser readback after reload. Its provider owns target setup, +usage and cleanup. Registration and local calibration do not establish a fresh +Cloud pass or inclusion in a recurring selection. + The explicit-only [necessary free-text question journey](../tests/runner-e2e/README.md#necessary-free-text-question) checks one essential input, the rendered prompt and context, the real answer, one bound continuation, and an exact saved confirmation. Its four development diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 117e358c263..ee715a8936d 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -25,6 +25,7 @@ never start a server or depend on a tenant filesystem. | --- | --- | --- | | `task-create` | Company and signed-in board user | Create a task through the shared production composer, assign it to yourself, reopen and reload it, verify the saved prompt and no agent work | | `ask-question` | Company, configured agent capability, positive model allowance and exactly one permitted work run | Reuse the existing cross-model Ask question prompt; require one successful run with matching company/task/agent/model/environment, one saved exact answer, and visible readback after reload | +| `question-answer-resume` | Native agent; up to three work runs | Reuse the classic two-question story: answer a choice and a free-text question through the browser, verify each native continuation, then read the exact saved answers in an agent-authored task document after reload | | `task-properties` | Company, signed-in board user, task-write capability, prepared run-owned backlog task | Open its properties, assign it to the signed-in user, reload, check both UI and saved assignment; verify no agent run | | `plan-revise-accept` | Native agent; three work runs | Reuse the classic Plan story: canonical draft, browser revision request, new revision-bound confirmation, approval and exact saved final response | | `build-revise` | Native agent; two work runs; qualified Docker artifact oracle | Browser-download both ZIP versions, independently exercise source/CLI behavior, verify different hashes and preserve the original download | @@ -33,6 +34,16 @@ never start a server or depend on a tenant filesystem. | `stop-redirect` | Native agent; three work runs; qualified Docker oracle | Download a verified checkpoint, Stop an active revision, require native cancellation acknowledgement, submit a new direction once and verify completion without original-run activity | | `cloud-onboarding` | Signed-in admitted Cloud account; fresh workspace; qualified bounded provider route | Create workspace in the portal, connect API key and native Codex agent in the wizard, complete a task, download and independently check its artifact | +The question-resumption journey uses the same natural-language scenario, browser +answer helper and behavioral grader as the classic `question-resume` suite. Native +provider questions can stay in one run; semantic questions create one exactly bound +continuation each. A target declares a three-run maximum and positive model allowance, +and a passing receipt must contain one to three attributed native runs. Other journeys +retain their exact run-count requirements. Both real board answers, no premature +output, the sole document's agent/company/task ownership and its reloaded revision +are independently checked. The private provider owns authentication, setup, accounting +and cleanup. Local calibration is not deployed-target qualification. + The shared Studio helper waits for the original saved file body before typing. An editor control can mount before the file query completes; editing that empty draft can otherwise lose the original content when the file arrives. diff --git a/tests/runner-e2e/continuation-flow.ts b/tests/runner-e2e/continuation-flow.ts index e667575ea90..89072dc005e 100644 --- a/tests/runner-e2e/continuation-flow.ts +++ b/tests/runner-e2e/continuation-flow.ts @@ -1,6 +1,7 @@ import { gradeNecessaryQuestion } from "./necessary-question.js"; import { necessaryQuestionContract } from "./continuation-cases.js"; import { validResumeQuestionForm } from "./question-resume-scoring.js"; +import { answerResumeQuestionThroughUi } from "./question-resume-ui.js"; import { gradeLifecycleBaseline, type LifecycleCheckpoint } from "./lifecycle-baseline.js"; import { lifecycleLiveCase, lifecycleLiveContinuation, gradeLifecycleNarrative } from "./lifecycle-live-cases.js"; import { prepareLegacyContinuationSkill, prepareContinuationBudget } from "./continuation-fixtures.js"; @@ -224,29 +225,7 @@ export async function runContinuationFlow(input: { expect(isSingleClaudeQuestion(set.questions), "one choice question with only the optional provider Other field").toBe(true); } else expect(set.questions, "ask only the requested next question").toHaveLength(1); const before = new Set(runs.map((r) => r.id)); - if (choice) { - expect(set.questions[0].answerMode, "choices must use radio controls").toBe("single_select"); - const options = questions[0].payload.questionSet?.questions[0]?.options - ?? questions[0].payload.questions[0]?.options ?? []; - expect(new Set(options.map((o: Row) => String(o.label).trim().toLowerCase())).size).toBeGreaterThanOrEqual(2); - await page.getByRole("radio", { name: new RegExp(`^${choice}\\b`, "i") }).last().click(); - } else { - expect(set.questions[0].answerMode, "open answers must render a text field, not a choice question").toBe("text"); - await page.getByTestId("question-text-answer-composer").last() - .locator('[contenteditable="true"],textarea').first().fill(scenario.answer); - } - // Claude may add a separate optional Other field after its choice page. - // Navigate every rendered page before submitting; do not invent an answer. - for (let index = 1; index < set.questions.length; index += 1) { - await page.getByRole("button", { name: "Next", exact: true }).last().click(); - } - await page - .getByRole("button", { - name: set.submitLabel ?? "Submit answers", - exact: true, - }) - .last() - .click(); + await answerResumeQuestionThroughUi(page, questions[0], choice ? { choice } : { text: scenario.answer }); await settle(before, false, questions[0].id); } async function reply(body: string) { diff --git a/tests/runner-e2e/product-journeys/agent-flows.ts b/tests/runner-e2e/product-journeys/agent-flows.ts index 57a7c772565..37ccb8995bf 100644 --- a/tests/runner-e2e/product-journeys/agent-flows.ts +++ b/tests/runner-e2e/product-journeys/agent-flows.ts @@ -18,7 +18,7 @@ import { agentJourneyRuns } from "./catalog.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; type Row = Record; -type AgentJourney = Exclude; +type AgentJourney = Exclude; interface TaskState { currentIssue: Row & { id: string; status: string }; taskRuns: Array; comments: Row[]; interactions: Array; attachments: StoryDelivery[] } interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } const active = (run: Row) => ["queued", "running"].includes(run.status); diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 9ac222f0079..86cdc9418cc 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -4,6 +4,7 @@ import type { JourneyId, TargetConfig } from "./types.js"; export const productJourneys = [ { id: "task-create", label: "Create a task for yourself and reopen it", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/create"], paid: false }, { id: "ask-question", label: "Ask an agent a question and read its answer", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/questions"], paid: true }, + { id: "question-answer-resume", label: "Answer agent questions and verify the saved result", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/questions"], paid: true }, { id: "task-properties", label: "Assign a task to the signed-in user", requires: ["company", "board-user", "task-write", "backlog-task"], capabilities: ["tasks/properties"], paid: false }, { id: "plan-revise-accept", label: "Revise and approve a saved Plan, then complete the task", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/plan-review"], paid: true }, { id: "build-revise", label: "Download and independently verify both artifact versions", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["documents-artifacts/attachments"], paid: true }, diff --git a/tests/runner-e2e/product-journeys/clarification-evidence.test.ts b/tests/runner-e2e/product-journeys/clarification-evidence.test.ts new file mode 100644 index 00000000000..044ef2df030 --- /dev/null +++ b/tests/runner-e2e/product-journeys/clarification-evidence.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; +import { parseTargetConfig, productJourneys } from "./catalog.js"; +import { agentJourneyRunCountValid } from "./success-contract.js"; + +describe("portable clarification journey admission", () => { + it("declares the existing two-question story as a bounded native journey", () => { + expect(productJourneys.find(j => j.id === "question-answer-resume")).toMatchObject({ paid: true, requires: ["company", "board-user", "task-write", "agent"] }); + const config = { version: 1, id: "attached", mode: "existing-cloud", adapterModule: "/private/provider.mjs", + executionEnvironment: "daytona", journeys: ["question-answer-resume"], baseline: { features: {}, connections: [] }, + limits: { durationMs: 600_000, modelBudgetCents: 500, maxRuns: 3 } }; + expect(parseTargetConfig(config).journeys).toEqual(["question-answer-resume"]); + for (const maxRuns of [0, 1, 2, 4]) expect(() => parseTargetConfig({ ...config, limits: { ...config.limits, maxRuns } })).toThrow(); + }); + it("admits exactly the provider/semantic bounds without changing existing exact-count stories", () => { + for (const count of [1, 2, 3]) expect(agentJourneyRunCountValid("question-answer-resume", count)).toBe(true); + for (const count of [undefined, 0, 4, 1.5]) expect(agentJourneyRunCountValid("question-answer-resume", count)).toBe(false); + expect(agentJourneyRunCountValid("plan-revise-accept", 2)).toBe(false); + expect(agentJourneyRunCountValid("plan-revise-accept", 3)).toBe(true); + }); +}); + +import { gradeClarificationEvidence } from "./clarification-evidence.js"; +import { continuationScenario } from "../continuation-cases.js"; +import { validateRetainedRunnerResult } from "../result-validation.js"; +import { journeyRequiredChecks } from "./success-contract.js"; +import type { JourneyContext } from "./types.js"; + +type Row = Record; +const context = { company: { id: "company" }, user: { id: "board" }, environmentId: "daytona", + agent: { id: "agent", provider: "opencode", model: "model", runtimeMode: "native" } } as JourneyContext; +function recording(paths: Array<"provider" | "semantic">) { + const runs: Row[] = [], cards: Row[] = [], checkpoints: any[] = []; + const createRun = (previous?: Row) => ({ id: `run-${runs.length}`, status: "running", runtimeMode: "native", companyId: "company", issueId: "task", agentId: "agent", + contextSnapshot: { paperclipEnvironment: { id: "daytona" }, ...(previous ? { issueId: "task", sourceRunId: previous.sourceRunId, + interactionId: previous.id, interactionKind: "ask_user_questions", interactionStatus: "answered" } : {}) }, + resultJson: { semanticToolReceipts: {} }, runnerProfileJson: { nativeExecutionInput: { task: { prompt: continuationScenario("question-answer-resume", "nonce").prompt }, provider: { kind: "opencode", model: "model" } } } }); + const document = { issueId: "task", companyId: "company", key: "welcome-note", latestRevisionId: "revision", createdByAgentId: "agent", updatedByAgentId: "agent", body: "Welcome to the afternoon meetup. Use AMBERnonce." }; + const snapshot = (phase: string) => checkpoints.push(structuredClone({ phase, + issue: { id: "task", companyId: "company", assigneeAgentId: "agent", status: phase === "final" ? "done" : "in_review" }, + runs, interactions: cards, children: [], attachments: [], comments: [], documents: phase === "final" ? [document] : [] })); + runs.push(createRun()); + for (const [index, path] of paths.entries()) { + const run = runs.at(-1)!; + const field = index === 0 ? { id: "choice", answerMode: "single_select", options: [{ id: "am", label: "Morning" }, { id: "pm", label: "Afternoon" }] } : { id: "reference", answerMode: "text" }; + const request = `request-${index}`, semantic = path === "semantic"; + const card: Row = { id: `question-${index}`, kind: "ask_user_questions", status: "pending", companyId: "company", issueId: "task", createdByAgentId: "agent", sourceRunId: run.id, + idempotencyKey: semantic ? `semantic-${index}` : `paperclip-runner-question:${run.id}:${request}`, continuationPolicy: semantic ? "wake_assignee" : "none", + payload: { ...(semantic ? {} : { runtimeRequestId: request }), questionSet: { questions: [field] } } }; + cards.push(card); + if (semantic) { run.status = "succeeded"; run.resultJson.semanticToolReceipts[card.idempotencyKey] = { operationId: "request_human_input", result: { disposition: "applied", interaction: structuredClone(card) } }; } + snapshot(index === 0 ? "initial" : "answered"); + card.status = "answered"; card.resolvedByUserId = "board"; + card.result = { answers: [{ questionId: field.id, optionIds: index === 0 ? ["pm"] : [], ...(index === 1 ? { otherText: "Use AMBERnonce." } : {}) }] }; + if (semantic) runs.push(createRun(card)); + } + runs.at(-1)!.status = "succeeded"; snapshot("final"); + return { checkpoints, readback: { issueId: "task", key: document.key, revisionId: document.latestRevisionId, body: document.body, visibleBody: document.body } }; +} +const failures = (f: ReturnType) => gradeClarificationEvidence(context, "task", "AMBERnonce", f.checkpoints, f.readback).filter(c => !c.passed); + +describe("attached clarification evidence", () => { + it.each([["provider", "provider"], ["provider", "semantic"], ["semantic", "provider"], ["semantic", "semantic"]] as const)("accepts the real %s/%s paths with native binding and reload proof", (...paths) => { + expect(failures(recording([...paths]))).toEqual([]); + }); + it.each(["foreign-user", "foreign-agent-document", "foreign-task-document", "lost-revision", "wrong-visible-answer", "changed-readback-body", "premature-document", "extra-child", "foreign-model", "foreign-environment", "missing-native-input", "duplicate-question", "wrong-answer"])("rejects a plausible false pass: %s", kind => { + const f = recording(["semantic", "semantic"]), first = f.checkpoints[0], final = f.checkpoints[2]; + if (kind === "foreign-user") final.interactions[1].resolvedByUserId = "other-board"; + if (kind === "foreign-agent-document") final.documents[0].updatedByAgentId = "other-agent"; + if (kind === "foreign-task-document") final.documents[0].issueId = "other-task"; + if (kind === "lost-revision") f.readback.revisionId = "old-revision"; + if (kind === "wrong-visible-answer") f.readback.visibleBody = "Welcome to the morning meetup."; + if (kind === "changed-readback-body") f.readback.body += " changed"; + if (kind === "premature-document") first.documents = [final.documents[0]]; + if (kind === "extra-child") final.children.push({ id: "extra-child" }); + if (kind === "foreign-model") final.runs[0].runnerProfileJson.nativeExecutionInput.provider.model = "other-model"; + if (kind === "foreign-environment") final.runs[0].contextSnapshot.paperclipEnvironment.id = "local"; + if (kind === "missing-native-input") delete final.runs[0].runnerProfileJson.nativeExecutionInput.task.prompt; + if (kind === "duplicate-question") final.interactions.push(structuredClone(final.interactions[1])); + if (kind === "wrong-answer") final.interactions[1].result.answers[0].otherText = "Use the old reference."; + expect(failures(f).length).toBeGreaterThan(0); + }); + it("requires reload evidence even for a correct API document", () => { + const f = recording(["provider", "provider"]); + expect(gradeClarificationEvidence(context, "task", "AMBERnonce", f.checkpoints).some(c => c.id === "question-document-reloaded" && !c.passed)).toBe(true); + }); + it("retains 1–3 bound native runs but rejects zero and excess work", () => { + const result = { schema: "paperclip.runner-e2e.result/v2", executionId: "product-journeys.attached.configured-agent.daytona.question-answer-resume", suiteId: "product-journeys", suiteDefinitionHash: "definition", + attempt: 1, status: "passed", profileId: "configured-agent", environmentId: "daytona", caseId: "question-answer-resume", provider: "opencode", model: "model", runtimeMode: "native", + startedAt: "2026-10-01T00:00:00Z", finishedAt: "2026-10-01T00:00:01Z", durationMs: 1000, cleanup: "passed", runIds: ["run-0"], + productJourney: { targetId: "attached", outcome: "passed", startingState: "recipe", privacy: "private", resourceDisposition: "retained", + phases: [{ phase: "journey", outcome: "passed" }], checks: journeyRequiredChecks["question-answer-resume"]!.map(id => ({ id, passed: true })), + deployment: { kind: "cloud", targetId: "attached", origin: "https://example.test", commit: "a".repeat(40), cloudRelease: "b".repeat(40), runtimeImage: "immutable-image", configurationHash: "config" } } }; + for (const count of [1, 2, 3]) expect(() => validateRetainedRunnerResult({ ...result, runIds: Array.from({ length: count }, (_, i) => `run-${i}`) })).not.toThrow(); + for (const count of [0, 4]) expect(() => validateRetainedRunnerResult({ ...result, runIds: Array.from({ length: count }, (_, i) => `run-${i}`) })).toThrow("result.agentJourneyRun"); + }); +}); diff --git a/tests/runner-e2e/product-journeys/clarification-evidence.ts b/tests/runner-e2e/product-journeys/clarification-evidence.ts new file mode 100644 index 00000000000..0cbec4bb17c --- /dev/null +++ b/tests/runner-e2e/product-journeys/clarification-evidence.ts @@ -0,0 +1,43 @@ +import { gradeQuestionResume } from "../question-resume-scoring.js"; +import type { ContinuationCheckpoint } from "../continuation-scoring.js"; +import { validateAgentTaskState } from "./agent-flows.js"; +import type { JourneyContext } from "./types.js"; + +type Row = Record; +export interface ClarificationReadback { + issueId: string; key: string; revisionId: string; body: string; visibleBody: string; +} + +export function gradeClarificationEvidence(context: JourneyContext, issueId: string, marker: string, + checkpoints: ContinuationCheckpoint[], readback?: ClarificationReadback) { + const checks = gradeQuestionResume(checkpoints, marker); + const final = checkpoints.find(point => point.phase === "final"); + const initial = checkpoints.filter(point => point.phase !== "final"); + const document = final?.documents[0] as Row | undefined; + let bound = !!context.agent && context.agent.runtimeMode === "native" && !!context.environmentId; + try { + for (const point of checkpoints) validateAgentTaskState(context, issueId, { + currentIssue: point.issue as Row & { id: string; status: string }, taskRuns: point.runs, + comments: point.comments as Row[], interactions: point.interactions as Array, attachments: [], + }, 3); + } catch { bound = false; } + checks.push({ id: "question-no-premature-output", passed: initial.length === 2 && initial.every(point => + point.documents.length === 0 && point.attachments.length === 0 && point.children.length === 0 && point.issue.status !== "done") + && !!final && final.issue.status === "done" && final.attachments.length === 0 && final.children.length === 0, + detail: "Wait for both real answers before producing the sole task document, with no extra task or download." }, + { id: "question-answers-owned-by-board", passed: !!final && final.interactions.length === 2 && final.interactions.every(value => { + const question = value as Row; + return question.status === "answered" && question.resolvedByUserId === context.user.id && !question.resolvedByAgentId; + }), detail: "Both retained responses must come from the signed-in board user." }, + { id: "question-document-attributed", passed: !!document && document.issueId === issueId && document.companyId === context.company.id + && document.createdByAgentId === context.agent?.id && document.updatedByAgentId === context.agent?.id, + detail: "The owned native agent created and last wrote the document on this exact task and company." }, + { id: "question-document-reloaded", passed: !!document && !!readback && readback.issueId === issueId + && readback.key === document.key && readback.revisionId === document.latestRevisionId && readback.body === document.body + && readback.visibleBody.includes(`Use ${marker}.`) && /\bafternoon\b/i.test(readback.visibleBody), + detail: "The reloaded document must show both saved answers at the independently read revision." }, + { id: "configured-model-and-environment", passed: bound && checkpoints.length === 3 && !!final + && final.runs.length >= 1 && final.runs.length <= 3 && final.runs.every(run => run.status === "succeeded"), + detail: "Every observed run belongs to the configured native agent, task, company, model and environment." }); + return checks; +} diff --git a/tests/runner-e2e/product-journeys/clarification-flow.ts b/tests/runner-e2e/product-journeys/clarification-flow.ts new file mode 100644 index 00000000000..4f62622227a --- /dev/null +++ b/tests/runner-e2e/product-journeys/clarification-flow.ts @@ -0,0 +1,108 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { continuationScenario } from "../continuation-cases.js"; +import { continuationAnswerCommitted, continuationCheckpointReady } from "../continuation-readiness.js"; +import type { ContinuationCheckpoint } from "../continuation-scoring.js"; +import { validResumeQuestionForm } from "../question-resume-scoring.js"; +import { answerResumeQuestionThroughUi } from "../question-resume-ui.js"; +import { createTaskThroughUi } from "../user-actions.js"; +import { validateAgentTaskState } from "./agent-flows.js"; +import { gradeClarificationEvidence, type ClarificationReadback } from "./clarification-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } + +export async function runClarificationJourney(context: JourneyContext, session: TargetSession, + campaignId: string, evidence: JourneyEvidence, options: Options) { + if (!context.agent || context.agent.runtimeMode !== "native" || !context.environmentId) { + throw new JourneyError("unsupported", "native_agent_and_environment_required"); + } + const agent = context.agent, scenario = continuationScenario("question-answer-resume", campaignId.replaceAll("-", "_")); + const checkpoints: ContinuationCheckpoint[] = []; + let issue: Row | undefined, readback: ClarificationReadback | undefined; + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + const retain = (name: string, value: unknown) => writeFile(path.join(options.directory, name), JSON.stringify(value, null, 2), { mode: 0o600 }); + const read = async () => { + const route = `/api/issues/${issue!.id}`; + const [currentIssue, summaries, interactions, documents, attachments, children] = await Promise.all([ + context.api.get(route), context.api.get(`${route}/runs`), context.api.get(`${route}/interactions`), + context.api.get(`${route}/documents`), context.api.get(`${route}/attachments`), + context.api.get(`/api/companies/${context.company.id}/issues?parentId=${encodeURIComponent(issue!.id)}&limit=2`), + ]); + if (!Array.isArray(summaries) || summaries.length > 3) throw new JourneyError("failed", "clarification_run_limit_exceeded"); + const runs = await Promise.all(summaries.map(summary => context.api.get(`/api/heartbeat-runs/${summary.runId}`))); + validateAgentTaskState(context, issue!.id, { currentIssue: currentIssue as Row & { id: string; status: string }, + taskRuns: runs as Array, interactions: interactions as Array, comments: [], attachments: [] }, 3); + if (!Array.isArray(children) || children.some(child => child.parentId !== issue!.id || child.companyId !== context.company.id)) { + throw new JourneyError("failed", "clarification_child_query_mismatch"); + } + return { currentIssue, runs, interactions, documents, attachments, children }; + }; + const settle = async (answeredId?: string, final = false) => { + let stable = ""; + while (Date.now() < options.deadlineAt) { + const state = await read(); + const pending = state.interactions.filter(i => i.kind === "ask_user_questions" && i.status === "pending"); + const ready = continuationAnswerCommitted(state.interactions, answeredId) + && continuationCheckpointReady({ issue: state.currentIssue as { status: string; executionRunId?: string }, runs: state.runs as Array, interactions: state.interactions }) + && (final ? state.currentIssue.status === "done" && !pending.length : pending.length === 1) + && !state.currentIssue.scheduledRetry && !state.currentIssue.activeRecoveryAction; + const key = ready ? JSON.stringify([state.currentIssue.status, state.currentIssue.executionRunId, + state.runs.map(run => [run.id, run.status]), state.interactions.map(i => [i.id, i.status])]) : ""; + if (key && key === stable) return state; + stable = key; + await new Promise(resolve => setTimeout(resolve, 1000)); + } + throw new JourneyError("failed", final ? "clarification_not_completed" : "clarification_question_not_ready"); + }; + const open = () => context.page.goto(`${context.api.origin}/${encodeURIComponent(context.company.issuePrefix)}/issues/${encodeURIComponent(issue!.identifier ?? issue!.id)}`, { waitUntil: "domcontentloaded" }); + const snapshot = async (phase: "initial" | "answered" | "final", state: Awaited>) => { + const documents = await Promise.all(state.documents.map(document => context.api.get(`/api/issues/${issue!.id}/documents/${encodeURIComponent(document.key)}`))); + const point = { phase, issue: state.currentIssue, runs: structuredClone(state.runs), interactions: structuredClone(state.interactions), + documents, attachments: state.attachments, children: state.children, comments: [] } as unknown as ContinuationCheckpoint; + checkpoints.push(point); + await retain("clarification.json", { checkpoints }); + await open(); + await options.capture(phase, `Clarification ${phase}`, `clarification-${phase}.png`); + return point; + }; + try { + const title = `Journey QA question-answer-resume ${campaignId}`; + const created = await createTaskThroughUi({ page: context.page, origin: context.api.origin, + issuePrefix: context.company.issuePrefix, agentName: agent.name, title, prompt: scenario.prompt, + workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: title, journeyId: "question-answer-resume" }) }); + await session.own({ kind: "issue", name: title, id: created.issueId, journeyId: "question-answer-resume" }); + issue = await context.api.get(`/api/issues/${created.issueId}`); + let state = await settle(); + for (const [index, input] of [{ choice: "Afternoon" }, { text: scenario.answer }].entries()) { + const point = await snapshot(index === 0 ? "initial" : "answered", state); + const question = state.interactions.find(i => i.kind === "ask_user_questions" && i.status === "pending")!; + if (!validResumeQuestionForm(point, question, index === 0)) throw new JourneyError("failed", "clarification_form_or_native_path_invalid"); + if (point.documents.length || point.attachments.length || point.children.length || point.issue.status === "done") { + throw new JourneyError("failed", "clarification_premature_output"); + } + await answerResumeQuestionThroughUi(context.page, question, input); + state = await settle(question.id, index === 1); + } + const final = await snapshot("final", state); + if (final.documents.length !== 1) throw new JourneyError("failed", "clarification_document_missing"); + const document = final.documents[0]!; + const href = `/${encodeURIComponent(context.company.issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}#document-${encodeURIComponent(document.key)}`; + await context.page.goto(new URL(href, context.api.origin).href, { waitUntil: "domcontentloaded" }); + await context.page.reload({ waitUntil: "domcontentloaded" }); + const visible = context.page.locator(`[id=${JSON.stringify(`document-${document.key}`)}]:visible, [id=${JSON.stringify(`side-panel-content-document:${document.key}`)}]:visible`); + await expect(visible).toHaveCount(1); await expect(visible).toBeVisible(); + await expect(visible).toContainText(scenario.answer); await expect(visible).toContainText(/afternoon/i); + const persisted = await context.api.get(`/api/issues/${issue.id}/documents/${encodeURIComponent(document.key)}`); + readback = { issueId: persisted.issueId, key: persisted.key, revisionId: persisted.latestRevisionId, body: persisted.body, visibleBody: await visible.innerText() }; + await options.capture("reloaded", "Saved answers after reload", "clarification-reloaded.png"); + } finally { + const checks = gradeClarificationEvidence(context, issue?.id ?? "", scenario.marker, checkpoints, readback); + evidence.checks.push(...checks.map(({ id, passed }) => ({ id, passed }))); + await retain("clarification.json", { checkpoints, readback, checks }); + } + if (evidence.checks.some(check => !check.passed)) throw new JourneyError("failed", "clarification_evidence_failed"); +} diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 601c1238832..99fd7cf2f03 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -11,6 +11,7 @@ import { journeyFixtures } from "./fixtures.js"; import { taskProperties } from "./task-properties.js"; import { askQuestion, taskCreate } from "./basic-flows.js"; import { runAgentJourney } from "./agent-flows.js"; +import { runClarificationJourney } from "./clarification-flow.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type JourneyOutcome, type TargetLifecycleInput, type TargetProvider, type TargetSession } from "./types.js"; import type { RunnerE2EResult } from "../types.js"; import { regenerateRunnerDashboard } from "../dashboard-regenerate.js"; @@ -102,7 +103,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["catalog.ts", "task-properties.ts", "basic-flows.ts", "basic-evidence.ts", "agent-flows.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of ["catalog.ts", "task-properties.ts", "basic-flows.ts", "basic-evidence.ts", "agent-flows.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); definitionHash.update(runnerSuites.flatMap(suite => suite.tasks).find(task => task.id === "ask-question")!.buildPrompt("definition")); const suiteDefinitionHash = definitionHash.digest("hex"); @@ -199,6 +200,13 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.model = ready.agent.model; result.runtimeMode = ready.agent.runtimeMode; if (id === "ask-question") await askQuestion(ready, session!, campaignId, evidence); + else if (id === "question-answer-resume") await runClarificationJourney(ready, session!, campaignId, evidence, { deadlineAt, + directory: path.join(outputDir, "private-stories", id), + capture: async (_id, _label, file) => { + const directory = path.join(outputDir, "private-stories", id); + await mkdir(directory, { recursive: true, mode: 0o700 }); + await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file)); + } }); else await runAgentJourney(id, ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id), capture: async (_id, _label, file) => { diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index 2c03d3de0ea..d31f58e23c1 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -1,10 +1,20 @@ /** Closed, target-independent success requirements used by dispatch and retained evidence. */ export const agentJourneyRuns = { "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, - "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3 } as const; + "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3 } as const; + +export function agentJourneyRunCountValid(id: string, count: unknown): boolean { + if (!Object.hasOwn(agentJourneyRuns, id) || !Number.isSafeInteger(count)) return false; + const maximum = agentJourneyRuns[id as keyof typeof agentJourneyRuns]; + const minimum = id === "question-answer-resume" ? 1 : maximum; + return (count as number) >= minimum && (count as number) <= maximum; +} export const journeyRequiredChecks: Record = { "task-properties": ["assignment-persisted", "no-agent-work"], "task-create": ["browser-created-task-persisted", "no-agent-work"], "ask-question": ["question-answer-persisted", "one-bound-successful-run", "configured-model-and-environment"], + "question-answer-resume": ["question-resume.forms", "question-resume.choice.answer", "question-resume.choice.path", + "question-resume.text.answer", "question-resume.text.path", "question-resume.complete-evidence", "question-no-premature-output", + "question-answers-owned-by-board", "question-document-attributed", "question-document-reloaded", "configured-model-and-environment"], "plan-revise-accept": ["plan-revision-and-approval", "plan-final-answer", "configured-model-and-environment"], "build-revise": ["initial-artifact-verified", "revised-artifact-verified", "artifact-versions-differ", "original-artifact-preserved", "user-reply-stored-once", "configured-model-and-environment"], "protected-actions": ["approve-waited-for-decision", "approve-decision-persisted", "approve-service-call-count", "approved-service-data-used", diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index 10e50287e0a..69b67eeabaa 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -3,7 +3,7 @@ import type { ConnectionApi } from "../connection-target.js"; export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; export type JourneyId = "task-properties" | "task-create" | "ask-question" | "cloud-onboarding" - | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect"; + | "question-answer-resume" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { kind: "self-hosted" | "cloud"; diff --git a/tests/runner-e2e/question-resume-ui.ts b/tests/runner-e2e/question-resume-ui.ts new file mode 100644 index 00000000000..ac6fa3f9de8 --- /dev/null +++ b/tests/runner-e2e/question-resume-ui.ts @@ -0,0 +1,23 @@ +import { expect, type Page } from "@playwright/test"; +import { chatQuestionPresentation } from "./chat-flow.js"; + +type Row = Record; + +export async function answerResumeQuestionThroughUi(page: Page, interaction: Row, input: { choice: string } | { text: string }) { + const set = chatQuestionPresentation(interaction.payload); + if ("choice" in input) { + expect(set.questions[0]?.answerMode, "choices must use radio controls").toBe("single_select"); + const options = interaction.payload.questionSet?.questions[0]?.options ?? interaction.payload.questions[0]?.options ?? []; + expect(new Set(options.map((o: Row) => String(o.label).trim().toLowerCase())).size).toBeGreaterThanOrEqual(2); + const choice = input.choice.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + await page.getByRole("radio", { name: new RegExp(`^${choice}\\b`, "i") }).last().click(); + } else { + expect(set.questions[0]?.answerMode, "open answers must render a text field, not a choice question").toBe("text"); + await page.getByTestId("question-text-answer-composer").last() + .locator('[contenteditable="true"],textarea').first().fill(input.text); + } + for (let index = 1; index < set.questions.length; index += 1) { + await page.getByRole("button", { name: "Next", exact: true }).last().click(); + } + await page.getByRole("button", { name: set.submitLabel ?? "Submit answers", exact: true }).last().click(); +} diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index b68c930b9fa..9b64b8bdd8d 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -1,4 +1,4 @@ -import { agentJourneyRuns, journeyRequiredChecks } from "./product-journeys/success-contract.js"; +import { agentJourneyRuns, agentJourneyRunCountValid, journeyRequiredChecks } from "./product-journeys/success-contract.js"; import { validateWorkerTiming } from "./worker-timing.js"; import { connectionCheckpoints, connectionEvidencePasses, connectionRunSignals } from "./connection-evidence.js"; import { githubJourneyCheckpoints, gradeGitHubReviewJourney } from "./github-review-evidence.js"; @@ -435,7 +435,7 @@ export function validateRetainedRunnerResult( const required = journeyRequiredChecks[checked.caseId]; if (evidence.outcome !== "passed" || !evidence.deployment || !required || !required.every(id => evidence.checks.some(c => c.id === id && c.passed)) || evidence.checks.some(check => !check.passed) || evidence.phases.some(p => p.outcome !== "passed") || checked.cleanup !== "passed") invalid("result.productJourney"); if (["task-properties", "task-create"].includes(checked.caseId) && (checked.runtimeMode !== "none" || checked.provider !== null || checked.model !== null || checked.runIds?.length)) invalid("result.browserOnly"); - if (Object.hasOwn(agentJourneyRuns, checked.caseId) && (checked.runIds?.length !== agentJourneyRuns[checked.caseId as keyof typeof agentJourneyRuns] + if (Object.hasOwn(agentJourneyRuns, checked.caseId) && (!agentJourneyRunCountValid(checked.caseId, checked.runIds?.length) || !checked.model || !checked.provider || checked.runtimeMode !== "native")) invalid("result.agentJourneyRun"); if (checked.caseId === "cloud-onboarding" && (!checked.runIds?.length || !evidence.operations || !evidence.operations.requestCount || !evidence.operations.traceIds.length || evidence.operations.campaignId !== evidence.campaignId || evidence.operations.attemptId !== evidence.attemptId)) invalid("result.paidJourneyReceipt"); }