From 23e716fcc754bb080352b15c0aa76380c317f395 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 12:42:17 -0500 Subject: [PATCH] Validate clarification receipts through campaign collection Co-Authored-By: Paperclip --- tests/runner-e2e/catalog.test.ts | 2 +- .../clarification-campaign.test.ts | 59 +++++++++++++++++++ .../clarification-flow.test.ts | 35 +++++++++++ .../product-journeys/clarification-flow.ts | 34 ++++++----- tests/runner-e2e/product-journeys/launch.ts | 5 +- 5 files changed, 116 insertions(+), 19 deletions(-) create mode 100644 tests/runner-e2e/product-journeys/clarification-campaign.test.ts create mode 100644 tests/runner-e2e/product-journeys/clarification-flow.test.ts diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 1bbde4a9cd4..c2855083c2a 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -59,7 +59,7 @@ describe("runner E2E catalog", () => { }); it("exports deployment journeys separately from the model matrix and default selection", () => { - expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "ask-question", "task-properties", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]); + expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "ask-question", "question-answer-resume", "task-properties", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); diff --git a/tests/runner-e2e/product-journeys/clarification-campaign.test.ts b/tests/runner-e2e/product-journeys/clarification-campaign.test.ts new file mode 100644 index 00000000000..95a93835e42 --- /dev/null +++ b/tests/runner-e2e/product-journeys/clarification-campaign.test.ts @@ -0,0 +1,59 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { journeyRequiredChecks } from "./success-contract.js"; + +vi.mock("@playwright/test", () => ({ chromium: { launch: async () => ({ + newContext: async () => ({ setDefaultTimeout() {}, pages: () => [] }), close: async () => {}, +}) }, expect: {} })); +vi.mock("./clarification-flow.js", () => ({ runClarificationJourney: async (_context: unknown, _session: unknown, + _campaignId: unknown, evidence: { checks: Array<{ id: string; passed: boolean }> }) => { + evidence.checks.push(...journeyRequiredChecks["question-answer-resume"]!.map(id => ({ id, passed: true }))); +} })); +vi.mock("../dashboard-regenerate.js", () => ({ regenerateRunnerDashboard: async ({ outputDirectory }: { outputDirectory: string }) => { + await Promise.all(["index.html", "dashboard.html"].map(file => writeFile(path.join(outputDirectory, file), "dashboard"))); +} })); +import { launchProductJourneys } from "./launch.js"; + +const roots: string[] = []; +afterEach(async () => { process.exitCode = 0; await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function campaign(count: number, omitReceipt = false) { + const root = await mkdtemp(path.join(os.tmpdir(), "clarification-collection-")); roots.push(root); + const module = path.join(root, "provider.mjs"), output = path.join(root, "report"); + const runIds = Array.from({ length: count }, (_, i) => `bound-run-${i}`); + const receipts = omitReceipt ? [] : [{ journeyId: "question-answer-resume", runIds, + usage: { runs: runIds.map(runId => ({ runId, usage: { costUsd: 0.01 } })) } }]; + await writeFile(module, ` +const identity = { kind: 'cloud', targetId: 'qa', origin: 'https://qa.example.test', commit: '${"a".repeat(40)}', cloudRelease: 'v1', runtimeImage: 'image', configurationHash: 'config' }; +const context = { identity, company: { id: 'company', issuePrefix: 'QA' }, user: { id: 'board', name: 'QA' }, + api: { origin: identity.origin }, startingState: 'recipe', capabilities: ['company', 'board-user', 'task-write', 'agent'], + agent: { id: 'agent', name: 'QA', provider: 'opencode', model: 'model', runtimeMode: 'native' }, environmentId: 'daytona', + page: { url: () => identity.origin + '/QA/issues/QA-1', locator: () => ({ isVisible: async () => true, screenshot: async () => {} }) } }; +export const provider = { version: 1, async acquire() { return { + async authenticate() {}, async prepare() { return context; }, async identity() { return identity; }, + async collect() { return ${JSON.stringify(receipts)}; }, async own() {}, async release() { return 'released'; }, async stop() {} +}; } };`); + const file = path.join(root, "target.json"); + await writeFile(file, JSON.stringify({ version: 1, id: "qa", mode: "existing-cloud", adapterModule: module, + journeys: ["question-answer-resume"], executionEnvironment: "daytona", baseline: { features: {}, connections: [] }, + limits: { durationMs: 60_000, modelBudgetCents: 100, maxRuns: 3 } })); + await launchProductJourneys(["--target-config", file, "--output", output], path.resolve(import.meta.dirname, "../../..")); + return JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8")).results[0]; +} + +it.each([1, 2, 3])("collects a successful clarification journey with %i native work runs", async count => { + const result = await campaign(count); + expect(result).toMatchObject({ status: "passed", cleanup: "passed", productJourney: { outcome: "passed" } }); + expect(result.runIds).toHaveLength(count); + expect(result.productJourney.phases).toContainEqual({ phase: "collection", outcome: "passed" }); + expect(result.usage.runs).toHaveLength(count); +}); +it.each([0, 4])("rejects a clarification collection with %i native work runs", async count => { + const result = await campaign(count); + expect(result).toMatchObject({ status: "failed", error: "paid_journey_receipt_missing" }); + expect(result.runIds).toHaveLength(count); +}); +it("rejects a passing story when its provider omits collection entirely", async () => { + expect(await campaign(1, true)).toMatchObject({ status: "failed", error: "paid_journey_receipt_missing" }); +}); diff --git a/tests/runner-e2e/product-journeys/clarification-flow.test.ts b/tests/runner-e2e/product-journeys/clarification-flow.test.ts new file mode 100644 index 00000000000..97711ca526a --- /dev/null +++ b/tests/runner-e2e/product-journeys/clarification-flow.test.ts @@ -0,0 +1,35 @@ +import { expect, it, vi } from "vitest"; +import { readClarificationState } from "./clarification-flow.js"; +import type { JourneyContext } from "./types.js"; + +const issueId = "11111111-1111-4111-8111-111111111111"; +const companyId = "22222222-2222-4222-8222-222222222222"; +const parent = { id: issueId, companyId, parentId: null, assigneeAgentId: "agent", status: "in_progress" }; +const child = { id: "child", companyId, parentId: issueId }; +function fixture(rows = [parent, child]) { + const get = vi.fn(async (route: string) => { + const url = new URL(route, "https://qa.example.test"); + if (url.pathname === `/api/companies/${companyId}/issues`) { + if (url.searchParams.get("view") !== "index") throw Error("unsupported child projection"); + return rows.filter(row => row.parentId === url.searchParams.get("parentIds")); + } + if (url.pathname === `/api/issues/${issueId}`) return parent; + return []; + }); + const context = { company: { id: companyId }, environmentId: "daytona", api: { get }, + agent: { id: "agent", provider: "opencode", model: "model", runtimeMode: "native" } } as unknown as JourneyContext; + return { context, get }; +} +it("reads the supported child projection while excluding the journey task and unrelated company work", async () => { + const { context, get } = fixture([parent, child, { id: "unrelated", companyId, parentId: "other" }]); + expect((await readClarificationState(context, issueId)).children).toEqual([child]); + expect(get).toHaveBeenCalledWith(`/api/companies/${companyId}/issues?view=index&parentIds=${issueId}&limit=2`); +}); +it("retains an empty child result for a task that did not delegate work", async () => { + expect((await readClarificationState(fixture([parent]).context, issueId)).children).toEqual([]); +}); +it.each([{ ...child, companyId: "other" }, { ...child, parentId: "other" }])("rejects a returned child outside the owned company or task", async foreign => { + const { context, get } = fixture(); + get.mockImplementation(async route => route.includes("/companies/") ? [foreign] : route === `/api/issues/${issueId}` ? parent : []); + await expect(readClarificationState(context, issueId)).rejects.toMatchObject({ code: "clarification_child_query_mismatch" }); +}); diff --git a/tests/runner-e2e/product-journeys/clarification-flow.ts b/tests/runner-e2e/product-journeys/clarification-flow.ts index 4f62622227a..e57167115f0 100644 --- a/tests/runner-e2e/product-journeys/clarification-flow.ts +++ b/tests/runner-e2e/product-journeys/clarification-flow.ts @@ -14,6 +14,23 @@ import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSes type Row = Record; interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } +export async function readClarificationState(context: JourneyContext, issueId: string) { + const route = `/api/issues/${issueId}`; + const [currentIssue, summaries, interactions, documents, attachments, children] = await Promise.all([ + context.api.get(route), context.api.get(`${route}/runs`), context.api.get(`${route}/interactions`), + context.api.get(`${route}/documents`), context.api.get(`${route}/attachments`), + context.api.get(`/api/companies/${context.company.id}/issues?view=index&parentIds=${encodeURIComponent(issueId)}&limit=2`), + ]); + if (!Array.isArray(summaries) || summaries.length > 3) throw new JourneyError("failed", "clarification_run_limit_exceeded"); + const runs = await Promise.all(summaries.map(summary => context.api.get(`/api/heartbeat-runs/${summary.runId}`))); + validateAgentTaskState(context, issueId, { currentIssue: currentIssue as Row & { id: string; status: string }, + taskRuns: runs as Array, interactions: interactions as Array, comments: [], attachments: [] }, 3); + if (!Array.isArray(children) || children.some(child => child.parentId !== issueId || child.companyId !== context.company.id)) { + throw new JourneyError("failed", "clarification_child_query_mismatch"); + } + return { currentIssue, runs, interactions, documents, attachments, children }; +} + export async function runClarificationJourney(context: JourneyContext, session: TargetSession, campaignId: string, evidence: JourneyEvidence, options: Options) { if (!context.agent || context.agent.runtimeMode !== "native" || !context.environmentId) { @@ -24,22 +41,7 @@ export async function runClarificationJourney(context: JourneyContext, session: let issue: Row | undefined, readback: ClarificationReadback | undefined; await mkdir(options.directory, { recursive: true, mode: 0o700 }); const retain = (name: string, value: unknown) => writeFile(path.join(options.directory, name), JSON.stringify(value, null, 2), { mode: 0o600 }); - const read = async () => { - const route = `/api/issues/${issue!.id}`; - const [currentIssue, summaries, interactions, documents, attachments, children] = await Promise.all([ - context.api.get(route), context.api.get(`${route}/runs`), context.api.get(`${route}/interactions`), - context.api.get(`${route}/documents`), context.api.get(`${route}/attachments`), - context.api.get(`/api/companies/${context.company.id}/issues?parentId=${encodeURIComponent(issue!.id)}&limit=2`), - ]); - if (!Array.isArray(summaries) || summaries.length > 3) throw new JourneyError("failed", "clarification_run_limit_exceeded"); - const runs = await Promise.all(summaries.map(summary => context.api.get(`/api/heartbeat-runs/${summary.runId}`))); - validateAgentTaskState(context, issue!.id, { currentIssue: currentIssue as Row & { id: string; status: string }, - taskRuns: runs as Array, interactions: interactions as Array, comments: [], attachments: [] }, 3); - if (!Array.isArray(children) || children.some(child => child.parentId !== issue!.id || child.companyId !== context.company.id)) { - throw new JourneyError("failed", "clarification_child_query_mismatch"); - } - return { currentIssue, runs, interactions, documents, attachments, children }; - }; + const read = () => readClarificationState(context, issue!.id); const settle = async (answeredId?: string, final = false) => { let stable = ""; while (Date.now() < options.deadlineAt) { diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 99fd7cf2f03..3f39f221870 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -6,7 +6,8 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { chromium, type Browser, type BrowserContext } from "@playwright/test"; import { runnerSuites } from "../catalog.js"; -import { productJourneys, parseTargetConfig, preparationPreview, isAgentJourney, agentJourneyRuns } from "./catalog.js"; +import { productJourneys, parseTargetConfig, preparationPreview, isAgentJourney } from "./catalog.js"; +import { agentJourneyRunCountValid } from "./success-contract.js"; import { journeyFixtures } from "./fixtures.js"; import { taskProperties } from "./task-properties.js"; import { askQuestion, taskCreate } from "./basic-flows.js"; @@ -269,7 +270,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri } for (const result of results) { if (isAgentJourney(result.caseId as import("./types.js").JourneyId) && result.status === "passed" - && (!seen.has(result.caseId) || result.runIds?.length !== agentJourneyRuns[result.caseId as keyof typeof agentJourneyRuns])) throw new JourneyError("failed", "paid_journey_receipt_missing"); + && (!seen.has(result.caseId) || !agentJourneyRunCountValid(result.caseId, result.runIds?.length))) throw new JourneyError("failed", "paid_journey_receipt_missing"); if (result.caseId === "cloud-onboarding" && result.status === "passed" && (!seen.has(result.caseId) || !result.runIds?.length || !result.productJourney?.operations || !result.productJourney.operations.traceIds.length)) throw new JourneyError("failed", "paid_journey_receipt_missing"); // Success-only checks must run before a disposable target can be deleted. // Cleanup is provisional here; the observed cleanup result is retained below.