Validate clarification receipts through campaign collection

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/catalog.test.ts

@@ -59,7 +59,7 @@ describe("runner E2E catalog", () => {
   });
 
   it("exports deployment journeys separately from the model matrix and default selection", () => {
-    expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "ask-question", "task-properties", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]);
+    expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "ask-question", "question-answer-resume", "task-properties", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]);
     expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false);
     expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false);
     expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false);

tests/runner-e2e/product-journeys/clarification-campaign.test.ts

@@ -0,0 +1,59 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+import { journeyRequiredChecks } from "./success-contract.js";
+
+vi.mock("@playwright/test", () => ({ chromium: { launch: async () => ({
+  newContext: async () => ({ setDefaultTimeout() {}, pages: () => [] }), close: async () => {},
+}) }, expect: {} }));
+vi.mock("./clarification-flow.js", () => ({ runClarificationJourney: async (_context: unknown, _session: unknown,
+  _campaignId: unknown, evidence: { checks: Array<{ id: string; passed: boolean }> }) => {
+  evidence.checks.push(...journeyRequiredChecks["question-answer-resume"]!.map(id => ({ id, passed: true })));
+} }));
+vi.mock("../dashboard-regenerate.js", () => ({ regenerateRunnerDashboard: async ({ outputDirectory }: { outputDirectory: string }) => {
+  await Promise.all(["index.html", "dashboard.html"].map(file => writeFile(path.join(outputDirectory, file), "dashboard")));
+} }));
+import { launchProductJourneys } from "./launch.js";
+
+const roots: string[] = [];
+afterEach(async () => { process.exitCode = 0; await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+async function campaign(count: number, omitReceipt = false) {
+  const root = await mkdtemp(path.join(os.tmpdir(), "clarification-collection-")); roots.push(root);
+  const module = path.join(root, "provider.mjs"), output = path.join(root, "report");
+  const runIds = Array.from({ length: count }, (_, i) => `bound-run-${i}`);
+  const receipts = omitReceipt ? [] : [{ journeyId: "question-answer-resume", runIds,
+    usage: { runs: runIds.map(runId => ({ runId, usage: { costUsd: 0.01 } })) } }];
+  await writeFile(module, `
+const identity = { kind: 'cloud', targetId: 'qa', origin: 'https://qa.example.test', commit: '${"a".repeat(40)}', cloudRelease: 'v1', runtimeImage: 'image', configurationHash: 'config' };
+const context = { identity, company: { id: 'company', issuePrefix: 'QA' }, user: { id: 'board', name: 'QA' },
+  api: { origin: identity.origin }, startingState: 'recipe', capabilities: ['company', 'board-user', 'task-write', 'agent'],
+  agent: { id: 'agent', name: 'QA', provider: 'opencode', model: 'model', runtimeMode: 'native' }, environmentId: 'daytona',
+  page: { url: () => identity.origin + '/QA/issues/QA-1', locator: () => ({ isVisible: async () => true, screenshot: async () => {} }) } };
+export const provider = { version: 1, async acquire() { return {
+  async authenticate() {}, async prepare() { return context; }, async identity() { return identity; },
+  async collect() { return ${JSON.stringify(receipts)}; }, async own() {}, async release() { return 'released'; }, async stop() {}
+}; } };`);
+  const file = path.join(root, "target.json");
+  await writeFile(file, JSON.stringify({ version: 1, id: "qa", mode: "existing-cloud", adapterModule: module,
+    journeys: ["question-answer-resume"], executionEnvironment: "daytona", baseline: { features: {}, connections: [] },
+    limits: { durationMs: 60_000, modelBudgetCents: 100, maxRuns: 3 } }));
+  await launchProductJourneys(["--target-config", file, "--output", output], path.resolve(import.meta.dirname, "../../.."));
+  return JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8")).results[0];
+}
+
+it.each([1, 2, 3])("collects a successful clarification journey with %i native work runs", async count => {
+  const result = await campaign(count);
+  expect(result).toMatchObject({ status: "passed", cleanup: "passed", productJourney: { outcome: "passed" } });
+  expect(result.runIds).toHaveLength(count);
+  expect(result.productJourney.phases).toContainEqual({ phase: "collection", outcome: "passed" });
+  expect(result.usage.runs).toHaveLength(count);
+});
+it.each([0, 4])("rejects a clarification collection with %i native work runs", async count => {
+  const result = await campaign(count);
+  expect(result).toMatchObject({ status: "failed", error: "paid_journey_receipt_missing" });
+  expect(result.runIds).toHaveLength(count);
+});
+it("rejects a passing story when its provider omits collection entirely", async () => {
+  expect(await campaign(1, true)).toMatchObject({ status: "failed", error: "paid_journey_receipt_missing" });
+});

tests/runner-e2e/product-journeys/clarification-flow.test.ts

@@ -0,0 +1,35 @@
+import { expect, it, vi } from "vitest";
+import { readClarificationState } from "./clarification-flow.js";
+import type { JourneyContext } from "./types.js";
+
+const issueId = "11111111-1111-4111-8111-111111111111";
+const companyId = "22222222-2222-4222-8222-222222222222";
+const parent = { id: issueId, companyId, parentId: null, assigneeAgentId: "agent", status: "in_progress" };
+const child = { id: "child", companyId, parentId: issueId };
+function fixture(rows = [parent, child]) {
+  const get = vi.fn(async (route: string) => {
+    const url = new URL(route, "https://qa.example.test");
+    if (url.pathname === `/api/companies/${companyId}/issues`) {
+      if (url.searchParams.get("view") !== "index") throw Error("unsupported child projection");
+      return rows.filter(row => row.parentId === url.searchParams.get("parentIds"));
+    }
+    if (url.pathname === `/api/issues/${issueId}`) return parent;
+    return [];
+  });
+  const context = { company: { id: companyId }, environmentId: "daytona", api: { get },
+    agent: { id: "agent", provider: "opencode", model: "model", runtimeMode: "native" } } as unknown as JourneyContext;
+  return { context, get };
+}
+it("reads the supported child projection while excluding the journey task and unrelated company work", async () => {
+  const { context, get } = fixture([parent, child, { id: "unrelated", companyId, parentId: "other" }]);
+  expect((await readClarificationState(context, issueId)).children).toEqual([child]);
+  expect(get).toHaveBeenCalledWith(`/api/companies/${companyId}/issues?view=index&parentIds=${issueId}&limit=2`);
+});
+it("retains an empty child result for a task that did not delegate work", async () => {
+  expect((await readClarificationState(fixture([parent]).context, issueId)).children).toEqual([]);
+});
+it.each([{ ...child, companyId: "other" }, { ...child, parentId: "other" }])("rejects a returned child outside the owned company or task", async foreign => {
+  const { context, get } = fixture();
+  get.mockImplementation(async route => route.includes("/companies/") ? [foreign] : route === `/api/issues/${issueId}` ? parent : []);
+  await expect(readClarificationState(context, issueId)).rejects.toMatchObject({ code: "clarification_child_query_mismatch" });
+});

tests/runner-e2e/product-journeys/clarification-flow.ts

@@ -14,6 +14,23 @@ import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSes
 type Row = Record<string, any>;
 interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise<void> }
 
+export async function readClarificationState(context: JourneyContext, issueId: string) {
+  const route = `/api/issues/${issueId}`;
+  const [currentIssue, summaries, interactions, documents, attachments, children] = await Promise.all([
+    context.api.get<Row>(route), context.api.get<Row[]>(`${route}/runs`), context.api.get<Row[]>(`${route}/interactions`),
+    context.api.get<Row[]>(`${route}/documents`), context.api.get<Row[]>(`${route}/attachments`),
+    context.api.get<Row[]>(`/api/companies/${context.company.id}/issues?view=index&parentIds=${encodeURIComponent(issueId)}&limit=2`),
+  ]);
+  if (!Array.isArray(summaries) || summaries.length > 3) throw new JourneyError("failed", "clarification_run_limit_exceeded");
+  const runs = await Promise.all(summaries.map(summary => context.api.get<Row>(`/api/heartbeat-runs/${summary.runId}`)));
+  validateAgentTaskState(context, issueId, { currentIssue: currentIssue as Row & { id: string; status: string },
+    taskRuns: runs as Array<Row & { status: string }>, interactions: interactions as Array<Row & { id: string; status: string }>, comments: [], attachments: [] }, 3);
+  if (!Array.isArray(children) || children.some(child => child.parentId !== issueId || child.companyId !== context.company.id)) {
+    throw new JourneyError("failed", "clarification_child_query_mismatch");
+  }
+  return { currentIssue, runs, interactions, documents, attachments, children };
+}
+
 export async function runClarificationJourney(context: JourneyContext, session: TargetSession,
   campaignId: string, evidence: JourneyEvidence, options: Options) {
   if (!context.agent || context.agent.runtimeMode !== "native" || !context.environmentId) {
@@ -24,22 +41,7 @@ export async function runClarificationJourney(context: JourneyContext, session:
   let issue: Row | undefined, readback: ClarificationReadback | undefined;
   await mkdir(options.directory, { recursive: true, mode: 0o700 });
   const retain = (name: string, value: unknown) => writeFile(path.join(options.directory, name), JSON.stringify(value, null, 2), { mode: 0o600 });
-  const read = async () => {
-    const route = `/api/issues/${issue!.id}`;
-    const [currentIssue, summaries, interactions, documents, attachments, children] = await Promise.all([
-      context.api.get<Row>(route), context.api.get<Row[]>(`${route}/runs`), context.api.get<Row[]>(`${route}/interactions`),
-      context.api.get<Row[]>(`${route}/documents`), context.api.get<Row[]>(`${route}/attachments`),
-      context.api.get<Row[]>(`/api/companies/${context.company.id}/issues?parentId=${encodeURIComponent(issue!.id)}&limit=2`),
-    ]);
-    if (!Array.isArray(summaries) || summaries.length > 3) throw new JourneyError("failed", "clarification_run_limit_exceeded");
-    const runs = await Promise.all(summaries.map(summary => context.api.get<Row>(`/api/heartbeat-runs/${summary.runId}`)));
-    validateAgentTaskState(context, issue!.id, { currentIssue: currentIssue as Row & { id: string; status: string },
-      taskRuns: runs as Array<Row & { status: string }>, interactions: interactions as Array<Row & { id: string; status: string }>, comments: [], attachments: [] }, 3);
-    if (!Array.isArray(children) || children.some(child => child.parentId !== issue!.id || child.companyId !== context.company.id)) {
-      throw new JourneyError("failed", "clarification_child_query_mismatch");
-    }
-    return { currentIssue, runs, interactions, documents, attachments, children };
-  };
+  const read = () => readClarificationState(context, issue!.id);
   const settle = async (answeredId?: string, final = false) => {
     let stable = "";
     while (Date.now() < options.deadlineAt) {

tests/runner-e2e/product-journeys/launch.ts

@@ -6,7 +6,8 @@ import path from "node:path";
 import { pathToFileURL } from "node:url";
 import { chromium, type Browser, type BrowserContext } from "@playwright/test";
 import { runnerSuites } from "../catalog.js";
-import { productJourneys, parseTargetConfig, preparationPreview, isAgentJourney, agentJourneyRuns } from "./catalog.js";
+import { productJourneys, parseTargetConfig, preparationPreview, isAgentJourney } from "./catalog.js";
+import { agentJourneyRunCountValid } from "./success-contract.js";
 import { journeyFixtures } from "./fixtures.js";
 import { taskProperties } from "./task-properties.js";
 import { askQuestion, taskCreate } from "./basic-flows.js";
@@ -269,7 +270,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri
       }
       for (const result of results) {
         if (isAgentJourney(result.caseId as import("./types.js").JourneyId) && result.status === "passed"
-          && (!seen.has(result.caseId) || result.runIds?.length !== agentJourneyRuns[result.caseId as keyof typeof agentJourneyRuns])) throw new JourneyError("failed", "paid_journey_receipt_missing");
+          && (!seen.has(result.caseId) || !agentJourneyRunCountValid(result.caseId, result.runIds?.length))) throw new JourneyError("failed", "paid_journey_receipt_missing");
         if (result.caseId === "cloud-onboarding" && result.status === "passed" && (!seen.has(result.caseId) || !result.runIds?.length || !result.productJourney?.operations || !result.productJourney.operations.traceIds.length)) throw new JourneyError("failed", "paid_journey_receipt_missing");
         // Success-only checks must run before a disposable target can be deleted.
         // Cleanup is provisional here; the observed cleanup result is retained below.