diff --git a/tests/runner-e2e/product-journeys/clarification-evidence.test.ts b/tests/runner-e2e/product-journeys/clarification-evidence.test.ts index 044ef2df030..d80af59931b 100644 --- a/tests/runner-e2e/product-journeys/clarification-evidence.test.ts +++ b/tests/runner-e2e/product-journeys/clarification-evidence.test.ts @@ -19,7 +19,7 @@ describe("portable clarification journey admission", () => { }); }); -import { gradeClarificationEvidence } from "./clarification-evidence.js"; +import { clarificationQuestionFailure, gradeClarificationEvidence } from "./clarification-evidence.js"; import { continuationScenario } from "../continuation-cases.js"; import { validateRetainedRunnerResult } from "../result-validation.js"; import { journeyRequiredChecks } from "./success-contract.js"; @@ -59,6 +59,23 @@ function recording(paths: Array<"provider" | "semantic">) { const failures = (f: ReturnType) => gradeClarificationEvidence(context, "task", "AMBERnonce", f.checkpoints, f.readback).filter(c => !c.passed); describe("attached clarification evidence", () => { + it.each([true, false])("reports API fallback without accepting it as native question evidence (choice=%s)", choice => { + const f = recording(["semantic", "semantic"]), point = f.checkpoints[choice ? 0 : 1]; + const question = point.interactions.find((i: Row) => i.status === "pending"), source = point.runs.at(-1); + source.resultJson = { apiToolReceipts: { call: { state: "completed", operationId: "POST /api/issues/{id}/interactions", + result: { ok: true, data: { id: question.id } } } } }; + expect(clarificationQuestionFailure(point, question, choice)).toBe(`clarification_${choice ? "choice" : "text"}_api_fallback`); + expect(failures(f).some(check => check.id === `question-resume.${choice ? "choice" : "text"}.path`)).toBe(true); + delete source.resultJson.apiToolReceipts; + expect(clarificationQuestionFailure(point, question, choice)).toBe(`clarification_${choice ? "choice" : "text"}_native_path_invalid`); + }); + it("distinguishes an invalid form from a correctly bound semantic question", () => { + const point = recording(["semantic", "semantic"]).checkpoints[0], question = point.interactions[0]; + expect(clarificationQuestionFailure(point, question, true)).toBeUndefined(); + question.payload.questionSet.questions[0].answerMode = "text"; + point.runs[0].resultJson.semanticToolReceipts[question.idempotencyKey].result.interaction.payload = structuredClone(question.payload); + expect(clarificationQuestionFailure(point, question, true)).toBe("clarification_choice_form_invalid"); + }); it.each([["provider", "provider"], ["provider", "semantic"], ["semantic", "provider"], ["semantic", "semantic"]] as const)("accepts the real %s/%s paths with native binding and reload proof", (...paths) => { expect(failures(recording([...paths]))).toEqual([]); }); diff --git a/tests/runner-e2e/product-journeys/clarification-evidence.ts b/tests/runner-e2e/product-journeys/clarification-evidence.ts index 0cbec4bb17c..69e39b7fa62 100644 --- a/tests/runner-e2e/product-journeys/clarification-evidence.ts +++ b/tests/runner-e2e/product-journeys/clarification-evidence.ts @@ -1,4 +1,4 @@ -import { gradeQuestionResume } from "../question-resume-scoring.js"; +import { gradeQuestionResume, questionPath, validResumeQuestionForm } from "../question-resume-scoring.js"; import type { ContinuationCheckpoint } from "../continuation-scoring.js"; import { validateAgentTaskState } from "./agent-flows.js"; import type { JourneyContext } from "./types.js"; @@ -8,6 +8,20 @@ export interface ClarificationReadback { issueId: string; key: string; revisionId: string; body: string; visibleBody: string; } +export function clarificationQuestionFailure(point: ContinuationCheckpoint, question: Row, choice: boolean) { + const stage = choice ? "choice" : "text"; + if (!questionPath(point, question)) { + const source = point.runs.find(run => run.id === question.sourceRunId) as Row | undefined; + const apiCreated = Object.values(source?.resultJson?.apiToolReceipts ?? {}).some(value => { + const receipt = value as Row; + return receipt.state === "completed" && receipt.operationId === "POST /api/issues/{id}/interactions" + && receipt.result?.ok === true && receipt.result?.data?.id === question.id; + }); + return apiCreated ? `clarification_${stage}_api_fallback` : `clarification_${stage}_native_path_invalid`; + } + return validResumeQuestionForm(point, question, choice) ? undefined : `clarification_${stage}_form_invalid`; +} + export function gradeClarificationEvidence(context: JourneyContext, issueId: string, marker: string, checkpoints: ContinuationCheckpoint[], readback?: ClarificationReadback) { const checks = gradeQuestionResume(checkpoints, marker); diff --git a/tests/runner-e2e/product-journeys/clarification-flow.ts b/tests/runner-e2e/product-journeys/clarification-flow.ts index e57167115f0..932efbebf92 100644 --- a/tests/runner-e2e/product-journeys/clarification-flow.ts +++ b/tests/runner-e2e/product-journeys/clarification-flow.ts @@ -4,11 +4,10 @@ import { expect } from "@playwright/test"; import { continuationScenario } from "../continuation-cases.js"; import { continuationAnswerCommitted, continuationCheckpointReady } from "../continuation-readiness.js"; import type { ContinuationCheckpoint } from "../continuation-scoring.js"; -import { validResumeQuestionForm } from "../question-resume-scoring.js"; import { answerResumeQuestionThroughUi } from "../question-resume-ui.js"; import { createTaskThroughUi } from "../user-actions.js"; import { validateAgentTaskState } from "./agent-flows.js"; -import { gradeClarificationEvidence, type ClarificationReadback } from "./clarification-evidence.js"; +import { clarificationQuestionFailure, gradeClarificationEvidence, type ClarificationReadback } from "./clarification-evidence.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; type Row = Record; @@ -82,7 +81,8 @@ export async function runClarificationJourney(context: JourneyContext, session: for (const [index, input] of [{ choice: "Afternoon" }, { text: scenario.answer }].entries()) { const point = await snapshot(index === 0 ? "initial" : "answered", state); const question = state.interactions.find(i => i.kind === "ask_user_questions" && i.status === "pending")!; - if (!validResumeQuestionForm(point, question, index === 0)) throw new JourneyError("failed", "clarification_form_or_native_path_invalid"); + const questionFailure = clarificationQuestionFailure(point, question, index === 0); + if (questionFailure) throw new JourneyError("failed", questionFailure); if (point.documents.length || point.attachments.length || point.children.length || point.issue.status === "done") { throw new JourneyError("failed", "clarification_premature_output"); }