Distinguish clarification form and native provenance failures

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/product-journeys/clarification-evidence.test.ts

@@ -19,7 +19,7 @@ describe("portable clarification journey admission", () => {
   });
 });
 
-import { gradeClarificationEvidence } from "./clarification-evidence.js";
+import { clarificationQuestionFailure, gradeClarificationEvidence } from "./clarification-evidence.js";
 import { continuationScenario } from "../continuation-cases.js";
 import { validateRetainedRunnerResult } from "../result-validation.js";
 import { journeyRequiredChecks } from "./success-contract.js";
@@ -59,6 +59,23 @@ function recording(paths: Array<"provider" | "semantic">) {
 const failures = (f: ReturnType<typeof recording>) => gradeClarificationEvidence(context, "task", "AMBERnonce", f.checkpoints, f.readback).filter(c => !c.passed);
 
 describe("attached clarification evidence", () => {
+  it.each([true, false])("reports API fallback without accepting it as native question evidence (choice=%s)", choice => {
+    const f = recording(["semantic", "semantic"]), point = f.checkpoints[choice ? 0 : 1];
+    const question = point.interactions.find((i: Row) => i.status === "pending"), source = point.runs.at(-1);
+    source.resultJson = { apiToolReceipts: { call: { state: "completed", operationId: "POST /api/issues/{id}/interactions",
+      result: { ok: true, data: { id: question.id } } } } };
+    expect(clarificationQuestionFailure(point, question, choice)).toBe(`clarification_${choice ? "choice" : "text"}_api_fallback`);
+    expect(failures(f).some(check => check.id === `question-resume.${choice ? "choice" : "text"}.path`)).toBe(true);
+    delete source.resultJson.apiToolReceipts;
+    expect(clarificationQuestionFailure(point, question, choice)).toBe(`clarification_${choice ? "choice" : "text"}_native_path_invalid`);
+  });
+  it("distinguishes an invalid form from a correctly bound semantic question", () => {
+    const point = recording(["semantic", "semantic"]).checkpoints[0], question = point.interactions[0];
+    expect(clarificationQuestionFailure(point, question, true)).toBeUndefined();
+    question.payload.questionSet.questions[0].answerMode = "text";
+    point.runs[0].resultJson.semanticToolReceipts[question.idempotencyKey].result.interaction.payload = structuredClone(question.payload);
+    expect(clarificationQuestionFailure(point, question, true)).toBe("clarification_choice_form_invalid");
+  });
   it.each([["provider", "provider"], ["provider", "semantic"], ["semantic", "provider"], ["semantic", "semantic"]] as const)("accepts the real %s/%s paths with native binding and reload proof", (...paths) => {
     expect(failures(recording([...paths]))).toEqual([]);
   });

tests/runner-e2e/product-journeys/clarification-evidence.ts

@@ -1,4 +1,4 @@
-import { gradeQuestionResume } from "../question-resume-scoring.js";
+import { gradeQuestionResume, questionPath, validResumeQuestionForm } from "../question-resume-scoring.js";
 import type { ContinuationCheckpoint } from "../continuation-scoring.js";
 import { validateAgentTaskState } from "./agent-flows.js";
 import type { JourneyContext } from "./types.js";
@@ -8,6 +8,20 @@ export interface ClarificationReadback {
   issueId: string; key: string; revisionId: string; body: string; visibleBody: string;
 }
 
+export function clarificationQuestionFailure(point: ContinuationCheckpoint, question: Row, choice: boolean) {
+  const stage = choice ? "choice" : "text";
+  if (!questionPath(point, question)) {
+    const source = point.runs.find(run => run.id === question.sourceRunId) as Row | undefined;
+    const apiCreated = Object.values(source?.resultJson?.apiToolReceipts ?? {}).some(value => {
+      const receipt = value as Row;
+      return receipt.state === "completed" && receipt.operationId === "POST /api/issues/{id}/interactions"
+        && receipt.result?.ok === true && receipt.result?.data?.id === question.id;
+    });
+    return apiCreated ? `clarification_${stage}_api_fallback` : `clarification_${stage}_native_path_invalid`;
+  }
+  return validResumeQuestionForm(point, question, choice) ? undefined : `clarification_${stage}_form_invalid`;
+}
+
 export function gradeClarificationEvidence(context: JourneyContext, issueId: string, marker: string,
   checkpoints: ContinuationCheckpoint[], readback?: ClarificationReadback) {
   const checks = gradeQuestionResume(checkpoints, marker);

tests/runner-e2e/product-journeys/clarification-flow.ts

@@ -4,11 +4,10 @@ import { expect } from "@playwright/test";
 import { continuationScenario } from "../continuation-cases.js";
 import { continuationAnswerCommitted, continuationCheckpointReady } from "../continuation-readiness.js";
 import type { ContinuationCheckpoint } from "../continuation-scoring.js";
-import { validResumeQuestionForm } from "../question-resume-scoring.js";
 import { answerResumeQuestionThroughUi } from "../question-resume-ui.js";
 import { createTaskThroughUi } from "../user-actions.js";
 import { validateAgentTaskState } from "./agent-flows.js";
-import { gradeClarificationEvidence, type ClarificationReadback } from "./clarification-evidence.js";
+import { clarificationQuestionFailure, gradeClarificationEvidence, type ClarificationReadback } from "./clarification-evidence.js";
 import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js";
 
 type Row = Record<string, any>;
@@ -82,7 +81,8 @@ export async function runClarificationJourney(context: JourneyContext, session:
     for (const [index, input] of [{ choice: "Afternoon" }, { text: scenario.answer }].entries()) {
       const point = await snapshot(index === 0 ? "initial" : "answered", state);
       const question = state.interactions.find(i => i.kind === "ask_user_questions" && i.status === "pending")!;
-      if (!validResumeQuestionForm(point, question, index === 0)) throw new JourneyError("failed", "clarification_form_or_native_path_invalid");
+      const questionFailure = clarificationQuestionFailure(point, question, index === 0);
+      if (questionFailure) throw new JourneyError("failed", questionFailure);
       if (point.documents.length || point.attachments.length || point.children.length || point.issue.status === "done") {
         throw new JourneyError("failed", "clarification_premature_output");
       }