test(evals): account for authenticated secure-entry question answers

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-contract.ts

@@ -1,3 +1,4 @@
+import type { AskUserQuestionsInteraction } from "../../packages/shared/src/types/issue.js";
 import type { ChatFlowInput, ChatIssue, ChatRun } from "./chat-flow.js";
 import type { CredentialName } from "./types.js";
 import type { ChatExternalInputRequirement } from "./chat-private-inputs.js";
@@ -194,8 +195,12 @@ export interface ChatCatalogCase {
   /** Known missing catalog conditions keep this executable diagnostic case from acceptance. */
   coverageGaps?: readonly string[];
   expectedRunCount: number;
-  /** Persisted human comments; explicit followups must be declared before execution. */
+  /** Persisted human comments by default. The two credential cases may count one
+   * independently verified UI question answer as a declared human action instead
+   * of a followup comment; raw comment/query evidence is never rewritten. */
   expectedHumanMessageCount?: number;
+  /** Only company/personal secure-entry cases; returns identity after full case proof, never authority. */
+  expectedHumanQuestionAnswer?(observation: unknown): ChatCatalogExpectedHumanQuestionAnswer | undefined;
   /** Genuine persisted human setup turns before the canonical query; included in the total above. */
   setupHumanMessageCount?: number;
   minimumRunCount?: number;
@@ -230,17 +235,54 @@ export function validChatCatalogHumanCounts(implementation: Pick<ChatCatalogCase
     && total >= setup + (implementation.queryOutcome === "rejected" ? 0 : 1);
 }
 export function chatCatalogHumanHistoryMatches(input: {
-  implementation: Pick<ChatCatalogCase, "setupHumanMessageCount" | "expectedHumanMessageCount" | "queryOutcome">;
+  implementation: Pick<ChatCatalogCase, "setupHumanMessageCount" | "expectedHumanMessageCount" | "queryOutcome"> & Partial<Pick<ChatCatalogCase, "scenarioId">>;
+  questionAnswer?: ChatCatalogVerifiedHumanQuestionAnswer;
   human: Array<{ id: string; body: string }>; runIds: string[];
   boundary: ChatCatalogSetupBoundary; renderedQuery: string;
 }): boolean {
   const count = input.implementation.setupHumanMessageCount ?? 0;
-  return validChatCatalogHumanCounts(input.implementation)
+  return (!input.questionAnswer || ["secrets.personal-secret", "secrets.company-secret"].includes(input.implementation.scenarioId ?? ""))
+    && validChatCatalogHumanCounts(input.implementation)
     && input.boundary.humanCommentIds.length === count
     && input.boundary.humanCommentIds.every((id, index) => input.human[index]?.id === id)
     && input.boundary.runIds.every(id => input.runIds.includes(id))
     && (input.implementation.queryOutcome === "rejected"
       ? input.human.length === count
       : input.human[count]?.body.trim() === input.renderedQuery.trim()
-        && input.human.length === (input.implementation.expectedHumanMessageCount ?? 1));
+        && input.human.length + (input.questionAnswer ? 1 : 0) === (input.implementation.expectedHumanMessageCount ?? 1));
+}
+
+export interface ChatCatalogExpectedHumanQuestionAnswer { interactionId: string; sourceRunId: string; userId: string }
+export interface ChatCatalogVerifiedHumanQuestionAnswer extends ChatCatalogExpectedHumanQuestionAnswer { continuationRunId: string; resolvedAt: string }
+export interface ChatCatalogHumanQuestionAudit {
+  companyId?: string; entityId?: string; action?: string; actorType?: string; actorId?: string;
+  details?: Record<string, unknown> | null;
+}
+/** Verify public resolver facts independently of the case's proposed identity. */
+export function verifyChatCatalogHumanQuestionAnswer(input: {
+  expected: ChatCatalogExpectedHumanQuestionAnswer; companyId: string; issueId: string; agentId: string;
+  canonicalComment: { id: string; authorUserId?: string };
+  interactions: Array<Partial<AskUserQuestionsInteraction>>; activity: ChatCatalogHumanQuestionAudit[];
+  runs: Array<{ id: string; companyId?: string; agentId: string; status: string; runtimeMode?: string; startedAt?: string | null; contextSnapshot?: Record<string, unknown> | null }>;
+}): ChatCatalogVerifiedHumanQuestionAnswer | undefined {
+  const b = input.expected, rows = input.interactions.filter(row => row.id === b.interactionId), row = rows[0];
+  if (rows.length !== 1 || !row || b.userId !== input.canonicalComment.authorUserId
+    || row.companyId !== input.companyId || row.issueId !== input.issueId || row.sourceRunId !== b.sourceRunId
+    || row.createdByAgentId !== input.agentId || row.createdByUserId || row.addresseeUserId !== b.userId || row.addresseeAgentId
+    || row.kind !== "ask_user_questions" || row.status !== "answered" || row.resolverPolicy !== "human_only"
+    || row.resolvedByUserId !== b.userId || row.resolvedByAgentId || row.result?.answers?.length !== 1) return;
+  const resolvedAt = Date.parse(String(row.resolvedAt));
+  const audits = input.activity.filter(event => event.details?.interactionId === b.interactionId && event.action === "issue.thread_interaction_answered");
+  const audit = audits[0];
+  if (!Number.isFinite(resolvedAt) || audits.length !== 1 || !audit || audit.companyId !== input.companyId || audit.entityId !== input.issueId
+    || audit.actorType !== "user" || audit.actorId !== b.userId || audit.details?.interactionKind !== "ask_user_questions" || audit.details.interactionStatus !== "answered") return;
+  const source = input.runs.find(run => run.id === b.sourceRunId);
+  const continuations = input.runs.filter(run => run.contextSnapshot?.interactionId === b.interactionId), run = continuations[0];
+  if (!source || source.companyId !== input.companyId || source.agentId !== input.agentId || source.runtimeMode !== "native" || source.status !== "succeeded"
+    || source.contextSnapshot?.issueId !== input.issueId || source.contextSnapshot.commentId !== input.canonicalComment.id
+    || !Number.isFinite(Date.parse(source.startedAt ?? "")) || Date.parse(source.startedAt!) > resolvedAt
+    || continuations.length !== 1 || !run || run.id === source.id || run.companyId !== input.companyId || run.agentId !== input.agentId
+    || run.runtimeMode !== "native" || run.status !== "succeeded" || run.contextSnapshot?.issueId !== input.issueId
+    || !Number.isFinite(Date.parse(run.startedAt ?? "")) || Date.parse(run.startedAt!) < resolvedAt) return;
+  return { ...b, continuationRunId: run.id, resolvedAt: new Date(resolvedAt).toISOString() };
 }

tests/runner-e2e/chat-catalog-flow.ts

@@ -1,3 +1,4 @@
+import type { AskUserQuestionsInteraction } from "../../packages/shared/src/types/issue.js";
 import { assertSdkCanaryFingerprintMatches } from "./sdk-canary-prefixture.js";
 import { collectConversationLinks, type ConversationLinkInput } from "./chat-rendered-links.js";
 import { chatCatalogAgentIsReady, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "./chat-catalog-contract.js";
@@ -11,7 +12,7 @@ import { expect } from "@playwright/test";
 import { pollUntil } from "./api.js";
 import { collectChatRunEvidence, sendChatMessage } from "./chat-flow.js";
 import { prepareNativeStopCanonicalBoundary } from "./chat-native-stop-boundary.js";
-import { chatCatalogHumanHistoryMatches, validateChatCatalogGrade, type ChatCatalogSetupBoundary, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js";
+import { chatCatalogHumanHistoryMatches, verifyChatCatalogHumanQuestionAnswer, type ChatCatalogHumanQuestionAudit, validateChatCatalogGrade, type ChatCatalogSetupBoundary, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js";
 import { findChatCatalogCase, loadChatCatalogImplementation } from "./chat-catalog-registry.js";
 import { chatCatalogInventoryDigest } from "./chat-catalog-inventory.js";
 import { resolveChatCatalogQuery } from "./chat-query-bindings.js";
@@ -63,14 +64,30 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
     grade = validateChatCatalogGrade(implementation!.grade(observation));
     const human = evidence.comments.filter(comment => !comment.authorAgentId);
     const canonicalHuman = human[implementation!.setupHumanMessageCount ?? 0];
+    const expectedAnswer = ["secrets.personal-secret", "secrets.company-secret"].includes(scenario.id)
+      ? implementation!.expectedHumanQuestionAnswer?.(observation) : undefined;
+    let questionAnswer: ReturnType<typeof verifyChatCatalogHumanQuestionAnswer>;
+    if (expectedAnswer && evidence.issue && canonicalHuman) {
+      const [interactions, activity] = await Promise.all([
+        input.api.get<AskUserQuestionsInteraction[]>(`/api/issues/${evidence.issue.id}/interactions`),
+        input.api.get<ChatCatalogHumanQuestionAudit[]>(`/api/issues/${evidence.issue.id}/activity`),
+      ]);
+      questionAnswer = verifyChatCatalogHumanQuestionAnswer({ expected: expectedAnswer, companyId: input.fixtures.company.id,
+        issueId: evidence.issue.id, agentId: input.fixtures.agent.id, canonicalComment: canonicalHuman,
+        interactions, activity, runs: evidence.runs });
+    }
+    const humanAccounting = { basis: expectedAnswer ? "comments_and_declared_credential_question_answer" : "comments", rawHumanCommentCount: human.length, verifiedHumanQuestionAnswerCount: questionAnswer ? 1 : 0,
+      accountedHumanActionCount: human.length + (questionAnswer ? 1 : 0), declaredHumanActionCount: implementation!.expectedHumanMessageCount ?? 1,
+      questionAnswer: questionAnswer ?? null, expectedQuestionAnswerVerified: expectedAnswer ? !!questionAnswer : null };
+
     const attachmentsValid = !queryAttachments.length || !!canonicalHuman && await verifyChatQueryAttachments(context, queryAttachments, canonicalHuman);
     if (queryAttachments.length) await input.evidence("chat-catalog-query-attachment-receipts.json", { receipts: queryAttachments,
       canonicalCommentId: canonicalHuman?.id ?? null, verified: attachmentsValid });
     const checks = [...grade.checks,
       { id: "catalog-complete-conditions", passed: !implementation!.coverageGaps?.length,
         detail: implementation!.coverageGaps?.length ? `Missing conditions: ${implementation!.coverageGaps.join(", ")}` : "No known catalog condition is omitted." },
-      { id: "catalog-original-human-query", passed: chatCatalogHumanHistoryMatches({ implementation: implementation!, human, runIds: evidence.runs.map(run => run.id), boundary: setupBoundary, renderedQuery: persistedQuery }),
-        detail: "The persisted request preserves the catalog query with only audited public-fixture identity bindings; all human turns were declared." },
+      { id: "catalog-original-human-query", passed: (!expectedAnswer || !!questionAnswer) && chatCatalogHumanHistoryMatches({ implementation: implementation!, human, questionAnswer, runIds: evidence.runs.map(run => run.id), boundary: setupBoundary, renderedQuery: persistedQuery }),
+        detail: "The original query/setup history is unchanged. Human actions are raw comments, plus one independently verified bound UI answer only for opted-in credential cases." },
       { id: "catalog-query-attachment-receipts", passed: attachmentsValid,
         detail: "Only declared production attachment suffixes are allowed, backed by same-source human receipts and independently retrieved matching bytes." },
       ...(slack ? [slackNativeRunCheck(context, evidence.runs, state)] : []),
@@ -79,7 +96,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
     ];
     grade = { passed: checks.every(check => check.passed), checks };
     await input.evidence("chat-catalog-observation.json", { scenarioId: scenario.id, inventoryDigest: chatCatalogInventoryDigest,
-      oracleVersion: implementation!.oracleVersion, observation, grade });
+      oracleVersion: implementation!.oracleVersion, observation, humanAccounting, grade });
     return { grade, runs: evidence.runs, issue: evidence.issue };
   }
   try {

tests/runner-e2e/chat-catalog-history.test.ts

@@ -1,5 +1,5 @@
 import { expect, it } from "vitest";
-import { chatCatalogHumanHistoryMatches, validChatCatalogHumanCounts } from "./chat-catalog-contract.js";
+import { chatCatalogHumanHistoryMatches, verifyChatCatalogHumanQuestionAnswer, validChatCatalogHumanCounts } from "./chat-catalog-contract.js";
 it("binds the canonical request after genuine setup history and retains native setup run identities", () => {
   const input = { implementation: { setupHumanMessageCount: 1, expectedHumanMessageCount: 2 },
     human: [{ id: "setup", body: "Remember this detail" }, { id: "query", body: "Use the detail" }],
@@ -17,3 +17,34 @@ it("preserves zero-setup semantics and refuses impossible setup/total declaratio
   expect(validChatCatalogHumanCounts({ setupHumanMessageCount: 0.5 })).toBe(false);
   expect(validChatCatalogHumanCounts({ setupHumanMessageCount: 0, expectedHumanMessageCount: 0, queryOutcome: "rejected" })).toBe(true);
 });
+
+function answerEvidence() {
+  return { expected: { interactionId: "question", sourceRunId: "initial", userId: "person" }, companyId: "company", issueId: "source", agentId: "ceo", canonicalComment: { id: "query", authorUserId: "person" },
+    interactions: [{ id: "question", companyId: "company", issueId: "source", sourceRunId: "initial", createdByAgentId: "ceo", addresseeUserId: "person", kind: "ask_user_questions" as const,
+      status: "answered" as const, resolverPolicy: "human_only" as const, resolvedByUserId: "person", resolvedAt: "2026-11-04T13:30:00Z", result: { version: 1 as const, answers: [{ questionId: "saved", optionIds: ["yes"] }], summaryMarkdown: null } }],
+    activity: [{ companyId: "company", entityId: "source", action: "issue.thread_interaction_answered", actorType: "user", actorId: "person", details: { interactionId: "question", interactionKind: "ask_user_questions", interactionStatus: "answered" } }],
+    runs: [{ id: "initial", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "succeeded", startedAt: "2026-11-04T13:20:00Z", contextSnapshot: { issueId: "source", commentId: "query" } },
+      { id: "confirmed", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "succeeded", startedAt: "2026-11-04T13:30:01Z", contextSnapshot: { issueId: "source", interactionId: "question" } }],
+  };
+}
+it("counts one independently verified UI answer as the second declared credential action without fabricating a comment", () => {
+  const raw = answerEvidence(), questionAnswer = verifyChatCatalogHumanQuestionAnswer(raw); expect(questionAnswer).toBeDefined();
+  const human = [{ id: "query", body: "Original query" }];
+  const input = { implementation: { scenarioId: "secrets.personal-secret", expectedHumanMessageCount: 2 }, human, questionAnswer,
+    runIds: ["initial", "confirmed"], boundary: { humanCommentIds: [], runIds: [] }, renderedQuery: "Original query" };
+  expect(chatCatalogHumanHistoryMatches(input)).toBe(true); expect(human).toHaveLength(1);
+  expect(chatCatalogHumanHistoryMatches({ ...input, questionAnswer: undefined })).toBe(false);
+  expect(chatCatalogHumanHistoryMatches({ ...input, human: [...human, { id: "extra", body: "Hidden nudge" }] })).toBe(false);
+  expect(chatCatalogHumanHistoryMatches({ ...input, implementation: { scenarioId: "questions-and-approvals.multi-part-question", expectedHumanMessageCount: 2 } })).toBe(false);
+});
+it.each(["wrong user", "missing audit", "duplicate audit", "foreign issue", "unrelated answer", "early successor", "extra successor"])("rejects %s as human-answer accounting authority", fault => {
+  const value = answerEvidence();
+  if (fault === "wrong user") value.interactions[0]!.resolvedByUserId = "other-person";
+  if (fault === "missing audit") value.activity = [];
+  if (fault === "duplicate audit") value.activity.push(value.activity[0]!);
+  if (fault === "foreign issue") value.interactions[0]!.issueId = "foreign";
+  if (fault === "unrelated answer") value.expected.interactionId = "unrelated";
+  if (fault === "early successor") value.runs[1]!.startedAt = "2026-11-04T13:29:00Z";
+  if (fault === "extra successor") value.runs.push({ ...value.runs[1]!, id: "extra" });
+  expect(verifyChatCatalogHumanQuestionAnswer(value)).toBeUndefined();
+});