fix(evals): scope question accounting to canonical credential cases

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-contract.ts

@@ -234,6 +234,9 @@ export function validChatCatalogHumanCounts(implementation: Pick<ChatCatalogCase
   return Number.isSafeInteger(setup) && setup >= 0 && Number.isSafeInteger(total)
     && total >= setup + (implementation.queryOutcome === "rejected" ? 0 : 1);
 }
+export function chatCatalogAllowsHumanQuestionAnswer(scenarioId: string): boolean {
+  return ["secrets.personal-secret", "secrets.configure-company-secret"].includes(scenarioId);
+}
 export function chatCatalogHumanHistoryMatches(input: {
   implementation: Pick<ChatCatalogCase, "setupHumanMessageCount" | "expectedHumanMessageCount" | "queryOutcome"> & Partial<Pick<ChatCatalogCase, "scenarioId">>;
   questionAnswer?: ChatCatalogVerifiedHumanQuestionAnswer;
@@ -241,7 +244,7 @@ export function chatCatalogHumanHistoryMatches(input: {
   boundary: ChatCatalogSetupBoundary; renderedQuery: string;
 }): boolean {
   const count = input.implementation.setupHumanMessageCount ?? 0;
-  return (!input.questionAnswer || ["secrets.personal-secret", "secrets.company-secret"].includes(input.implementation.scenarioId ?? ""))
+  return (!input.questionAnswer || chatCatalogAllowsHumanQuestionAnswer(input.implementation.scenarioId ?? ""))
     && validChatCatalogHumanCounts(input.implementation)
     && input.boundary.humanCommentIds.length === count
     && input.boundary.humanCommentIds.every((id, index) => input.human[index]?.id === id)

tests/runner-e2e/chat-catalog-flow.ts

@@ -12,7 +12,7 @@ import { expect } from "@playwright/test";
 import { pollUntil } from "./api.js";
 import { collectChatRunEvidence, sendChatMessage } from "./chat-flow.js";
 import { prepareNativeStopCanonicalBoundary } from "./chat-native-stop-boundary.js";
-import { chatCatalogHumanHistoryMatches, verifyChatCatalogHumanQuestionAnswer, type ChatCatalogHumanQuestionAudit, validateChatCatalogGrade, type ChatCatalogSetupBoundary, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js";
+import { chatCatalogHumanHistoryMatches, chatCatalogAllowsHumanQuestionAnswer, verifyChatCatalogHumanQuestionAnswer, type ChatCatalogHumanQuestionAudit, validateChatCatalogGrade, type ChatCatalogSetupBoundary, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js";
 import { findChatCatalogCase, loadChatCatalogImplementation } from "./chat-catalog-registry.js";
 import { chatCatalogInventoryDigest } from "./chat-catalog-inventory.js";
 import { resolveChatCatalogQuery } from "./chat-query-bindings.js";
@@ -64,7 +64,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
     grade = validateChatCatalogGrade(implementation!.grade(observation));
     const human = evidence.comments.filter(comment => !comment.authorAgentId);
     const canonicalHuman = human[implementation!.setupHumanMessageCount ?? 0];
-    const expectedAnswer = ["secrets.personal-secret", "secrets.company-secret"].includes(scenario.id)
+    const expectedAnswer = chatCatalogAllowsHumanQuestionAnswer(scenario.id)
       ? implementation!.expectedHumanQuestionAnswer?.(observation) : undefined;
     let questionAnswer: ReturnType<typeof verifyChatCatalogHumanQuestionAnswer>;
     if (expectedAnswer && evidence.issue && canonicalHuman) {

tests/runner-e2e/chat-catalog-history.test.ts

@@ -33,6 +33,8 @@ it("counts one independently verified UI answer as the second declared credentia
   const input = { implementation: { scenarioId: "secrets.personal-secret", expectedHumanMessageCount: 2 }, human, questionAnswer,
     runIds: ["initial", "confirmed"], boundary: { humanCommentIds: [], runIds: [] }, renderedQuery: "Original query" };
   expect(chatCatalogHumanHistoryMatches(input)).toBe(true); expect(human).toHaveLength(1);
+  expect(chatCatalogHumanHistoryMatches({ ...input, implementation: { scenarioId: "secrets.configure-company-secret", expectedHumanMessageCount: 2 } })).toBe(true);
+  expect(chatCatalogHumanHistoryMatches({ ...input, implementation: { scenarioId: "secrets.company-secret", expectedHumanMessageCount: 2 } })).toBe(false);
   expect(chatCatalogHumanHistoryMatches({ ...input, questionAnswer: undefined })).toBe(false);
   expect(chatCatalogHumanHistoryMatches({ ...input, human: [...human, { id: "extra", body: "Hidden nudge" }] })).toBe(false);
   expect(chatCatalogHumanHistoryMatches({ ...input, implementation: { scenarioId: "questions-and-approvals.multi-part-question", expectedHumanMessageCount: 2 } })).toBe(false);