From 5e8e36958be73bdf3801113c714de0092861cb68 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 08:03:55 -0600 Subject: [PATCH] fix(evals): scope question accounting to canonical credential cases Co-Authored-By: Paperclip --- tests/runner-e2e/chat-catalog-contract.ts | 5 ++++- tests/runner-e2e/chat-catalog-flow.ts | 4 ++-- tests/runner-e2e/chat-catalog-history.test.ts | 2 ++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 9f2e7d23cad..f4df4bc6f21 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -234,6 +234,9 @@ export function validChatCatalogHumanCounts(implementation: Pick= 0 && Number.isSafeInteger(total) && total >= setup + (implementation.queryOutcome === "rejected" ? 0 : 1); } +export function chatCatalogAllowsHumanQuestionAnswer(scenarioId: string): boolean { + return ["secrets.personal-secret", "secrets.configure-company-secret"].includes(scenarioId); +} export function chatCatalogHumanHistoryMatches(input: { implementation: Pick & Partial>; questionAnswer?: ChatCatalogVerifiedHumanQuestionAnswer; @@ -241,7 +244,7 @@ export function chatCatalogHumanHistoryMatches(input: { boundary: ChatCatalogSetupBoundary; renderedQuery: string; }): boolean { const count = input.implementation.setupHumanMessageCount ?? 0; - return (!input.questionAnswer || ["secrets.personal-secret", "secrets.company-secret"].includes(input.implementation.scenarioId ?? "")) + return (!input.questionAnswer || chatCatalogAllowsHumanQuestionAnswer(input.implementation.scenarioId ?? "")) && validChatCatalogHumanCounts(input.implementation) && input.boundary.humanCommentIds.length === count && input.boundary.humanCommentIds.every((id, index) => input.human[index]?.id === id) diff --git a/tests/runner-e2e/chat-catalog-flow.ts b/tests/runner-e2e/chat-catalog-flow.ts index 137dde26bc8..2b8b88a7ff9 100644 --- a/tests/runner-e2e/chat-catalog-flow.ts +++ b/tests/runner-e2e/chat-catalog-flow.ts @@ -12,7 +12,7 @@ import { expect } from "@playwright/test"; import { pollUntil } from "./api.js"; import { collectChatRunEvidence, sendChatMessage } from "./chat-flow.js"; import { prepareNativeStopCanonicalBoundary } from "./chat-native-stop-boundary.js"; -import { chatCatalogHumanHistoryMatches, verifyChatCatalogHumanQuestionAnswer, type ChatCatalogHumanQuestionAudit, validateChatCatalogGrade, type ChatCatalogSetupBoundary, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js"; +import { chatCatalogHumanHistoryMatches, chatCatalogAllowsHumanQuestionAnswer, verifyChatCatalogHumanQuestionAnswer, type ChatCatalogHumanQuestionAudit, validateChatCatalogGrade, type ChatCatalogSetupBoundary, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js"; import { findChatCatalogCase, loadChatCatalogImplementation } from "./chat-catalog-registry.js"; import { chatCatalogInventoryDigest } from "./chat-catalog-inventory.js"; import { resolveChatCatalogQuery } from "./chat-query-bindings.js"; @@ -64,7 +64,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { grade = validateChatCatalogGrade(implementation!.grade(observation)); const human = evidence.comments.filter(comment => !comment.authorAgentId); const canonicalHuman = human[implementation!.setupHumanMessageCount ?? 0]; - const expectedAnswer = ["secrets.personal-secret", "secrets.company-secret"].includes(scenario.id) + const expectedAnswer = chatCatalogAllowsHumanQuestionAnswer(scenario.id) ? implementation!.expectedHumanQuestionAnswer?.(observation) : undefined; let questionAnswer: ReturnType; if (expectedAnswer && evidence.issue && canonicalHuman) { diff --git a/tests/runner-e2e/chat-catalog-history.test.ts b/tests/runner-e2e/chat-catalog-history.test.ts index c6747f7c993..7abf70bb271 100644 --- a/tests/runner-e2e/chat-catalog-history.test.ts +++ b/tests/runner-e2e/chat-catalog-history.test.ts @@ -33,6 +33,8 @@ it("counts one independently verified UI answer as the second declared credentia const input = { implementation: { scenarioId: "secrets.personal-secret", expectedHumanMessageCount: 2 }, human, questionAnswer, runIds: ["initial", "confirmed"], boundary: { humanCommentIds: [], runIds: [] }, renderedQuery: "Original query" }; expect(chatCatalogHumanHistoryMatches(input)).toBe(true); expect(human).toHaveLength(1); + expect(chatCatalogHumanHistoryMatches({ ...input, implementation: { scenarioId: "secrets.configure-company-secret", expectedHumanMessageCount: 2 } })).toBe(true); + expect(chatCatalogHumanHistoryMatches({ ...input, implementation: { scenarioId: "secrets.company-secret", expectedHumanMessageCount: 2 } })).toBe(false); expect(chatCatalogHumanHistoryMatches({ ...input, questionAnswer: undefined })).toBe(false); expect(chatCatalogHumanHistoryMatches({ ...input, human: [...human, { id: "extra", body: "Hidden nudge" }] })).toBe(false); expect(chatCatalogHumanHistoryMatches({ ...input, implementation: { scenarioId: "questions-and-approvals.multi-part-question", expectedHumanMessageCount: 2 } })).toBe(false);