From 3fcf06adc85ecb83022177177c66ca912209ab5b Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 07:59:45 -0600 Subject: [PATCH] test(evals): account for authenticated secure-entry question answers Co-Authored-By: Paperclip --- tests/runner-e2e/chat-catalog-contract.ts | 50 +++++++++++++++++-- tests/runner-e2e/chat-catalog-flow.ts | 25 ++++++++-- tests/runner-e2e/chat-catalog-history.test.ts | 33 +++++++++++- 3 files changed, 99 insertions(+), 9 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index d6c74713e05..9f2e7d23cad 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -1,3 +1,4 @@ +import type { AskUserQuestionsInteraction } from "../../packages/shared/src/types/issue.js"; import type { ChatFlowInput, ChatIssue, ChatRun } from "./chat-flow.js"; import type { CredentialName } from "./types.js"; import type { ChatExternalInputRequirement } from "./chat-private-inputs.js"; @@ -194,8 +195,12 @@ export interface ChatCatalogCase { /** Known missing catalog conditions keep this executable diagnostic case from acceptance. */ coverageGaps?: readonly string[]; expectedRunCount: number; - /** Persisted human comments; explicit followups must be declared before execution. */ + /** Persisted human comments by default. The two credential cases may count one + * independently verified UI question answer as a declared human action instead + * of a followup comment; raw comment/query evidence is never rewritten. */ expectedHumanMessageCount?: number; + /** Only company/personal secure-entry cases; returns identity after full case proof, never authority. */ + expectedHumanQuestionAnswer?(observation: unknown): ChatCatalogExpectedHumanQuestionAnswer | undefined; /** Genuine persisted human setup turns before the canonical query; included in the total above. */ setupHumanMessageCount?: number; minimumRunCount?: number; @@ -230,17 +235,54 @@ export function validChatCatalogHumanCounts(implementation: Pick= setup + (implementation.queryOutcome === "rejected" ? 0 : 1); } export function chatCatalogHumanHistoryMatches(input: { - implementation: Pick; + implementation: Pick & Partial>; + questionAnswer?: ChatCatalogVerifiedHumanQuestionAnswer; human: Array<{ id: string; body: string }>; runIds: string[]; boundary: ChatCatalogSetupBoundary; renderedQuery: string; }): boolean { const count = input.implementation.setupHumanMessageCount ?? 0; - return validChatCatalogHumanCounts(input.implementation) + return (!input.questionAnswer || ["secrets.personal-secret", "secrets.company-secret"].includes(input.implementation.scenarioId ?? "")) + && validChatCatalogHumanCounts(input.implementation) && input.boundary.humanCommentIds.length === count && input.boundary.humanCommentIds.every((id, index) => input.human[index]?.id === id) && input.boundary.runIds.every(id => input.runIds.includes(id)) && (input.implementation.queryOutcome === "rejected" ? input.human.length === count : input.human[count]?.body.trim() === input.renderedQuery.trim() - && input.human.length === (input.implementation.expectedHumanMessageCount ?? 1)); + && input.human.length + (input.questionAnswer ? 1 : 0) === (input.implementation.expectedHumanMessageCount ?? 1)); +} + +export interface ChatCatalogExpectedHumanQuestionAnswer { interactionId: string; sourceRunId: string; userId: string } +export interface ChatCatalogVerifiedHumanQuestionAnswer extends ChatCatalogExpectedHumanQuestionAnswer { continuationRunId: string; resolvedAt: string } +export interface ChatCatalogHumanQuestionAudit { + companyId?: string; entityId?: string; action?: string; actorType?: string; actorId?: string; + details?: Record | null; +} +/** Verify public resolver facts independently of the case's proposed identity. */ +export function verifyChatCatalogHumanQuestionAnswer(input: { + expected: ChatCatalogExpectedHumanQuestionAnswer; companyId: string; issueId: string; agentId: string; + canonicalComment: { id: string; authorUserId?: string }; + interactions: Array>; activity: ChatCatalogHumanQuestionAudit[]; + runs: Array<{ id: string; companyId?: string; agentId: string; status: string; runtimeMode?: string; startedAt?: string | null; contextSnapshot?: Record | null }>; +}): ChatCatalogVerifiedHumanQuestionAnswer | undefined { + const b = input.expected, rows = input.interactions.filter(row => row.id === b.interactionId), row = rows[0]; + if (rows.length !== 1 || !row || b.userId !== input.canonicalComment.authorUserId + || row.companyId !== input.companyId || row.issueId !== input.issueId || row.sourceRunId !== b.sourceRunId + || row.createdByAgentId !== input.agentId || row.createdByUserId || row.addresseeUserId !== b.userId || row.addresseeAgentId + || row.kind !== "ask_user_questions" || row.status !== "answered" || row.resolverPolicy !== "human_only" + || row.resolvedByUserId !== b.userId || row.resolvedByAgentId || row.result?.answers?.length !== 1) return; + const resolvedAt = Date.parse(String(row.resolvedAt)); + const audits = input.activity.filter(event => event.details?.interactionId === b.interactionId && event.action === "issue.thread_interaction_answered"); + const audit = audits[0]; + if (!Number.isFinite(resolvedAt) || audits.length !== 1 || !audit || audit.companyId !== input.companyId || audit.entityId !== input.issueId + || audit.actorType !== "user" || audit.actorId !== b.userId || audit.details?.interactionKind !== "ask_user_questions" || audit.details.interactionStatus !== "answered") return; + const source = input.runs.find(run => run.id === b.sourceRunId); + const continuations = input.runs.filter(run => run.contextSnapshot?.interactionId === b.interactionId), run = continuations[0]; + if (!source || source.companyId !== input.companyId || source.agentId !== input.agentId || source.runtimeMode !== "native" || source.status !== "succeeded" + || source.contextSnapshot?.issueId !== input.issueId || source.contextSnapshot.commentId !== input.canonicalComment.id + || !Number.isFinite(Date.parse(source.startedAt ?? "")) || Date.parse(source.startedAt!) > resolvedAt + || continuations.length !== 1 || !run || run.id === source.id || run.companyId !== input.companyId || run.agentId !== input.agentId + || run.runtimeMode !== "native" || run.status !== "succeeded" || run.contextSnapshot?.issueId !== input.issueId + || !Number.isFinite(Date.parse(run.startedAt ?? "")) || Date.parse(run.startedAt!) < resolvedAt) return; + return { ...b, continuationRunId: run.id, resolvedAt: new Date(resolvedAt).toISOString() }; } diff --git a/tests/runner-e2e/chat-catalog-flow.ts b/tests/runner-e2e/chat-catalog-flow.ts index 5787989efc6..137dde26bc8 100644 --- a/tests/runner-e2e/chat-catalog-flow.ts +++ b/tests/runner-e2e/chat-catalog-flow.ts @@ -1,3 +1,4 @@ +import type { AskUserQuestionsInteraction } from "../../packages/shared/src/types/issue.js"; import { assertSdkCanaryFingerprintMatches } from "./sdk-canary-prefixture.js"; import { collectConversationLinks, type ConversationLinkInput } from "./chat-rendered-links.js"; import { chatCatalogAgentIsReady, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "./chat-catalog-contract.js"; @@ -11,7 +12,7 @@ import { expect } from "@playwright/test"; import { pollUntil } from "./api.js"; import { collectChatRunEvidence, sendChatMessage } from "./chat-flow.js"; import { prepareNativeStopCanonicalBoundary } from "./chat-native-stop-boundary.js"; -import { chatCatalogHumanHistoryMatches, validateChatCatalogGrade, type ChatCatalogSetupBoundary, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js"; +import { chatCatalogHumanHistoryMatches, verifyChatCatalogHumanQuestionAnswer, type ChatCatalogHumanQuestionAudit, validateChatCatalogGrade, type ChatCatalogSetupBoundary, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js"; import { findChatCatalogCase, loadChatCatalogImplementation } from "./chat-catalog-registry.js"; import { chatCatalogInventoryDigest } from "./chat-catalog-inventory.js"; import { resolveChatCatalogQuery } from "./chat-query-bindings.js"; @@ -63,14 +64,30 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { grade = validateChatCatalogGrade(implementation!.grade(observation)); const human = evidence.comments.filter(comment => !comment.authorAgentId); const canonicalHuman = human[implementation!.setupHumanMessageCount ?? 0]; + const expectedAnswer = ["secrets.personal-secret", "secrets.company-secret"].includes(scenario.id) + ? implementation!.expectedHumanQuestionAnswer?.(observation) : undefined; + let questionAnswer: ReturnType; + if (expectedAnswer && evidence.issue && canonicalHuman) { + const [interactions, activity] = await Promise.all([ + input.api.get(`/api/issues/${evidence.issue.id}/interactions`), + input.api.get(`/api/issues/${evidence.issue.id}/activity`), + ]); + questionAnswer = verifyChatCatalogHumanQuestionAnswer({ expected: expectedAnswer, companyId: input.fixtures.company.id, + issueId: evidence.issue.id, agentId: input.fixtures.agent.id, canonicalComment: canonicalHuman, + interactions, activity, runs: evidence.runs }); + } + const humanAccounting = { basis: expectedAnswer ? "comments_and_declared_credential_question_answer" : "comments", rawHumanCommentCount: human.length, verifiedHumanQuestionAnswerCount: questionAnswer ? 1 : 0, + accountedHumanActionCount: human.length + (questionAnswer ? 1 : 0), declaredHumanActionCount: implementation!.expectedHumanMessageCount ?? 1, + questionAnswer: questionAnswer ?? null, expectedQuestionAnswerVerified: expectedAnswer ? !!questionAnswer : null }; + const attachmentsValid = !queryAttachments.length || !!canonicalHuman && await verifyChatQueryAttachments(context, queryAttachments, canonicalHuman); if (queryAttachments.length) await input.evidence("chat-catalog-query-attachment-receipts.json", { receipts: queryAttachments, canonicalCommentId: canonicalHuman?.id ?? null, verified: attachmentsValid }); const checks = [...grade.checks, { id: "catalog-complete-conditions", passed: !implementation!.coverageGaps?.length, detail: implementation!.coverageGaps?.length ? `Missing conditions: ${implementation!.coverageGaps.join(", ")}` : "No known catalog condition is omitted." }, - { id: "catalog-original-human-query", passed: chatCatalogHumanHistoryMatches({ implementation: implementation!, human, runIds: evidence.runs.map(run => run.id), boundary: setupBoundary, renderedQuery: persistedQuery }), - detail: "The persisted request preserves the catalog query with only audited public-fixture identity bindings; all human turns were declared." }, + { id: "catalog-original-human-query", passed: (!expectedAnswer || !!questionAnswer) && chatCatalogHumanHistoryMatches({ implementation: implementation!, human, questionAnswer, runIds: evidence.runs.map(run => run.id), boundary: setupBoundary, renderedQuery: persistedQuery }), + detail: "The original query/setup history is unchanged. Human actions are raw comments, plus one independently verified bound UI answer only for opted-in credential cases." }, { id: "catalog-query-attachment-receipts", passed: attachmentsValid, detail: "Only declared production attachment suffixes are allowed, backed by same-source human receipts and independently retrieved matching bytes." }, ...(slack ? [slackNativeRunCheck(context, evidence.runs, state)] : []), @@ -79,7 +96,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { ]; grade = { passed: checks.every(check => check.passed), checks }; await input.evidence("chat-catalog-observation.json", { scenarioId: scenario.id, inventoryDigest: chatCatalogInventoryDigest, - oracleVersion: implementation!.oracleVersion, observation, grade }); + oracleVersion: implementation!.oracleVersion, observation, humanAccounting, grade }); return { grade, runs: evidence.runs, issue: evidence.issue }; } try { diff --git a/tests/runner-e2e/chat-catalog-history.test.ts b/tests/runner-e2e/chat-catalog-history.test.ts index 93ab2b5bab4..c6747f7c993 100644 --- a/tests/runner-e2e/chat-catalog-history.test.ts +++ b/tests/runner-e2e/chat-catalog-history.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { chatCatalogHumanHistoryMatches, validChatCatalogHumanCounts } from "./chat-catalog-contract.js"; +import { chatCatalogHumanHistoryMatches, verifyChatCatalogHumanQuestionAnswer, validChatCatalogHumanCounts } from "./chat-catalog-contract.js"; it("binds the canonical request after genuine setup history and retains native setup run identities", () => { const input = { implementation: { setupHumanMessageCount: 1, expectedHumanMessageCount: 2 }, human: [{ id: "setup", body: "Remember this detail" }, { id: "query", body: "Use the detail" }], @@ -17,3 +17,34 @@ it("preserves zero-setup semantics and refuses impossible setup/total declaratio expect(validChatCatalogHumanCounts({ setupHumanMessageCount: 0.5 })).toBe(false); expect(validChatCatalogHumanCounts({ setupHumanMessageCount: 0, expectedHumanMessageCount: 0, queryOutcome: "rejected" })).toBe(true); }); + +function answerEvidence() { + return { expected: { interactionId: "question", sourceRunId: "initial", userId: "person" }, companyId: "company", issueId: "source", agentId: "ceo", canonicalComment: { id: "query", authorUserId: "person" }, + interactions: [{ id: "question", companyId: "company", issueId: "source", sourceRunId: "initial", createdByAgentId: "ceo", addresseeUserId: "person", kind: "ask_user_questions" as const, + status: "answered" as const, resolverPolicy: "human_only" as const, resolvedByUserId: "person", resolvedAt: "2026-11-04T13:30:00Z", result: { version: 1 as const, answers: [{ questionId: "saved", optionIds: ["yes"] }], summaryMarkdown: null } }], + activity: [{ companyId: "company", entityId: "source", action: "issue.thread_interaction_answered", actorType: "user", actorId: "person", details: { interactionId: "question", interactionKind: "ask_user_questions", interactionStatus: "answered" } }], + runs: [{ id: "initial", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "succeeded", startedAt: "2026-11-04T13:20:00Z", contextSnapshot: { issueId: "source", commentId: "query" } }, + { id: "confirmed", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "succeeded", startedAt: "2026-11-04T13:30:01Z", contextSnapshot: { issueId: "source", interactionId: "question" } }], + }; +} +it("counts one independently verified UI answer as the second declared credential action without fabricating a comment", () => { + const raw = answerEvidence(), questionAnswer = verifyChatCatalogHumanQuestionAnswer(raw); expect(questionAnswer).toBeDefined(); + const human = [{ id: "query", body: "Original query" }]; + const input = { implementation: { scenarioId: "secrets.personal-secret", expectedHumanMessageCount: 2 }, human, questionAnswer, + runIds: ["initial", "confirmed"], boundary: { humanCommentIds: [], runIds: [] }, renderedQuery: "Original query" }; + expect(chatCatalogHumanHistoryMatches(input)).toBe(true); expect(human).toHaveLength(1); + expect(chatCatalogHumanHistoryMatches({ ...input, questionAnswer: undefined })).toBe(false); + expect(chatCatalogHumanHistoryMatches({ ...input, human: [...human, { id: "extra", body: "Hidden nudge" }] })).toBe(false); + expect(chatCatalogHumanHistoryMatches({ ...input, implementation: { scenarioId: "questions-and-approvals.multi-part-question", expectedHumanMessageCount: 2 } })).toBe(false); +}); +it.each(["wrong user", "missing audit", "duplicate audit", "foreign issue", "unrelated answer", "early successor", "extra successor"])("rejects %s as human-answer accounting authority", fault => { + const value = answerEvidence(); + if (fault === "wrong user") value.interactions[0]!.resolvedByUserId = "other-person"; + if (fault === "missing audit") value.activity = []; + if (fault === "duplicate audit") value.activity.push(value.activity[0]!); + if (fault === "foreign issue") value.interactions[0]!.issueId = "foreign"; + if (fault === "unrelated answer") value.expected.interactionId = "unrelated"; + if (fault === "early successor") value.runs[1]!.startedAt = "2026-11-04T13:29:00Z"; + if (fault === "extra successor") value.runs.push({ ...value.runs[1]!, id: "extra" }); + expect(verifyChatCatalogHumanQuestionAnswer(value)).toBeUndefined(); +});