From 639ddf3ad8d6447eb33207e09f3a79ad6f10639d Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 11:03:07 -0600 Subject: [PATCH] fix(evals): honor addressed human question authority Co-Authored-By: Paperclip --- tests/runner-e2e/chat-catalog-contract.ts | 10 ++++++++-- .../external-secret-question.ts | 7 ++++--- tests/runner-e2e/chat-catalog-history.test.ts | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 7519ab6a476..fe0a01f0c5e 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -252,6 +252,11 @@ export function chatCatalogHumanHistoryMatches(input: { && input.human.length + (input.questionAnswer ? 1 : 0) === (input.implementation.expectedHumanMessageCount ?? 1)); } +export function chatCatalogHumanQuestionPolicies(row: { resolverPolicy?: unknown; requestedResolverPolicy?: unknown; effectiveResolverPolicy?: unknown }): boolean { + return [row.resolverPolicy, row.requestedResolverPolicy ?? row.resolverPolicy, row.effectiveResolverPolicy ?? row.resolverPolicy] + .every(policy => policy === "anyone" || policy === "human_only"); +} + export interface ChatCatalogExpectedHumanQuestionAnswer { interactionId: string; sourceRunId: string; userId: string } export interface ChatCatalogVerifiedHumanQuestionAnswer extends ChatCatalogExpectedHumanQuestionAnswer { continuationRunId: string; resolvedAt: string } export interface ChatCatalogHumanQuestionAudit { @@ -268,8 +273,9 @@ export function verifyChatCatalogHumanQuestionAnswer(input: { if (rows.length !== 1 || !row || b.userId !== input.canonicalComment.authorUserId || row.companyId !== input.companyId || row.issueId !== input.issueId || row.sourceRunId !== b.sourceRunId || row.createdByAgentId !== input.agentId || row.createdByUserId || row.addresseeUserId !== b.userId || row.addresseeAgentId - || row.kind !== "ask_user_questions" || row.status !== "answered" || row.resolverPolicy !== "human_only" - || row.resolvedByUserId !== b.userId || row.resolvedByAgentId || row.result?.answers?.length !== 1) return; + || row.kind !== "ask_user_questions" || row.status !== "answered" || !chatCatalogHumanQuestionPolicies(row) + || row.resolvedByUserId !== b.userId || row.resolvedByAgentId || row.result?.answers?.length !== 1 + || row.result.answers.some(answer => !!answer.otherText)) return; const resolvedAt = Date.parse(String(row.resolvedAt)); const audits = input.activity.filter(event => event.details?.interactionId === b.interactionId && event.action === "issue.thread_interaction_answered"); const audit = audits[0]; diff --git a/tests/runner-e2e/chat-catalog-domains/external-secret-question.ts b/tests/runner-e2e/chat-catalog-domains/external-secret-question.ts index 3c2594e7f9a..bcab4a3d7d2 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-secret-question.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-secret-question.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { chatCatalogHumanQuestionPolicies } from "../chat-catalog-contract.js"; import type { AskUserQuestionsInteraction } from "../../../packages/shared/src/types/issue.js"; import { askUserQuestionsPayloadSchema } from "../../../packages/shared/src/validators/issue.js"; @@ -10,13 +11,13 @@ export function boundSecretQuestion(value: unknown, b: SecretQuestionBinding, st if (!row || typeof row.id !== "string" || row.kind !== "ask_user_questions" || row.status !== status || row.companyId !== b.companyId || row.issueId !== b.issueId || row.sourceRunId !== b.runId || row.createdByAgentId !== b.agentId || row.createdByUserId || row.addresseeUserId !== b.userId || row.addresseeAgentId - || row.resolverPolicy !== "human_only" || row.continuationPolicy !== "wake_assignee") return; + || !chatCatalogHumanQuestionPolicies(row) || row.continuationPolicy !== "wake_assignee") return; if (status === "pending" ? row.resolvedByUserId || row.resolvedByAgentId : row.resolvedByUserId !== b.userId || row.resolvedByAgentId) return; const parsed = askUserQuestionsPayloadSchema.safeParse(row.payload), qs = parsed.success ? parsed.data.questionSet : undefined; if (!qs || qs.questions.length !== 1) return; const q = qs.questions[0]!; - if (q.answerMode !== "single_select" || q.customAnswer?.enabled || !q.required || !q.options?.length) return; + if (q.answerMode !== "single_select" || !q.required || !q.options?.length) return; return { row: { ...row, id: row.id }, questionSet: qs, question: { ...q, options: q.options }, - text: [qs.title, qs.description, q.header, q.prompt, q.helpText, qs.submitLabel].filter(Boolean).join("\n"), + text: [qs.title, qs.description, q.header, q.prompt, q.helpText, q.customAnswer?.enabled ? q.customAnswer.label : undefined, qs.submitLabel].filter(Boolean).join("\n"), payloadHash: createHash("sha256").update(JSON.stringify(row.payload)).digest("hex") }; } diff --git a/tests/runner-e2e/chat-catalog-history.test.ts b/tests/runner-e2e/chat-catalog-history.test.ts index 7abf70bb271..00276bbb51b 100644 --- a/tests/runner-e2e/chat-catalog-history.test.ts +++ b/tests/runner-e2e/chat-catalog-history.test.ts @@ -50,3 +50,21 @@ it.each(["wrong user", "missing audit", "duplicate audit", "foreign issue", "unr if (fault === "extra successor") value.runs.push({ ...value.runs[1]!, id: "extra" }); expect(verifyChatCatalogHumanQuestionAnswer(value)).toBeUndefined(); }); + +it.each(["anyone", "human_only"] as const)("independently verifies current-human answered %s questions", policy => { + const raw = answerEvidence(); + const value = { ...raw, interactions: raw.interactions.map(row => ({ ...row, resolverPolicy: policy, requestedResolverPolicy: policy, effectiveResolverPolicy: policy })) }; + expect(verifyChatCatalogHumanQuestionAnswer(value)).toBeDefined(); + expect(verifyChatCatalogHumanQuestionAnswer({ ...value, interactions: value.interactions.map(row => ({ ...row, addresseeUserId: "other" })) })).toBeUndefined(); + expect(verifyChatCatalogHumanQuestionAnswer({ ...value, interactions: value.interactions.map(row => ({ ...row, resolvedByUserId: undefined, resolvedByAgentId: "ceo" })) })).toBeUndefined(); +}); + +it("rejects unknown or agent-only policies and submitted text in independent accounting", () => { + const raw = answerEvidence(); + for (const effectiveResolverPolicy of ["unknown", "agent_only"] as const) { + expect(verifyChatCatalogHumanQuestionAnswer({ ...raw, interactions: raw.interactions.map(row => ({ ...row, + effectiveResolverPolicy: effectiveResolverPolicy as unknown as typeof row.resolverPolicy })) })).toBeUndefined(); + } + expect(verifyChatCatalogHumanQuestionAnswer({ ...raw, interactions: raw.interactions.map(row => ({ ...row, + result: { ...row.result, answers: [{ ...row.result.answers[0]!, otherText: "Do not submit a note" }] } })) })).toBeUndefined(); +});