fix(evals): honor addressed human question authority

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-contract.ts

@@ -252,6 +252,11 @@ export function chatCatalogHumanHistoryMatches(input: {
         && input.human.length + (input.questionAnswer ? 1 : 0) === (input.implementation.expectedHumanMessageCount ?? 1));
 }
 
+export function chatCatalogHumanQuestionPolicies(row: { resolverPolicy?: unknown; requestedResolverPolicy?: unknown; effectiveResolverPolicy?: unknown }): boolean {
+  return [row.resolverPolicy, row.requestedResolverPolicy ?? row.resolverPolicy, row.effectiveResolverPolicy ?? row.resolverPolicy]
+    .every(policy => policy === "anyone" || policy === "human_only");
+}
+
 export interface ChatCatalogExpectedHumanQuestionAnswer { interactionId: string; sourceRunId: string; userId: string }
 export interface ChatCatalogVerifiedHumanQuestionAnswer extends ChatCatalogExpectedHumanQuestionAnswer { continuationRunId: string; resolvedAt: string }
 export interface ChatCatalogHumanQuestionAudit {
@@ -268,8 +273,9 @@ export function verifyChatCatalogHumanQuestionAnswer(input: {
   if (rows.length !== 1 || !row || b.userId !== input.canonicalComment.authorUserId
     || row.companyId !== input.companyId || row.issueId !== input.issueId || row.sourceRunId !== b.sourceRunId
     || row.createdByAgentId !== input.agentId || row.createdByUserId || row.addresseeUserId !== b.userId || row.addresseeAgentId
-    || row.kind !== "ask_user_questions" || row.status !== "answered" || row.resolverPolicy !== "human_only"
-    || row.resolvedByUserId !== b.userId || row.resolvedByAgentId || row.result?.answers?.length !== 1) return;
+    || row.kind !== "ask_user_questions" || row.status !== "answered" || !chatCatalogHumanQuestionPolicies(row)
+    || row.resolvedByUserId !== b.userId || row.resolvedByAgentId || row.result?.answers?.length !== 1
+    || row.result.answers.some(answer => !!answer.otherText)) return;
   const resolvedAt = Date.parse(String(row.resolvedAt));
   const audits = input.activity.filter(event => event.details?.interactionId === b.interactionId && event.action === "issue.thread_interaction_answered");
   const audit = audits[0];

tests/runner-e2e/chat-catalog-domains/external-secret-question.ts

@@ -1,4 +1,5 @@
 import { createHash } from "node:crypto";
+import { chatCatalogHumanQuestionPolicies } from "../chat-catalog-contract.js";
 import type { AskUserQuestionsInteraction } from "../../../packages/shared/src/types/issue.js";
 import { askUserQuestionsPayloadSchema } from "../../../packages/shared/src/validators/issue.js";
 
@@ -10,13 +11,13 @@ export function boundSecretQuestion(value: unknown, b: SecretQuestionBinding, st
   if (!row || typeof row.id !== "string" || row.kind !== "ask_user_questions" || row.status !== status
     || row.companyId !== b.companyId || row.issueId !== b.issueId || row.sourceRunId !== b.runId
     || row.createdByAgentId !== b.agentId || row.createdByUserId || row.addresseeUserId !== b.userId || row.addresseeAgentId
-    || row.resolverPolicy !== "human_only" || row.continuationPolicy !== "wake_assignee") return;
+    || !chatCatalogHumanQuestionPolicies(row) || row.continuationPolicy !== "wake_assignee") return;
   if (status === "pending" ? row.resolvedByUserId || row.resolvedByAgentId : row.resolvedByUserId !== b.userId || row.resolvedByAgentId) return;
   const parsed = askUserQuestionsPayloadSchema.safeParse(row.payload), qs = parsed.success ? parsed.data.questionSet : undefined;
   if (!qs || qs.questions.length !== 1) return;
   const q = qs.questions[0]!;
-  if (q.answerMode !== "single_select" || q.customAnswer?.enabled || !q.required || !q.options?.length) return;
+  if (q.answerMode !== "single_select" || !q.required || !q.options?.length) return;
   return { row: { ...row, id: row.id }, questionSet: qs, question: { ...q, options: q.options },
-    text: [qs.title, qs.description, q.header, q.prompt, q.helpText, qs.submitLabel].filter(Boolean).join("\n"),
+    text: [qs.title, qs.description, q.header, q.prompt, q.helpText, q.customAnswer?.enabled ? q.customAnswer.label : undefined, qs.submitLabel].filter(Boolean).join("\n"),
     payloadHash: createHash("sha256").update(JSON.stringify(row.payload)).digest("hex") };
 }

tests/runner-e2e/chat-catalog-history.test.ts

@@ -50,3 +50,21 @@ it.each(["wrong user", "missing audit", "duplicate audit", "foreign issue", "unr
   if (fault === "extra successor") value.runs.push({ ...value.runs[1]!, id: "extra" });
   expect(verifyChatCatalogHumanQuestionAnswer(value)).toBeUndefined();
 });
+
+it.each(["anyone", "human_only"] as const)("independently verifies current-human answered %s questions", policy => {
+  const raw = answerEvidence();
+  const value = { ...raw, interactions: raw.interactions.map(row => ({ ...row, resolverPolicy: policy, requestedResolverPolicy: policy, effectiveResolverPolicy: policy })) };
+  expect(verifyChatCatalogHumanQuestionAnswer(value)).toBeDefined();
+  expect(verifyChatCatalogHumanQuestionAnswer({ ...value, interactions: value.interactions.map(row => ({ ...row, addresseeUserId: "other" })) })).toBeUndefined();
+  expect(verifyChatCatalogHumanQuestionAnswer({ ...value, interactions: value.interactions.map(row => ({ ...row, resolvedByUserId: undefined, resolvedByAgentId: "ceo" })) })).toBeUndefined();
+});
+
+it("rejects unknown or agent-only policies and submitted text in independent accounting", () => {
+  const raw = answerEvidence();
+  for (const effectiveResolverPolicy of ["unknown", "agent_only"] as const) {
+    expect(verifyChatCatalogHumanQuestionAnswer({ ...raw, interactions: raw.interactions.map(row => ({ ...row,
+      effectiveResolverPolicy: effectiveResolverPolicy as unknown as typeof row.resolverPolicy })) })).toBeUndefined();
+  }
+  expect(verifyChatCatalogHumanQuestionAnswer({ ...raw, interactions: raw.interactions.map(row => ({ ...row,
+    result: { ...row.result, answers: [{ ...row.result.answers[0]!, otherText: "Do not submit a note" }] } })) })).toBeUndefined();
+});