feat(evals): retain genuine pre-query chat history boundaries

tests/runner-e2e/chat-catalog-contract.ts

@@ -44,6 +44,8 @@ export interface ChatCatalogCase {
   expectedRunCount: number;
   /** Persisted human comments; explicit followups must be declared before execution. */
   expectedHumanMessageCount?: number;
+  /** Genuine persisted human setup turns before the canonical query; included in the total above. */
+  setupHumanMessageCount?: number;
   minimumRunCount?: number;
   turnTimeoutMs: number;
   attemptTimeoutMs: number;
@@ -68,3 +70,25 @@ export function validateChatCatalogGrade(grade: ChatCatalogGrade): ChatCatalogGr
   }
   return grade;
 }
+
+export interface ChatCatalogSetupBoundary { humanCommentIds: string[]; runIds: string[] }
+export function validChatCatalogHumanCounts(implementation: Pick<ChatCatalogCase, "setupHumanMessageCount" | "expectedHumanMessageCount" | "queryOutcome">): boolean {
+  const setup = implementation.setupHumanMessageCount ?? 0, total = implementation.expectedHumanMessageCount ?? 1;
+  return Number.isSafeInteger(setup) && setup >= 0 && Number.isSafeInteger(total)
+    && total >= setup + (implementation.queryOutcome === "rejected" ? 0 : 1);
+}
+export function chatCatalogHumanHistoryMatches(input: {
+  implementation: Pick<ChatCatalogCase, "setupHumanMessageCount" | "expectedHumanMessageCount" | "queryOutcome">;
+  human: Array<{ id: string; body: string }>; runIds: string[];
+  boundary: ChatCatalogSetupBoundary; renderedQuery: string;
+}): boolean {
+  const count = input.implementation.setupHumanMessageCount ?? 0;
+  return validChatCatalogHumanCounts(input.implementation)
+    && input.boundary.humanCommentIds.length === count
+    && input.boundary.humanCommentIds.every((id, index) => input.human[index]?.id === id)
+    && input.boundary.runIds.every(id => input.runIds.includes(id))
+    && (input.implementation.queryOutcome === "rejected"
+      ? input.human.length === count
+      : input.human[count]?.body.trim() === input.renderedQuery.trim()
+        && input.human.length === (input.implementation.expectedHumanMessageCount ?? 1));
+}

tests/runner-e2e/chat-catalog-fingerprint.test.ts

@@ -67,7 +67,7 @@ it("binds the real selected area and reviewed worker closure without importing u
   expect(files).toContain("tests/runner-e2e/chat-catalog-domains/internal-support.ts");
   expect(files).toContain("packages/adapter-utils/src/workspace-snapshot-thread-worker.ts");
   expect(files).not.toContain("tests/runner-e2e/chat-catalog-domains/internal-goals.ts");
-  expect(files).not.toContain("tests/runner-e2e/chat-catalog-domains/identity-auth.ts");
+  expect(files).toContain("tests/runner-e2e/chat-catalog-domains/identity-auth.ts");
   expect(result.unknownDynamicImports).toEqual([]);
 });
 it("fails closed when preserved prerequisites or a rejected query lack their independent verifier", async () => {

tests/runner-e2e/chat-catalog-flow.ts

@@ -3,7 +3,7 @@ import { chatCatalogCaseFingerprint, normalizedChatCatalogExecution } from "./ch
 import { expect } from "@playwright/test";
 import { pollUntil } from "./api.js";
 import { collectChatRunEvidence, sendChatMessage } from "./chat-flow.js";
-import { validateChatCatalogGrade, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js";
+import { chatCatalogHumanHistoryMatches, validateChatCatalogGrade, type ChatCatalogSetupBoundary, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js";
 import { findChatCatalogCase, loadChatCatalogImplementation } from "./chat-catalog-registry.js";
 import { chatCatalogInventoryDigest } from "./chat-catalog-inventory.js";
 import { resolveChatCatalogQuery } from "./chat-query-bindings.js";
@@ -21,6 +21,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
   let grade: ChatCatalogGrade | undefined;
   let failed = false;
   let renderedQuery = scenario.query;
+  let setupBoundary: ChatCatalogSetupBoundary = { humanCommentIds: [], runIds: [] };
   async function conversation() {
     await context.refreshIssue();
     const issue = context.issue();
@@ -46,8 +47,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
     const checks = [...grade.checks,
       { id: "catalog-complete-conditions", passed: !implementation!.coverageGaps?.length,
         detail: implementation!.coverageGaps?.length ? `Missing conditions: ${implementation!.coverageGaps.join(", ")}` : "No known catalog condition is omitted." },
-      { id: "catalog-original-human-query", passed: (implementation!.queryOutcome === "rejected" ? human.length === 0 : human[0]?.body.trim() === renderedQuery.trim()
-        && human.length === (implementation!.expectedHumanMessageCount ?? 1)),
+      { id: "catalog-original-human-query", passed: chatCatalogHumanHistoryMatches({ implementation: implementation!, human, runIds: evidence.runs.map(run => run.id), boundary: setupBoundary, renderedQuery }),
         detail: "The persisted request preserves the catalog query with only audited public-fixture identity bindings; all human turns were declared." },
       ...gradeChatCatalogRuns({ implementation: implementation!, companyId: input.fixtures.company.id,
         leadId: input.fixtures.agent.id, source: evidence.issue, runs: evidence.runs }).checks,
@@ -90,6 +90,13 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
       await input.evidence("chat-catalog-prepared-state.json", setupGrade);
       expect(setupGrade.passed && (implementation.setupPolicy?.invariantCheckIds ?? []).every(id => setupGrade.checks.some(check => check.id === id && check.passed)), "Independent scenario prerequisites").toBe(true);
     }
+    const setupConversation = await conversation();
+    const setupHuman = setupConversation.comments.filter(comment => !comment.authorAgentId);
+    expect(setupHuman.length, "Only declared genuine pre-query human turns exist").toBe(implementation.setupHumanMessageCount ?? 0);
+    expect(setupConversation.runs.length <= implementation.expectedRunCount, "Setup runs count toward the total native run bound").toBe(true);
+    setupBoundary = { humanCommentIds: setupHuman.map(comment => comment.id), runIds: setupConversation.runs.map(run => run.id) };
+    await input.evidence("chat-catalog-setup-boundary.json", { ...setupBoundary, setupHumanMessageCount: implementation.setupHumanMessageCount ?? 0,
+      expectedHumanMessageCount: implementation.expectedHumanMessageCount ?? 1, expectedRunCount: implementation.expectedRunCount });
     await input.evidence("chat-catalog-query.json", { query: scenario.query, expectedOutcome: implementation.queryOutcome ?? "persisted", journeyKind: "agent-chat" });
     await sendChatMessage(input.page, renderedQuery);
     if (implementation.queryOutcome === "rejected") {

tests/runner-e2e/chat-catalog-history.test.ts

@@ -0,0 +1,19 @@
+import { expect, it } from "vitest";
+import { chatCatalogHumanHistoryMatches, validChatCatalogHumanCounts } from "./chat-catalog-contract.js";
+it("binds the canonical request after genuine setup history and retains native setup run identities", () => {
+  const input = { implementation: { setupHumanMessageCount: 1, expectedHumanMessageCount: 2 },
+    human: [{ id: "setup", body: "Remember this detail" }, { id: "query", body: "Use the detail" }],
+    runIds: ["setup-run", "query-run"], boundary: { humanCommentIds: ["setup"], runIds: ["setup-run"] }, renderedQuery: "Use the detail" };
+  expect(chatCatalogHumanHistoryMatches(input)).toBe(true);
+  expect(chatCatalogHumanHistoryMatches({ ...input, human: [...input.human].reverse() })).toBe(false);
+  expect(chatCatalogHumanHistoryMatches({ ...input, runIds: ["query-run"] })).toBe(false);
+  expect(chatCatalogHumanHistoryMatches({ ...input, boundary: { ...input.boundary, humanCommentIds: ["injected"] } })).toBe(false);
+  expect(chatCatalogHumanHistoryMatches({ ...input, human: [...input.human, { id: "hidden", body: "Please finish" }] })).toBe(false);
+});
+it("preserves zero-setup semantics and refuses impossible setup/total declarations", () => {
+  expect(chatCatalogHumanHistoryMatches({ implementation: {}, human: [{ id: "query", body: "Original query" }], runIds: ["run"], boundary: { humanCommentIds: [], runIds: [] }, renderedQuery: "Original query" })).toBe(true);
+  expect(validChatCatalogHumanCounts({ setupHumanMessageCount: 1, expectedHumanMessageCount: 1 })).toBe(false);
+  expect(validChatCatalogHumanCounts({ setupHumanMessageCount: -1 })).toBe(false);
+  expect(validChatCatalogHumanCounts({ setupHumanMessageCount: 0.5 })).toBe(false);
+  expect(validChatCatalogHumanCounts({ setupHumanMessageCount: 0, expectedHumanMessageCount: 0, queryOutcome: "rejected" })).toBe(true);
+});

tests/runner-e2e/chat-catalog-registry.ts

@@ -1,4 +1,5 @@
 import { existsSync, readFileSync } from "node:fs";
+import { validChatCatalogHumanCounts } from "./chat-catalog-contract.js";
 import type { ChatCatalogCase } from "./chat-catalog-contract.js";
 import { chatCatalogScenarios } from "./chat-catalog-inventory.js";
 import type { IdentityBoundaryCase } from "./chat-catalog-domains/identity-boundary-contract.js";
@@ -21,7 +22,7 @@ export function findChatCatalogCase(scenarioId: string): ChatCatalogDescriptor |
   if (!descriptor) return;
   if (!known.has(scenarioId) || !descriptor.oracleVersion || !descriptor.sourceFiles.length
     || !/^\.\/chat-catalog-domains\/[a-z0-9-]+\.js$/.test(descriptor.modulePath)
-    || !descriptor.exportName || descriptor.expectedRunCount < 0 || descriptor.turnTimeoutMs <= 0 || descriptor.attemptTimeoutMs <= 0
+    || !descriptor.exportName || (descriptor.journeyKind !== "browser-boundary" && !validChatCatalogHumanCounts(descriptor)) || descriptor.expectedRunCount < 0 || descriptor.turnTimeoutMs <= 0 || descriptor.attemptTimeoutMs <= 0
     || (descriptor.minimumRunCount ?? descriptor.expectedRunCount) > descriptor.expectedRunCount)
     throw new Error(`Incomplete bounds/provenance for ${scenarioId}`);
   return descriptor;
@@ -43,7 +44,7 @@ export async function loadChatCatalogImplementation(
   const selected = implementations.filter((row: ChatCatalogCase) => row.scenarioId === descriptor.scenarioId);
   if (selected.length !== 1) throw new Error(`Missing or duplicate runtime case: ${descriptor.scenarioId}`);
   const implementation = selected[0] as ChatCatalogCase;
-  for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings"] as const) {
+  for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings"] as const) {
     if (JSON.stringify(implementation[key]) !== JSON.stringify(descriptor[key])) throw new Error(`Runtime descriptor drift: ${descriptor.scenarioId}.${key}`);
   }
   if ((descriptor.setupPolicy?.funding === "preserve-scenario" || descriptor.setupPolicy?.readiness === "preserve-scenario")
@@ -61,14 +62,14 @@ export async function loadIdentityBoundaryImplementation(descriptor: ChatCatalog
   importer: (specifier: string) => Promise<Record<string, unknown>> = specifier => import(specifier)): Promise<IdentityBoundaryCase> {
   if (descriptor.journeyKind !== "browser-boundary" || !["access.sign-in", "onboarding.new-instance", "onboarding.choose-first-runtime", "onboarding.resume-wizard"].includes(descriptor.scenarioId))
     throw new Error("Only explicit catalog browser boundaries may bypass Agent Chat prerequisites");
-  if (descriptor.expectedRunCount !== 0 || (descriptor.minimumRunCount ?? 0) !== 0 || descriptor.expectedHumanMessageCount !== 0)
+  if (descriptor.expectedRunCount !== 0 || (descriptor.minimumRunCount ?? 0) !== 0 || descriptor.expectedHumanMessageCount !== 0 || (descriptor.setupHumanMessageCount ?? 0) !== 0)
     throw new Error("Pre-chat boundaries require zero agent admissions and zero persisted chat queries");
   const module = await importer(chatCatalogModuleUrl(descriptor.modulePath));
   const rows = module[descriptor.exportName];
   const selected = Array.isArray(rows) ? rows.filter(row => row.scenarioId === descriptor.scenarioId) : [];
   if (selected.length !== 1) throw new Error("Missing or duplicate browser boundary implementation");
   const implementation = selected[0] as IdentityBoundaryCase;
-  for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "boundaryTarget", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings"] as const) {
+  for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "boundaryTarget", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings"] as const) {
     if (JSON.stringify(implementation[key]) !== JSON.stringify(descriptor[key])) throw new Error(`Runtime boundary descriptor drift: ${descriptor.scenarioId}.${key}`);
   }
   if ([implementation.run, implementation.grade, implementation.cleanup].some(value => typeof value !== "function") || descriptor.queryBindings?.length