From e652b0de6eff21473a6d9b4915846c9851f292fa Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 17:24:54 -0500 Subject: [PATCH] feat(evals): retain genuine pre-query chat history boundaries --- tests/runner-e2e/chat-catalog-contract.ts | 24 +++++++++++++++++++ .../chat-catalog-fingerprint.test.ts | 2 +- tests/runner-e2e/chat-catalog-flow.ts | 13 +++++++--- tests/runner-e2e/chat-catalog-history.test.ts | 19 +++++++++++++++ tests/runner-e2e/chat-catalog-registry.ts | 9 +++---- 5 files changed, 59 insertions(+), 8 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-history.test.ts diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 2e4779fb80c..96b045a575d 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -44,6 +44,8 @@ export interface ChatCatalogCase { expectedRunCount: number; /** Persisted human comments; explicit followups must be declared before execution. */ expectedHumanMessageCount?: number; + /** Genuine persisted human setup turns before the canonical query; included in the total above. */ + setupHumanMessageCount?: number; minimumRunCount?: number; turnTimeoutMs: number; attemptTimeoutMs: number; @@ -68,3 +70,25 @@ export function validateChatCatalogGrade(grade: ChatCatalogGrade): ChatCatalogGr } return grade; } + +export interface ChatCatalogSetupBoundary { humanCommentIds: string[]; runIds: string[] } +export function validChatCatalogHumanCounts(implementation: Pick): boolean { + const setup = implementation.setupHumanMessageCount ?? 0, total = implementation.expectedHumanMessageCount ?? 1; + return Number.isSafeInteger(setup) && setup >= 0 && Number.isSafeInteger(total) + && total >= setup + (implementation.queryOutcome === "rejected" ? 0 : 1); +} +export function chatCatalogHumanHistoryMatches(input: { + implementation: Pick; + human: Array<{ id: string; body: string }>; runIds: string[]; + boundary: ChatCatalogSetupBoundary; renderedQuery: string; +}): boolean { + const count = input.implementation.setupHumanMessageCount ?? 0; + return validChatCatalogHumanCounts(input.implementation) + && input.boundary.humanCommentIds.length === count + && input.boundary.humanCommentIds.every((id, index) => input.human[index]?.id === id) + && input.boundary.runIds.every(id => input.runIds.includes(id)) + && (input.implementation.queryOutcome === "rejected" + ? input.human.length === count + : input.human[count]?.body.trim() === input.renderedQuery.trim() + && input.human.length === (input.implementation.expectedHumanMessageCount ?? 1)); +} diff --git a/tests/runner-e2e/chat-catalog-fingerprint.test.ts b/tests/runner-e2e/chat-catalog-fingerprint.test.ts index dfba6e12deb..cce867c4640 100644 --- a/tests/runner-e2e/chat-catalog-fingerprint.test.ts +++ b/tests/runner-e2e/chat-catalog-fingerprint.test.ts @@ -67,7 +67,7 @@ it("binds the real selected area and reviewed worker closure without importing u expect(files).toContain("tests/runner-e2e/chat-catalog-domains/internal-support.ts"); expect(files).toContain("packages/adapter-utils/src/workspace-snapshot-thread-worker.ts"); expect(files).not.toContain("tests/runner-e2e/chat-catalog-domains/internal-goals.ts"); - expect(files).not.toContain("tests/runner-e2e/chat-catalog-domains/identity-auth.ts"); + expect(files).toContain("tests/runner-e2e/chat-catalog-domains/identity-auth.ts"); expect(result.unknownDynamicImports).toEqual([]); }); it("fails closed when preserved prerequisites or a rejected query lack their independent verifier", async () => { diff --git a/tests/runner-e2e/chat-catalog-flow.ts b/tests/runner-e2e/chat-catalog-flow.ts index 897b256aa9b..b5b053d394f 100644 --- a/tests/runner-e2e/chat-catalog-flow.ts +++ b/tests/runner-e2e/chat-catalog-flow.ts @@ -3,7 +3,7 @@ import { chatCatalogCaseFingerprint, normalizedChatCatalogExecution } from "./ch import { expect } from "@playwright/test"; import { pollUntil } from "./api.js"; import { collectChatRunEvidence, sendChatMessage } from "./chat-flow.js"; -import { validateChatCatalogGrade, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js"; +import { chatCatalogHumanHistoryMatches, validateChatCatalogGrade, type ChatCatalogSetupBoundary, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js"; import { findChatCatalogCase, loadChatCatalogImplementation } from "./chat-catalog-registry.js"; import { chatCatalogInventoryDigest } from "./chat-catalog-inventory.js"; import { resolveChatCatalogQuery } from "./chat-query-bindings.js"; @@ -21,6 +21,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { let grade: ChatCatalogGrade | undefined; let failed = false; let renderedQuery = scenario.query; + let setupBoundary: ChatCatalogSetupBoundary = { humanCommentIds: [], runIds: [] }; async function conversation() { await context.refreshIssue(); const issue = context.issue(); @@ -46,8 +47,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { const checks = [...grade.checks, { id: "catalog-complete-conditions", passed: !implementation!.coverageGaps?.length, detail: implementation!.coverageGaps?.length ? `Missing conditions: ${implementation!.coverageGaps.join(", ")}` : "No known catalog condition is omitted." }, - { id: "catalog-original-human-query", passed: (implementation!.queryOutcome === "rejected" ? human.length === 0 : human[0]?.body.trim() === renderedQuery.trim() - && human.length === (implementation!.expectedHumanMessageCount ?? 1)), + { id: "catalog-original-human-query", passed: chatCatalogHumanHistoryMatches({ implementation: implementation!, human, runIds: evidence.runs.map(run => run.id), boundary: setupBoundary, renderedQuery }), detail: "The persisted request preserves the catalog query with only audited public-fixture identity bindings; all human turns were declared." }, ...gradeChatCatalogRuns({ implementation: implementation!, companyId: input.fixtures.company.id, leadId: input.fixtures.agent.id, source: evidence.issue, runs: evidence.runs }).checks, @@ -90,6 +90,13 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { await input.evidence("chat-catalog-prepared-state.json", setupGrade); expect(setupGrade.passed && (implementation.setupPolicy?.invariantCheckIds ?? []).every(id => setupGrade.checks.some(check => check.id === id && check.passed)), "Independent scenario prerequisites").toBe(true); } + const setupConversation = await conversation(); + const setupHuman = setupConversation.comments.filter(comment => !comment.authorAgentId); + expect(setupHuman.length, "Only declared genuine pre-query human turns exist").toBe(implementation.setupHumanMessageCount ?? 0); + expect(setupConversation.runs.length <= implementation.expectedRunCount, "Setup runs count toward the total native run bound").toBe(true); + setupBoundary = { humanCommentIds: setupHuman.map(comment => comment.id), runIds: setupConversation.runs.map(run => run.id) }; + await input.evidence("chat-catalog-setup-boundary.json", { ...setupBoundary, setupHumanMessageCount: implementation.setupHumanMessageCount ?? 0, + expectedHumanMessageCount: implementation.expectedHumanMessageCount ?? 1, expectedRunCount: implementation.expectedRunCount }); await input.evidence("chat-catalog-query.json", { query: scenario.query, expectedOutcome: implementation.queryOutcome ?? "persisted", journeyKind: "agent-chat" }); await sendChatMessage(input.page, renderedQuery); if (implementation.queryOutcome === "rejected") { diff --git a/tests/runner-e2e/chat-catalog-history.test.ts b/tests/runner-e2e/chat-catalog-history.test.ts new file mode 100644 index 00000000000..93ab2b5bab4 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-history.test.ts @@ -0,0 +1,19 @@ +import { expect, it } from "vitest"; +import { chatCatalogHumanHistoryMatches, validChatCatalogHumanCounts } from "./chat-catalog-contract.js"; +it("binds the canonical request after genuine setup history and retains native setup run identities", () => { + const input = { implementation: { setupHumanMessageCount: 1, expectedHumanMessageCount: 2 }, + human: [{ id: "setup", body: "Remember this detail" }, { id: "query", body: "Use the detail" }], + runIds: ["setup-run", "query-run"], boundary: { humanCommentIds: ["setup"], runIds: ["setup-run"] }, renderedQuery: "Use the detail" }; + expect(chatCatalogHumanHistoryMatches(input)).toBe(true); + expect(chatCatalogHumanHistoryMatches({ ...input, human: [...input.human].reverse() })).toBe(false); + expect(chatCatalogHumanHistoryMatches({ ...input, runIds: ["query-run"] })).toBe(false); + expect(chatCatalogHumanHistoryMatches({ ...input, boundary: { ...input.boundary, humanCommentIds: ["injected"] } })).toBe(false); + expect(chatCatalogHumanHistoryMatches({ ...input, human: [...input.human, { id: "hidden", body: "Please finish" }] })).toBe(false); +}); +it("preserves zero-setup semantics and refuses impossible setup/total declarations", () => { + expect(chatCatalogHumanHistoryMatches({ implementation: {}, human: [{ id: "query", body: "Original query" }], runIds: ["run"], boundary: { humanCommentIds: [], runIds: [] }, renderedQuery: "Original query" })).toBe(true); + expect(validChatCatalogHumanCounts({ setupHumanMessageCount: 1, expectedHumanMessageCount: 1 })).toBe(false); + expect(validChatCatalogHumanCounts({ setupHumanMessageCount: -1 })).toBe(false); + expect(validChatCatalogHumanCounts({ setupHumanMessageCount: 0.5 })).toBe(false); + expect(validChatCatalogHumanCounts({ setupHumanMessageCount: 0, expectedHumanMessageCount: 0, queryOutcome: "rejected" })).toBe(true); +}); diff --git a/tests/runner-e2e/chat-catalog-registry.ts b/tests/runner-e2e/chat-catalog-registry.ts index 2a336bc9799..826957cb4f8 100644 --- a/tests/runner-e2e/chat-catalog-registry.ts +++ b/tests/runner-e2e/chat-catalog-registry.ts @@ -1,4 +1,5 @@ import { existsSync, readFileSync } from "node:fs"; +import { validChatCatalogHumanCounts } from "./chat-catalog-contract.js"; import type { ChatCatalogCase } from "./chat-catalog-contract.js"; import { chatCatalogScenarios } from "./chat-catalog-inventory.js"; import type { IdentityBoundaryCase } from "./chat-catalog-domains/identity-boundary-contract.js"; @@ -21,7 +22,7 @@ export function findChatCatalogCase(scenarioId: string): ChatCatalogDescriptor | if (!descriptor) return; if (!known.has(scenarioId) || !descriptor.oracleVersion || !descriptor.sourceFiles.length || !/^\.\/chat-catalog-domains\/[a-z0-9-]+\.js$/.test(descriptor.modulePath) - || !descriptor.exportName || descriptor.expectedRunCount < 0 || descriptor.turnTimeoutMs <= 0 || descriptor.attemptTimeoutMs <= 0 + || !descriptor.exportName || (descriptor.journeyKind !== "browser-boundary" && !validChatCatalogHumanCounts(descriptor)) || descriptor.expectedRunCount < 0 || descriptor.turnTimeoutMs <= 0 || descriptor.attemptTimeoutMs <= 0 || (descriptor.minimumRunCount ?? descriptor.expectedRunCount) > descriptor.expectedRunCount) throw new Error(`Incomplete bounds/provenance for ${scenarioId}`); return descriptor; @@ -43,7 +44,7 @@ export async function loadChatCatalogImplementation( const selected = implementations.filter((row: ChatCatalogCase) => row.scenarioId === descriptor.scenarioId); if (selected.length !== 1) throw new Error(`Missing or duplicate runtime case: ${descriptor.scenarioId}`); const implementation = selected[0] as ChatCatalogCase; - for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings"] as const) { + for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings"] as const) { if (JSON.stringify(implementation[key]) !== JSON.stringify(descriptor[key])) throw new Error(`Runtime descriptor drift: ${descriptor.scenarioId}.${key}`); } if ((descriptor.setupPolicy?.funding === "preserve-scenario" || descriptor.setupPolicy?.readiness === "preserve-scenario") @@ -61,14 +62,14 @@ export async function loadIdentityBoundaryImplementation(descriptor: ChatCatalog importer: (specifier: string) => Promise> = specifier => import(specifier)): Promise { if (descriptor.journeyKind !== "browser-boundary" || !["access.sign-in", "onboarding.new-instance", "onboarding.choose-first-runtime", "onboarding.resume-wizard"].includes(descriptor.scenarioId)) throw new Error("Only explicit catalog browser boundaries may bypass Agent Chat prerequisites"); - if (descriptor.expectedRunCount !== 0 || (descriptor.minimumRunCount ?? 0) !== 0 || descriptor.expectedHumanMessageCount !== 0) + if (descriptor.expectedRunCount !== 0 || (descriptor.minimumRunCount ?? 0) !== 0 || descriptor.expectedHumanMessageCount !== 0 || (descriptor.setupHumanMessageCount ?? 0) !== 0) throw new Error("Pre-chat boundaries require zero agent admissions and zero persisted chat queries"); const module = await importer(chatCatalogModuleUrl(descriptor.modulePath)); const rows = module[descriptor.exportName]; const selected = Array.isArray(rows) ? rows.filter(row => row.scenarioId === descriptor.scenarioId) : []; if (selected.length !== 1) throw new Error("Missing or duplicate browser boundary implementation"); const implementation = selected[0] as IdentityBoundaryCase; - for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "boundaryTarget", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings"] as const) { + for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "boundaryTarget", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings"] as const) { if (JSON.stringify(implementation[key]) !== JSON.stringify(descriptor[key])) throw new Error(`Runtime boundary descriptor drift: ${descriptor.scenarioId}.${key}`); } if ([implementation.run, implementation.grade, implementation.cleanup].some(value => typeof value !== "function") || descriptor.queryBindings?.length