test(evals): guard real native Stop canonical boundary

tests/runner-e2e/chat-catalog-contract.ts

@@ -46,7 +46,11 @@ export interface ChatSlackIngressReceipt {
   provider: { workspaceId: string; channelId: string; threadId: string; canonicalMessageId: string; senderUserId: string };
 }
 /** One concrete scenario, with production API setup and a case-specific independent oracle. */
+export interface NativeStopBoundaryReceipt { runId: string; issueId: string; intentId: string; intentRecordedAt: string }
 export interface ChatCatalogCase {
+  /** Only post-stop-message: actual human Stop after bookkeeping, before unchanged canonical send. */
+  canonicalBoundary?: "native-stop-cleanup";
+  beforeCanonicalSend?(context: ChatCatalogContext, state: Record<string, unknown>): Promise<NativeStopBoundaryReceipt>;
   scenarioId: string;
   /** Real browser locale context for the canonical person's schedule. */
   browserTimezone?: "America/Chicago";

tests/runner-e2e/chat-catalog-flow.ts

@@ -6,6 +6,7 @@ import { chatCatalogCaseFingerprint, normalizedChatCatalogExecution } from "./ch
 import { expect } from "@playwright/test";
 import { pollUntil } from "./api.js";
 import { collectChatRunEvidence, sendChatMessage } from "./chat-flow.js";
+import { prepareNativeStopCanonicalBoundary } from "./chat-native-stop-boundary.js";
 import { chatCatalogHumanHistoryMatches, validateChatCatalogGrade, type ChatCatalogSetupBoundary, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js";
 import { findChatCatalogCase, loadChatCatalogImplementation } from "./chat-catalog-registry.js";
 import { chatCatalogInventoryDigest } from "./chat-catalog-inventory.js";
@@ -125,7 +126,10 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
       state.slackIngressReceipt = await dispatchNativeSlackIngress(context, implementation, state, deadlineAt);
       await input.page.goto(new URL(`/${input.fixtures.company.issuePrefix}/issues/${context.issue()!.id}`, input.api.baseURL).href);
       if (implementation.scenarioId === "chat-channels.channel-file") await completeNativeSlackFileConditional(context, implementation, state, deadlineAt);
-    } else await sendChatMessage(input.page, renderedQuery);
+    } else {
+      await prepareNativeStopCanonicalBoundary(context, implementation, state, renderedQuery, setupBoundary);
+      await sendChatMessage(input.page, renderedQuery);
+    }
     if (implementation.queryOutcome === "rejected") {
       const rejection = validateChatCatalogGrade(await implementation.verifyQueryRejection!(context, state));
       await input.evidence("chat-catalog-query-rejection.json", rejection);

tests/runner-e2e/chat-catalog-registry.ts

@@ -1,3 +1,4 @@
+import { validNativeStopDeclaration, assertNativeStopCallback } from "./chat-native-stop-boundary.js";
 import { validSlackIngressDeclaration } from "./chat-slack-ingress.js";
 import { validCompanionDeclaration } from "./chat-companion-surface.js";
 import { installedChatScenarioIds } from "./chat-installed-lane.js";
@@ -8,7 +9,7 @@ import { chatCatalogScenarios } from "./chat-catalog-inventory.js";
 import type { IdentityBoundaryCase } from "./chat-catalog-domains/identity-boundary-contract.js";
 import { identityBoundaryScenarioIds } from "./chat-catalog-domains/identity-boundary-contract.js";
 
-export type ChatCatalogDescriptor = Omit<ChatCatalogCase, "verifySlackSupportedTransfer" | "dispatchSlackUnsupportedFile" | "dispatchSlackQuery" | "runCompanionSurface" | "journeyKind" | "setup" | "act" | "observe" | "grade" | "cleanup" | "verifyResult" | "verifySetup" | "verifyQueryRejection"> & {
+export type ChatCatalogDescriptor = Omit<ChatCatalogCase, "beforeCanonicalSend" | "dispatchSlackOtherThread" | "verifySlackSupportedTransfer" | "dispatchSlackUnsupportedFile" | "dispatchSlackQuery" | "runCompanionSurface" | "journeyKind" | "setup" | "act" | "observe" | "grade" | "cleanup" | "verifyResult" | "verifySetup" | "verifyQueryRejection"> & {
   journeyKind?: "agent-chat" | "browser-boundary" | "scoped-external-assistant" | "native-slack-thread";
   boundaryTarget?: "authenticated-harness" | "empty-installed-target";
   modulePath: string;
@@ -26,6 +27,7 @@ export function findChatCatalogCase(scenarioId: string): ChatCatalogDescriptor |
   if (!descriptor) return;
   if ((descriptor.companionSurface || descriptor.journeyKind === "scoped-external-assistant") && !validCompanionDeclaration(descriptor)) throw new Error("Only the declared scoped-client catalog case may select an external assistant");
   if ((descriptor.slackIngress || descriptor.journeyKind === "native-slack-thread") && !validSlackIngressDeclaration(descriptor)) throw new Error("Only two declared Slack thread cases may use native ingress");
+  if (descriptor.canonicalBoundary && !validNativeStopDeclaration(descriptor)) throw new Error("Native Stop boundary is exclusive to the exact human post-stop scenario");
   if (descriptor.browserTimezone && (scenarioId !== "routines.create-daily-brief" || descriptor.browserTimezone !== "America/Chicago"))
     throw new Error("Browser timezone is declared only for the canonical Chicago weekday routine");
   if (descriptor.respondingAgent && (scenarioId !== "cases.handoff-same-task" || Object.keys(descriptor.respondingAgent).length !== 4
@@ -57,13 +59,16 @@ export async function loadChatCatalogImplementation(
   const selected = implementations.filter((row: ChatCatalogCase) => row.scenarioId === descriptor.scenarioId);
   if (selected.length !== 1) throw new Error(`Missing or duplicate runtime case: ${descriptor.scenarioId}`);
   const implementation = selected[0] as ChatCatalogCase;
-  for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments", "requiredCredentials", "externalInput", "installedSource", "browserTimezone", "respondingAgent", "companionSurface", "slackIngress"] as const) {
+  for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments", "requiredCredentials", "externalInput", "installedSource", "browserTimezone", "respondingAgent", "companionSurface", "slackIngress", "canonicalBoundary"] as const) {
     if (JSON.stringify(implementation[key]) !== JSON.stringify(descriptor[key])) throw new Error(`Runtime descriptor drift: ${descriptor.scenarioId}.${key}`);
   }
   if ((descriptor.companionSurface || descriptor.journeyKind === "scoped-external-assistant") && (!validCompanionDeclaration(implementation) || typeof implementation.runCompanionSurface !== "function" || implementation.act)) throw new Error("Missing actual finite scoped assistant driver");
   if ((descriptor.slackIngress || descriptor.journeyKind === "native-slack-thread") && (!validSlackIngressDeclaration(implementation) || typeof implementation.dispatchSlackQuery !== "function" || implementation.act)) throw new Error("Missing finite actual Slack ingress driver");
   if (descriptor.scenarioId === "chat-channels.channel-file" && descriptor.slackIngress && (typeof implementation.verifySlackSupportedTransfer !== "function" || typeof implementation.dispatchSlackUnsupportedFile !== "function")) throw new Error("Slack file case requires actual supported and unsupported transfer phases");
   if ((descriptor.scenarioId !== "chat-channels.channel-file" || !descriptor.slackIngress) && (implementation.verifySlackSupportedTransfer || implementation.dispatchSlackUnsupportedFile)) throw new Error("Undeclared Slack file conditional");
+  const declaredOtherThread = descriptor.scenarioId === "chat-channels.reply-in-thread" && descriptor.slackIngress?.setupThread === "same-agent-distractor";
+  if (declaredOtherThread ? typeof implementation.dispatchSlackOtherThread !== "function" : implementation.dispatchSlackOtherThread !== undefined) throw new Error("Missing or undeclared Slack other-thread callback");
+  assertNativeStopCallback(implementation);
   if (!descriptor.slackIngress && implementation.dispatchSlackQuery) throw new Error("Undeclared Slack ingress driver");
   if (!descriptor.companionSurface && implementation.runCompanionSurface) throw new Error("Undeclared external assistant driver");
   if ((descriptor.setupPolicy?.funding === "preserve-scenario" || descriptor.setupPolicy?.readiness === "preserve-scenario")
@@ -79,7 +84,7 @@ export async function loadChatCatalogImplementation(
 /** Separate typed dispatch: no company, agent, or ChatFlowInput is fabricated. */
 export async function loadIdentityBoundaryImplementation(descriptor: ChatCatalogDescriptor,
   importer: (specifier: string) => Promise<Record<string, unknown>> = specifier => import(specifier)): Promise<IdentityBoundaryCase> {
-  if (descriptor.journeyKind !== "browser-boundary" || !(identityBoundaryScenarioIds as readonly string[]).includes(descriptor.scenarioId))
+  if (descriptor.canonicalBoundary || descriptor.journeyKind !== "browser-boundary" || !(identityBoundaryScenarioIds as readonly string[]).includes(descriptor.scenarioId))
     throw new Error("Only explicit catalog browser boundaries may bypass Agent Chat prerequisites");
   if (descriptor.scenarioId === "agent-chat.chat-disabled" && (descriptor.boundaryTarget !== "authenticated-harness" || descriptor.authentication !== "human-session"))
     throw new Error("Disabled Agent Chat requires the actual owned authenticated company target");

tests/runner-e2e/chat-native-stop-boundary.test.ts

@@ -0,0 +1,55 @@
+import { expect, it, vi } from "vitest";
+import type { ChatCatalogCase, ChatCatalogContext, NativeStopBoundaryReceipt } from "./chat-catalog-contract.js";
+import { loadChatCatalogImplementation, type ChatCatalogDescriptor } from "./chat-catalog-registry.js";
+import { assertNativeStopCallback, nativeStopPendingReceiptMatches, prepareNativeStopCanonicalBoundary, validNativeStopDeclaration } from "./chat-native-stop-boundary.js";
+const receipt: NativeStopBoundaryReceipt = { runId: "11111111-1111-4111-8111-111111111111", issueId: "22222222-2222-4222-8222-222222222222", intentId: "33333333-3333-4333-8333-333333333333", intentRecordedAt: "2026-10-26T12:00:00.000Z" };
+const metadata: ChatCatalogDescriptor = { scenarioId: "recovery.post-stop-message", canonicalBoundary: "native-stop-cleanup", authentication: "human-session", sourceFiles: ["runtime-post-stop.ts"], modulePath: "./chat-catalog-domains/runtime-post-stop.js", exportName: "cases", oracleVersion: "control", expectedRunCount: 3, expectedHumanMessageCount: 1, turnTimeoutMs: 120000, attemptTimeoutMs: 600000 };
+function fixture() {
+  const run = { id: receipt.runId, companyId: "company", nativeIssueId: receipt.issueId, status: "running", resultJson: { nativeCancellation: { schema: "paperclip.native-cancellation.v1", companyId: "company", runId: receipt.runId, issueId: receipt.issueId, scope: "run", intentId: receipt.intentId, recordedAt: receipt.intentRecordedAt, dispatchState: "pending", dispatched: false } } };
+  const comments: Array<{ id: string; authorAgentId?: string }> = [], runs = [{ id: receipt.runId }]; let url = "http://fixture/source";
+  const ctx = { scenario: { id: metadata.scenarioId, query: "Make the page simpler." }, issue: () => ({ id: "source" }), allRuns: async () => runs,
+    input: { fixtures: { company: { id: "company" } }, page: { url: () => url }, evidence: vi.fn(async () => {}), api: { get: async (path: string) => path.startsWith("/api/heartbeat-runs/") ? run : comments } } } as unknown as ChatCatalogContext;
+  const implementation = { ...metadata, setup: async () => ({}), observe: async () => ({}), grade: () => ({ passed: false, checks: [] }), verifyResult: async () => {}, beforeCanonicalSend: vi.fn(async () => ({ ...receipt })) } as unknown as ChatCatalogCase;
+  return { run, comments, runs, ctx, implementation, move: () => { url += "/other"; }, setup: { humanCommentIds: [], runIds: [receipt.runId] } };
+}
+it("admits only the exact unchanged human post-stop boundary declaration", () => {
+  expect(validNativeStopDeclaration(metadata)).toBe(true);
+  for (const patch of [{ scenarioId: "recovery.explain-stall" }, { journeyKind: "browser-boundary" }, { authentication: "local-trusted" }, { expectedHumanMessageCount: 2 }, { setupHumanMessageCount: 1 }, { queryOutcome: "rejected" }, { queryAttachments: [{ stateKey: "file", filename: "brief.md" }] }]) expect(validNativeStopDeclaration({ ...metadata, ...patch } as typeof metadata)).toBe(false);
+  expect(() => assertNativeStopCallback({ ...fixture().implementation, beforeCanonicalSend: undefined })).toThrow("callback");
+  expect(() => assertNativeStopCallback({ ...fixture().implementation, canonicalBoundary: undefined })).toThrow("callback");
+});
+it("preflight rejects declaration drift and absent or hidden callbacks before any admission", async () => {
+  const f = fixture(), load = (row: ChatCatalogCase, descriptor = metadata) => loadChatCatalogImplementation(descriptor, async () => ({ cases: [row] }));
+  await expect(load(f.implementation)).resolves.toBe(f.implementation);
+  await expect(load({ ...f.implementation, beforeCanonicalSend: undefined })).rejects.toThrow("callback");
+  await expect(load({ ...f.implementation, canonicalBoundary: undefined })).rejects.toThrow("drift");
+  await expect(load({ ...f.implementation, canonicalBoundary: undefined }, { ...metadata, canonicalBoundary: undefined })).rejects.toThrow("callback");
+});
+it("requires actual same-company admitted running setup work and a pending unacknowledged persisted intent", () => {
+  const input = () => { const f = fixture(); return { receipt: { ...receipt }, run: f.run, companyId: "company", setupRunIds: [receipt.runId] }; };
+  expect(nativeStopPendingReceiptMatches(input())).toBe(true);
+  for (const change of [(v: ReturnType<typeof input>) => { v.run.companyId = "foreign"; }, (v: ReturnType<typeof input>) => { v.run.status = "cancelled"; }, (v: ReturnType<typeof input>) => { v.setupRunIds = []; }, (v: ReturnType<typeof input>) => { v.run.resultJson.nativeCancellation.dispatchState = "acknowledged"; }, (v: ReturnType<typeof input>) => { v.run.resultJson.nativeCancellation.dispatched = true; }, (v: ReturnType<typeof input>) => { v.receipt.intentId = receipt.runId; }, (v: ReturnType<typeof input>) => { v.receipt.intentRecordedAt = "invalid"; }]) { const value = input(); change(value); expect(nativeStopPendingReceiptMatches(value)).toBe(false); }
+});
+it("keeps the canonical human message untouched and rejects hidden turns, source navigation or extra admissions", async () => {
+  const success = fixture(), state = {};
+  await prepareNativeStopCanonicalBoundary(success.ctx, success.implementation, state, success.ctx.scenario.query, success.setup);
+  expect(state).toEqual({ nativeStopBoundary: receipt }); expect(success.implementation.beforeCanonicalSend).toHaveBeenCalledOnce();
+  for (const change of [(f: ReturnType<typeof fixture>) => { f.comments.push({ id: "hidden-human" }); }, (f: ReturnType<typeof fixture>) => { f.runs.push({ id: "extra-run" }); }, (f: ReturnType<typeof fixture>) => { f.move(); }, (f: ReturnType<typeof fixture>) => { f.ctx.scenario.query += " Hurry"; }]) {
+    const f = fixture(); f.implementation.beforeCanonicalSend = async () => { change(f); return receipt; };
+    await expect(prepareNativeStopCanonicalBoundary(f.ctx, f.implementation, {}, f.ctx.scenario.query, f.setup)).rejects.toThrow("changed source/query/history");
+  }
+  const f = fixture(); await expect(prepareNativeStopCanonicalBoundary(f.ctx, f.implementation, {}, "Alternate direction", f.setup)).rejects.toThrow("alter"); expect(f.implementation.beforeCanonicalSend).not.toHaveBeenCalled();
+});
+it("does no boundary work in an ordinary undeclared catalog scenario", async () => {
+  const f = fixture(); await prepareNativeStopCanonicalBoundary(f.ctx, { ...f.implementation, canonicalBoundary: undefined, beforeCanonicalSend: undefined }, {}, "Exact ordinary query", f.setup);
+  expect(f.ctx.input.evidence).not.toHaveBeenCalled();
+});
+it("preflights the finite Slack distractor callback only for its declared reply thread", async () => {
+  const descriptor: ChatCatalogDescriptor = { ...metadata, scenarioId: "chat-channels.reply-in-thread", canonicalBoundary: undefined, journeyKind: "native-slack-thread", slackIngress: { kind: "native-slack-thread", maximumProviderCalls: 20, ingressTimeoutMs: 120000, setupThread: "same-agent-distractor" }, expectedRunCount: 4, minimumRunCount: 2 };
+  const row = { ...fixture().implementation, ...descriptor, beforeCanonicalSend: undefined, dispatchSlackQuery: async () => ({}), dispatchSlackOtherThread: async () => ({}) } as unknown as ChatCatalogCase;
+  const load = (implementation: ChatCatalogCase, selected = descriptor) => loadChatCatalogImplementation(selected, async () => ({ cases: [implementation] }));
+  await expect(load(row)).resolves.toBe(row);
+  await expect(load({ ...row, dispatchSlackOtherThread: undefined })).rejects.toThrow("other-thread callback");
+  const ordinary = { ...metadata, canonicalBoundary: undefined };
+  await expect(load({ ...fixture().implementation, canonicalBoundary: undefined, beforeCanonicalSend: undefined, dispatchSlackOtherThread: row.dispatchSlackOtherThread }, ordinary)).rejects.toThrow("other-thread callback");
+});

tests/runner-e2e/chat-native-stop-boundary.ts

@@ -0,0 +1,33 @@
+import type { ChatCatalogCase, ChatCatalogContext, ChatCatalogSetupBoundary, NativeStopBoundaryReceipt } from "./chat-catalog-contract.js";
+export function validNativeStopDeclaration(value: Omit<Pick<ChatCatalogCase, "scenarioId" | "canonicalBoundary" | "authentication" | "journeyKind" | "expectedHumanMessageCount" | "setupHumanMessageCount" | "queryOutcome" | "queryBindings" | "queryAttachments">, "journeyKind"> & { journeyKind?: string }) {
+  return value.canonicalBoundary === "native-stop-cleanup" && value.scenarioId === "recovery.post-stop-message" && value.authentication === "human-session"
+    && (value.journeyKind ?? "agent-chat") === "agent-chat" && (value.expectedHumanMessageCount ?? 1) === 1 && (value.setupHumanMessageCount ?? 0) === 0
+    && (value.queryOutcome ?? "persisted") === "persisted" && !value.queryBindings?.length && !value.queryAttachments?.length;
+}
+export function assertNativeStopCallback(implementation: ChatCatalogCase) {
+  if (implementation.canonicalBoundary ? !validNativeStopDeclaration(implementation) || typeof implementation.beforeCanonicalSend !== "function" : implementation.beforeCanonicalSend !== undefined)
+    throw new Error("Missing, wrong-case or undeclared native Stop canonical-boundary callback");
+}
+export function nativeStopPendingReceiptMatches(input: { receipt: NativeStopBoundaryReceipt; run: { id?: string; companyId?: string; nativeIssueId?: string; status?: string; resultJson?: { nativeCancellation?: Record<string, unknown> } }; companyId: string; setupRunIds: string[] }) {
+  const { receipt: r, run } = input, intent = run.resultJson?.nativeCancellation;
+  return !!r && Object.keys(r).every(key => ["runId", "issueId", "intentId", "intentRecordedAt"].includes(key)) && [r.runId, r.issueId, r.intentId].every(id => typeof id === "string" && /^[a-f0-9-]{36}$/i.test(id))
+    && Number.isFinite(Date.parse(r.intentRecordedAt)) && input.setupRunIds.includes(r.runId) && run.id === r.runId && run.companyId === input.companyId && run.nativeIssueId === r.issueId
+    && run.status === "running" && intent?.schema === "paperclip.native-cancellation.v1" && intent.companyId === input.companyId && intent.runId === r.runId && intent.issueId === r.issueId
+    && intent.intentId === r.intentId && intent.recordedAt === r.intentRecordedAt && intent.scope === "run" && intent.dispatchState === "pending" && intent.dispatched === false && intent.acknowledgedAt == null;
+}
+export async function prepareNativeStopCanonicalBoundary(ctx: ChatCatalogContext, implementation: ChatCatalogCase, state: Record<string, unknown>, canonicalQuery: string, setup: ChatCatalogSetupBoundary) {
+  assertNativeStopCallback(implementation); if (!implementation.canonicalBoundary) return;
+  if (canonicalQuery !== ctx.scenario.query || setup.humanCommentIds.length) throw new Error("Native Stop boundary cannot alter the canonical query or introduce setup chat turns");
+  const source = ctx.issue(); if (!source) throw new Error("Native Stop boundary requires the existing actual source chat");
+  const sourceURL = ctx.input.page.url(), originalQuery = ctx.scenario.query;
+  const receipt = await implementation.beforeCanonicalSend!(ctx, state);
+  const [run, comments, runs] = await Promise.all([
+    ctx.input.api.get<Parameters<typeof nativeStopPendingReceiptMatches>[0]["run"]>(`/api/heartbeat-runs/${receipt.runId}`),
+    ctx.input.api.get<Array<{ id: string; authorAgentId?: string }>>(`/api/issues/${source.id}/comments?order=asc`), ctx.allRuns(),
+  ]);
+  if (!nativeStopPendingReceiptMatches({ receipt, run, companyId: ctx.input.fixtures.company.id, setupRunIds: setup.runIds }) || ctx.issue()?.id !== source.id || ctx.input.page.url() !== sourceURL
+    || ctx.scenario.query !== originalQuery || canonicalQuery !== originalQuery || comments.some(row => !row.authorAgentId)
+    || runs.length !== setup.runIds.length || runs.some(row => !setup.runIds.includes(row.id))) throw new Error("Native Stop boundary changed source/query/history or missed the genuine pending cleanup interval");
+  state.nativeStopBoundary = receipt;
+  await ctx.input.evidence("chat-catalog-native-stop-boundary.json", { kind: implementation.canonicalBoundary, sourceIssueId: source.id, ...receipt, canonicalQueryUnchanged: true, extraHumanMessages: 0, extraRuns: 0 });
+}