diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index beb6001c2b9..23d5f5ebfcc 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -46,7 +46,11 @@ export interface ChatSlackIngressReceipt { provider: { workspaceId: string; channelId: string; threadId: string; canonicalMessageId: string; senderUserId: string }; } /** One concrete scenario, with production API setup and a case-specific independent oracle. */ +export interface NativeStopBoundaryReceipt { runId: string; issueId: string; intentId: string; intentRecordedAt: string } export interface ChatCatalogCase { + /** Only post-stop-message: actual human Stop after bookkeeping, before unchanged canonical send. */ + canonicalBoundary?: "native-stop-cleanup"; + beforeCanonicalSend?(context: ChatCatalogContext, state: Record): Promise; scenarioId: string; /** Real browser locale context for the canonical person's schedule. */ browserTimezone?: "America/Chicago"; diff --git a/tests/runner-e2e/chat-catalog-flow.ts b/tests/runner-e2e/chat-catalog-flow.ts index 60d5125d71b..bc4def856af 100644 --- a/tests/runner-e2e/chat-catalog-flow.ts +++ b/tests/runner-e2e/chat-catalog-flow.ts @@ -6,6 +6,7 @@ import { chatCatalogCaseFingerprint, normalizedChatCatalogExecution } from "./ch import { expect } from "@playwright/test"; import { pollUntil } from "./api.js"; import { collectChatRunEvidence, sendChatMessage } from "./chat-flow.js"; +import { prepareNativeStopCanonicalBoundary } from "./chat-native-stop-boundary.js"; import { chatCatalogHumanHistoryMatches, validateChatCatalogGrade, type ChatCatalogSetupBoundary, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js"; import { findChatCatalogCase, loadChatCatalogImplementation } from "./chat-catalog-registry.js"; import { chatCatalogInventoryDigest } from "./chat-catalog-inventory.js"; @@ -125,7 +126,10 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { state.slackIngressReceipt = await dispatchNativeSlackIngress(context, implementation, state, deadlineAt); await input.page.goto(new URL(`/${input.fixtures.company.issuePrefix}/issues/${context.issue()!.id}`, input.api.baseURL).href); if (implementation.scenarioId === "chat-channels.channel-file") await completeNativeSlackFileConditional(context, implementation, state, deadlineAt); - } else await sendChatMessage(input.page, renderedQuery); + } else { + await prepareNativeStopCanonicalBoundary(context, implementation, state, renderedQuery, setupBoundary); + await sendChatMessage(input.page, renderedQuery); + } if (implementation.queryOutcome === "rejected") { const rejection = validateChatCatalogGrade(await implementation.verifyQueryRejection!(context, state)); await input.evidence("chat-catalog-query-rejection.json", rejection); diff --git a/tests/runner-e2e/chat-catalog-registry.ts b/tests/runner-e2e/chat-catalog-registry.ts index 5227b6fd7d0..c4e415d80b1 100644 --- a/tests/runner-e2e/chat-catalog-registry.ts +++ b/tests/runner-e2e/chat-catalog-registry.ts @@ -1,3 +1,4 @@ +import { validNativeStopDeclaration, assertNativeStopCallback } from "./chat-native-stop-boundary.js"; import { validSlackIngressDeclaration } from "./chat-slack-ingress.js"; import { validCompanionDeclaration } from "./chat-companion-surface.js"; import { installedChatScenarioIds } from "./chat-installed-lane.js"; @@ -8,7 +9,7 @@ import { chatCatalogScenarios } from "./chat-catalog-inventory.js"; import type { IdentityBoundaryCase } from "./chat-catalog-domains/identity-boundary-contract.js"; import { identityBoundaryScenarioIds } from "./chat-catalog-domains/identity-boundary-contract.js"; -export type ChatCatalogDescriptor = Omit & { +export type ChatCatalogDescriptor = Omit & { journeyKind?: "agent-chat" | "browser-boundary" | "scoped-external-assistant" | "native-slack-thread"; boundaryTarget?: "authenticated-harness" | "empty-installed-target"; modulePath: string; @@ -26,6 +27,7 @@ export function findChatCatalogCase(scenarioId: string): ChatCatalogDescriptor | if (!descriptor) return; if ((descriptor.companionSurface || descriptor.journeyKind === "scoped-external-assistant") && !validCompanionDeclaration(descriptor)) throw new Error("Only the declared scoped-client catalog case may select an external assistant"); if ((descriptor.slackIngress || descriptor.journeyKind === "native-slack-thread") && !validSlackIngressDeclaration(descriptor)) throw new Error("Only two declared Slack thread cases may use native ingress"); + if (descriptor.canonicalBoundary && !validNativeStopDeclaration(descriptor)) throw new Error("Native Stop boundary is exclusive to the exact human post-stop scenario"); if (descriptor.browserTimezone && (scenarioId !== "routines.create-daily-brief" || descriptor.browserTimezone !== "America/Chicago")) throw new Error("Browser timezone is declared only for the canonical Chicago weekday routine"); if (descriptor.respondingAgent && (scenarioId !== "cases.handoff-same-task" || Object.keys(descriptor.respondingAgent).length !== 4 @@ -57,13 +59,16 @@ export async function loadChatCatalogImplementation( const selected = implementations.filter((row: ChatCatalogCase) => row.scenarioId === descriptor.scenarioId); if (selected.length !== 1) throw new Error(`Missing or duplicate runtime case: ${descriptor.scenarioId}`); const implementation = selected[0] as ChatCatalogCase; - for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments", "requiredCredentials", "externalInput", "installedSource", "browserTimezone", "respondingAgent", "companionSurface", "slackIngress"] as const) { + for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments", "requiredCredentials", "externalInput", "installedSource", "browserTimezone", "respondingAgent", "companionSurface", "slackIngress", "canonicalBoundary"] as const) { if (JSON.stringify(implementation[key]) !== JSON.stringify(descriptor[key])) throw new Error(`Runtime descriptor drift: ${descriptor.scenarioId}.${key}`); } if ((descriptor.companionSurface || descriptor.journeyKind === "scoped-external-assistant") && (!validCompanionDeclaration(implementation) || typeof implementation.runCompanionSurface !== "function" || implementation.act)) throw new Error("Missing actual finite scoped assistant driver"); if ((descriptor.slackIngress || descriptor.journeyKind === "native-slack-thread") && (!validSlackIngressDeclaration(implementation) || typeof implementation.dispatchSlackQuery !== "function" || implementation.act)) throw new Error("Missing finite actual Slack ingress driver"); if (descriptor.scenarioId === "chat-channels.channel-file" && descriptor.slackIngress && (typeof implementation.verifySlackSupportedTransfer !== "function" || typeof implementation.dispatchSlackUnsupportedFile !== "function")) throw new Error("Slack file case requires actual supported and unsupported transfer phases"); if ((descriptor.scenarioId !== "chat-channels.channel-file" || !descriptor.slackIngress) && (implementation.verifySlackSupportedTransfer || implementation.dispatchSlackUnsupportedFile)) throw new Error("Undeclared Slack file conditional"); + const declaredOtherThread = descriptor.scenarioId === "chat-channels.reply-in-thread" && descriptor.slackIngress?.setupThread === "same-agent-distractor"; + if (declaredOtherThread ? typeof implementation.dispatchSlackOtherThread !== "function" : implementation.dispatchSlackOtherThread !== undefined) throw new Error("Missing or undeclared Slack other-thread callback"); + assertNativeStopCallback(implementation); if (!descriptor.slackIngress && implementation.dispatchSlackQuery) throw new Error("Undeclared Slack ingress driver"); if (!descriptor.companionSurface && implementation.runCompanionSurface) throw new Error("Undeclared external assistant driver"); if ((descriptor.setupPolicy?.funding === "preserve-scenario" || descriptor.setupPolicy?.readiness === "preserve-scenario") @@ -79,7 +84,7 @@ export async function loadChatCatalogImplementation( /** Separate typed dispatch: no company, agent, or ChatFlowInput is fabricated. */ export async function loadIdentityBoundaryImplementation(descriptor: ChatCatalogDescriptor, importer: (specifier: string) => Promise> = specifier => import(specifier)): Promise { - if (descriptor.journeyKind !== "browser-boundary" || !(identityBoundaryScenarioIds as readonly string[]).includes(descriptor.scenarioId)) + if (descriptor.canonicalBoundary || descriptor.journeyKind !== "browser-boundary" || !(identityBoundaryScenarioIds as readonly string[]).includes(descriptor.scenarioId)) throw new Error("Only explicit catalog browser boundaries may bypass Agent Chat prerequisites"); if (descriptor.scenarioId === "agent-chat.chat-disabled" && (descriptor.boundaryTarget !== "authenticated-harness" || descriptor.authentication !== "human-session")) throw new Error("Disabled Agent Chat requires the actual owned authenticated company target"); diff --git a/tests/runner-e2e/chat-native-stop-boundary.test.ts b/tests/runner-e2e/chat-native-stop-boundary.test.ts new file mode 100644 index 00000000000..3b81fa167a4 --- /dev/null +++ b/tests/runner-e2e/chat-native-stop-boundary.test.ts @@ -0,0 +1,55 @@ +import { expect, it, vi } from "vitest"; +import type { ChatCatalogCase, ChatCatalogContext, NativeStopBoundaryReceipt } from "./chat-catalog-contract.js"; +import { loadChatCatalogImplementation, type ChatCatalogDescriptor } from "./chat-catalog-registry.js"; +import { assertNativeStopCallback, nativeStopPendingReceiptMatches, prepareNativeStopCanonicalBoundary, validNativeStopDeclaration } from "./chat-native-stop-boundary.js"; +const receipt: NativeStopBoundaryReceipt = { runId: "11111111-1111-4111-8111-111111111111", issueId: "22222222-2222-4222-8222-222222222222", intentId: "33333333-3333-4333-8333-333333333333", intentRecordedAt: "2026-10-26T12:00:00.000Z" }; +const metadata: ChatCatalogDescriptor = { scenarioId: "recovery.post-stop-message", canonicalBoundary: "native-stop-cleanup", authentication: "human-session", sourceFiles: ["runtime-post-stop.ts"], modulePath: "./chat-catalog-domains/runtime-post-stop.js", exportName: "cases", oracleVersion: "control", expectedRunCount: 3, expectedHumanMessageCount: 1, turnTimeoutMs: 120000, attemptTimeoutMs: 600000 }; +function fixture() { + const run = { id: receipt.runId, companyId: "company", nativeIssueId: receipt.issueId, status: "running", resultJson: { nativeCancellation: { schema: "paperclip.native-cancellation.v1", companyId: "company", runId: receipt.runId, issueId: receipt.issueId, scope: "run", intentId: receipt.intentId, recordedAt: receipt.intentRecordedAt, dispatchState: "pending", dispatched: false } } }; + const comments: Array<{ id: string; authorAgentId?: string }> = [], runs = [{ id: receipt.runId }]; let url = "http://fixture/source"; + const ctx = { scenario: { id: metadata.scenarioId, query: "Make the page simpler." }, issue: () => ({ id: "source" }), allRuns: async () => runs, + input: { fixtures: { company: { id: "company" } }, page: { url: () => url }, evidence: vi.fn(async () => {}), api: { get: async (path: string) => path.startsWith("/api/heartbeat-runs/") ? run : comments } } } as unknown as ChatCatalogContext; + const implementation = { ...metadata, setup: async () => ({}), observe: async () => ({}), grade: () => ({ passed: false, checks: [] }), verifyResult: async () => {}, beforeCanonicalSend: vi.fn(async () => ({ ...receipt })) } as unknown as ChatCatalogCase; + return { run, comments, runs, ctx, implementation, move: () => { url += "/other"; }, setup: { humanCommentIds: [], runIds: [receipt.runId] } }; +} +it("admits only the exact unchanged human post-stop boundary declaration", () => { + expect(validNativeStopDeclaration(metadata)).toBe(true); + for (const patch of [{ scenarioId: "recovery.explain-stall" }, { journeyKind: "browser-boundary" }, { authentication: "local-trusted" }, { expectedHumanMessageCount: 2 }, { setupHumanMessageCount: 1 }, { queryOutcome: "rejected" }, { queryAttachments: [{ stateKey: "file", filename: "brief.md" }] }]) expect(validNativeStopDeclaration({ ...metadata, ...patch } as typeof metadata)).toBe(false); + expect(() => assertNativeStopCallback({ ...fixture().implementation, beforeCanonicalSend: undefined })).toThrow("callback"); + expect(() => assertNativeStopCallback({ ...fixture().implementation, canonicalBoundary: undefined })).toThrow("callback"); +}); +it("preflight rejects declaration drift and absent or hidden callbacks before any admission", async () => { + const f = fixture(), load = (row: ChatCatalogCase, descriptor = metadata) => loadChatCatalogImplementation(descriptor, async () => ({ cases: [row] })); + await expect(load(f.implementation)).resolves.toBe(f.implementation); + await expect(load({ ...f.implementation, beforeCanonicalSend: undefined })).rejects.toThrow("callback"); + await expect(load({ ...f.implementation, canonicalBoundary: undefined })).rejects.toThrow("drift"); + await expect(load({ ...f.implementation, canonicalBoundary: undefined }, { ...metadata, canonicalBoundary: undefined })).rejects.toThrow("callback"); +}); +it("requires actual same-company admitted running setup work and a pending unacknowledged persisted intent", () => { + const input = () => { const f = fixture(); return { receipt: { ...receipt }, run: f.run, companyId: "company", setupRunIds: [receipt.runId] }; }; + expect(nativeStopPendingReceiptMatches(input())).toBe(true); + for (const change of [(v: ReturnType) => { v.run.companyId = "foreign"; }, (v: ReturnType) => { v.run.status = "cancelled"; }, (v: ReturnType) => { v.setupRunIds = []; }, (v: ReturnType) => { v.run.resultJson.nativeCancellation.dispatchState = "acknowledged"; }, (v: ReturnType) => { v.run.resultJson.nativeCancellation.dispatched = true; }, (v: ReturnType) => { v.receipt.intentId = receipt.runId; }, (v: ReturnType) => { v.receipt.intentRecordedAt = "invalid"; }]) { const value = input(); change(value); expect(nativeStopPendingReceiptMatches(value)).toBe(false); } +}); +it("keeps the canonical human message untouched and rejects hidden turns, source navigation or extra admissions", async () => { + const success = fixture(), state = {}; + await prepareNativeStopCanonicalBoundary(success.ctx, success.implementation, state, success.ctx.scenario.query, success.setup); + expect(state).toEqual({ nativeStopBoundary: receipt }); expect(success.implementation.beforeCanonicalSend).toHaveBeenCalledOnce(); + for (const change of [(f: ReturnType) => { f.comments.push({ id: "hidden-human" }); }, (f: ReturnType) => { f.runs.push({ id: "extra-run" }); }, (f: ReturnType) => { f.move(); }, (f: ReturnType) => { f.ctx.scenario.query += " Hurry"; }]) { + const f = fixture(); f.implementation.beforeCanonicalSend = async () => { change(f); return receipt; }; + await expect(prepareNativeStopCanonicalBoundary(f.ctx, f.implementation, {}, f.ctx.scenario.query, f.setup)).rejects.toThrow("changed source/query/history"); + } + const f = fixture(); await expect(prepareNativeStopCanonicalBoundary(f.ctx, f.implementation, {}, "Alternate direction", f.setup)).rejects.toThrow("alter"); expect(f.implementation.beforeCanonicalSend).not.toHaveBeenCalled(); +}); +it("does no boundary work in an ordinary undeclared catalog scenario", async () => { + const f = fixture(); await prepareNativeStopCanonicalBoundary(f.ctx, { ...f.implementation, canonicalBoundary: undefined, beforeCanonicalSend: undefined }, {}, "Exact ordinary query", f.setup); + expect(f.ctx.input.evidence).not.toHaveBeenCalled(); +}); +it("preflights the finite Slack distractor callback only for its declared reply thread", async () => { + const descriptor: ChatCatalogDescriptor = { ...metadata, scenarioId: "chat-channels.reply-in-thread", canonicalBoundary: undefined, journeyKind: "native-slack-thread", slackIngress: { kind: "native-slack-thread", maximumProviderCalls: 20, ingressTimeoutMs: 120000, setupThread: "same-agent-distractor" }, expectedRunCount: 4, minimumRunCount: 2 }; + const row = { ...fixture().implementation, ...descriptor, beforeCanonicalSend: undefined, dispatchSlackQuery: async () => ({}), dispatchSlackOtherThread: async () => ({}) } as unknown as ChatCatalogCase; + const load = (implementation: ChatCatalogCase, selected = descriptor) => loadChatCatalogImplementation(selected, async () => ({ cases: [implementation] })); + await expect(load(row)).resolves.toBe(row); + await expect(load({ ...row, dispatchSlackOtherThread: undefined })).rejects.toThrow("other-thread callback"); + const ordinary = { ...metadata, canonicalBoundary: undefined }; + await expect(load({ ...fixture().implementation, canonicalBoundary: undefined, beforeCanonicalSend: undefined, dispatchSlackOtherThread: row.dispatchSlackOtherThread }, ordinary)).rejects.toThrow("other-thread callback"); +}); diff --git a/tests/runner-e2e/chat-native-stop-boundary.ts b/tests/runner-e2e/chat-native-stop-boundary.ts new file mode 100644 index 00000000000..a9059095dcf --- /dev/null +++ b/tests/runner-e2e/chat-native-stop-boundary.ts @@ -0,0 +1,33 @@ +import type { ChatCatalogCase, ChatCatalogContext, ChatCatalogSetupBoundary, NativeStopBoundaryReceipt } from "./chat-catalog-contract.js"; +export function validNativeStopDeclaration(value: Omit, "journeyKind"> & { journeyKind?: string }) { + return value.canonicalBoundary === "native-stop-cleanup" && value.scenarioId === "recovery.post-stop-message" && value.authentication === "human-session" + && (value.journeyKind ?? "agent-chat") === "agent-chat" && (value.expectedHumanMessageCount ?? 1) === 1 && (value.setupHumanMessageCount ?? 0) === 0 + && (value.queryOutcome ?? "persisted") === "persisted" && !value.queryBindings?.length && !value.queryAttachments?.length; +} +export function assertNativeStopCallback(implementation: ChatCatalogCase) { + if (implementation.canonicalBoundary ? !validNativeStopDeclaration(implementation) || typeof implementation.beforeCanonicalSend !== "function" : implementation.beforeCanonicalSend !== undefined) + throw new Error("Missing, wrong-case or undeclared native Stop canonical-boundary callback"); +} +export function nativeStopPendingReceiptMatches(input: { receipt: NativeStopBoundaryReceipt; run: { id?: string; companyId?: string; nativeIssueId?: string; status?: string; resultJson?: { nativeCancellation?: Record } }; companyId: string; setupRunIds: string[] }) { + const { receipt: r, run } = input, intent = run.resultJson?.nativeCancellation; + return !!r && Object.keys(r).every(key => ["runId", "issueId", "intentId", "intentRecordedAt"].includes(key)) && [r.runId, r.issueId, r.intentId].every(id => typeof id === "string" && /^[a-f0-9-]{36}$/i.test(id)) + && Number.isFinite(Date.parse(r.intentRecordedAt)) && input.setupRunIds.includes(r.runId) && run.id === r.runId && run.companyId === input.companyId && run.nativeIssueId === r.issueId + && run.status === "running" && intent?.schema === "paperclip.native-cancellation.v1" && intent.companyId === input.companyId && intent.runId === r.runId && intent.issueId === r.issueId + && intent.intentId === r.intentId && intent.recordedAt === r.intentRecordedAt && intent.scope === "run" && intent.dispatchState === "pending" && intent.dispatched === false && intent.acknowledgedAt == null; +} +export async function prepareNativeStopCanonicalBoundary(ctx: ChatCatalogContext, implementation: ChatCatalogCase, state: Record, canonicalQuery: string, setup: ChatCatalogSetupBoundary) { + assertNativeStopCallback(implementation); if (!implementation.canonicalBoundary) return; + if (canonicalQuery !== ctx.scenario.query || setup.humanCommentIds.length) throw new Error("Native Stop boundary cannot alter the canonical query or introduce setup chat turns"); + const source = ctx.issue(); if (!source) throw new Error("Native Stop boundary requires the existing actual source chat"); + const sourceURL = ctx.input.page.url(), originalQuery = ctx.scenario.query; + const receipt = await implementation.beforeCanonicalSend!(ctx, state); + const [run, comments, runs] = await Promise.all([ + ctx.input.api.get[0]["run"]>(`/api/heartbeat-runs/${receipt.runId}`), + ctx.input.api.get>(`/api/issues/${source.id}/comments?order=asc`), ctx.allRuns(), + ]); + if (!nativeStopPendingReceiptMatches({ receipt, run, companyId: ctx.input.fixtures.company.id, setupRunIds: setup.runIds }) || ctx.issue()?.id !== source.id || ctx.input.page.url() !== sourceURL + || ctx.scenario.query !== originalQuery || canonicalQuery !== originalQuery || comments.some(row => !row.authorAgentId) + || runs.length !== setup.runIds.length || runs.some(row => !setup.runIds.includes(row.id))) throw new Error("Native Stop boundary changed source/query/history or missed the genuine pending cleanup interval"); + state.nativeStopBoundary = receipt; + await ctx.input.evidence("chat-catalog-native-stop-boundary.json", { kind: implementation.canonicalBoundary, sourceIssueId: source.id, ...receipt, canonicalQueryUnchanged: true, extraHumanMessages: 0, extraRuns: 0 }); +}