From 83ed9d147723c9862722bc5c8d9a8b8833655a1e Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 23:10:33 -0500 Subject: [PATCH] test(evals): require actual unsupported Slack file phase after transfer --- tests/runner-e2e/chat-catalog-contract.ts | 3 ++ tests/runner-e2e/chat-catalog-flow.ts | 5 +-- tests/runner-e2e/chat-catalog-registry.ts | 4 ++- tests/runner-e2e/chat-slack-ingress.test.ts | 21 ++++++++++- tests/runner-e2e/chat-slack-ingress.ts | 39 ++++++++++++++++----- 5 files changed, 60 insertions(+), 12 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index cc6077cbad4..418526ff04d 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -58,6 +58,9 @@ export interface ChatCatalogCase { dispatchSlackQuery?(context: ChatCatalogContext, state: Record, input: { query: string; signal: AbortSignal }): Promise; companionSurface?: ChatCompanionSurface; runCompanionSurface?(context: ChatCatalogContext, state: Record, input: { query: string; signal: AbortSignal }): Promise; + /** Only channel-file: independently settle the supported transfer before a second actual upload. */ + verifySlackSupportedTransfer?(context: ChatCatalogContext, state: Record): Promise; + dispatchSlackUnsupportedFile?(context: ChatCatalogContext, state: Record, input: { query: string; signal: AbortSignal }): Promise; authentication?: "local-trusted" | "human-session"; setupPolicy?: ChatCatalogSetupPolicy; runPolicy?: ChatCatalogRunPolicy; diff --git a/tests/runner-e2e/chat-catalog-flow.ts b/tests/runner-e2e/chat-catalog-flow.ts index c312b69be8f..dacb82dc9da 100644 --- a/tests/runner-e2e/chat-catalog-flow.ts +++ b/tests/runner-e2e/chat-catalog-flow.ts @@ -1,4 +1,4 @@ -import { dispatchNativeSlackIngress, slackNativeRunCheck } from "./chat-slack-ingress.js"; +import { completeNativeSlackFileConditional, dispatchNativeSlackIngress, slackNativeRunCheck } from "./chat-slack-ingress.js"; import { runScopedCompanionSurface } from "./chat-companion-surface.js"; import { chatQueryWithAttachments, resolveChatQueryAttachments, verifyChatQueryAttachments, type ChatQueryAttachmentReceipt } from "./chat-query-attachments.js"; import { chatCatalogAllowedRunStatuses, gradeChatCatalogRuns } from "./chat-catalog-run-policy.js"; @@ -62,7 +62,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { detail: "The persisted request preserves the catalog query with only audited public-fixture identity bindings; all human turns were declared." }, { id: "catalog-query-attachment-receipts", passed: attachmentsValid, detail: "Only declared production attachment suffixes are allowed, backed by same-source human receipts and independently retrieved matching bytes." }, - ...(slack ? [slackNativeRunCheck(context, evidence.runs)] : []), + ...(slack ? [slackNativeRunCheck(context, evidence.runs, state)] : []), ...gradeChatCatalogRuns({ implementation: implementation!, companyId: input.fixtures.company.id, leadId: input.fixtures.agent.id, source: evidence.issue, runs: evidence.runs }).checks, ]; @@ -121,6 +121,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { if (setupBoundary.runIds.length || setupBoundary.humanCommentIds.length) throw new Error("Native Slack setup cannot admit canonical or unrelated work"); state.slackIngressReceipt = await dispatchNativeSlackIngress(context, implementation, state, deadlineAt); await input.page.goto(new URL(`/${input.fixtures.company.issuePrefix}/issues/${context.issue()!.id}`, input.api.baseURL).href); + if (implementation.scenarioId === "chat-channels.channel-file") await completeNativeSlackFileConditional(context, implementation, state, deadlineAt); } else await sendChatMessage(input.page, renderedQuery); if (implementation.queryOutcome === "rejected") { const rejection = validateChatCatalogGrade(await implementation.verifyQueryRejection!(context, state)); diff --git a/tests/runner-e2e/chat-catalog-registry.ts b/tests/runner-e2e/chat-catalog-registry.ts index fdee905712e..5227b6fd7d0 100644 --- a/tests/runner-e2e/chat-catalog-registry.ts +++ b/tests/runner-e2e/chat-catalog-registry.ts @@ -8,7 +8,7 @@ import { chatCatalogScenarios } from "./chat-catalog-inventory.js"; import type { IdentityBoundaryCase } from "./chat-catalog-domains/identity-boundary-contract.js"; import { identityBoundaryScenarioIds } from "./chat-catalog-domains/identity-boundary-contract.js"; -export type ChatCatalogDescriptor = Omit & { +export type ChatCatalogDescriptor = Omit & { journeyKind?: "agent-chat" | "browser-boundary" | "scoped-external-assistant" | "native-slack-thread"; boundaryTarget?: "authenticated-harness" | "empty-installed-target"; modulePath: string; @@ -62,6 +62,8 @@ export async function loadChatCatalogImplementation( } if ((descriptor.companionSurface || descriptor.journeyKind === "scoped-external-assistant") && (!validCompanionDeclaration(implementation) || typeof implementation.runCompanionSurface !== "function" || implementation.act)) throw new Error("Missing actual finite scoped assistant driver"); if ((descriptor.slackIngress || descriptor.journeyKind === "native-slack-thread") && (!validSlackIngressDeclaration(implementation) || typeof implementation.dispatchSlackQuery !== "function" || implementation.act)) throw new Error("Missing finite actual Slack ingress driver"); + if (descriptor.scenarioId === "chat-channels.channel-file" && descriptor.slackIngress && (typeof implementation.verifySlackSupportedTransfer !== "function" || typeof implementation.dispatchSlackUnsupportedFile !== "function")) throw new Error("Slack file case requires actual supported and unsupported transfer phases"); + if ((descriptor.scenarioId !== "chat-channels.channel-file" || !descriptor.slackIngress) && (implementation.verifySlackSupportedTransfer || implementation.dispatchSlackUnsupportedFile)) throw new Error("Undeclared Slack file conditional"); if (!descriptor.slackIngress && implementation.dispatchSlackQuery) throw new Error("Undeclared Slack ingress driver"); if (!descriptor.companionSurface && implementation.runCompanionSurface) throw new Error("Undeclared external assistant driver"); if ((descriptor.setupPolicy?.funding === "preserve-scenario" || descriptor.setupPolicy?.readiness === "preserve-scenario") diff --git a/tests/runner-e2e/chat-slack-ingress.test.ts b/tests/runner-e2e/chat-slack-ingress.test.ts index df3d94bac38..d305801de1b 100644 --- a/tests/runner-e2e/chat-slack-ingress.test.ts +++ b/tests/runner-e2e/chat-slack-ingress.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { loadChatCatalogImplementation, type ChatCatalogDescriptor } from "./chat-catalog-registry.js"; -import { slackIngressReceiptMatches, validSlackIngressDeclaration, slackNativeRunCheck } from "./chat-slack-ingress.js"; +import { sameSlackThreadNewMessage, slackIngressReceiptMatches, validSlackIngressDeclaration, slackNativeRunCheck } from "./chat-slack-ingress.js"; import type { ChatCatalogContext, ChatSlackIngressReceipt } from "./chat-catalog-contract.js"; import type { ChatConversation, ChatEndpoint, ChatIdentityLink, ExternalChannelBindingSummary } from "../../packages/shared/src/types/chat-channels.js"; import type { IssueComment } from "../../packages/shared/src/types/issue.js"; @@ -44,11 +44,30 @@ describe("only real declared Slack canonical ingress", () => { const actual = { ...implementation, dispatchSlackQuery: async () => input().receipt }; await expect(loadChatCatalogImplementation(descriptor, async () => ({ slackCases: [actual] }))).resolves.toBe(actual); await expect(loadChatCatalogImplementation(descriptor, async () => ({ slackCases: [{ ...actual, act: async () => {} }] }))).rejects.toThrow("actual Slack ingress"); + const fileDescriptor: ChatCatalogDescriptor = { ...descriptor, scenarioId: "chat-channels.channel-file", expectedRunCount: 6, minimumRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 900000 }; + const file = { ...actual, ...fileDescriptor }; + await expect(loadChatCatalogImplementation(fileDescriptor, async () => ({ slackCases: [file] }))).rejects.toThrow("supported and unsupported transfer phases"); + const completeFile = { ...file, verifySlackSupportedTransfer: async () => ({ passed: true, checks: [{ id: "actual-transfer", passed: true, detail: "Independent phase-specific byte/egress evidence" }] }), dispatchSlackUnsupportedFile: async () => input().receipt }; + await expect(loadChatCatalogImplementation(fileDescriptor, async () => ({ slackCases: [completeFile] }))).resolves.toBe(completeFile); + + }); + it("requires a separate same-thread canonical upload and declared two-phase bounds for the file conditional", () => { + const file = { ...metadata, scenarioId: "chat-channels.channel-file", expectedRunCount: 6, minimumRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 900000 }; + expect(validSlackIngressDeclaration(file)).toBe(true); + expect(validSlackIngressDeclaration({ ...file, expectedHumanMessageCount: 1 })).toBe(false); + expect(validSlackIngressDeclaration({ ...metadata, scenarioId: "chat-channels.channel-file" })).toBe(false); + const first = input().receipt, next = structuredClone(first); next.humanCommentId = "second-comment"; next.provider.canonicalMessageId = "1234567892.123456"; + expect(sameSlackThreadNewMessage(first, next)).toBe(true); + expect(sameSlackThreadNewMessage(first, first)).toBe(false); + expect(sameSlackThreadNewMessage(first, { ...next, query: "Use a different hidden instruction" })).toBe(false); + expect(sameSlackThreadNewMessage(first, { ...next, provider: { ...next.provider, threadId: "9999999999.123456" } })).toBe(false); }); it("cannot qualify a fast response, wrong agent or unrelated native run as the source execution", () => { const ctx = { issue: () => ({ id: "issue", companyId: "company", assigneeAgentId: "agent" }), input: { fixtures: { company: { id: "company" }, agent: { id: "agent" } } } } as unknown as ChatCatalogContext; const run = { id: "run", companyId: "company", agentId: "agent", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: "issue" } }; expect(slackNativeRunCheck(ctx, [run]).passed).toBe(true); + expect(slackNativeRunCheck(ctx, [run], { slackUnsupportedIngressReceipt: {}, slackSupportedTransferRunIds: [run.id] }).passed).toBe(false); + expect(slackNativeRunCheck(ctx, [run, { ...run, id: "second-native" }], { slackUnsupportedIngressReceipt: {}, slackSupportedTransferRunIds: [run.id] }).passed).toBe(true); expect(slackNativeRunCheck(ctx, []).passed).toBe(false); expect(slackNativeRunCheck(ctx, [{ ...run, agentId: "other" }]).passed).toBe(false); expect(slackNativeRunCheck(ctx, [{ ...run, contextSnapshot: { issueId: "other" } }]).passed).toBe(false); diff --git a/tests/runner-e2e/chat-slack-ingress.ts b/tests/runner-e2e/chat-slack-ingress.ts index 61c244ec16c..e98bc2bc251 100644 --- a/tests/runner-e2e/chat-slack-ingress.ts +++ b/tests/runner-e2e/chat-slack-ingress.ts @@ -1,3 +1,5 @@ +import { pollUntil } from "./api.js"; +import { validateChatCatalogGrade } from "./chat-catalog-contract.js"; import type { ChatConversation, ChatEndpoint, ChatIdentityLink, ExternalChannelBindingSummary } from "../../packages/shared/src/types/chat-channels.js"; import type { IssueComment } from "../../packages/shared/src/types/issue.js"; import type { ChatCatalogCase, ChatCatalogContext, ChatSlackIngressReceipt } from "./chat-catalog-contract.js"; @@ -7,8 +9,9 @@ export function validSlackIngressDeclaration(c: Pick section.rows.some(row => row.type === "key_value" && row.label === "Provider ID" && row.value === r.provider.senderUserId)) && input.links.some(link => link.companyId === input.companyId && link.endpointId === e.id && link.status === "linked" && link.paperclipUserId === r.linkedPaperclipUserId); } +export function sameSlackThreadNewMessage(first: ChatSlackIngressReceipt, next: ChatSlackIngressReceipt) { + return next.query === first.query && next.endpointId === first.endpointId && next.conversationId === first.conversationId && next.issueId === first.issueId + && next.linkedPaperclipUserId === first.linkedPaperclipUserId && next.humanCommentId !== first.humanCommentId + && next.provider.workspaceId === first.provider.workspaceId && next.provider.channelId === first.provider.channelId && next.provider.threadId === first.provider.threadId + && next.provider.senderUserId === first.provider.senderUserId && next.provider.canonicalMessageId !== first.provider.canonicalMessageId; +} /** Provider transport belongs to the selected domain; public product identity * and exact canonical persistence are independently checked here. */ -export async function dispatchNativeSlackIngress(ctx: ChatCatalogContext, c: ChatCatalogCase, state: Record, deadlineAt: number) { +export async function dispatchNativeSlackIngress(ctx: ChatCatalogContext, c: ChatCatalogCase, state: Record, deadlineAt: number, phase: "canonical" | "unsupported-file" = "canonical") { if (!validSlackIngressDeclaration(c) || !c.dispatchSlackQuery || c.act || !ctx.selectSlackSource) throw new Error("Invalid actual native Slack ingress contract"); + const driver = phase === "canonical" ? c.dispatchSlackQuery : c.scenarioId === "chat-channels.channel-file" ? c.dispatchSlackUnsupportedFile : undefined; + if (!driver) throw new Error("Undeclared Slack ingress phase"); const remaining = Math.min(c.slackIngress!.ingressTimeoutMs, deadlineAt - Date.now()); if (remaining <= 0) throw new Error("Slack setup exhausted the actual attempt deadline"); const signal = AbortSignal.timeout(remaining); let listener: (() => void) | undefined; const interrupted = new Promise((_resolve, reject) => { listener = () => reject(new Error("Actual Slack ingress exceeded its declared deadline")); signal.addEventListener("abort", listener, { once: true }); }); let receipt: ChatSlackIngressReceipt; - try { receipt = await Promise.race([c.dispatchSlackQuery(ctx, state, { query: ctx.scenario.query, signal }), interrupted]); } + try { receipt = await Promise.race([driver(ctx, state, { query: ctx.scenario.query, signal }), interrupted]); } finally { if (listener) signal.removeEventListener("abort", listener); } // Fail before interpreting identifiers as API routes. for (const id of [receipt?.endpointId, receipt?.conversationId, receipt?.issueId, receipt?.humanCommentId]) if (typeof id !== "string" || !/^[a-zA-Z0-9-]{1,100}$/.test(id)) throw new Error("Invalid Slack product receipt identity"); @@ -47,17 +58,29 @@ export async function dispatchNativeSlackIngress(ctx: ChatCatalogContext, c: Cha const links = await api.get(`/api/chat-endpoints/${receipt.endpointId}/principals`); const matched = slackIngressReceiptMatches({ receipt, query: ctx.scenario.query, companyId: ctx.input.fixtures.company.id, agentId: ctx.input.fixtures.agent.id, endpoint, conversation: conversations.find(row => row.id === receipt.conversationId), binding, comment: comments.find(row => row.id === receipt.humanCommentId), links }); - await ctx.input.evidence("chat-slack-ingress-binding.json", { matched, canonicalQuery: ctx.scenario.query, receipt, assignedAgentId: endpoint.assignedAgentId, + const first = state.slackIngressReceipt as ChatSlackIngressReceipt | undefined; + const sameThreadNewMessage = phase === "canonical" || !!first && sameSlackThreadNewMessage(first, receipt); + await ctx.input.evidence(`chat-slack-ingress-${phase}-binding.json`, { matched, sameThreadNewMessage, phase, canonicalQuery: ctx.scenario.query, receipt, assignedAgentId: endpoint.assignedAgentId, linkedIdentityConfirmed: matched, privateProviderConfiguration: "withheld", providerSenderMapping: "independent case oracle required" }); - if (signal.aborted || !matched) throw new Error("Slack ingress did not independently bind exact canonical human, linked identity, company and provider thread"); + if (signal.aborted || !matched || !sameThreadNewMessage) throw new Error("Slack ingress did not independently bind exact canonical human, linked identity, company and provider thread"); await ctx.selectSlackSource(receipt.issueId); return receipt; } -export function slackNativeRunCheck(ctx: ChatCatalogContext, runs: ChatRun[]) { +export async function completeNativeSlackFileConditional(ctx: ChatCatalogContext, c: ChatCatalogCase, state: Record, deadlineAt: number) { + if (c.scenarioId !== "chat-channels.channel-file" || !c.verifySlackSupportedTransfer || !c.dispatchSlackUnsupportedFile) throw new Error("Missing actual Slack file conditional"); + const first = await pollUntil({ label: "real supported Slack transfer before unsupported upload", deadlineAt, intervalMs: 1000, + load: async () => { const grade = validateChatCatalogGrade(await c.verifySlackSupportedTransfer!(ctx, state)); const runs = await ctx.allRuns(); return { grade, runs }; }, + accept: value => value.grade.passed && value.runs.length >= 1 && value.runs.length <= 3 && value.runs.every(run => run.runtimeMode === "native" && run.companyId === ctx.input.fixtures.company.id && run.status === "succeeded") && slackNativeRunCheck(ctx, value.runs).passed }); + state.slackSupportedTransferRunIds = first.runs.map(run => run.id); + await ctx.input.evidence("chat-slack-supported-transfer-checkpoint.json", { grade: first.grade, runIds: state.slackSupportedTransferRunIds, beforeUnsupportedUpload: true }); + state.slackUnsupportedIngressReceipt = await dispatchNativeSlackIngress(ctx, c, state, deadlineAt, "unsupported-file"); +} +export function slackNativeRunCheck(ctx: ChatCatalogContext, runs: ChatRun[], state?: Record) { const source = ctx.issue(); const sourceRuns = runs.filter(run => run.contextSnapshot?.issueId === source?.id); return { id: "catalog-real-slack-source-native", passed: !!source && source.companyId === ctx.input.fixtures.company.id && source.assigneeAgentId === ctx.input.fixtures.agent.id && sourceRuns.length >= 1 && sourceRuns.every(run => run.agentId === ctx.input.fixtures.agent.id && run.runtimeMode === "native" && run.status === "succeeded") - && runs.every(run => typeof run.contextSnapshot?.issueId === "string"), + && runs.every(run => typeof run.contextSnapshot?.issueId === "string") + && (!state?.slackUnsupportedIngressReceipt || Array.isArray(state.slackSupportedTransferRunIds) && sourceRuns.some(run => !(state.slackSupportedTransferRunIds as unknown[]).includes(run.id))), detail: "The actual inbound source has a genuine assigned-agent native admission; fast responses cannot substitute. The independent case oracle binds every downstream task and automatic source follow-up to this provider exchange." }; }