test(evals): require actual unsupported Slack file phase after transfer

tests/runner-e2e/chat-catalog-contract.ts

@@ -58,6 +58,9 @@ export interface ChatCatalogCase {
   dispatchSlackQuery?(context: ChatCatalogContext, state: Record<string, unknown>, input: { query: string; signal: AbortSignal }): Promise<ChatSlackIngressReceipt>;
   companionSurface?: ChatCompanionSurface;
   runCompanionSurface?(context: ChatCatalogContext, state: Record<string, unknown>, input: { query: string; signal: AbortSignal }): Promise<ChatCompanionResult>;
+  /** Only channel-file: independently settle the supported transfer before a second actual upload. */
+  verifySlackSupportedTransfer?(context: ChatCatalogContext, state: Record<string, unknown>): Promise<ChatCatalogGrade>;
+  dispatchSlackUnsupportedFile?(context: ChatCatalogContext, state: Record<string, unknown>, input: { query: string; signal: AbortSignal }): Promise<ChatSlackIngressReceipt>;
   authentication?: "local-trusted" | "human-session";
   setupPolicy?: ChatCatalogSetupPolicy;
   runPolicy?: ChatCatalogRunPolicy;

tests/runner-e2e/chat-catalog-flow.ts

@@ -1,4 +1,4 @@
-import { dispatchNativeSlackIngress, slackNativeRunCheck } from "./chat-slack-ingress.js";
+import { completeNativeSlackFileConditional, dispatchNativeSlackIngress, slackNativeRunCheck } from "./chat-slack-ingress.js";
 import { runScopedCompanionSurface } from "./chat-companion-surface.js";
 import { chatQueryWithAttachments, resolveChatQueryAttachments, verifyChatQueryAttachments, type ChatQueryAttachmentReceipt } from "./chat-query-attachments.js";
 import { chatCatalogAllowedRunStatuses, gradeChatCatalogRuns } from "./chat-catalog-run-policy.js";
@@ -62,7 +62,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
         detail: "The persisted request preserves the catalog query with only audited public-fixture identity bindings; all human turns were declared." },
       { id: "catalog-query-attachment-receipts", passed: attachmentsValid,
         detail: "Only declared production attachment suffixes are allowed, backed by same-source human receipts and independently retrieved matching bytes." },
-      ...(slack ? [slackNativeRunCheck(context, evidence.runs)] : []),
+      ...(slack ? [slackNativeRunCheck(context, evidence.runs, state)] : []),
       ...gradeChatCatalogRuns({ implementation: implementation!, companyId: input.fixtures.company.id,
         leadId: input.fixtures.agent.id, source: evidence.issue, runs: evidence.runs }).checks,
     ];
@@ -121,6 +121,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
       if (setupBoundary.runIds.length || setupBoundary.humanCommentIds.length) throw new Error("Native Slack setup cannot admit canonical or unrelated work");
       state.slackIngressReceipt = await dispatchNativeSlackIngress(context, implementation, state, deadlineAt);
       await input.page.goto(new URL(`/${input.fixtures.company.issuePrefix}/issues/${context.issue()!.id}`, input.api.baseURL).href);
+      if (implementation.scenarioId === "chat-channels.channel-file") await completeNativeSlackFileConditional(context, implementation, state, deadlineAt);
     } else await sendChatMessage(input.page, renderedQuery);
     if (implementation.queryOutcome === "rejected") {
       const rejection = validateChatCatalogGrade(await implementation.verifyQueryRejection!(context, state));

tests/runner-e2e/chat-catalog-registry.ts

@@ -8,7 +8,7 @@ import { chatCatalogScenarios } from "./chat-catalog-inventory.js";
 import type { IdentityBoundaryCase } from "./chat-catalog-domains/identity-boundary-contract.js";
 import { identityBoundaryScenarioIds } from "./chat-catalog-domains/identity-boundary-contract.js";
 
-export type ChatCatalogDescriptor = Omit<ChatCatalogCase, "dispatchSlackQuery" | "runCompanionSurface" | "journeyKind" | "setup" | "act" | "observe" | "grade" | "cleanup" | "verifyResult" | "verifySetup" | "verifyQueryRejection"> & {
+export type ChatCatalogDescriptor = Omit<ChatCatalogCase, "verifySlackSupportedTransfer" | "dispatchSlackUnsupportedFile" | "dispatchSlackQuery" | "runCompanionSurface" | "journeyKind" | "setup" | "act" | "observe" | "grade" | "cleanup" | "verifyResult" | "verifySetup" | "verifyQueryRejection"> & {
   journeyKind?: "agent-chat" | "browser-boundary" | "scoped-external-assistant" | "native-slack-thread";
   boundaryTarget?: "authenticated-harness" | "empty-installed-target";
   modulePath: string;
@@ -62,6 +62,8 @@ export async function loadChatCatalogImplementation(
   }
   if ((descriptor.companionSurface || descriptor.journeyKind === "scoped-external-assistant") && (!validCompanionDeclaration(implementation) || typeof implementation.runCompanionSurface !== "function" || implementation.act)) throw new Error("Missing actual finite scoped assistant driver");
   if ((descriptor.slackIngress || descriptor.journeyKind === "native-slack-thread") && (!validSlackIngressDeclaration(implementation) || typeof implementation.dispatchSlackQuery !== "function" || implementation.act)) throw new Error("Missing finite actual Slack ingress driver");
+  if (descriptor.scenarioId === "chat-channels.channel-file" && descriptor.slackIngress && (typeof implementation.verifySlackSupportedTransfer !== "function" || typeof implementation.dispatchSlackUnsupportedFile !== "function")) throw new Error("Slack file case requires actual supported and unsupported transfer phases");
+  if ((descriptor.scenarioId !== "chat-channels.channel-file" || !descriptor.slackIngress) && (implementation.verifySlackSupportedTransfer || implementation.dispatchSlackUnsupportedFile)) throw new Error("Undeclared Slack file conditional");
   if (!descriptor.slackIngress && implementation.dispatchSlackQuery) throw new Error("Undeclared Slack ingress driver");
   if (!descriptor.companionSurface && implementation.runCompanionSurface) throw new Error("Undeclared external assistant driver");
   if ((descriptor.setupPolicy?.funding === "preserve-scenario" || descriptor.setupPolicy?.readiness === "preserve-scenario")

tests/runner-e2e/chat-slack-ingress.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from "vitest";
 import { loadChatCatalogImplementation, type ChatCatalogDescriptor } from "./chat-catalog-registry.js";
-import { slackIngressReceiptMatches, validSlackIngressDeclaration, slackNativeRunCheck } from "./chat-slack-ingress.js";
+import { sameSlackThreadNewMessage, slackIngressReceiptMatches, validSlackIngressDeclaration, slackNativeRunCheck } from "./chat-slack-ingress.js";
 import type { ChatCatalogContext, ChatSlackIngressReceipt } from "./chat-catalog-contract.js";
 import type { ChatConversation, ChatEndpoint, ChatIdentityLink, ExternalChannelBindingSummary } from "../../packages/shared/src/types/chat-channels.js";
 import type { IssueComment } from "../../packages/shared/src/types/issue.js";
@@ -44,11 +44,30 @@ describe("only real declared Slack canonical ingress", () => {
     const actual = { ...implementation, dispatchSlackQuery: async () => input().receipt };
     await expect(loadChatCatalogImplementation(descriptor, async () => ({ slackCases: [actual] }))).resolves.toBe(actual);
     await expect(loadChatCatalogImplementation(descriptor, async () => ({ slackCases: [{ ...actual, act: async () => {} }] }))).rejects.toThrow("actual Slack ingress");
+    const fileDescriptor: ChatCatalogDescriptor = { ...descriptor, scenarioId: "chat-channels.channel-file", expectedRunCount: 6, minimumRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 900000 };
+    const file = { ...actual, ...fileDescriptor };
+    await expect(loadChatCatalogImplementation(fileDescriptor, async () => ({ slackCases: [file] }))).rejects.toThrow("supported and unsupported transfer phases");
+    const completeFile = { ...file, verifySlackSupportedTransfer: async () => ({ passed: true, checks: [{ id: "actual-transfer", passed: true, detail: "Independent phase-specific byte/egress evidence" }] }), dispatchSlackUnsupportedFile: async () => input().receipt };
+    await expect(loadChatCatalogImplementation(fileDescriptor, async () => ({ slackCases: [completeFile] }))).resolves.toBe(completeFile);
+
+  });
+  it("requires a separate same-thread canonical upload and declared two-phase bounds for the file conditional", () => {
+    const file = { ...metadata, scenarioId: "chat-channels.channel-file", expectedRunCount: 6, minimumRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 900000 };
+    expect(validSlackIngressDeclaration(file)).toBe(true);
+    expect(validSlackIngressDeclaration({ ...file, expectedHumanMessageCount: 1 })).toBe(false);
+    expect(validSlackIngressDeclaration({ ...metadata, scenarioId: "chat-channels.channel-file" })).toBe(false);
+    const first = input().receipt, next = structuredClone(first); next.humanCommentId = "second-comment"; next.provider.canonicalMessageId = "1234567892.123456";
+    expect(sameSlackThreadNewMessage(first, next)).toBe(true);
+    expect(sameSlackThreadNewMessage(first, first)).toBe(false);
+    expect(sameSlackThreadNewMessage(first, { ...next, query: "Use a different hidden instruction" })).toBe(false);
+    expect(sameSlackThreadNewMessage(first, { ...next, provider: { ...next.provider, threadId: "9999999999.123456" } })).toBe(false);
   });
   it("cannot qualify a fast response, wrong agent or unrelated native run as the source execution", () => {
     const ctx = { issue: () => ({ id: "issue", companyId: "company", assigneeAgentId: "agent" }), input: { fixtures: { company: { id: "company" }, agent: { id: "agent" } } } } as unknown as ChatCatalogContext;
     const run = { id: "run", companyId: "company", agentId: "agent", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: "issue" } };
     expect(slackNativeRunCheck(ctx, [run]).passed).toBe(true);
+    expect(slackNativeRunCheck(ctx, [run], { slackUnsupportedIngressReceipt: {}, slackSupportedTransferRunIds: [run.id] }).passed).toBe(false);
+    expect(slackNativeRunCheck(ctx, [run, { ...run, id: "second-native" }], { slackUnsupportedIngressReceipt: {}, slackSupportedTransferRunIds: [run.id] }).passed).toBe(true);
     expect(slackNativeRunCheck(ctx, []).passed).toBe(false);
     expect(slackNativeRunCheck(ctx, [{ ...run, agentId: "other" }]).passed).toBe(false);
     expect(slackNativeRunCheck(ctx, [{ ...run, contextSnapshot: { issueId: "other" } }]).passed).toBe(false);

tests/runner-e2e/chat-slack-ingress.ts

@@ -1,3 +1,5 @@
+import { pollUntil } from "./api.js";
+import { validateChatCatalogGrade } from "./chat-catalog-contract.js";
 import type { ChatConversation, ChatEndpoint, ChatIdentityLink, ExternalChannelBindingSummary } from "../../packages/shared/src/types/chat-channels.js";
 import type { IssueComment } from "../../packages/shared/src/types/issue.js";
 import type { ChatCatalogCase, ChatCatalogContext, ChatSlackIngressReceipt } from "./chat-catalog-contract.js";
@@ -7,8 +9,9 @@ export function validSlackIngressDeclaration(c: Pick<ChatCatalogCase, "scenarioI
   return (nativeSlackScenarioIds as readonly string[]).includes(c.scenarioId) && c.journeyKind === "native-slack-thread"
     && c.slackIngress?.kind === "native-slack-thread" && c.slackIngress.maximumProviderCalls === 20 && c.slackIngress.ingressTimeoutMs === 120000
     && Object.keys(c.slackIngress).sort().join(",") === "ingressTimeoutMs,kind,maximumProviderCalls"
-    && c.turnTimeoutMs === 120000 && c.attemptTimeoutMs === 600000
-    && c.expectedRunCount === 3 && c.minimumRunCount === 1 && c.expectedHumanMessageCount === 1 && (c.setupHumanMessageCount ?? 0) === 0
+    && c.turnTimeoutMs === 120000
+    && (c.scenarioId === "chat-channels.channel-file" ? c.attemptTimeoutMs === 900000 && c.expectedRunCount === 6 && c.minimumRunCount === 2 && c.expectedHumanMessageCount === 2
+      : c.attemptTimeoutMs === 600000 && c.expectedRunCount === 3 && c.minimumRunCount === 1 && c.expectedHumanMessageCount === 1) && (c.setupHumanMessageCount ?? 0) === 0
     && !c.queryBindings?.length && !c.queryAttachments?.length && !c.queryOutcome && !c.companionSurface && !c.runPolicy;
 }
 export function slackIngressReceiptMatches(input: { receipt: ChatSlackIngressReceipt; query: string; companyId: string; agentId: string;
@@ -26,16 +29,24 @@ export function slackIngressReceiptMatches(input: { receipt: ChatSlackIngressRec
     && Array.isArray(comment.metadata.sections) && comment.metadata.sections.some(section => section.rows.some(row => row.type === "key_value" && row.label === "Provider ID" && row.value === r.provider.senderUserId))
     && input.links.some(link => link.companyId === input.companyId && link.endpointId === e.id && link.status === "linked" && link.paperclipUserId === r.linkedPaperclipUserId);
 }
+export function sameSlackThreadNewMessage(first: ChatSlackIngressReceipt, next: ChatSlackIngressReceipt) {
+  return next.query === first.query && next.endpointId === first.endpointId && next.conversationId === first.conversationId && next.issueId === first.issueId
+    && next.linkedPaperclipUserId === first.linkedPaperclipUserId && next.humanCommentId !== first.humanCommentId
+    && next.provider.workspaceId === first.provider.workspaceId && next.provider.channelId === first.provider.channelId && next.provider.threadId === first.provider.threadId
+    && next.provider.senderUserId === first.provider.senderUserId && next.provider.canonicalMessageId !== first.provider.canonicalMessageId;
+}
 /** Provider transport belongs to the selected domain; public product identity
  * and exact canonical persistence are independently checked here. */
-export async function dispatchNativeSlackIngress(ctx: ChatCatalogContext, c: ChatCatalogCase, state: Record<string, unknown>, deadlineAt: number) {
+export async function dispatchNativeSlackIngress(ctx: ChatCatalogContext, c: ChatCatalogCase, state: Record<string, unknown>, deadlineAt: number, phase: "canonical" | "unsupported-file" = "canonical") {
   if (!validSlackIngressDeclaration(c) || !c.dispatchSlackQuery || c.act || !ctx.selectSlackSource) throw new Error("Invalid actual native Slack ingress contract");
+  const driver = phase === "canonical" ? c.dispatchSlackQuery : c.scenarioId === "chat-channels.channel-file" ? c.dispatchSlackUnsupportedFile : undefined;
+  if (!driver) throw new Error("Undeclared Slack ingress phase");
   const remaining = Math.min(c.slackIngress!.ingressTimeoutMs, deadlineAt - Date.now());
   if (remaining <= 0) throw new Error("Slack setup exhausted the actual attempt deadline");
   const signal = AbortSignal.timeout(remaining); let listener: (() => void) | undefined;
   const interrupted = new Promise<never>((_resolve, reject) => { listener = () => reject(new Error("Actual Slack ingress exceeded its declared deadline")); signal.addEventListener("abort", listener, { once: true }); });
   let receipt: ChatSlackIngressReceipt;
-  try { receipt = await Promise.race([c.dispatchSlackQuery(ctx, state, { query: ctx.scenario.query, signal }), interrupted]); }
+  try { receipt = await Promise.race([driver(ctx, state, { query: ctx.scenario.query, signal }), interrupted]); }
   finally { if (listener) signal.removeEventListener("abort", listener); }
   // Fail before interpreting identifiers as API routes.
   for (const id of [receipt?.endpointId, receipt?.conversationId, receipt?.issueId, receipt?.humanCommentId]) if (typeof id !== "string" || !/^[a-zA-Z0-9-]{1,100}$/.test(id)) throw new Error("Invalid Slack product receipt identity");
@@ -47,17 +58,29 @@ export async function dispatchNativeSlackIngress(ctx: ChatCatalogContext, c: Cha
   const links = await api.get<ChatIdentityLink[]>(`/api/chat-endpoints/${receipt.endpointId}/principals`);
   const matched = slackIngressReceiptMatches({ receipt, query: ctx.scenario.query, companyId: ctx.input.fixtures.company.id, agentId: ctx.input.fixtures.agent.id,
     endpoint, conversation: conversations.find(row => row.id === receipt.conversationId), binding, comment: comments.find(row => row.id === receipt.humanCommentId), links });
-  await ctx.input.evidence("chat-slack-ingress-binding.json", { matched, canonicalQuery: ctx.scenario.query, receipt, assignedAgentId: endpoint.assignedAgentId,
+  const first = state.slackIngressReceipt as ChatSlackIngressReceipt | undefined;
+  const sameThreadNewMessage = phase === "canonical" || !!first && sameSlackThreadNewMessage(first, receipt);
+  await ctx.input.evidence(`chat-slack-ingress-${phase}-binding.json`, { matched, sameThreadNewMessage, phase, canonicalQuery: ctx.scenario.query, receipt, assignedAgentId: endpoint.assignedAgentId,
     linkedIdentityConfirmed: matched, privateProviderConfiguration: "withheld", providerSenderMapping: "independent case oracle required" });
-  if (signal.aborted || !matched) throw new Error("Slack ingress did not independently bind exact canonical human, linked identity, company and provider thread");
+  if (signal.aborted || !matched || !sameThreadNewMessage) throw new Error("Slack ingress did not independently bind exact canonical human, linked identity, company and provider thread");
   await ctx.selectSlackSource(receipt.issueId);
   return receipt;
 }
-export function slackNativeRunCheck(ctx: ChatCatalogContext, runs: ChatRun[]) {
+export async function completeNativeSlackFileConditional(ctx: ChatCatalogContext, c: ChatCatalogCase, state: Record<string, unknown>, deadlineAt: number) {
+  if (c.scenarioId !== "chat-channels.channel-file" || !c.verifySlackSupportedTransfer || !c.dispatchSlackUnsupportedFile) throw new Error("Missing actual Slack file conditional");
+  const first = await pollUntil({ label: "real supported Slack transfer before unsupported upload", deadlineAt, intervalMs: 1000,
+    load: async () => { const grade = validateChatCatalogGrade(await c.verifySlackSupportedTransfer!(ctx, state)); const runs = await ctx.allRuns(); return { grade, runs }; },
+    accept: value => value.grade.passed && value.runs.length >= 1 && value.runs.length <= 3 && value.runs.every(run => run.runtimeMode === "native" && run.companyId === ctx.input.fixtures.company.id && run.status === "succeeded") && slackNativeRunCheck(ctx, value.runs).passed });
+  state.slackSupportedTransferRunIds = first.runs.map(run => run.id);
+  await ctx.input.evidence("chat-slack-supported-transfer-checkpoint.json", { grade: first.grade, runIds: state.slackSupportedTransferRunIds, beforeUnsupportedUpload: true });
+  state.slackUnsupportedIngressReceipt = await dispatchNativeSlackIngress(ctx, c, state, deadlineAt, "unsupported-file");
+}
+export function slackNativeRunCheck(ctx: ChatCatalogContext, runs: ChatRun[], state?: Record<string, unknown>) {
   const source = ctx.issue();
   const sourceRuns = runs.filter(run => run.contextSnapshot?.issueId === source?.id);
   return { id: "catalog-real-slack-source-native", passed: !!source && source.companyId === ctx.input.fixtures.company.id && source.assigneeAgentId === ctx.input.fixtures.agent.id
     && sourceRuns.length >= 1 && sourceRuns.every(run => run.agentId === ctx.input.fixtures.agent.id && run.runtimeMode === "native" && run.status === "succeeded")
-    && runs.every(run => typeof run.contextSnapshot?.issueId === "string"),
+    && runs.every(run => typeof run.contextSnapshot?.issueId === "string")
+    && (!state?.slackUnsupportedIngressReceipt || Array.isArray(state.slackSupportedTransferRunIds) && sourceRuns.some(run => !(state.slackSupportedTransferRunIds as unknown[]).includes(run.id))),
     detail: "The actual inbound source has a genuine assigned-agent native admission; fast responses cannot substitute. The independent case oracle binds every downstream task and automatic source follow-up to this provider exchange." };
 }