diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index fc8e4a6ed1a..f6934b44be1 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -17,7 +17,11 @@ export interface ChatCatalogContext { export type ChatCatalogTerminalRunStatus = "succeeded" | "cancelled" | "failed" | "timed_out"; export type ChatCatalogRunPolicy = | { kind: "terminal-counts"; terminalStatusCounts: Partial> } - | { kind: "source-settled-with-active-work"; maximumActiveWorkerRuns: number }; + | { kind: "source-settled-with-active-work"; maximumActiveWorkerRuns: number; + expectedWorkerCancellation?: "owned-task-reassignment" }; +export interface ChatCatalogExpectedWorkerCancellation { + runId: string; taskId: string; workerId: string; sourceRunId: string; +} export interface ChatCatalogSetupPolicy { funding?: "standard" | "preserve-scenario"; readiness?: "all-ready" | "preserve-scenario"; @@ -71,6 +75,8 @@ export interface ChatCatalogCase { authentication?: "local-trusted" | "human-session"; setupPolicy?: ChatCatalogSetupPolicy; runPolicy?: ChatCatalogRunPolicy; + /** Only edit-properties: independent observation proof of the exact reassigned worker. */ + expectedCancelledWorkerRun?(observation: unknown): ChatCatalogExpectedWorkerCancellation | undefined; queryOutcome?: "persisted" | "rejected"; /** Audited substitution of an incidental catalog fixture token, never other prose. */ queryBindings?: readonly ChatCatalogQueryBinding[]; diff --git a/tests/runner-e2e/chat-catalog-flow.ts b/tests/runner-e2e/chat-catalog-flow.ts index bc4def856af..b3809592a19 100644 --- a/tests/runner-e2e/chat-catalog-flow.ts +++ b/tests/runner-e2e/chat-catalog-flow.ts @@ -1,7 +1,7 @@ import { completeNativeSlackFileConditional, dispatchNativeSlackIngress, prepareNativeSlackOtherThread, slackOtherThreadBoundaryMatches, slackNativeRunCheck } from "./chat-slack-ingress.js"; import { runScopedCompanionSurface } from "./chat-companion-surface.js"; import { chatQueryWithAttachments, resolveChatQueryAttachments, verifyChatQueryAttachments, type ChatQueryAttachmentReceipt } from "./chat-query-attachments.js"; -import { chatCatalogAllowedRunStatuses, gradeChatCatalogRuns } from "./chat-catalog-run-policy.js"; +import { chatCatalogRunStatusAllowed, gradeChatCatalogRuns } from "./chat-catalog-run-policy.js"; import { chatCatalogCaseFingerprint, normalizedChatCatalogExecution } from "./chat-catalog-fingerprint.js"; import { expect } from "@playwright/test"; import { pollUntil } from "./api.js"; @@ -65,7 +65,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { detail: "Only declared production attachment suffixes are allowed, backed by same-source human receipts and independently retrieved matching bytes." }, ...(slack ? [slackNativeRunCheck(context, evidence.runs, state)] : []), ...gradeChatCatalogRuns({ implementation: implementation!, companyId: input.fixtures.company.id, - leadId: input.fixtures.agent.id, source: evidence.issue, runs: evidence.runs }).checks, + leadId: input.fixtures.agent.id, source: evidence.issue, runs: evidence.runs, observation }).checks, ]; grade = { passed: checks.every(check => check.passed), checks }; await input.evidence("chat-catalog-observation.json", { scenarioId: scenario.id, inventoryDigest: chatCatalogInventoryDigest, @@ -146,7 +146,8 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { }, reject: result => { if (result.runs.length > implementation.expectedRunCount) return "Scenario exceeded its declared run bound"; - if (result.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status) && !chatCatalogAllowedRunStatuses(implementation).includes(run.status))) return "Observed provider run failed"; + if (result.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status) && !chatCatalogRunStatusAllowed({ implementation, + run, companyId: input.fixtures.company.id, leadId: input.fixtures.agent.id, source: result.issue, observation }))) return "Observed provider run failed"; if (result.runs.length && result.runs.every(run => !["queued", "running"].includes(run.status)) && result.issue?.conversationState === "waiting") settledAt ??= Date.now(); else settledAt = undefined; return settledAt && Date.now() - settledAt >= 15_000 diff --git a/tests/runner-e2e/chat-catalog-registry.ts b/tests/runner-e2e/chat-catalog-registry.ts index c4e415d80b1..16d441f237c 100644 --- a/tests/runner-e2e/chat-catalog-registry.ts +++ b/tests/runner-e2e/chat-catalog-registry.ts @@ -1,6 +1,7 @@ import { validNativeStopDeclaration, assertNativeStopCallback } from "./chat-native-stop-boundary.js"; import { validSlackIngressDeclaration } from "./chat-slack-ingress.js"; import { validCompanionDeclaration } from "./chat-companion-surface.js"; +import { assertExpectedWorkerCancellation } from "./chat-catalog-run-policy.js"; import { installedChatScenarioIds } from "./chat-installed-lane.js"; import { existsSync, readFileSync } from "node:fs"; import { validChatCatalogHumanCounts } from "./chat-catalog-contract.js"; @@ -9,7 +10,7 @@ import { chatCatalogScenarios } from "./chat-catalog-inventory.js"; import type { IdentityBoundaryCase } from "./chat-catalog-domains/identity-boundary-contract.js"; import { identityBoundaryScenarioIds } from "./chat-catalog-domains/identity-boundary-contract.js"; -export type ChatCatalogDescriptor = Omit & { +export type ChatCatalogDescriptor = Omit & { journeyKind?: "agent-chat" | "browser-boundary" | "scoped-external-assistant" | "native-slack-thread"; boundaryTarget?: "authenticated-harness" | "empty-installed-target"; modulePath: string; @@ -25,6 +26,9 @@ export function findChatCatalogCase(scenarioId: string): ChatCatalogDescriptor | if (matches.length > 1) throw new Error(`Duplicate chat implementation: ${scenarioId}`); const descriptor = matches[0]; if (!descriptor) return; + if (descriptor.runPolicy?.kind === "source-settled-with-active-work" && descriptor.runPolicy.expectedWorkerCancellation !== undefined + && (scenarioId !== "tasks.edit-properties" || descriptor.runPolicy.expectedWorkerCancellation !== "owned-task-reassignment")) + throw new Error("Worker reassignment cancellation is declared only for edit-properties"); if ((descriptor.companionSurface || descriptor.journeyKind === "scoped-external-assistant") && !validCompanionDeclaration(descriptor)) throw new Error("Only the declared scoped-client catalog case may select an external assistant"); if ((descriptor.slackIngress || descriptor.journeyKind === "native-slack-thread") && !validSlackIngressDeclaration(descriptor)) throw new Error("Only two declared Slack thread cases may use native ingress"); if (descriptor.canonicalBoundary && !validNativeStopDeclaration(descriptor)) throw new Error("Native Stop boundary is exclusive to the exact human post-stop scenario"); @@ -59,6 +63,7 @@ export async function loadChatCatalogImplementation( const selected = implementations.filter((row: ChatCatalogCase) => row.scenarioId === descriptor.scenarioId); if (selected.length !== 1) throw new Error(`Missing or duplicate runtime case: ${descriptor.scenarioId}`); const implementation = selected[0] as ChatCatalogCase; + assertExpectedWorkerCancellation(implementation); for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments", "requiredCredentials", "externalInput", "installedSource", "browserTimezone", "respondingAgent", "companionSurface", "slackIngress", "canonicalBoundary"] as const) { if (JSON.stringify(implementation[key]) !== JSON.stringify(descriptor[key])) throw new Error(`Runtime descriptor drift: ${descriptor.scenarioId}.${key}`); } diff --git a/tests/runner-e2e/chat-catalog-run-policy.test.ts b/tests/runner-e2e/chat-catalog-run-policy.test.ts index 0c00894ddd7..72aec7542b7 100644 --- a/tests/runner-e2e/chat-catalog-run-policy.test.ts +++ b/tests/runner-e2e/chat-catalog-run-policy.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { gradeChatCatalogRuns } from "./chat-catalog-run-policy.js"; +import { gradeChatCatalogRuns, chatCatalogAllowedRunStatuses, assertExpectedWorkerCancellation, chatCatalogRunStatusAllowed } from "./chat-catalog-run-policy.js"; import type { ChatRun } from "./chat-flow.js"; const succeeded: ChatRun = { id: "source-run", companyId: "company", agentId: "ceo", status: "succeeded", runtimeMode: "native", contextSnapshot: { issueId: "source" } }; const worker: ChatRun = { id: "worker-run", companyId: "company", agentId: "worker", status: "running", runtimeMode: "native", contextSnapshot: { issueId: "task" } }; @@ -19,3 +19,63 @@ it("admits declared separate active work while rejecting active CEO, wrong scope expect(gradeChatCatalogRuns({ ...scope, implementation, runs: [succeeded, bad] }).passed).toBe(false); expect(gradeChatCatalogRuns({ ...scope, implementation, runs: [succeeded, worker, { ...worker, id: "extra" }] }).passed).toBe(false); }); + +function reassignment() { + const receipt = { runId: "worker-run", taskId: "task", workerId: "worker", sourceRunId: "final-source" }; + const stopped: ChatRun = { ...worker, status: "cancelled", errorCode: "issue_reassigned", resultJson: { + reassignmentStopConfirmed: true, nativeCancellation: { schema: "paperclip.native-cancellation.v1", runId: "worker-run", issueId: "task", companyId: "company", dispatchState: "acknowledged", acknowledgedAt: "2026-10-27T10:00:00Z", intentAuditId: "intent", acknowledgementAuditId: "ack" }, + } }; + const implementation = { scenarioId: "tasks.edit-properties", expectedRunCount: 4, minimumRunCount: 3, + runPolicy: { kind: "source-settled-with-active-work" as const, maximumActiveWorkerRuns: 1, expectedWorkerCancellation: "owned-task-reassignment" as const }, + expectedCancelledWorkerRun: (observation: unknown) => (observation as { proofComplete?: boolean } | undefined)?.proofComplete ? receipt : undefined }; + const observation = { proofComplete: true, state: { heldRun: { ...worker }, heldTask: { id: "task", companyId: "company", assigneeAgentId: "worker", executionRunId: "worker-run" }, riley: { id: "worker" } } }; + return { implementation, stopped, receipt, observation, runs: [succeeded, { ...succeeded, id: "final-source" }, stopped] }; +} +it("defers only the declared worker cancellation during polling and requires final independent proof", () => { + const v = reassignment(); + expect(chatCatalogAllowedRunStatuses(v.implementation)).toContain("cancelled"); + expect(chatCatalogRunStatusAllowed({ ...scope, implementation: v.implementation, run: v.stopped, observation: { ...v.observation, proofComplete: false } })).toBe(true); + for (const run of [{ ...v.stopped, id: "other" }, { ...v.stopped, agentId: "ceo" }, { ...v.stopped, companyId: "foreign" }, { ...v.stopped, errorCode: "user_cancelled" }]) + expect(chatCatalogRunStatusAllowed({ ...scope, implementation: v.implementation, run, observation: v.observation })).toBe(false); + expect(gradeChatCatalogRuns({ ...scope, implementation: v.implementation, runs: v.runs }).passed).toBe(false); + expect(gradeChatCatalogRuns({ ...scope, implementation: v.implementation, runs: v.runs, observation: v.observation }).passed).toBe(true); + // No cancellation remains a valid genuine ownership-denied branch; no proof is invented. + expect(gradeChatCatalogRuns({ ...scope, implementation: v.implementation, runs: [succeeded, { ...succeeded, id: "final-source" }, worker] }).passed).toBe(true); +}); +it("rejects wrong cancellation identities, scope, reason, acknowledgement and unrelated cancelled work", () => { + const mutations = [ + (v: ReturnType) => { v.receipt.runId = "unrelated"; }, + (v: ReturnType) => { v.receipt.taskId = "other-task"; }, + (v: ReturnType) => { v.receipt.workerId = "other-worker"; }, + (v: ReturnType) => { v.receipt.sourceRunId = "unrelated-source"; }, + (v: ReturnType) => { v.stopped.companyId = "foreign"; }, + (v: ReturnType) => { v.stopped.agentId = "ceo"; }, + (v: ReturnType) => { v.stopped.errorCode = "user_cancelled"; }, + (v: ReturnType) => { v.stopped.resultJson!.reassignmentStopConfirmed = false; }, + (v: ReturnType) => { (v.stopped.resultJson!.nativeCancellation as Record).dispatchState = "pending"; }, + (v: ReturnType) => { (v.stopped.resultJson!.nativeCancellation as Record).issueId = "other-task"; }, + (v: ReturnType) => { v.runs.push({ ...v.stopped, id: "another-cancelled-run" }); }, + ]; + for (const mutate of mutations) { const v = reassignment(); mutate(v); expect(gradeChatCatalogRuns({ ...scope, implementation: v.implementation, runs: v.runs, observation: v.observation }).passed).toBe(false); } +}); +it("requires exact scenario/flag/callback and leaves every undeclared active-work case unchanged", () => { + const v = reassignment(); + expect(() => assertExpectedWorkerCancellation(v.implementation)).not.toThrow(); + for (const implementation of [ + { ...v.implementation, scenarioId: "tasks.assignment-conflict" }, + { ...v.implementation, expectedCancelledWorkerRun: undefined }, + { ...v.implementation, runPolicy: { kind: "source-settled-with-active-work" as const, maximumActiveWorkerRuns: 1 } }, + ]) { expect(() => assertExpectedWorkerCancellation(implementation)).toThrow(); expect(gradeChatCatalogRuns({ ...scope, implementation, runs: v.runs, observation: v.observation }).passed).toBe(false); } + expect(chatCatalogAllowedRunStatuses({ runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 } })).not.toContain("cancelled"); +}); + +it("binds cancellation proof presence through the actual lazy registry loader", async () => { + const { loadChatCatalogImplementation } = await import("./chat-catalog-registry.js"); + const v = reassignment(); const { expectedCancelledWorkerRun, ...metadata } = v.implementation; + const descriptor = { ...metadata, modulePath: "./chat-catalog-domains/internal-tasks.js", exportName: "cases", sourceFiles: ["chat-catalog-domains/internal-edit-ownership.ts"], oracleVersion: "test", turnTimeoutMs: 120_000, attemptTimeoutMs: 600_000 }; + const implementation = { ...descriptor, expectedCancelledWorkerRun, setup: async () => ({}), observe: async () => ({}), grade: () => ({ passed: true, checks: [{ id: "actual-proof", passed: true, detail: "Fixture-only registry control" }] }), verifyResult: async () => {} }; + await expect(loadChatCatalogImplementation(descriptor, async () => ({ cases: [implementation] }))).resolves.toBe(implementation); + await expect(loadChatCatalogImplementation(descriptor, async () => ({ cases: [{ ...implementation, expectedCancelledWorkerRun: undefined }] }))).rejects.toThrow("cancellation proof"); + const wrongDescriptor = { ...descriptor, scenarioId: "tasks.assignment-conflict" }; + await expect(loadChatCatalogImplementation(wrongDescriptor, async () => ({ cases: [{ ...implementation, scenarioId: wrongDescriptor.scenarioId }] }))).rejects.toThrow("cancellation proof"); +}); diff --git a/tests/runner-e2e/chat-catalog-run-policy.ts b/tests/runner-e2e/chat-catalog-run-policy.ts index 2933e6c47bd..67e7e2abb68 100644 --- a/tests/runner-e2e/chat-catalog-run-policy.ts +++ b/tests/runner-e2e/chat-catalog-run-policy.ts @@ -1,13 +1,40 @@ import type { ChatCatalogCase, ChatCatalogGrade } from "./chat-catalog-contract.js"; import type { ChatIssue, ChatRun } from "./chat-flow.js"; -export function chatCatalogAllowedRunStatuses(implementation: Pick): string[] { +type CancellationCase = Pick & Partial>; +export function assertExpectedWorkerCancellation(implementation: CancellationCase) { + const flag = implementation.runPolicy?.kind === "source-settled-with-active-work" + ? implementation.runPolicy.expectedWorkerCancellation : undefined; + if (flag === undefined ? implementation.expectedCancelledWorkerRun !== undefined + : flag !== "owned-task-reassignment" || implementation.scenarioId !== "tasks.edit-properties" || typeof implementation.expectedCancelledWorkerRun !== "function") + throw new Error("Missing or undeclared exact task-reassignment cancellation proof"); +} +function permitsReassignment(implementation: CancellationCase) { + return implementation.scenarioId === "tasks.edit-properties" && implementation.runPolicy?.kind === "source-settled-with-active-work" + && implementation.runPolicy.expectedWorkerCancellation === "owned-task-reassignment" && typeof implementation.expectedCancelledWorkerRun === "function"; +} +export function chatCatalogAllowedRunStatuses(implementation: CancellationCase): string[] { const policy = implementation.runPolicy; return policy?.kind === "terminal-counts" ? Object.keys(policy.terminalStatusCounts) - : policy?.kind === "source-settled-with-active-work" ? ["queued", "running", "succeeded"] : ["succeeded"]; + : policy?.kind === "source-settled-with-active-work" ? ["queued", "running", "succeeded", ...(permitsReassignment(implementation) ? ["cancelled"] : [])] : ["succeeded"]; +} +const taskId = (run: ChatRun) => (run as ChatRun & { nativeIssueId?: string }).nativeIssueId ?? run.contextSnapshot?.issueId; +const record = (value: unknown) => value && typeof value === "object" ? value as Record : {}; +/** Early polling can defer only the saved, independently held worker, never a source or unrelated cancellation. */ +export function chatCatalogRunStatusAllowed(input: { + implementation: CancellationCase; run: ChatRun; companyId: string; leadId: string; source?: ChatIssue; observation?: unknown; +}): boolean { + const { implementation, run } = input; + if (run.status !== "cancelled" || !permitsReassignment(implementation)) return chatCatalogAllowedRunStatuses(implementation).includes(run.status); + const state = record(record(input.observation).state), held = record(state.heldRun), task = record(state.heldTask), worker = record(state.riley); + return held.id === run.id && held.status === "running" && held.companyId === input.companyId && held.agentId === worker.id + && run.companyId === input.companyId && run.agentId === worker.id && run.agentId !== input.leadId + && task.companyId === input.companyId && task.assigneeAgentId === worker.id && task.executionRunId === run.id + && task.id === taskId(run) && task.id !== input.source?.id && run.errorCode === "issue_reassigned"; } export function gradeChatCatalogRuns(input: { - implementation: Pick; + implementation: Pick & CancellationCase; companyId: string; leadId: string; source?: ChatIssue; runs: ChatRun[]; + observation?: unknown; }): ChatCatalogGrade { const { implementation: c, runs, source } = input; const checks = [ @@ -15,13 +42,36 @@ export function gradeChatCatalogRuns(input: { && runs.every(run => run.companyId === input.companyId && run.runtimeMode === "native" && chatCatalogAllowedRunStatuses(c).includes(run.status)), detail: "Observed provider runs satisfy declared count, company, native runtime and allowed outcome bounds." }, ]; + if (c.expectedCancelledWorkerRun !== undefined || c.runPolicy?.kind === "source-settled-with-active-work" && c.runPolicy.expectedWorkerCancellation !== undefined) { + let valid = true; try { assertExpectedWorkerCancellation(c); } catch { valid = false; } + checks.push({ id: "catalog-worker-cancellation-declaration", passed: valid, + detail: "Only edit-properties may declare the exact independently proved worker reassignment cancellation." }); + } if (c.runPolicy?.kind === "terminal-counts") { checks.push({ id: "catalog-exact-run-outcomes", passed: Object.entries(c.runPolicy.terminalStatusCounts).every(([status, count]) => runs.filter(run => run.status === status).length === count), detail: "The independently observed ledger matches every declared successful/cancelled/failed terminal outcome count." }); } else if (c.runPolicy?.kind === "source-settled-with-active-work") { - const taskId = (run: ChatRun) => (run as ChatRun & { nativeIssueId?: string }).nativeIssueId ?? run.contextSnapshot?.issueId; const sourceRuns = runs.filter(run => taskId(run) === source?.id); const workers = runs.filter(run => taskId(run) !== source?.id); + const cancelled = runs.filter(run => run.status === "cancelled"); + const declared = permitsReassignment(c); + let expected: ReturnType>; + try { expected = declared ? c.expectedCancelledWorkerRun!(input.observation) : undefined; } catch { expected = undefined; } + const actual = expected && workers.find(run => run.id === expected.runId); + const acknowledgement = actual?.resultJson?.nativeCancellation as Record | undefined; + const exact = !!expected && cancelled.length === 1 && cancelled[0]?.id === expected.runId && !!actual + && chatCatalogRunStatusAllowed({ ...input, run: actual }) + && actual.companyId === input.companyId && actual.agentId === expected.workerId && actual.agentId !== input.leadId + && taskId(actual) === expected.taskId && expected.taskId !== source?.id && actual.errorCode === "issue_reassigned" + && actual.resultJson?.reassignmentStopConfirmed === true + && acknowledgement?.schema === "paperclip.native-cancellation.v1" && acknowledgement.runId === actual.id + && acknowledgement.issueId === expected.taskId && acknowledgement.companyId === input.companyId + && acknowledgement.dispatchState === "acknowledged" && typeof acknowledgement.acknowledgedAt === "string" && Number.isFinite(Date.parse(acknowledgement.acknowledgedAt)) + && typeof acknowledgement.intentAuditId === "string" && acknowledgement.intentAuditId.length > 0 + && typeof acknowledgement.acknowledgementAuditId === "string" && acknowledgement.acknowledgementAuditId.length > 0 + && sourceRuns.some(run => run.id === expected.sourceRunId && run.agentId === input.leadId && run.status === "succeeded"); + checks.push({ id: "catalog-exact-owned-worker-cancellation", passed: cancelled.length === 0 ? expected === undefined : declared && exact, + detail: "Only the exact independently proved task-reassignment cancellation is accepted; final actor, chronology and saved-state proof remains mandatory in the case callback." }); checks.push({ id: "catalog-settled-source-separate-work", passed: !!source && source.companyId === input.companyId && source.conversationState === "waiting" && sourceRuns.length > 0 && sourceRuns.every(run => run.agentId === input.leadId && run.status === "succeeded") && workers.every(run => typeof taskId(run) === "string" && run.agentId !== input.leadId)