diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 46291421865..a117bbf0f84 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -1410,14 +1410,16 @@ "scenarioId": "cases.handoff-same-task", "sourceFiles": [ "chat-catalog-domains/integrator-case-handoff.ts", + "chat-catalog-domains/run-roles.ts", "chat-catalog-domains/internal-run-brief.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/cases.yaml" ], - "oracleVersion": "paperclip.chat-catalog.case-handoff.structural.v2", + "oracleVersion": "paperclip.chat-catalog.case-handoff.structural.v3", "expectedRunCount": 4, + "minimumRunCount": 3, "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 900000, @@ -1425,6 +1427,9 @@ "kind": "terminal-counts", "terminalStatusCounts": { "cancelled": 1, + "succeeded": 2 + }, + "maximumTerminalStatusCounts": { "succeeded": 3 } }, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.test.ts index 9ee9385a5ad..eadf326bc13 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.test.ts @@ -6,14 +6,15 @@ function valid() { const row = { id: "case", workIssueId: "primary", handoffVersion: 1, status: "in_progress" }; const runs = [{ id: "stopped", agentId: "morgan", status: "cancelled", contextSnapshot: { issueId: "primary" }, runtimeMode: "native", startedAt: "2026-10-26T10:00:00Z" }, { id: "source", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "chat" }, runtimeMode: "native", startedAt: "2026-10-26T11:00:00Z" }, - { id: "owner", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "primary" }, runtimeMode: "native", startedAt: "2026-10-26T11:01:00Z" }, + { id: "owner", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "primary", wakeCommentId: "resume" }, runtimeMode: "native", startedAt: "2026-10-26T11:01:00Z" }, { id: "review", agentId: "riley", status: "succeeded", contextSnapshot: { issueId: "primary" }, runtimeMode: "native", startedAt: "2026-10-26T11:02:00Z" }]; const base = { scenarioId: "cases.handoff-same-task", origin: "https://fixture.test", companyId: "company", leadId: "morgan", state: { case: row, reviewer: { id: "riley" }, respondentId: "morgan", findings, interruptedRunId: "stopped", interruptedTask: { id: "primary", executionRunId: "stopped", assigneeAgentId: "morgan" } }, sourceIssueId: "chat", actingAgent: { id: "morgan", name: "Morgan", role: "engineer" }, reviewer: { id: "riley", name: "Riley", role: "engineer" }, tasks: [{ id: "primary", assigneeAgentId: "riley", status: "done" }], projects: [], goals: [], documents: [], revisions: [], humanQueries: [], activity: [], links: [], runs, - comments: [], sourceReplies: [{ id: "reply", authorAgentId: "morgan", createdAt: "2026-10-26T11:00:30Z", createdByRunId: "source", body: "The review direction is queued for the primary task." }], + comments: [{ id: "resume", issueId: "primary", authorAgentId: "morgan", createdByRunId: "source" }, { id: "handoff-comment", issueId: "primary", authorAgentId: "morgan", createdByRunId: "owner" }], + sourceReplies: [{ id: "request", authorAgentId: null, createdAt: "2026-10-26T10:59:50Z", body: "Hand this case to Riley for review with the findings you saved." }, { id: "reply", authorAgentId: "morgan", createdAt: "2026-10-26T11:00:30Z", createdByRunId: "source", body: "The review direction is queued for the primary task." }], caseRecord: { ...row, handoffVersion: 2 }, findings, findingVersions: [{ id: "original", issueId: "primary", body: findings.body, createdByAgentId: "morgan", createdByRunId: "stopped" }], - handoffs: [{ id: "kickoff", version: 1 }, { id: "handoff", version: 2, issueId: "primary", actorType: "agent", actorId: "morgan", fromAgentId: "morgan", toAgentId: "riley", phase: "review", sourceRunId: "owner", instructions: findings.body, state: "completed", createdAt: "2026-10-26T11:01:30Z" }], + handoffs: [{ id: "kickoff", version: 1 }, { id: "handoff", version: 2, issueId: "primary", actorType: "agent", actorId: "morgan", fromAgentId: "morgan", toAgentId: "riley", phase: "review", sourceRunId: "owner", commentId: "handoff-comment", instructions: findings.body, state: "completed", createdAt: "2026-10-26T11:01:30Z" }], reply: "The review direction is queued for the primary task." }; return { ...base, state: { ...base.state, handoffCheckpoint: structuredClone(base) } }; } @@ -40,6 +41,39 @@ describe("real Morgan same-primary review handoff", () => { const wrong = valid(); wrong.handoffs[1]!.instructions = "Mobile Safari tapping Pay works. Keyboard fails."; expect(gradeIntegratorCaseHandoff(wrong).passed).toBe(false); const negated = valid(); negated.handoffs[1]!.instructions = "On mobile Safari, tapping Pay does not fail. Keyboard activation works."; expect(gradeIntegratorCaseHandoff(negated).passed).toBe(false); }); + /** Retained paid run 43413553210: Morgan's answering chat run handed the released primary task straight to Riley. */ + function direct() { + const o = valid(); + o.runs = o.runs.filter(run => run.id !== "owner"); + o.comments = [{ id: "handoff-comment", issueId: "primary", authorAgentId: "morgan", createdByRunId: "source" }]; + o.handoffs[1]!.sourceRunId = null as unknown as string; o.handoffs[1]!.state = "delivered"; + return o; + } + it("accepts the answering chat run handing off its own released primary task under ordinary owner authority", () => { + const grade = gradeIntegratorCaseHandoff(direct()); + expect(grade.checks.filter(check => !check.passed)).toEqual([]); + }); + it("rejects a handoff recorded by a stale, foreign or superseded execution, and extra unexplained runs", () => { + const stale = direct(); stale.comments[0]!.createdByRunId = "stopped"; expect(gradeIntegratorCaseHandoff(stale).passed).toBe(false); + const foreign = direct(); foreign.comments[0]!.authorAgentId = "riley"; expect(gradeIntegratorCaseHandoff(foreign).passed).toBe(false); + const concurrent = direct(); concurrent.handoffs[1]!.sourceRunId = "other-execution"; expect(gradeIntegratorCaseHandoff(concurrent).passed).toBe(false); + const early = direct(); early.runs[1]!.startedAt = "2026-10-26T10:59:00Z"; expect(gradeIntegratorCaseHandoff(early).passed).toBe(false); + const unwoken = valid(); unwoken.runs[2]!.contextSnapshot = { issueId: "primary" }; expect(gradeIntegratorCaseHandoff(unwoken).passed).toBe(false); + const extra = direct(); extra.runs.push({ id: "again", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "primary" }, runtimeMode: "native", startedAt: "2026-10-26T11:03:00Z" }); + expect(gradeIntegratorCaseHandoff(extra).passed).toBe(false); + const twice = direct(); twice.runs.push({ id: "review-2", agentId: "riley", status: "succeeded", contextSnapshot: { issueId: "primary" }, runtimeMode: "native", startedAt: "2026-10-26T11:03:00Z" }); + expect(gradeIntegratorCaseHandoff(twice).passed).toBe(false); + }); + it("bounds the ledger to the direct or routed run shape", async () => { + const { gradeChatCatalogRuns } = await import("../chat-catalog-run-policy.js"); + const { integratorCaseHandoffCases } = await import("./integrator-case-handoff.js"); + const grade = (runs: ReturnType["runs"]) => gradeChatCatalogRuns({ implementation: integratorCaseHandoffCases[0]!, companyId: "company", leadId: "morgan", + source: { id: "chat" } as never, runs: runs.map(run => ({ ...run, companyId: "company" })) as never }).checks.every(check => check.passed); + expect(grade(direct().runs)).toBe(true); + expect(grade(valid().runs)).toBe(true); + expect(grade(direct().runs.filter(run => run.id !== "review"))).toBe(false); + expect(grade([...valid().runs, { id: "extra", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "primary" }, runtimeMode: "native", startedAt: "2026-10-26T11:04:00Z" }])).toBe(false); + }); it("rejects fictional delivery before the actual handoff and task-done-as-case-closed", () => { const early = valid(); early.state.handoffCheckpoint.reply = "I handed this case to Riley."; expect(gradeIntegratorCaseHandoff(early).passed).toBe(false); const closed = valid(); closed.caseRecord.status = "done"; expect(gradeIntegratorCaseHandoff(closed).passed).toBe(false); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts index c57127455b5..6620480803f 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts @@ -10,6 +10,7 @@ import { checks, observation, project, type Entity, type InternalObservation } f import { fixtureWorker } from "./internal-definition.js"; import { startRunBriefGate } from "./internal-run-brief.js"; import { integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js"; +import { commentWokenRun, runIssueId } from "./run-roles.js"; type CaseRecord = typeof cases.$inferSelect; type OwnedTask = Entity & { executionRunId?: string | null; checkoutRunId?: string | null }; const resources = new WeakMap> }>(); @@ -41,6 +42,23 @@ function referencesSavedFindings(instructions: string, findings: IssueDocument | const saved = findings?.body?.trim(); return /(? row.id === handoff.commentId && row.issueId === primaryId && row.authorAgentId === o.leadId); + const run = o.runs.find(row => row.id === comment?.createdByRunId); + if (!run || run.agentId !== o.leadId || run.status !== "succeeded" || run.id === o.state.interruptedRunId || !run.startedAt) return undefined; + // A recorded source execution must be the recording run itself; another execution then still owned the task. + if (handoff.sourceRunId && handoff.sourceRunId !== run.id) return undefined; + const sourceRuns = o.runs.filter(row => row.agentId === o.leadId && runIssueId(row) === o.sourceIssueId && !!row.startedAt); + const request = o.sourceReplies.find(row => !row.authorAgentId && !!row.createdAt); + if (!request || Date.parse(String(run.startedAt)) < Date.parse(request.createdAt!)) return undefined; + if (runIssueId(run) === o.sourceIssueId) return sourceRuns.some(row => row.id === run.id) ? run : undefined; + return runIssueId(run) === primaryId && commentWokenRun(run, { agentId: o.leadId, issueId: primaryId, comments: o.comments, causeRunIds: sourceRuns.map(row => row.id) }) ? run : undefined; +} export function gradeIntegratorCaseHandoff(value: unknown) { const c = checks(); const o = handoffObservation(value); c.add("real-owner-primary-and-handoff-evidence", !!o, "Requires actual Morgan respondent, original primary, saved finding revisions and real handoff source execution."); if (!o) return c.result(); const original = o.state.case as CaseRecord; const primary = o.tasks.find(task => task.id === original?.workIssueId); const riley = o.state.reviewer as Entity; @@ -50,15 +68,19 @@ export function gradeIntegratorCaseHandoff(value: unknown) { && stopped.contextSnapshot?.issueId === primary?.id && stoppedTask?.executionRunId === stopped.id && stoppedTask.assigneeAgentId === o.leadId && o.findingVersions.some(revision => revision.id === initial?.latestRevisionId && revision.body === initial.body && revision.issueId === primary?.id && revision.createdByAgentId === o.leadId && revision.createdByRunId === stopped.id), "The real responding Morgan saved the genuine primary-task finding revision before an actual operator stop released its checkout; no fixture status or roleplay substitutes."); - const handoff = o.handoffs.find(row => row.version === 2); const primaryRun = o.runs.find(run => run.id === handoff?.sourceRunId); + const handoff = o.handoffs.find(row => row.version === 2); + const authority = handoffAuthorRun(o, handoff, primary?.id); c.add("one-same-primary-real-morgan-to-riley", o.handoffs.length === 2 && !!handoff && o.caseRecord.id === original.id && o.caseRecord.workIssueId === original.workIssueId && handoff.issueId === primary?.id && handoff.actorType === "agent" && handoff.actorId === o.leadId && handoff.fromAgentId === o.leadId && handoff.toAgentId === riley?.id && /review/i.test(handoff.phase) && primary?.assigneeAgentId === riley.id && o.reviewer.id === riley.id && o.reviewer.name === "Riley" && o.tasks.length === 1, "One new review handoff changes the eligible owner of the original ordinary primary task, preserving case identity and creating no replacement task."); - c.add("normal-primary-execution-authority", !!primaryRun && primaryRun.agentId === o.leadId && primaryRun.contextSnapshot?.issueId === primary?.id && primaryRun.status === "succeeded" - && primaryRun.contextSnapshot?.issueId !== o.sourceIssueId - && o.runs.some(run => run.agentId === o.leadId && run.contextSnapshot?.issueId === o.sourceIssueId && !!run.startedAt && !!primaryRun.startedAt && Date.parse(String(primaryRun.startedAt)) >= Date.parse(String(run.startedAt))), - "The source Morgan chat routes the explicit request to a genuinely admitted primary Morgan execution, whose normal JWT/checkout authority performs the handoff."); + c.add("owner-execution-authority", !!authority, + "A Morgan run started for this request records the handoff under the product's ordinary owner authority: the answering chat run while no other run owns the released primary task, or a primary-task continuation that chat run woke. A stale, foreign or concurrent execution does not qualify."); + const answers = o.runs.filter(run => run.agentId === o.leadId && runIssueId(run) === o.sourceIssueId); + const reviews = o.runs.filter(run => run.agentId === riley?.id && runIssueId(run) === primary?.id); + const roles = new Set([stopped?.id, authority?.id, ...answers.map(run => run.id), ...reviews.map(run => run.id)]); + c.add("every-run-has-a-handoff-role", answers.length === 1 && reviews.length === 1 && o.runs.every(run => roles.has(run.id)), + "Each run is the stopped investigation, the single answering chat, the run that recorded the handoff, or Riley's single review; anything else is extra work."); c.add("saved-findings-carried-forward", !!handoff && (referencesSavedFindings(handoff.instructions, initial) || checkoutObservations(handoff.instructions)) && !!initial?.body?.trim() && o.findings.issueId === primary?.id && o.findingVersions.some(revision => revision.id === initial?.latestRevisionId), "The review instructions point the reviewer at the saved findings document on the same task, and the original task-owned revision remains inspectable."); @@ -107,11 +129,12 @@ export const integratorCaseHandoffCases = [integratorDefinition("cases", "cases. const task = state.task as Entity; const row = state.case as CaseRecord; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, task.title!); await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/cases/${row.identifier}`, row.title); -}, { oracleVersion: "paperclip.chat-catalog.case-handoff.structural.v2", sourceFiles: ["chat-catalog-domains/integrator-case-handoff.ts", "chat-catalog-domains/internal-run-brief.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/cases.yaml"], +}, { oracleVersion: "paperclip.chat-catalog.case-handoff.structural.v3", sourceFiles: ["chat-catalog-domains/integrator-case-handoff.ts", "chat-catalog-domains/run-roles.ts", "chat-catalog-domains/internal-run-brief.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/cases.yaml"], respondingAgent: { kind: "case-primary-owner", name: "Morgan", role: "engineer", canAssignTasks: true }, - expectedRunCount: 4, attemptTimeoutMs: 900_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 3 } }, + expectedRunCount: 4, minimumRunCount: 3, attemptTimeoutMs: 900_000, + runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 2 }, maximumTerminalStatusCounts: { succeeded: 3 } }, async act(ctx, state) { - const first = await pollUntil({ label: "actual Morgan source reply and normal primary continuation", deadlineAt: Date.now() + 180_000, intervalMs: 1000, + const first = await pollUntil({ label: "actual Morgan source reply", deadlineAt: Date.now() + 180_000, intervalMs: 1000, load: () => observeHandoff(ctx, state), accept: o => o.runs.some(run => run.agentId === o.leadId && run.contextSnapshot?.issueId === o.sourceIssueId && run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); state.handoffCheckpoint = { ...first, state: structuredClone(state) }; await ctx.input.evidence("chat-morgan-review-direction.json", state.handoffCheckpoint); }, cleanup,