From 2d53c3952f0704c8601497a980e5f4fb65e24234 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 21:19:08 -0500 Subject: [PATCH] test(evals): hand off Morgan primary through genuine native authority --- .../integrator-case-handoff.test.ts | 44 ++++++++ .../integrator-case-handoff.ts | 105 ++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.test.ts new file mode 100644 index 00000000000..9a3f4a04f32 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { gradeIntegratorCaseHandoff } from "./integrator-case-handoff.js"; +import { shouldWakeAssigneeForIssueComment } from "../../../server/src/services/issue-comment-wakeup.js"; +function valid() { + const findings = { id: "findings", issueId: "primary", latestRevisionId: "original", body: "On mobile Safari, tapping Pay has no effect. Keyboard activation succeeds." }; + const row = { id: "case", workIssueId: "primary", handoffVersion: 1, status: "in_progress" }; + const runs = [{ id: "stopped", agentId: "morgan", status: "cancelled", contextSnapshot: { issueId: "primary" }, runtimeMode: "native", startedAt: "2026-10-26T10:00:00Z" }, + { id: "source", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "chat" }, runtimeMode: "native", startedAt: "2026-10-26T11:00:00Z" }, + { id: "owner", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "primary" }, runtimeMode: "native", startedAt: "2026-10-26T11:01:00Z" }, + { id: "review", agentId: "riley", status: "succeeded", contextSnapshot: { issueId: "primary" }, runtimeMode: "native", startedAt: "2026-10-26T11:02:00Z" }]; + const base = { scenarioId: "cases.handoff-same-task", origin: "https://fixture.test", companyId: "company", leadId: "morgan", state: { case: row, reviewer: { id: "riley" }, respondentId: "morgan", findings, + interruptedRunId: "stopped", interruptedTask: { id: "primary", executionRunId: "stopped", assigneeAgentId: "morgan" } }, sourceIssueId: "chat", actingAgent: { id: "morgan", name: "Morgan", role: "engineer" }, reviewer: { id: "riley", name: "Riley", role: "engineer" }, + tasks: [{ id: "primary", assigneeAgentId: "riley", status: "done" }], projects: [], goals: [], documents: [], revisions: [], humanQueries: [], activity: [], links: [], runs, + comments: [], sourceReplies: [{ id: "reply", authorAgentId: "morgan", createdAt: "2026-10-26T11:00:30Z", createdByRunId: "source", body: "The review direction is queued for the primary task." }], + caseRecord: { ...row, handoffVersion: 2 }, findings, findingVersions: [{ id: "original", issueId: "primary", body: findings.body, createdByAgentId: "morgan", createdByRunId: "stopped" }], + handoffs: [{ id: "kickoff", version: 1 }, { id: "handoff", version: 2, issueId: "primary", actorType: "agent", actorId: "morgan", fromAgentId: "morgan", toAgentId: "riley", phase: "review", sourceRunId: "owner", instructions: findings.body, state: "completed", createdAt: "2026-10-26T11:01:30Z" }], + reply: "The review direction is queued for the primary task." }; + return { ...base, state: { ...base.state, handoffCheckpoint: structuredClone(base) } }; +} +describe("real Morgan same-primary review handoff", () => { + it("source-qualifies an explicit self resume from a different chat execution without bypassing current-owner exclusion", () => { + const request = { selfComment: true, resumeRequested: true, commentCreatedByRunId: "source-chat", issueAtCommentStart: { checkoutRunId: null, executionRunId: null }, reopened: false, currentStatus: "in_progress" }; + expect(shouldWakeAssigneeForIssueComment(request)).toBe(true); + expect(shouldWakeAssigneeForIssueComment({ ...request, resumeRequested: false })).toBe(false); + expect(shouldWakeAssigneeForIssueComment({ ...request, issueAtCommentStart: { executionRunId: "source-chat", checkoutRunId: null } })).toBe(false); + }); + it("accepts genuine saved findings, actual primary continuation and eligible Riley review", () => { + expect(gradeIntegratorCaseHandoff(valid()).passed).toBe(true); expect(gradeIntegratorCaseHandoff(undefined).passed).toBe(false); + const natural = valid(); natural.handoffs[1]!.instructions = "Mobile Safari: touch on the checkout button produces no response. The keyboard works."; + expect(gradeIntegratorCaseHandoff(natural).passed).toBe(true); + }); + it("rejects CEO roleplay, a replacement primary, direct chat authority and missing original findings", () => { + const ceo = valid(); ceo.actingAgent.name = "CEO"; ceo.actingAgent.role = "ceo"; expect(gradeIntegratorCaseHandoff(ceo).passed).toBe(false); + const replacement = valid(); replacement.caseRecord.workIssueId = "new-primary"; expect(gradeIntegratorCaseHandoff(replacement).passed).toBe(false); + const chat = valid(); chat.handoffs[1]!.sourceRunId = "source"; expect(gradeIntegratorCaseHandoff(chat).passed).toBe(false); + const missing = valid(); missing.findingVersions = []; expect(gradeIntegratorCaseHandoff(missing).passed).toBe(false); + const wrong = valid(); wrong.handoffs[1]!.instructions = "Mobile Safari tapping Pay works. Keyboard fails."; expect(gradeIntegratorCaseHandoff(wrong).passed).toBe(false); + }); + it("rejects fictional delivery before the actual handoff and task-done-as-case-closed", () => { + const early = valid(); early.state.handoffCheckpoint.reply = "I handed this case to Riley."; expect(gradeIntegratorCaseHandoff(early).passed).toBe(false); + const closed = valid(); closed.caseRecord.status = "done"; expect(gradeIntegratorCaseHandoff(closed).passed).toBe(false); + const wrongActor = valid(); wrongActor.handoffs[1]!.actorId = "ceo"; expect(gradeIntegratorCaseHandoff(wrongActor).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts new file mode 100644 index 00000000000..9707d938ea1 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-handoff.ts @@ -0,0 +1,105 @@ +import { randomUUID } from "node:crypto"; +import type { cases } from "../../../packages/db/src/schema/cases.js"; +import type { CaseHandoff, HandoffCaseResult } from "../../../packages/shared/src/case-handoffs.js"; +import type { IssueDocument, DocumentRevision } from "../../../packages/shared/src/types/issue.js"; +import type { Agent } from "../../../packages/shared/src/types/agent.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { checks, observation, project, type Entity, type InternalObservation } from "./internal-support.js"; +import { fixtureWorker } from "./internal-definition.js"; +import { startRunBriefGate } from "./internal-run-brief.js"; +import { integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js"; +type CaseRecord = typeof cases.$inferSelect; +type OwnedTask = Entity & { executionRunId?: string | null; checkoutRunId?: string | null }; +const resources = new WeakMap> }>(); +interface HandoffObservation extends InternalObservation { caseRecord: CaseRecord; handoffs: CaseHandoff[]; findings: IssueDocument; findingVersions: DocumentRevision[]; sourceReplies: Entity[]; sourceIssueId?: string; actingAgent: Pick; reviewer: Pick } +async function cleanup(ctx: ChatCatalogContext) { const value = resources.get(ctx); if (!value) return; try { await value.gate?.close(); } finally { await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: value.experimental }); resources.delete(ctx); } } +async function observeHandoff(ctx: ChatCatalogContext, state: Record): Promise { + const base = await observeInternal(ctx, state); const row = state.case as CaseRecord; const id = (state.task as Entity).id; + const [acting, reviewer] = await Promise.all([ctx.input.api.get(`/api/agents/${base.leadId}`), ctx.input.api.get(`/api/agents/${(state.reviewer as Entity).id}`)]); + return { ...base, sourceIssueId: ctx.issue()?.id, caseRecord: await ctx.input.api.get(`/api/cases/${row.id}`), + actingAgent: { id: acting.id, name: acting.name, role: acting.role }, reviewer: { id: reviewer.id, name: reviewer.name, role: reviewer.role }, + handoffs: (await ctx.input.api.get<{ items: CaseHandoff[] }>(`/api/cases/${row.id}/handoffs?limit=100`)).items, + findings: await ctx.input.api.get(`/api/issues/${id}/documents/findings`), findingVersions: await ctx.input.api.get(`/api/issues/${id}/documents/findings/revisions`), + sourceReplies: ctx.issue() ? await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`) : [] }; +} +function handoffObservation(value: unknown): HandoffObservation | undefined { const o = value as HandoffObservation; return observation(value) && !!o.caseRecord && Array.isArray(o.handoffs) && !!o.findings && Array.isArray(o.findingVersions) && Array.isArray(o.sourceReplies) && !!o.actingAgent && !!o.reviewer ? o : undefined; } +function checkoutObservations(body: string) { + const parts = body.split(/\n+|;|(?<=[.!?])\s+/); + return /mobile Safari/i.test(body) && parts.some(part => /\b(?:tap|tapping|touch)\b/i.test(part) && /\b(?:Pay|checkout|button)\b/i.test(part) && /\b(?:no effect|no response|does not|doesn't|fails?|unresponsive|nothing)\b/i.test(part)) + && parts.some(part => /keyboard/i.test(part) && /\b(?:succeeds?|successful|works?|responds?)\b/i.test(part) && !/\b(?:not|fails?|doesn't|does not)\b/i.test(part)); +} +export function gradeIntegratorCaseHandoff(value: unknown) { + const c = checks(); const o = handoffObservation(value); c.add("real-owner-primary-and-handoff-evidence", !!o, "Requires actual Morgan respondent, original primary, saved finding revisions and real handoff source execution."); if (!o) return c.result(); + const original = o.state.case as CaseRecord; const primary = o.tasks.find(task => task.id === original?.workIssueId); const riley = o.state.reviewer as Entity; + const initial = o.state.findings as IssueDocument; const stoppedTask = o.state.interruptedTask as OwnedTask; + const stopped = o.runs.find(run => run.id === o.state.interruptedRunId); + c.add("genuine-morgan-findings-before-human-direction", o.actingAgent.id === o.leadId && o.actingAgent.name === "Morgan" && o.actingAgent.role === "engineer" && o.state.respondentId === o.leadId && stopped?.agentId === o.leadId && stopped.status === "cancelled" + && stopped.contextSnapshot?.issueId === primary?.id && stoppedTask?.executionRunId === stopped.id && stoppedTask.assigneeAgentId === o.leadId + && o.findingVersions.some(revision => revision.id === initial?.latestRevisionId && revision.body === initial.body && revision.issueId === primary?.id && revision.createdByAgentId === o.leadId && revision.createdByRunId === stopped.id), + "The real responding Morgan saved the genuine primary-task finding revision before an actual operator stop released its checkout; no fixture status or roleplay substitutes."); + const handoff = o.handoffs.find(row => row.version === 2); const primaryRun = o.runs.find(run => run.id === handoff?.sourceRunId); + c.add("one-same-primary-real-morgan-to-riley", o.handoffs.length === 2 && !!handoff && o.caseRecord.id === original.id && o.caseRecord.workIssueId === original.workIssueId + && handoff.issueId === primary?.id && handoff.actorType === "agent" && handoff.actorId === o.leadId && handoff.fromAgentId === o.leadId && handoff.toAgentId === riley?.id + && /review/i.test(handoff.phase) && primary?.assigneeAgentId === riley.id && o.reviewer.id === riley.id && o.reviewer.name === "Riley" && o.tasks.length === 1, + "One new review handoff changes the eligible owner of the original ordinary primary task, preserving case identity and creating no replacement task."); + c.add("normal-primary-execution-authority", !!primaryRun && primaryRun.agentId === o.leadId && primaryRun.contextSnapshot?.issueId === primary?.id && primaryRun.status === "succeeded" + && primaryRun.contextSnapshot?.issueId !== o.sourceIssueId + && o.runs.some(run => run.agentId === o.leadId && run.contextSnapshot?.issueId === o.sourceIssueId && !!run.startedAt && !!primaryRun.startedAt && Date.parse(String(primaryRun.startedAt)) >= Date.parse(String(run.startedAt))), + "The source Morgan chat routes the explicit request to a genuinely admitted primary Morgan execution, whose normal JWT/checkout authority performs the handoff."); + c.add("saved-findings-carried-forward", !!handoff && checkoutObservations(handoff.instructions) && checkoutObservations(initial?.body ?? "") + && o.findings.issueId === primary?.id && o.findingVersions.some(revision => revision.id === initial?.latestRevisionId), + "The review instructions retain the saved observed mobile Safari checkout findings, and the original task-owned revision remains inspectable."); + const reviewRun = o.runs.find(run => run.agentId === riley?.id && run.contextSnapshot?.issueId === primary?.id); + c.add("review-through-normal-admission-not-case-closure", reviewRun?.status === "succeeded" && reviewRun.runtimeMode === "native" && primary?.status === "done" + && ["delivered", "completed"].includes(handoff?.state ?? "") && o.caseRecord.status !== "done", + "Riley's review runs through ordinary native admission on that same task. Completing review alone does not fabricate terminal case closure."); + const first = handoffObservation(o.state.handoffCheckpoint); + const firstReply = first?.sourceReplies.filter(comment => comment.authorAgentId === o.leadId).at(-1); + const claimsDelivered = /\b(?:handed|transferred|assigned|sent)\b[^.!?\n]{0,50}\bRiley\b/i.test(first?.reply ?? "") + && !/\b(?:not yet|pending|queued|will|request(?:ed)?|waiting)\b/i.test(first?.reply ?? ""); + c.add("no-fictional-immediate-handoff", !!first && (!claimsDelivered || !!handoff && !!firstReply?.createdAt && Date.parse(String(handoff.createdAt)) <= Date.parse(firstReply.createdAt)), + "An early pending/resume acknowledgement is allowed; a claim of delivered handoff must already be supported when that actual chat reply was saved."); + return c.result(); +} +export const integratorCaseHandoffCases = [integratorDefinition("cases", "cases.handoff-same-task", async ctx => { + const settings = await ctx.input.api.get<{ enableCases: boolean }>("/api/instance/settings/experimental"); resources.set(ctx, { experimental: settings.enableCases }); + try { + await ctx.input.api.patch("/api/instance/settings/experimental", { enableCases: true }); + const p = await project(ctx, "Launch"); const reviewer = await fixtureWorker(ctx); await ctx.input.api.patch(`/api/agents/${reviewer.id}`, { name: "Riley", title: "Case reviewer", capabilities: "Reviews the same ordinary case task using saved findings; completes review without inventing terminal case closure." }); + for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, ...[ctx.input.fixtures.agent.id, reviewer.id].map(scopeId => ({ scopeType: "agent", scopeId }))]) + await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); + const gate = await startRunBriefGate(ctx, ctx.input.fixtures.agent.id); resources.get(ctx)!.gate = gate; + const kickoff = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/case-handoffs`, { + idempotencyKey: `morgan-primary-${ctx.input.nonce}`, case: { create: { caseType: "customer", title: "Mobile checkout investigation", projectId: p.id, fields: { browser: "mobile Safari", observed: "Tapping Pay does nothing; keyboard activation succeeds" } } }, toAgentId: ctx.input.fixtures.agent.id, phase: "investigation", + instructions: "Read the saved customer observations and save a document with key findings on this ordinary task, keeping observation distinct from an unproven root cause. Then call the connected Website Brief read_website_brief tool to wait for the remaining source evidence. Do not complete or hand off this investigation until the person's next direction.", + workflow_document_text: `# Owners and review\nMorgan investigates this same ordinary primary task and saves findings. Riley (${reviewer.id}) is the eligible next reviewer. A later explicit human review request must preserve this task and its findings. Case handoff authority belongs to the actual primary task execution. Riley reviews the saved findings, saves a review comment, and marks this review task done without a terminal case handoff. Case closure requires a later explicit accepted-fix decision.`, + initialDocuments: [{ key: "brief", title: "Customer observations", body: "Synthetic test observations: on mobile Safari, tapping the checkout Pay button has no effect. Activating the same button using a keyboard succeeds. These are observations, not proof of a specific JavaScript root cause." }], + }); + const phase = await pollUntil({ label: "genuine Morgan saved findings and source wait", deadlineAt: Date.now() + 180_000, intervalMs: 1000, + load: async () => ({ task: await ctx.input.api.get(`/api/issues/${kickoff.taskId}`), documents: await ctx.input.api.get(`/api/issues/${kickoff.taskId}/documents`), runs: await ctx.allRuns() }), + accept: value => value.runs.length === 1 && value.task.executionRunId === value.runs[0]?.id && value.runs[0]?.status === "running" && value.documents.some(doc => doc.key === "findings") && gate.calls.some(call => call.authorized) }); + const findings = await ctx.input.api.get(`/api/issues/${kickoff.taskId}/documents/findings`); + const revisions = await ctx.input.api.get(`/api/issues/${kickoff.taskId}/documents/findings/revisions`); + if (!revisions.some(revision => revision.id === findings.latestRevisionId && revision.createdByRunId === phase.runs[0]!.id && revision.createdByAgentId === ctx.input.fixtures.agent.id)) throw new Error("Morgan prerequisite findings have no actual native revision provenance"); + await ctx.input.api.post(`/api/heartbeat-runs/${phase.runs[0]!.id}/cancel`, { cancellationRequestId: randomUUID() }); + await pollUntil({ label: "actual Morgan stop releases ordinary primary ownership", deadlineAt: Date.now() + 90_000, intervalMs: 1000, + load: async () => ({ runs: await ctx.allRuns(), task: await ctx.input.api.get(`/api/issues/${kickoff.taskId}`) }), + accept: value => value.runs.length === 1 && value.runs[0]?.status === "cancelled" && value.task.status === "in_progress" && !value.task.executionRunId && !value.task.checkoutRunId }); + gate.release(); + return { project: p, reviewer: { id: reviewer.id, name: "Riley" }, respondentId: ctx.input.fixtures.agent.id, task: await ctx.input.api.get(`/api/issues/${kickoff.taskId}`), + case: await ctx.input.api.get(`/api/cases/${kickoff.caseId}`), findings, interruptedTask: phase.task, interruptedRunId: phase.runs[0]!.id }; + } catch (error) { await cleanup(ctx); throw error; } +}, observeHandoff, gradeIntegratorCaseHandoff, async (ctx, state) => { + const task = state.task as Entity; const row = state.case as CaseRecord; + await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, task.title!); + await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/cases/${row.identifier}`, row.title); +}, { sourceFiles: ["chat-catalog-domains/integrator-case-handoff.ts", "chat-catalog-domains/internal-run-brief.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/cases.yaml"], + respondingAgent: { kind: "case-primary-owner", name: "Morgan", role: "engineer", canAssignTasks: true }, + expectedRunCount: 4, attemptTimeoutMs: 900_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 3 } }, + async act(ctx, state) { + const first = await pollUntil({ label: "actual Morgan source reply and normal primary continuation", deadlineAt: Date.now() + 180_000, intervalMs: 1000, + load: () => observeHandoff(ctx, state), accept: o => o.runs.some(run => run.agentId === o.leadId && run.contextSnapshot?.issueId === o.sourceIssueId && run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); + state.handoffCheckpoint = { ...first, state: structuredClone(state) }; await ctx.input.evidence("chat-morgan-review-direction.json", state.handoffCheckpoint); + }, cleanup, +})];