From 910bdb5905fd923b7c85e77ee8522792c18556ba Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 05:06:19 -0600 Subject: [PATCH] test(evals): start multiple-handoffs without a brief cue and grade the reply by the task it names The baseline lead never replied in time: the setup message called the Website task "actively waiting for the brief", so the lead read the headline change as that brief and spent its turn inspecting the worker's workspace. Describe the task as in progress instead. Replace the reply keyword checks ("queue|pending| wait..." and an "already implemented" deny-list) with the setup-assigned task identifier or link the reply names; the saved-for-later state is graded by the queue entry, the still-running worker at reply time and the later native delivery. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../runtime-multiple-handoffs.test.ts | 25 +++++++++++++------ .../runtime-multiple-handoffs.ts | 15 +++++++---- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index cbd93dc7ca6..dd586de7b22 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -320,7 +320,7 @@ "registered-artifact.ts", "chat-scenarios/agent-chat.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime-multiple-handoffs.v3", "coverageGaps": [], "expectedRunCount": 5, "setupHumanMessageCount": 1, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts index 1a357a740ac..3a3dd7ebbf8 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts @@ -1,21 +1,32 @@ import { expect, it } from "vitest"; import { gradeMultipleHandoffs } from "./runtime-multiple-handoffs.js"; function valid() { - const logo = { id: "logo", title: "Logo", companyId: "company", status: "done", assigneeAgentId: "riley" }; + const logo = { id: "logo", identifier: "RUN-1", title: "Logo", companyId: "company", status: "done", assigneeAgentId: "riley" }; const artifact = { contentVerified: true, attachmentId: "logo-file", sha256: "same", content: "Acorn real-marker" }; return { scenarioId: "agent-chat.multiple-handoffs", companyId: "company", leadId: "ceo", agents: [], links: [], activity: [], humanQueries: [], reply: "", - tasks: [logo, { id: "website", assigneeAgentId: "morgan", status: "done" }], + tasks: [logo, { id: "8ba8b5a2-066a-4bb2-b591-e396942ecdf5", assigneeAgentId: "morgan", status: "done" }], runs: [{ id: "logo-run", agentId: "riley", status: "succeeded" }, { id: "prior", agentId: "morgan", status: "succeeded" }, { id: "setup", agentId: "ceo", status: "succeeded" }, { id: "canonical", agentId: "ceo", status: "succeeded" }, { id: "consumer", agentId: "morgan", status: "succeeded" }], - state: { task: { id: "website" }, worker: { id: "morgan" }, logoWorker: { id: "riley" }, workerRunId: "prior", canonicalRunId: "canonical", - queueAtReply: { issueId: "website", targetRunId: "prior", state: "queued", entries: [{ comment: { id: "headline", authorAgentId: "ceo", createdByRunId: "canonical", body: "Change the headline to Launch sooner." } }] }, + state: { task: { id: "8ba8b5a2-066a-4bb2-b591-e396942ecdf5", identifier: "RUN-2" }, worker: { id: "morgan" }, logoWorker: { id: "riley" }, workerRunId: "prior", canonicalRunId: "canonical", + queueAtReply: { issueId: "8ba8b5a2-066a-4bb2-b591-e396942ecdf5", targetRunId: "prior", state: "queued", entries: [{ comment: { id: "headline", authorAgentId: "ceo", createdByRunId: "canonical", body: "Change the headline to Launch sooner." } }] }, sourceHistoryBound: true, workerRunningAtReply: true, nativeDeliveryBound: true, queueEmptyAfterDelivery: true, logoBefore: logo, logoAfter: structuredClone(logo), logoArtifactBefore: artifact, logoArtifactAfter: { ...artifact }, logoMarker: "real-marker", logoRevisionsUnchanged: true, - queryReply: "I saved the headline feedback in the existing Website queue while Morgan is busy. The logo remains unchanged.", + queryReply: "I saved the headline feedback on [RUN-2](/RUN/issues/RUN-2) while Morgan is busy. The logo [RUN-1](/RUN/issues/RUN-1) remains unchanged.", finalHeadlineRevision: true, originalWebsiteRevisionRetained: true } }; } it("accepts one Website delivery with real completed Logo preservation", () => expect(gradeMultipleHandoffs(valid()).passed).toBe(true)); -it("accepts natural pending-delivery wording for the exact affected work", () => { const o = valid(); o.state.queryReply = "Website's new headline requirement will wait for Morgan's next turn; I left Riley's completed logo alone."; expect(gradeMultipleHandoffs(o).passed).toBe(true); }); +it("accepts reworded replies that name the affected task by identifier or link", () => { + for (const reply of ["RUN-2's new headline requirement will wait for Morgan's next turn; I left Riley's completed logo alone.", + "Morgan will pick up \"Launch sooner\" when the current turn ends: [Website task](/RUN/issues/RUN-2#comment-headline).", + "Queued for Morgan on RUN-2. Nothing changed on RUN-1."]) { + const o = valid(); o.state.queryReply = reply; expect(gradeMultipleHandoffs(o).passed).toBe(true); + } +}); +it("rejects a reply that does not name the affected Website task", () => { + for (const reply of ["I saved the headline change for the website.", "Saved on RUN-20 for Morgan.", "The logo RUN-1 is unchanged."]) { + const o = valid(); o.state.queryReply = reply; expect(gradeMultipleHandoffs(o).passed).toBe(false); + } +}); it("rejects missing independent two-task evidence", () => expect(gradeMultipleHandoffs(undefined).passed).toBe(false)); it("rejects routing headline feedback to Logo instead", () => { const o = valid(); o.state.queueAtReply.issueId = "logo"; expect(gradeMultipleHandoffs(o).passed).toBe(false); }); it("rejects duplicate saved headline requirements", () => { const o = valid(); o.state.queueAtReply.entries.push({ ...o.state.queueAtReply.entries[0]! }); expect(gradeMultipleHandoffs(o).passed).toBe(false); }); @@ -24,6 +35,6 @@ it("rejects a consuming run without the exact saved comment", () => { const o = it("rejects changing the actual delivered Logo bytes", () => { const o = valid(); o.state.logoArtifactAfter.content = "new"; expect(gradeMultipleHandoffs(o).passed).toBe(false); }); it("rejects retaining a Logo filename but replacing its artifact", () => { const o = valid(); o.state.logoArtifactAfter.attachmentId = "replacement"; expect(gradeMultipleHandoffs(o).passed).toBe(false); }); it("rejects erasing an earlier Logo revision", () => { const o = valid(); o.state.logoRevisionsUnchanged = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); }); -it("rejects claiming implementation before active-run consumption", () => { const o = valid(); o.state.queryReply = "The Website headline has been changed; the queued request is complete."; expect(gradeMultipleHandoffs(o).passed).toBe(false); }); +it("rejects a reply written after the active worker stopped holding the saved instruction", () => { const o = valid(); o.state.workerRunningAtReply = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); }); it("rejects redirection to a different worker", () => { const o = valid(); o.tasks[1]!.assigneeAgentId = "riley"; expect(gradeMultipleHandoffs(o).passed).toBe(false); }); it("rejects losing the original Website work", () => { const o = valid(); o.state.originalWebsiteRevisionRetained = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts index 1fa123d6f94..87013476c2d 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts @@ -10,6 +10,11 @@ import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.j import { runtimeQueueCases } from "./runtime-queue.js"; import { runtimeDefinition, runtimeObservation, observeRuntime, runtimeWorker } from "./runtime-support.js"; type Artifact = Awaited>[number]; +/** A task is named by its setup-assigned identifier (which chat links) or by a link to its page. */ +function replyNamesTask(reply: string, task: Entity) { + const keys = [task.identifier, task.id].filter((key): key is string => !!key); + return keys.some(key => new RegExp(`(? row.id === (s.task as Entity).id && row.status === "done" && row.assigneeAgentId === (s.worker as Entity).id) && o.runs.length === 5 && o.runs.every(row => row.status === "succeeded") && o.runs.filter(row => row.agentId === (s.worker as Entity).id).length === 2 @@ -48,7 +53,7 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip const documents = await ctx.input.api.get(`/api/issues/${logo.id}/documents`), revisions: Entity[] = []; for (const doc of documents) revisions.push(...await ctx.input.api.get(`/api/issues/${logo.id}/documents/${encodeURIComponent(doc.key!)}/revisions`)); const busy = await busyRuntimeWorker(ctx); await openStandingChat(ctx); - const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan and actively waiting for the brief; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered logo is ${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}. Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`; + const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan, who is working on it now; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered logo is ${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}. Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`; await sendChatMessage(ctx.input.page, message); const saved = await pollUntil({ label: "Real source history retains separate active Website and completed Logo", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, @@ -58,7 +63,7 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip return { ...busy, logoWorker, logoBefore: complete.task, logoRunId: complete.runs[0]!.id, logoMarker, logoArtifactBefore: artifact, logoDocumentsBefore: documents, logoRevisionsBefore: revisions, sourceHistoryBound: comments.filter(row => !row.authorAgentId && row.body === message).length === 1, setupRunIds: saved.runs.map(row => row.id) }; }, gradeMultipleHandoffs, { - setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, + setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-multiple-handoffs.v3", sourceFiles: ["chat-catalog-domains/runtime-multiple-handoffs.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/agent-chat.yaml"], setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, async act(ctx, state) {