From 834c4f6cfcbb7e3df80b2546bf433972b8427ef3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 05:48:39 -0600 Subject: [PATCH] test(evals): bind multiple-handoffs setup and delivery the way the product records them Run 42475944787 delivered the headline change correctly but failed two harness checks. The composer stores the setup message's bare URL as a Markdown link (swallowing the trailing period), so exact-body equality never matched; send the logo as a Markdown link and bind the one human setup message by its task identifiers and attachment id. The native continuation embeds new comments as JSON, so a body with quotes appears escaped; accept the comment id plus its raw or JSON-escaped body. Regraded offline, the retained run passes, a reworded reply passes, and a reply that names no task fails. Co-Authored-By: Claude Opus 5.5 --- .../runtime-multiple-handoffs.test.ts | 11 ++++++++++- .../runtime-multiple-handoffs.ts | 13 ++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts index 3a3dd7ebbf8..650406ae041 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { gradeMultipleHandoffs } from "./runtime-multiple-handoffs.js"; +import { gradeMultipleHandoffs, promptCarriesComment } from "./runtime-multiple-handoffs.js"; function valid() { const logo = { id: "logo", identifier: "RUN-1", title: "Logo", companyId: "company", status: "done", assigneeAgentId: "riley" }; const artifact = { contentVerified: true, attachmentId: "logo-file", sha256: "same", content: "Acorn real-marker" }; @@ -38,3 +38,12 @@ it("rejects erasing an earlier Logo revision", () => { const o = valid(); o.stat it("rejects a reply written after the active worker stopped holding the saved instruction", () => { const o = valid(); o.state.workerRunningAtReply = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); }); it("rejects redirection to a different worker", () => { const o = valid(); o.tasks[1]!.assigneeAgentId = "riley"; expect(gradeMultipleHandoffs(o).passed).toBe(false); }); it("rejects losing the original Website work", () => { const o = valid(); o.state.originalWebsiteRevisionRetained = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); }); +it("finds a delivered comment in a native continuation that embeds it as JSON", () => { + const comment = { id: "b9e3ab32-934e-4d07-a5cf-e06fc28d574a", body: 'The headline must be **"Launch sooner"** — nothing else.\nKeep the logo.' }; + const prompt = `## Paperclip Wake Payload\n${JSON.stringify({ messages: [{ id: comment.id, authorType: "agent", body: comment.body }] })}`; + expect(promptCarriesComment(prompt, comment)).toBe(true); + expect(promptCarriesComment(`plain ${comment.id}\n${comment.body}`, comment)).toBe(true); + expect(promptCarriesComment(prompt.replace(comment.id, "other"), comment)).toBe(false); + expect(promptCarriesComment(prompt.replace("Launch sooner", "Launch later"), comment)).toBe(false); + expect(promptCarriesComment(undefined, comment)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts index 87013476c2d..2d7767ecdf4 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts @@ -15,6 +15,11 @@ function replyNamesTask(reply: string, task: Entity) { const keys = [task.identifier, task.id].filter((key): key is string => !!key); return keys.some(key => new RegExp(`(?(`/api/issues/${logo.id}/documents`), revisions: Entity[] = []; for (const doc of documents) revisions.push(...await ctx.input.api.get(`/api/issues/${logo.id}/documents/${encodeURIComponent(doc.key!)}/revisions`)); const busy = await busyRuntimeWorker(ctx); await openStandingChat(ctx); - const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan, who is working on it now; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered logo is ${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}. Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`; + const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan, who is working on it now; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered logo is [logo.html](${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}). Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`; await sendChatMessage(ctx.input.page, message); const saved = await pollUntil({ label: "Real source history retains separate active Website and completed Logo", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, @@ -61,7 +66,9 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") }); const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); return { ...busy, logoWorker, logoBefore: complete.task, logoRunId: complete.runs[0]!.id, logoMarker, logoArtifactBefore: artifact, logoDocumentsBefore: documents, logoRevisionsBefore: revisions, - sourceHistoryBound: comments.filter(row => !row.authorAgentId && row.body === message).length === 1, setupRunIds: saved.runs.map(row => row.id) }; + // The composer may normalize Markdown, so bind the one human setup message by its setup-chosen references. + sourceHistoryBound: comments.filter(row => !row.authorAgentId).length === 1 && comments.some(row => !row.authorAgentId + && row.body?.includes(busy.task.identifier ?? busy.task.id) && row.body.includes(logo.identifier ?? logo.id) && row.body.includes(artifact.attachmentId)), setupRunIds: saved.runs.map(row => row.id) }; }, gradeMultipleHandoffs, { setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-multiple-handoffs.v3", sourceFiles: ["chat-catalog-domains/runtime-multiple-handoffs.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/agent-chat.yaml"], @@ -84,7 +91,7 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip const consumer = final.runs.find(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId)!; const detail = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumer.id}`), saved = first.queue.entries[0]!.comment; state.nativeDeliveryBound = detail.contextSnapshot?.wakeCommentIds?.length === 1 && detail.contextSnapshot.wakeCommentIds[0] === saved.id - && detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(saved.body) === true; + && promptCarriesComment(detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt, saved); state.queueEmptyAfterDelivery = true; state.finalWebsiteRunId = consumer.id; }, async observe(ctx, state) {