test(evals): bind multiple-handoffs setup and delivery the way the product records them

Run 42475944787 delivered the headline change correctly but failed two harness
checks. The composer stores the setup message's bare URL as a Markdown link
(swallowing the trailing period), so exact-body equality never matched; send
the logo as a Markdown link and bind the one human setup message by its task
identifiers and attachment id. The native continuation embeds new comments as
JSON, so a body with quotes appears escaped; accept the comment id plus its raw
or JSON-escaped body. Regraded offline, the retained run passes, a reworded
reply passes, and a reply that names no task fails.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts

@@ -1,5 +1,5 @@
 import { expect, it } from "vitest";
-import { gradeMultipleHandoffs } from "./runtime-multiple-handoffs.js";
+import { gradeMultipleHandoffs, promptCarriesComment } from "./runtime-multiple-handoffs.js";
 function valid() {
   const logo = { id: "logo", identifier: "RUN-1", title: "Logo", companyId: "company", status: "done", assigneeAgentId: "riley" };
   const artifact = { contentVerified: true, attachmentId: "logo-file", sha256: "same", content: "<svg><text>Acorn real-marker</text></svg>" };
@@ -38,3 +38,12 @@ it("rejects erasing an earlier Logo revision", () => { const o = valid(); o.stat
 it("rejects a reply written after the active worker stopped holding the saved instruction", () => { const o = valid(); o.state.workerRunningAtReply = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
 it("rejects redirection to a different worker", () => { const o = valid(); o.tasks[1]!.assigneeAgentId = "riley"; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
 it("rejects losing the original Website work", () => { const o = valid(); o.state.originalWebsiteRevisionRetained = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
+it("finds a delivered comment in a native continuation that embeds it as JSON", () => {
+  const comment = { id: "b9e3ab32-934e-4d07-a5cf-e06fc28d574a", body: 'The headline must be **"Launch sooner"** — nothing else.\nKeep the logo.' };
+  const prompt = `## Paperclip Wake Payload\n${JSON.stringify({ messages: [{ id: comment.id, authorType: "agent", body: comment.body }] })}`;
+  expect(promptCarriesComment(prompt, comment)).toBe(true);
+  expect(promptCarriesComment(`plain ${comment.id}\n${comment.body}`, comment)).toBe(true);
+  expect(promptCarriesComment(prompt.replace(comment.id, "other"), comment)).toBe(false);
+  expect(promptCarriesComment(prompt.replace("Launch sooner", "Launch later"), comment)).toBe(false);
+  expect(promptCarriesComment(undefined, comment)).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts

@@ -15,6 +15,11 @@ function replyNamesTask(reply: string, task: Entity) {
   const keys = [task.identifier, task.id].filter((key): key is string => !!key);
   return keys.some(key => new RegExp(`(?<![A-Za-z0-9-])${key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?![0-9])`).test(reply));
 }
+/** Native continuations embed new comments as JSON, so a body with quotes or newlines appears JSON-escaped. */
+export function promptCarriesComment(prompt: string | undefined, comment: { id: string; body: string }) {
+  return typeof prompt === "string" && prompt.includes(comment.id)
+    && (prompt.includes(comment.body) || prompt.includes(JSON.stringify(comment.body).slice(1, -1)));
+}
 export function gradeMultipleHandoffs(value: unknown) {
   const c = checks(), o = runtimeObservation(value);
   c.add("actual-two-task-history-and-delivery-evidence", !!o, "Separate real website/logo work, saved source references and exact consuming native input are required."); if (!o) return c.result();
@@ -53,7 +58,7 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip
   const documents = await ctx.input.api.get<Entity[]>(`/api/issues/${logo.id}/documents`), revisions: Entity[] = [];
   for (const doc of documents) revisions.push(...await ctx.input.api.get<Entity[]>(`/api/issues/${logo.id}/documents/${encodeURIComponent(doc.key!)}/revisions`));
   const busy = await busyRuntimeWorker(ctx); await openStandingChat(ctx);
-  const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan, who is working on it now; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered logo is ${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}. Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`;
+  const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan, who is working on it now; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered logo is [logo.html](${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}). Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`;
   await sendChatMessage(ctx.input.page, message);
   const saved = await pollUntil({ label: "Real source history retains separate active Website and completed Logo", deadlineAt: Date.now() + 150_000,
     load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
@@ -61,7 +66,9 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip
       && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") });
   const comments = await ctx.input.api.get<Entity[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`);
   return { ...busy, logoWorker, logoBefore: complete.task, logoRunId: complete.runs[0]!.id, logoMarker, logoArtifactBefore: artifact, logoDocumentsBefore: documents, logoRevisionsBefore: revisions,
-    sourceHistoryBound: comments.filter(row => !row.authorAgentId && row.body === message).length === 1, setupRunIds: saved.runs.map(row => row.id) };
+    // The composer may normalize Markdown, so bind the one human setup message by its setup-chosen references.
+    sourceHistoryBound: comments.filter(row => !row.authorAgentId).length === 1 && comments.some(row => !row.authorAgentId
+      && row.body?.includes(busy.task.identifier ?? busy.task.id) && row.body.includes(logo.identifier ?? logo.id) && row.body.includes(artifact.attachmentId)), setupRunIds: saved.runs.map(row => row.id) };
 }, gradeMultipleHandoffs, {
   setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-multiple-handoffs.v3",
   sourceFiles: ["chat-catalog-domains/runtime-multiple-handoffs.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/agent-chat.yaml"],
@@ -84,7 +91,7 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip
     const consumer = final.runs.find(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId)!;
     const detail = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumer.id}`), saved = first.queue.entries[0]!.comment;
     state.nativeDeliveryBound = detail.contextSnapshot?.wakeCommentIds?.length === 1 && detail.contextSnapshot.wakeCommentIds[0] === saved.id
-      && detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(saved.body) === true;
+      && promptCarriesComment(detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt, saved);
     state.queueEmptyAfterDelivery = true; state.finalWebsiteRunId = consumer.id;
   },
   async observe(ctx, state) {