test(evals): start multiple-handoffs without a brief cue and grade the reply by the task it names

The baseline lead never replied in time: the setup message called the Website
task "actively waiting for the brief", so the lead read the headline change as
that brief and spent its turn inspecting the worker's workspace. Describe the
task as in progress instead. Replace the reply keyword checks ("queue|pending|
wait..." and an "already implemented" deny-list) with the setup-assigned task
identifier or link the reply names; the saved-for-later state is graded by the
queue entry, the still-running worker at reply time and the later native
delivery.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -320,7 +320,7 @@
       "registered-artifact.ts",
       "chat-scenarios/agent-chat.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.runtime.v2",
+    "oracleVersion": "paperclip.chat-catalog.runtime-multiple-handoffs.v3",
     "coverageGaps": [],
     "expectedRunCount": 5,
     "setupHumanMessageCount": 1,

tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts

@@ -1,21 +1,32 @@
 import { expect, it } from "vitest";
 import { gradeMultipleHandoffs } from "./runtime-multiple-handoffs.js";
 function valid() {
-  const logo = { id: "logo", title: "Logo", companyId: "company", status: "done", assigneeAgentId: "riley" };
+  const logo = { id: "logo", identifier: "RUN-1", title: "Logo", companyId: "company", status: "done", assigneeAgentId: "riley" };
   const artifact = { contentVerified: true, attachmentId: "logo-file", sha256: "same", content: "<svg><text>Acorn real-marker</text></svg>" };
   return { scenarioId: "agent-chat.multiple-handoffs", companyId: "company", leadId: "ceo", agents: [], links: [], activity: [], humanQueries: [], reply: "",
-    tasks: [logo, { id: "website", assigneeAgentId: "morgan", status: "done" }],
+    tasks: [logo, { id: "8ba8b5a2-066a-4bb2-b591-e396942ecdf5", assigneeAgentId: "morgan", status: "done" }],
     runs: [{ id: "logo-run", agentId: "riley", status: "succeeded" }, { id: "prior", agentId: "morgan", status: "succeeded" },
       { id: "setup", agentId: "ceo", status: "succeeded" }, { id: "canonical", agentId: "ceo", status: "succeeded" }, { id: "consumer", agentId: "morgan", status: "succeeded" }],
-    state: { task: { id: "website" }, worker: { id: "morgan" }, logoWorker: { id: "riley" }, workerRunId: "prior", canonicalRunId: "canonical",
-      queueAtReply: { issueId: "website", targetRunId: "prior", state: "queued", entries: [{ comment: { id: "headline", authorAgentId: "ceo", createdByRunId: "canonical", body: "Change the headline to Launch sooner." } }] },
+    state: { task: { id: "8ba8b5a2-066a-4bb2-b591-e396942ecdf5", identifier: "RUN-2" }, worker: { id: "morgan" }, logoWorker: { id: "riley" }, workerRunId: "prior", canonicalRunId: "canonical",
+      queueAtReply: { issueId: "8ba8b5a2-066a-4bb2-b591-e396942ecdf5", targetRunId: "prior", state: "queued", entries: [{ comment: { id: "headline", authorAgentId: "ceo", createdByRunId: "canonical", body: "Change the headline to Launch sooner." } }] },
       sourceHistoryBound: true, workerRunningAtReply: true, nativeDeliveryBound: true, queueEmptyAfterDelivery: true, logoBefore: logo, logoAfter: structuredClone(logo),
       logoArtifactBefore: artifact, logoArtifactAfter: { ...artifact }, logoMarker: "real-marker", logoRevisionsUnchanged: true,
-      queryReply: "I saved the headline feedback in the existing Website queue while Morgan is busy. The logo remains unchanged.",
+      queryReply: "I saved the headline feedback on [RUN-2](/RUN/issues/RUN-2) while Morgan is busy. The logo [RUN-1](/RUN/issues/RUN-1) remains unchanged.",
       finalHeadlineRevision: true, originalWebsiteRevisionRetained: true } };
 }
 it("accepts one Website delivery with real completed Logo preservation", () => expect(gradeMultipleHandoffs(valid()).passed).toBe(true));
-it("accepts natural pending-delivery wording for the exact affected work", () => { const o = valid(); o.state.queryReply = "Website's new headline requirement will wait for Morgan's next turn; I left Riley's completed logo alone."; expect(gradeMultipleHandoffs(o).passed).toBe(true); });
+it("accepts reworded replies that name the affected task by identifier or link", () => {
+  for (const reply of ["RUN-2's new headline requirement will wait for Morgan's next turn; I left Riley's completed logo alone.",
+    "Morgan will pick up \"Launch sooner\" when the current turn ends: [Website task](/RUN/issues/RUN-2#comment-headline).",
+    "Queued for Morgan on RUN-2. Nothing changed on RUN-1."]) {
+    const o = valid(); o.state.queryReply = reply; expect(gradeMultipleHandoffs(o).passed).toBe(true);
+  }
+});
+it("rejects a reply that does not name the affected Website task", () => {
+  for (const reply of ["I saved the headline change for the website.", "Saved on RUN-20 for Morgan.", "The logo RUN-1 is unchanged."]) {
+    const o = valid(); o.state.queryReply = reply; expect(gradeMultipleHandoffs(o).passed).toBe(false);
+  }
+});
 it("rejects missing independent two-task evidence", () => expect(gradeMultipleHandoffs(undefined).passed).toBe(false));
 it("rejects routing headline feedback to Logo instead", () => { const o = valid(); o.state.queueAtReply.issueId = "logo"; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
 it("rejects duplicate saved headline requirements", () => { const o = valid(); o.state.queueAtReply.entries.push({ ...o.state.queueAtReply.entries[0]! }); expect(gradeMultipleHandoffs(o).passed).toBe(false); });
@@ -24,6 +35,6 @@ it("rejects a consuming run without the exact saved comment", () => { const o =
 it("rejects changing the actual delivered Logo bytes", () => { const o = valid(); o.state.logoArtifactAfter.content = "<svg>new</svg>"; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
 it("rejects retaining a Logo filename but replacing its artifact", () => { const o = valid(); o.state.logoArtifactAfter.attachmentId = "replacement"; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
 it("rejects erasing an earlier Logo revision", () => { const o = valid(); o.state.logoRevisionsUnchanged = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
-it("rejects claiming implementation before active-run consumption", () => { const o = valid(); o.state.queryReply = "The Website headline has been changed; the queued request is complete."; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
+it("rejects a reply written after the active worker stopped holding the saved instruction", () => { const o = valid(); o.state.workerRunningAtReply = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
 it("rejects redirection to a different worker", () => { const o = valid(); o.tasks[1]!.assigneeAgentId = "riley"; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
 it("rejects losing the original Website work", () => { const o = valid(); o.state.originalWebsiteRevisionRetained = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); });

tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts

@@ -10,6 +10,11 @@ import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.j
 import { runtimeQueueCases } from "./runtime-queue.js";
 import { runtimeDefinition, runtimeObservation, observeRuntime, runtimeWorker } from "./runtime-support.js";
 type Artifact = Awaited<ReturnType<typeof readRegisteredArtifacts>>[number];
+/** A task is named by its setup-assigned identifier (which chat links) or by a link to its page. */
+function replyNamesTask(reply: string, task: Entity) {
+  const keys = [task.identifier, task.id].filter((key): key is string => !!key);
+  return keys.some(key => new RegExp(`(?<![A-Za-z0-9-])${key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?![0-9])`).test(reply));
+}
 export function gradeMultipleHandoffs(value: unknown) {
   const c = checks(), o = runtimeObservation(value);
   c.add("actual-two-task-history-and-delivery-evidence", !!o, "Separate real website/logo work, saved source references and exact consuming native input are required."); if (!o) return c.result();
@@ -24,9 +29,9 @@ export function gradeMultipleHandoffs(value: unknown) {
     && before?.contentVerified === true && after?.contentVerified === true && before.attachmentId === after.attachmentId && before.sha256 === after.sha256
     && before.content === after.content && after.content.includes(String(s.logoMarker)) && /<svg\b/i.test(after.content) && s.logoRevisionsUnchanged === true,
   "The separate real Logo task, logo bytes, artifact identity, saved revisions and original worker attribution remain unchanged.");
-  c.add("canonical-reply-honestly-identifies-affected-pending-work", /website/i.test(String(s.queryReply)) && /queue|pending|wait|next|forward|saved/i.test(String(s.queryReply))
-    && !/already (?:implemented|consumed)|headline (?:is|has been) (?:already )?(?:changed|updated)|I(?:'ve| have)? implemented/i.test(String(s.queryReply)),
-  "The canonical reply identifies Website and accurately describes feedback saved behind active work rather than claiming it already executed.");
+  // The reply's account of pending versus applied work is free-form; it is read from the evidence, not matched by wording.
+  c.add("canonical-reply-identifies-affected-task", replyNamesTask(String(s.queryReply), s.task as Entity),
+  "The canonical reply, written while the saved instruction still waits behind active work, names the affected Website task.");
   c.add("real-final-headline-and-original-work-retained", s.finalHeadlineRevision === true && s.originalWebsiteRevisionRetained === true
     && o.tasks.length === 2 && o.tasks.some(row => row.id === (s.task as Entity).id && row.status === "done" && row.assigneeAgentId === (s.worker as Entity).id)
     && o.runs.length === 5 && o.runs.every(row => row.status === "succeeded") && o.runs.filter(row => row.agentId === (s.worker as Entity).id).length === 2
@@ -48,7 +53,7 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip
   const documents = await ctx.input.api.get<Entity[]>(`/api/issues/${logo.id}/documents`), revisions: Entity[] = [];
   for (const doc of documents) revisions.push(...await ctx.input.api.get<Entity[]>(`/api/issues/${logo.id}/documents/${encodeURIComponent(doc.key!)}/revisions`));
   const busy = await busyRuntimeWorker(ctx); await openStandingChat(ctx);
-  const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan and actively waiting for the brief; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered logo is ${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}. Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`;
+  const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan, who is working on it now; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered logo is ${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}. Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`;
   await sendChatMessage(ctx.input.page, message);
   const saved = await pollUntil({ label: "Real source history retains separate active Website and completed Logo", deadlineAt: Date.now() + 150_000,
     load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
@@ -58,7 +63,7 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip
   return { ...busy, logoWorker, logoBefore: complete.task, logoRunId: complete.runs[0]!.id, logoMarker, logoArtifactBefore: artifact, logoDocumentsBefore: documents, logoRevisionsBefore: revisions,
     sourceHistoryBound: comments.filter(row => !row.authorAgentId && row.body === message).length === 1, setupRunIds: saved.runs.map(row => row.id) };
 }, gradeMultipleHandoffs, {
-  setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000,
+  setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-multiple-handoffs.v3",
   sourceFiles: ["chat-catalog-domains/runtime-multiple-handoffs.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/agent-chat.yaml"],
   setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup,
   async act(ctx, state) {