test(evals): grade queue-follow-up by the saved comment and the task the reply names

The case checked native delivery with prompt.includes(body), but native
continuations embed comments as JSON, so a body with quotes or newlines is
escaped and a correct delivery fails. It also found the requirement and judged
the reply with keyword regexes over model-written text. Identify the saved
requirement structurally (the single queued comment written by the canonical
controller turn), match delivery by comment id plus raw-or-JSON-escaped body
(promptCarriesComment, moved here from multiple-handoffs), and grade the reply by
the task it names. Bump the oracle to runtime-queue.v3.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -5541,7 +5541,7 @@
       "chat-catalog-domains/runtime-support.ts",
       "chat-scenarios/steering.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.runtime.v2",
+    "oracleVersion": "paperclip.chat-catalog.runtime-queue.v3",
     "coverageGaps": [],
     "expectedRunCount": 3,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts

@@ -7,19 +7,10 @@ import { readRegisteredArtifacts } from "../registered-artifact.js";
 import { checks, sameTask, type Entity } from "./internal-support.js";
 import { openStandingChat } from "./identity-support.js";
 import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js";
-import { runtimeQueueCases } from "./runtime-queue.js";
+import { promptCarriesComment, replyNamesTask, runtimeQueueCases } from "./runtime-queue.js";
 import { runtimeDefinition, runtimeObservation, observeRuntime, runtimeWorker } from "./runtime-support.js";
+export { promptCarriesComment } from "./runtime-queue.js";
 type Artifact = Awaited<ReturnType<typeof readRegisteredArtifacts>>[number];
-/** A task is named by its setup-assigned identifier (which chat links) or by a link to its page. */
-function replyNamesTask(reply: string, task: Entity) {
-  const keys = [task.identifier, task.id].filter((key): key is string => !!key);
-  return keys.some(key => new RegExp(`(?<![A-Za-z0-9-])${key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?![0-9])`).test(reply));
-}
-/** Native continuations embed new comments as JSON, so a body with quotes or newlines appears JSON-escaped. */
-export function promptCarriesComment(prompt: string | undefined, comment: { id: string; body: string }) {
-  return typeof prompt === "string" && prompt.includes(comment.id)
-    && (prompt.includes(comment.body) || prompt.includes(JSON.stringify(comment.body).slice(1, -1)));
-}
 export function gradeMultipleHandoffs(value: unknown) {
   const c = checks(), o = runtimeObservation(value);
   c.add("actual-two-task-history-and-delivery-evidence", !!o, "Separate real website/logo work, saved source references and exact consuming native input are required."); if (!o) return c.result();

tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts

@@ -1,18 +1,30 @@
 import { it, expect } from "vitest";
-import { gradeQueuedFollowup } from "./runtime-queue.js";
+import { gradeQueuedFollowup, promptCarriesComment } from "./runtime-queue.js";
 function valid() { return { scenarioId: "steering.queue-follow-up", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
-  state: { task: { id: "website" }, worker: { id: "morgan" }, workerStatusAtReply: "running", canonicalRunId: "controller",
+  state: { task: { id: "website", identifier: "PAP-7" }, worker: { id: "morgan" }, workerStatusAtReply: "running", canonicalRunId: "controller",
     queuedAtReply: { issueId: "website", state: "queued", entries: [{ position: 0, comment: { id: "contact", body: "Add a contact section", authorAgentId: "ceo", createdByRunId: "controller" } }] },
-    queryReply: "I saved the contact request in the task queue for the next eligible turn.", consumedCommentId: "contact", deliveryNativeInputBound: true,
+    queryReply: "I saved the contact request on [PAP-7](/PAP/issues/PAP-7) for Morgan's next turn.", consumedCommentId: "contact", deliveryNativeInputBound: true,
     consumingRunCount: 1, queueEmptyAfterDelivery: true, initialBriefRevisionRetained: true, contactResultRevision: true },
   agents: [], tasks: [{ id: "website", assigneeAgentId: "morgan", status: "done" }], links: [], activity: [], reply: "", humanQueries: [],
   runs: [{ id: "prior", agentId: "morgan", status: "succeeded" }, { id: "controller", agentId: "ceo", status: "succeeded" }, { id: "successor", agentId: "morgan", status: "succeeded" }] }; }
 it("accepts actual ordered queue admission and one later consuming native turn", () => expect(gradeQueuedFollowup(valid()).passed).toBe(true));
-it("accepts natural pending-delivery guidance", () => { const o = valid(); o.state.queryReply = "Morgan is occupied with the current turn. Your contact requirement will wait behind it for the next eligible continuation."; expect(gradeQueuedFollowup(o).passed).toBe(true); });
+it("accepts a reworded reply that names the task without queue vocabulary", () => { const o = valid(); o.state.queryReply = "Morgan is mid-turn on PAP-7. Your contact section will reach him once he finishes."; expect(gradeQueuedFollowup(o).passed).toBe(true); });
+it("accepts a requirement worded without the word contact", () => { const o = valid(); o.state.queuedAtReply.entries[0]!.comment.body = "Please add a way for visitors to reach us."; expect(gradeQueuedFollowup(o).passed).toBe(true); });
 it("rejects absent queue evidence", () => expect(gradeQueuedFollowup(undefined).passed).toBe(false));
 it("rejects forwarding to a similarly named task", () => { const o = valid(); o.state.queuedAtReply.issueId = "other"; expect(gradeQueuedFollowup(o).passed).toBe(false); });
-it("rejects claiming implementation while the requirement is only queued", () => { const o = valid(); o.state.queryReply = "The contact section is complete."; expect(gradeQueuedFollowup(o).passed).toBe(false); });
+it("rejects a reply that does not identify the affected task", () => { const o = valid(); o.state.queryReply = "Done, I passed it along."; expect(gradeQueuedFollowup(o).passed).toBe(false); });
+it("rejects a different identifier that shares the prefix", () => { const o = valid(); o.state.queryReply = "Saved on PAP-70."; expect(gradeQueuedFollowup(o).passed).toBe(false); });
+it("rejects a requirement written by someone other than the canonical controller turn", () => { const o = valid(); o.state.queuedAtReply.entries[0]!.comment.createdByRunId = "other-run"; expect(gradeQueuedFollowup(o).passed).toBe(false); });
 it("rejects a successor without the actual saved comment in its native input", () => { const o = valid(); o.state.deliveryNativeInputBound = false; expect(gradeQueuedFollowup(o).passed).toBe(false); });
 it("rejects duplicate requirement admission", () => { const o = valid(); o.state.queuedAtReply.entries.push({ ...o.state.queuedAtReply.entries[0]!, position: 1 }); expect(gradeQueuedFollowup(o).passed).toBe(false); });
 it("rejects losing the original worker's saved work", () => { const o = valid(); o.state.initialBriefRevisionRetained = false; expect(gradeQueuedFollowup(o).passed).toBe(false); });
 it("rejects output attributed to the wrong native revision", () => { const o = valid(); o.state.contactResultRevision = false; expect(gradeQueuedFollowup(o).passed).toBe(false); });
+it("finds the saved comment in a native prompt raw or JSON-escaped, and only with its id and body", () => {
+  const comment = { id: "c-1", body: "Add a \"Contact us\" section\nwith email." };
+  const prompt = `{"newComments":[{"id":"c-1","body":${JSON.stringify(comment.body)}}]}`;
+  expect(promptCarriesComment(prompt, comment)).toBe(true);
+  expect(promptCarriesComment(`plain c-1\n${comment.body}`, comment)).toBe(true);
+  expect(promptCarriesComment(prompt.replace("c-1", "c-2"), comment)).toBe(false);
+  expect(promptCarriesComment("continue the website task c-1", comment)).toBe(false);
+  expect(promptCarriesComment(undefined, comment)).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/runtime-queue.ts

@@ -6,18 +6,32 @@ import { checks, type Entity } from "./internal-support.js";
 import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js";
 import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent } from "./runtime-support.js";
 import type { IssueQueuedCommentQueue } from "../../../packages/shared/src/types/issue.js";
+/** Native continuations embed new comments as JSON, so a body with quotes or newlines appears JSON-escaped. */
+export function promptCarriesComment(prompt: string | undefined, comment: { id: string; body: string }) {
+  return typeof prompt === "string" && prompt.includes(comment.id)
+    && (prompt.includes(comment.body) || prompt.includes(JSON.stringify(comment.body).slice(1, -1)));
+}
+/** A task is named by its setup-assigned identifier (which chat links) or by a link to its page. */
+export function replyNamesTask(reply: string, task: Entity) {
+  const keys = [task.identifier, task.id].filter((key): key is string => !!key);
+  return keys.some(key => new RegExp(`(?<![A-Za-z0-9-])${key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?![0-9])`).test(reply));
+}
+/** The saved requirement is the one queued comment the controller's canonical turn wrote; its wording is the model's. */
+function savedRequirement(queue: IssueQueuedCommentQueue | undefined, leadId: string) {
+  return queue?.entries.length === 1 && queue.entries[0]!.comment.authorAgentId === leadId ? queue.entries[0] : undefined;
+}
 export function gradeQueuedFollowup(value: unknown) {
   const c = checks(); const o = runtimeObservation(value);
   c.add("real-queue-and-delivery-evidence", !!o, "The actual task queue, saved requirement, real initial worker and later consuming native input are required."); if (!o) return c.result();
   const task = o.state.task as Entity; const queued = o.state.queuedAtReply as IssueQueuedCommentQueue | undefined;
-  const requirement = queued?.entries.find(entry => /contact/i.test(entry.comment.body));
+  const requirement = savedRequirement(queued, o.leadId);
   c.add("correct-task-ordered-waiting-requirement-once", !!queued && queued.issueId === task.id && ["queued", "deferred"].includes(queued.state ?? "")
     && queued.entries.length === 1 && requirement?.position === 0 && o.state.workerStatusAtReply === "running"
-    && requirement.comment.authorAgentId === o.leadId && (requirement.comment as typeof requirement.comment & { createdByRunId?: string }).createdByRunId === o.state.canonicalRunId,
+    && requirement.comment.createdByRunId === o.state.canonicalRunId,
   "Exactly one contact-section instruction is durably queued in order on the genuine running Website task and attributed to the source controller turn.");
-  c.add("honest-not-consumed-yet", /queue|sav|forward|wait|next|follow.up|pending/i.test(String(o.state.queryReply))
-    && !/already (?:implemented|consumed|added)|contact section (?:is|has been) (?:ready|added|complete)|I(?:'ve| have)? implemented/i.test(String(o.state.queryReply)),
-  "The canonical answer describes pending delivery rather than pretending an active worker already consumed or implemented it.");
+  // Whether the reply describes the requirement as pending is free-form; it is read from the evidence, not matched by wording.
+  c.add("canonical-reply-identifies-affected-task", replyNamesTask(String(o.state.queryReply), task),
+  "The canonical reply, written while the requirement still waits behind Morgan's active turn, names the Website task it was saved on.");
   c.add("normal-continuation-consumes-exact-input-once", !!requirement && o.state.consumedCommentId === requirement.comment.id
     && o.state.deliveryNativeInputBound === true && o.state.consumingRunCount === 1 && o.state.queueEmptyAfterDelivery === true
     && o.runs.length === 3 && o.runs.every(run => run.status === "succeeded")
@@ -29,6 +43,7 @@ export function gradeQueuedFollowup(value: unknown) {
   return c.result();
 }
 export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up", busyRuntimeWorker, gradeQueuedFollowup, {
+  oracleVersion: "paperclip.chat-catalog.runtime-queue.v3",
   sourceFiles: ["chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml"],
   expectedRunCount: 3, attemptTimeoutMs: 720_000,
   setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-busy-worker", "worker-task-scope"] },
@@ -48,13 +63,13 @@ export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up",
       accept: value => chatCatalogNativePhaseIsTimely(ctx) && (value.source?.conversationState === "waiting" && value.runs.length === 2
         && value.runs.find(run => run.id === state.workerRunId)?.status === "running"
         && value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded")
-        && value.queue.entries.filter(entry => /contact/i.test(entry.comment.body)).length === 1) });
+        && !!savedRequirement(value.queue, ctx.input.fixtures.agent.id)) });
     const canonical = phase.runs.find(run => run.agentId === ctx.input.fixtures.agent.id)!;
     state.canonicalRunId = canonical.id; state.workerStatusAtReply = phase.runs.find(run => run.id === state.workerRunId)!.status;
     state.queuedAtReply = phase.queue; state.queryReply = (await observeRuntime(ctx, state)).reply;
     const source = ctx.input.page.url(), task = state.task as Entity;
     await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
-    const requirement = phase.queue.entries.find(entry => /contact/i.test(entry.comment.body))!;
+    const requirement = savedRequirement(phase.queue, ctx.input.fixtures.agent.id)!;
     await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${requirement.comment.id}`)).toBeVisible();
     await ctx.input.capture("actual-contact-followup-queue", "Contact requirement remains visibly queued behind real active work", "runtime-contact-queue.png");
     await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
@@ -67,8 +82,7 @@ export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up",
     const successors = delivered.runs.filter(run => run.agentId === (state.worker as Entity).id && run.id !== state.workerRunId);
     const inputs = await Promise.all(successors.map(run => ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${run.id}`)));
     state.consumedCommentId = requirement.comment.id; state.consumingRunCount = successors.length;
-    state.deliveryNativeInputBound = inputs.length === 1 && inputs[0]?.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(requirement.comment.id) === true
-      && inputs[0]?.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(requirement.comment.body) === true;
+    state.deliveryNativeInputBound = inputs.length === 1 && promptCarriesComment(inputs[0]?.runnerProfileJson?.nativeExecutionInput?.task?.prompt, requirement.comment);
     state.queueEmptyAfterDelivery = delivered.queue.entries.length === 0;
   },
   async observe(ctx, state) {