test(evals): read discard-queued-message evidence the way the product records it

Run 42456573611 completed the whole journey but failed two checks. The guidance
reply linked the task in its saved Markdown before the link rendered, and the
worker's native prompt embeds the multi-line direction comment as JSON, with
escaped newlines. Read saved reply links and accept the JSON-escaped body.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -5423,7 +5423,7 @@
       "chat-catalog-domains/identity-support.ts",
       "chat-scenarios/steering.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.runtime-discard.v3",
+    "oracleVersion": "paperclip.chat-catalog.runtime-discard.v4",
     "coverageGaps": [],
     "expectedRunCount": 5,
     "expectedHumanMessageCount": 2,

tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts

@@ -1,5 +1,5 @@
 import { expect, it } from "vitest";
-import { gradeDiscard } from "./runtime-discard.js";
+import { gradeDiscard, promptCarries, replyLinkHrefs } from "./runtime-discard.js";
 function valid() {
   const neighbour = { position: 0, comment: { id: "font", body: "Use a readable font.", authorUserId: "human", authorAgentId: null } };
   return { scenarioId: "steering.discard-queued-message", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "",
@@ -44,3 +44,15 @@ it("rejects overwriting the already-delivered historical request", () => { const
 it.each([403, 500])("rejects late discard status %s even when the human fixture otherwise looks valid", status => {
   const o = valid(); o.state.lateDiscardStatus = status; expect(gradeDiscard(o).passed).toBe(false);
 });
+
+it("recognises a multi-line direction carried as JSON in the native prompt, and rejects one that is absent", () => {
+  const body = "New direction.\n\n- Remove the contact section.\n- Keep the font guidance.";
+  const prompt = `{"messages":[{"body":${JSON.stringify(body)}}]}`;
+  expect(promptCarries(prompt, body)).toBe(true);
+  expect(promptCarries(prompt.replace("Remove", "Add"), body)).toBe(false);
+  expect(promptCarries(undefined, body)).toBe(false);
+});
+it("reads task links saved in the reply even before they render", () => {
+  expect(replyLinkHrefs("Discard it on [RUN-1](/RUN/issues/RUN-1).", [])).toEqual(["/RUN/issues/RUN-1"]);
+  expect(replyLinkHrefs("Discard it on RUN-1.", [])).toEqual([]);
+});

tests/runner-e2e/chat-catalog-domains/runtime-discard.ts

@@ -13,6 +13,14 @@ import { runtimeQueueCases } from "./runtime-queue.js";
 import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js";
 
 const contactText = "Also add a contact section to the website.";
+/** Native prompts embed task comments as JSON, so a multi-line body appears with escaped newlines. */
+export function promptCarries(prompt: string | undefined, body: string) {
+  return !!prompt && (prompt.includes(body) || prompt.includes(JSON.stringify(body).slice(1, -1)));
+}
+/** Links the reply saved in Markdown, plus links already rendered when the reply was observed. */
+export function replyLinkHrefs(reply: string, rendered: readonly string[]) {
+  return [...new Set([...rendered, ...[...reply.matchAll(/\]\(([^)\s]+)\)/g)].map(match => match[1]!)])];
+}
 export function gradeDiscard(value: unknown) {
   const c = checks(), o = runtimeObservation(value);
   c.add("actual-pending-and-consumed-cancellation-evidence", !!o, "Both actual human discard and separately consumed input with later new direction are required."); if (!o) return c.result();
@@ -41,7 +49,7 @@ export function gradeDiscard(value: unknown) {
   return c.result();
 }
 export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-message", ctx => humanQueuedRuntime(ctx, contactText), gradeDiscard, {
-  authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v3",
+  authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v4",
   sourceFiles: ["chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml"],
   setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup,
   async act(ctx, state) {
@@ -52,7 +60,7 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m
     const answered = await observeRuntime(ctx, state);
     const source = ctx.input.page.url(), task = state.task as Entity, root = `/api/issues/${task.id}`, queuePath = `${root}/queued-comments`;
     const initial = await ctx.input.api.get<IssueQueuedCommentQueue>(queuePath), targetId = String(state.targetId);
-    state.queryReply = answered.reply; state.queryLinkedTask = hasLink(answered, "issues", task);
+    state.queryReply = answered.reply; state.queryLinkedTask = hasLink({ ...answered, links: replyLinkHrefs(answered.reply, answered.links) }, "issues", task);
     state.targetPendingAfterReply = initial.entries.some(row => row.comment.id === targetId && row.comment.body === contactText);
     state.agentDiscardStatus = await probeAgentPermission(ctx, `${queuePath}/${targetId}`, "DELETE", { queueId: initial.queueId, revision: initial.revision });
     await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
@@ -80,7 +88,7 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m
     state.consumerRunId = consumer.id;
     const detail = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumer.id}`);
     state.consumedBeforeCancellation = detail.contextSnapshot?.wakeCommentIds?.includes(second.comment.id) === true
-      && detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(secondText) === true;
+      && promptCarries(detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt, secondText);
     state.discardedNeverDelivered = consumed.runs.every(row => !((row.contextSnapshot?.wakeCommentIds as string[] | undefined) ?? []).includes(targetId));
     state.lateDiscardStatus = await probeConsumedHumanQueue(ctx, String(state.humanId), `${queuePath}/${second.comment.id}`, "DELETE", { queueId: secondQueue.queueId, revision: secondQueue.revision });
     await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
@@ -95,13 +103,13 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m
     const sourceComments = await ctx.input.api.get<Entity[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), human = sourceComments.filter(row => !row.authorAgentId && row.body === message);
     const replies = sourceComments.filter(row => row.createdByRunId === lead?.id && row.authorAgentId === ctx.input.fixtures.agent.id);
     const leadInput = lead && await ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${lead.id}`);
-    state.latePhaseAttributed = human.length === 1 && !!lead && replies.length > 0 && leadInput?.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(human[0]!.id) === true;
+    state.latePhaseAttributed = human.length === 1 && !!lead && replies.length > 0 && promptCarries(leadInput?.runnerProfileJson?.nativeExecutionInput?.task?.prompt, human[0]!.id);
     state.lateReply = replies.map(row => row.body).join("\n");
     const comments = await ctx.input.api.get<IssueComment[]>(`${root}/comments?order=asc`), direction = comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.createdByRunId === lead?.id);
     const workerInput = worker && await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[]; wakeCommentId?: string }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${worker.id}`);
     state.cancelDirectionRunBound = later.length === 2 && !!worker && direction.length === 1
       && [...(workerInput?.contextSnapshot?.wakeCommentIds ?? []), workerInput?.contextSnapshot?.wakeCommentId].includes(direction[0]!.id)
-      && workerInput?.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(direction[0]!.body) === true;
+      && promptCarries(workerInput?.runnerProfileJson?.nativeExecutionInput?.task?.prompt, direction[0]!.body);
     state.finalWorkerRunId = worker?.id; state.consumedStillReadable = comments.some(row => row.id === second.comment.id && row.body === secondText && !row.deletedAt);
     state.finalQueueEmpty = final.queue.entries.length === 0;
   },