From d3102c577c2d14fc76638dacff4238b6720ce75d Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 01:18:29 -0600 Subject: [PATCH] test(evals): read discard-queued-message evidence the way the product records it Run 42456573611 completed the whole journey but failed two checks. The guidance reply linked the task in its saved Markdown before the link rendered, and the worker's native prompt embeds the multi-line direction comment as JSON, with escaped newlines. Read saved reply links and accept the JSON-escaped body. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../runtime-discard.test.ts | 14 +++++++++++++- .../chat-catalog-domains/runtime-discard.ts | 18 +++++++++++++----- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 7ba2b9ecee9..ea11c0842ac 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5423,7 +5423,7 @@ "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime-discard.v3", + "oracleVersion": "paperclip.chat-catalog.runtime-discard.v4", "coverageGaps": [], "expectedRunCount": 5, "expectedHumanMessageCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts index 64b2b1688ee..96b5088736b 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { gradeDiscard } from "./runtime-discard.js"; +import { gradeDiscard, promptCarries, replyLinkHrefs } from "./runtime-discard.js"; function valid() { const neighbour = { position: 0, comment: { id: "font", body: "Use a readable font.", authorUserId: "human", authorAgentId: null } }; return { scenarioId: "steering.discard-queued-message", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", @@ -44,3 +44,15 @@ it("rejects overwriting the already-delivered historical request", () => { const it.each([403, 500])("rejects late discard status %s even when the human fixture otherwise looks valid", status => { const o = valid(); o.state.lateDiscardStatus = status; expect(gradeDiscard(o).passed).toBe(false); }); + +it("recognises a multi-line direction carried as JSON in the native prompt, and rejects one that is absent", () => { + const body = "New direction.\n\n- Remove the contact section.\n- Keep the font guidance."; + const prompt = `{"messages":[{"body":${JSON.stringify(body)}}]}`; + expect(promptCarries(prompt, body)).toBe(true); + expect(promptCarries(prompt.replace("Remove", "Add"), body)).toBe(false); + expect(promptCarries(undefined, body)).toBe(false); +}); +it("reads task links saved in the reply even before they render", () => { + expect(replyLinkHrefs("Discard it on [RUN-1](/RUN/issues/RUN-1).", [])).toEqual(["/RUN/issues/RUN-1"]); + expect(replyLinkHrefs("Discard it on RUN-1.", [])).toEqual([]); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts index 1a879746ef2..3da3f4928dc 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts @@ -13,6 +13,14 @@ import { runtimeQueueCases } from "./runtime-queue.js"; import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; const contactText = "Also add a contact section to the website."; +/** Native prompts embed task comments as JSON, so a multi-line body appears with escaped newlines. */ +export function promptCarries(prompt: string | undefined, body: string) { + return !!prompt && (prompt.includes(body) || prompt.includes(JSON.stringify(body).slice(1, -1))); +} +/** Links the reply saved in Markdown, plus links already rendered when the reply was observed. */ +export function replyLinkHrefs(reply: string, rendered: readonly string[]) { + return [...new Set([...rendered, ...[...reply.matchAll(/\]\(([^)\s]+)\)/g)].map(match => match[1]!)])]; +} export function gradeDiscard(value: unknown) { const c = checks(), o = runtimeObservation(value); c.add("actual-pending-and-consumed-cancellation-evidence", !!o, "Both actual human discard and separately consumed input with later new direction are required."); if (!o) return c.result(); @@ -41,7 +49,7 @@ export function gradeDiscard(value: unknown) { return c.result(); } export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-message", ctx => humanQueuedRuntime(ctx, contactText), gradeDiscard, { - authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v3", + authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v4", sourceFiles: ["chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml"], setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, async act(ctx, state) { @@ -52,7 +60,7 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m const answered = await observeRuntime(ctx, state); const source = ctx.input.page.url(), task = state.task as Entity, root = `/api/issues/${task.id}`, queuePath = `${root}/queued-comments`; const initial = await ctx.input.api.get(queuePath), targetId = String(state.targetId); - state.queryReply = answered.reply; state.queryLinkedTask = hasLink(answered, "issues", task); + state.queryReply = answered.reply; state.queryLinkedTask = hasLink({ ...answered, links: replyLinkHrefs(answered.reply, answered.links) }, "issues", task); state.targetPendingAfterReply = initial.entries.some(row => row.comment.id === targetId && row.comment.body === contactText); state.agentDiscardStatus = await probeAgentPermission(ctx, `${queuePath}/${targetId}`, "DELETE", { queueId: initial.queueId, revision: initial.revision }); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); @@ -80,7 +88,7 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m state.consumerRunId = consumer.id; const detail = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumer.id}`); state.consumedBeforeCancellation = detail.contextSnapshot?.wakeCommentIds?.includes(second.comment.id) === true - && detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(secondText) === true; + && promptCarries(detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt, secondText); state.discardedNeverDelivered = consumed.runs.every(row => !((row.contextSnapshot?.wakeCommentIds as string[] | undefined) ?? []).includes(targetId)); state.lateDiscardStatus = await probeConsumedHumanQueue(ctx, String(state.humanId), `${queuePath}/${second.comment.id}`, "DELETE", { queueId: secondQueue.queueId, revision: secondQueue.revision }); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); @@ -95,13 +103,13 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m const sourceComments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), human = sourceComments.filter(row => !row.authorAgentId && row.body === message); const replies = sourceComments.filter(row => row.createdByRunId === lead?.id && row.authorAgentId === ctx.input.fixtures.agent.id); const leadInput = lead && await ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${lead.id}`); - state.latePhaseAttributed = human.length === 1 && !!lead && replies.length > 0 && leadInput?.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(human[0]!.id) === true; + state.latePhaseAttributed = human.length === 1 && !!lead && replies.length > 0 && promptCarries(leadInput?.runnerProfileJson?.nativeExecutionInput?.task?.prompt, human[0]!.id); state.lateReply = replies.map(row => row.body).join("\n"); const comments = await ctx.input.api.get(`${root}/comments?order=asc`), direction = comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.createdByRunId === lead?.id); const workerInput = worker && await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[]; wakeCommentId?: string }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${worker.id}`); state.cancelDirectionRunBound = later.length === 2 && !!worker && direction.length === 1 && [...(workerInput?.contextSnapshot?.wakeCommentIds ?? []), workerInput?.contextSnapshot?.wakeCommentId].includes(direction[0]!.id) - && workerInput?.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(direction[0]!.body) === true; + && promptCarries(workerInput?.runnerProfileJson?.nativeExecutionInput?.task?.prompt, direction[0]!.body); state.finalWorkerRunId = worker?.id; state.consumedStillReadable = comments.some(row => row.id === second.comment.id && row.body === secondText && !row.deletedAt); state.finalQueueEmpty = final.queue.entries.length === 0; },