From 6c4cba0806d264981bd37867ce25e1b9d469c474 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 06:14:55 -0600 Subject: [PATCH] test(evals): grade queue-follow-up by the saved comment and the task the reply names The case checked native delivery with prompt.includes(body), but native continuations embed comments as JSON, so a body with quotes or newlines is escaped and a correct delivery fails. It also found the requirement and judged the reply with keyword regexes over model-written text. Identify the saved requirement structurally (the single queued comment written by the canonical controller turn), match delivery by comment id plus raw-or-JSON-escaped body (promptCarriesComment, moved here from multiple-handoffs), and grade the reply by the task it names. Bump the oracle to runtime-queue.v3. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../runtime-multiple-handoffs.ts | 13 ++------ .../runtime-queue.test.ts | 22 ++++++++++--- .../chat-catalog-domains/runtime-queue.ts | 32 +++++++++++++------ 4 files changed, 43 insertions(+), 26 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 2d4ee0c2c36..35c207abadb 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5541,7 +5541,7 @@ "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime-queue.v3", "coverageGaps": [], "expectedRunCount": 3, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts index 2d7767ecdf4..6ee340545e2 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts @@ -7,19 +7,10 @@ import { readRegisteredArtifacts } from "../registered-artifact.js"; import { checks, sameTask, type Entity } from "./internal-support.js"; import { openStandingChat } from "./identity-support.js"; import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; -import { runtimeQueueCases } from "./runtime-queue.js"; +import { promptCarriesComment, replyNamesTask, runtimeQueueCases } from "./runtime-queue.js"; import { runtimeDefinition, runtimeObservation, observeRuntime, runtimeWorker } from "./runtime-support.js"; +export { promptCarriesComment } from "./runtime-queue.js"; type Artifact = Awaited>[number]; -/** A task is named by its setup-assigned identifier (which chat links) or by a link to its page. */ -function replyNamesTask(reply: string, task: Entity) { - const keys = [task.identifier, task.id].filter((key): key is string => !!key); - return keys.some(key => new RegExp(`(? expect(gradeQueuedFollowup(valid()).passed).toBe(true)); -it("accepts natural pending-delivery guidance", () => { const o = valid(); o.state.queryReply = "Morgan is occupied with the current turn. Your contact requirement will wait behind it for the next eligible continuation."; expect(gradeQueuedFollowup(o).passed).toBe(true); }); +it("accepts a reworded reply that names the task without queue vocabulary", () => { const o = valid(); o.state.queryReply = "Morgan is mid-turn on PAP-7. Your contact section will reach him once he finishes."; expect(gradeQueuedFollowup(o).passed).toBe(true); }); +it("accepts a requirement worded without the word contact", () => { const o = valid(); o.state.queuedAtReply.entries[0]!.comment.body = "Please add a way for visitors to reach us."; expect(gradeQueuedFollowup(o).passed).toBe(true); }); it("rejects absent queue evidence", () => expect(gradeQueuedFollowup(undefined).passed).toBe(false)); it("rejects forwarding to a similarly named task", () => { const o = valid(); o.state.queuedAtReply.issueId = "other"; expect(gradeQueuedFollowup(o).passed).toBe(false); }); -it("rejects claiming implementation while the requirement is only queued", () => { const o = valid(); o.state.queryReply = "The contact section is complete."; expect(gradeQueuedFollowup(o).passed).toBe(false); }); +it("rejects a reply that does not identify the affected task", () => { const o = valid(); o.state.queryReply = "Done, I passed it along."; expect(gradeQueuedFollowup(o).passed).toBe(false); }); +it("rejects a different identifier that shares the prefix", () => { const o = valid(); o.state.queryReply = "Saved on PAP-70."; expect(gradeQueuedFollowup(o).passed).toBe(false); }); +it("rejects a requirement written by someone other than the canonical controller turn", () => { const o = valid(); o.state.queuedAtReply.entries[0]!.comment.createdByRunId = "other-run"; expect(gradeQueuedFollowup(o).passed).toBe(false); }); it("rejects a successor without the actual saved comment in its native input", () => { const o = valid(); o.state.deliveryNativeInputBound = false; expect(gradeQueuedFollowup(o).passed).toBe(false); }); it("rejects duplicate requirement admission", () => { const o = valid(); o.state.queuedAtReply.entries.push({ ...o.state.queuedAtReply.entries[0]!, position: 1 }); expect(gradeQueuedFollowup(o).passed).toBe(false); }); it("rejects losing the original worker's saved work", () => { const o = valid(); o.state.initialBriefRevisionRetained = false; expect(gradeQueuedFollowup(o).passed).toBe(false); }); it("rejects output attributed to the wrong native revision", () => { const o = valid(); o.state.contactResultRevision = false; expect(gradeQueuedFollowup(o).passed).toBe(false); }); +it("finds the saved comment in a native prompt raw or JSON-escaped, and only with its id and body", () => { + const comment = { id: "c-1", body: "Add a \"Contact us\" section\nwith email." }; + const prompt = `{"newComments":[{"id":"c-1","body":${JSON.stringify(comment.body)}}]}`; + expect(promptCarriesComment(prompt, comment)).toBe(true); + expect(promptCarriesComment(`plain c-1\n${comment.body}`, comment)).toBe(true); + expect(promptCarriesComment(prompt.replace("c-1", "c-2"), comment)).toBe(false); + expect(promptCarriesComment("continue the website task c-1", comment)).toBe(false); + expect(promptCarriesComment(undefined, comment)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts index f6ed2b47715..5ef32e3ed4d 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts @@ -6,18 +6,32 @@ import { checks, type Entity } from "./internal-support.js"; import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent } from "./runtime-support.js"; import type { IssueQueuedCommentQueue } from "../../../packages/shared/src/types/issue.js"; +/** Native continuations embed new comments as JSON, so a body with quotes or newlines appears JSON-escaped. */ +export function promptCarriesComment(prompt: string | undefined, comment: { id: string; body: string }) { + return typeof prompt === "string" && prompt.includes(comment.id) + && (prompt.includes(comment.body) || prompt.includes(JSON.stringify(comment.body).slice(1, -1))); +} +/** A task is named by its setup-assigned identifier (which chat links) or by a link to its page. */ +export function replyNamesTask(reply: string, task: Entity) { + const keys = [task.identifier, task.id].filter((key): key is string => !!key); + return keys.some(key => new RegExp(`(? /contact/i.test(entry.comment.body)); + const requirement = savedRequirement(queued, o.leadId); c.add("correct-task-ordered-waiting-requirement-once", !!queued && queued.issueId === task.id && ["queued", "deferred"].includes(queued.state ?? "") && queued.entries.length === 1 && requirement?.position === 0 && o.state.workerStatusAtReply === "running" - && requirement.comment.authorAgentId === o.leadId && (requirement.comment as typeof requirement.comment & { createdByRunId?: string }).createdByRunId === o.state.canonicalRunId, + && requirement.comment.createdByRunId === o.state.canonicalRunId, "Exactly one contact-section instruction is durably queued in order on the genuine running Website task and attributed to the source controller turn."); - c.add("honest-not-consumed-yet", /queue|sav|forward|wait|next|follow.up|pending/i.test(String(o.state.queryReply)) - && !/already (?:implemented|consumed|added)|contact section (?:is|has been) (?:ready|added|complete)|I(?:'ve| have)? implemented/i.test(String(o.state.queryReply)), - "The canonical answer describes pending delivery rather than pretending an active worker already consumed or implemented it."); + // Whether the reply describes the requirement as pending is free-form; it is read from the evidence, not matched by wording. + c.add("canonical-reply-identifies-affected-task", replyNamesTask(String(o.state.queryReply), task), + "The canonical reply, written while the requirement still waits behind Morgan's active turn, names the Website task it was saved on."); c.add("normal-continuation-consumes-exact-input-once", !!requirement && o.state.consumedCommentId === requirement.comment.id && o.state.deliveryNativeInputBound === true && o.state.consumingRunCount === 1 && o.state.queueEmptyAfterDelivery === true && o.runs.length === 3 && o.runs.every(run => run.status === "succeeded") @@ -29,6 +43,7 @@ export function gradeQueuedFollowup(value: unknown) { return c.result(); } export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up", busyRuntimeWorker, gradeQueuedFollowup, { + oracleVersion: "paperclip.chat-catalog.runtime-queue.v3", sourceFiles: ["chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml"], expectedRunCount: 3, attemptTimeoutMs: 720_000, setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-busy-worker", "worker-task-scope"] }, @@ -48,13 +63,13 @@ export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up", accept: value => chatCatalogNativePhaseIsTimely(ctx) && (value.source?.conversationState === "waiting" && value.runs.length === 2 && value.runs.find(run => run.id === state.workerRunId)?.status === "running" && value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded") - && value.queue.entries.filter(entry => /contact/i.test(entry.comment.body)).length === 1) }); + && !!savedRequirement(value.queue, ctx.input.fixtures.agent.id)) }); const canonical = phase.runs.find(run => run.agentId === ctx.input.fixtures.agent.id)!; state.canonicalRunId = canonical.id; state.workerStatusAtReply = phase.runs.find(run => run.id === state.workerRunId)!.status; state.queuedAtReply = phase.queue; state.queryReply = (await observeRuntime(ctx, state)).reply; const source = ctx.input.page.url(), task = state.task as Entity; await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); - const requirement = phase.queue.entries.find(entry => /contact/i.test(entry.comment.body))!; + const requirement = savedRequirement(phase.queue, ctx.input.fixtures.agent.id)!; await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${requirement.comment.id}`)).toBeVisible(); await ctx.input.capture("actual-contact-followup-queue", "Contact requirement remains visibly queued behind real active work", "runtime-contact-queue.png"); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); @@ -67,8 +82,7 @@ export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up", const successors = delivered.runs.filter(run => run.agentId === (state.worker as Entity).id && run.id !== state.workerRunId); const inputs = await Promise.all(successors.map(run => ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${run.id}`))); state.consumedCommentId = requirement.comment.id; state.consumingRunCount = successors.length; - state.deliveryNativeInputBound = inputs.length === 1 && inputs[0]?.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(requirement.comment.id) === true - && inputs[0]?.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(requirement.comment.body) === true; + state.deliveryNativeInputBound = inputs.length === 1 && promptCarriesComment(inputs[0]?.runnerProfileJson?.nativeExecutionInput?.task?.prompt, requirement.comment); state.queueEmptyAfterDelivery = delivered.queue.entries.length === 0; }, async observe(ctx, state) {