From 886a36aeba7022eac885af957ef864c2cca417da Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 20:09:08 -0600 Subject: [PATCH] test(evals): grade the ambiguous-task clarification by state and links, not wording The clarify-before-write check required a question mark or one of which/choose/select/clarify in the free-form reply. A valid question phrased as "Tell me the one to raise" failed it, and the keywords proved nothing the saved state and links do not. The check now relies on no early mutation of either task and links to both candidates; the later checks still reject picking the first match, changing both, or the wrong task after the selection. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../chat-catalog-domains/internal-tasks.ts | 6 +++--- tests/runner-e2e/chat-linked-question-grade.test.ts | 13 +++++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index e395ae84384..a602e150f1b 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5737,7 +5737,7 @@ "chat-catalog-domains/internal-distinct-fixtures.ts", "chat-linked-question.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.internal.ambiguous-task.v2", "coverageGaps": [], "expectedRunCount": 2, "expectedHumanMessageCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts index a12fe9ed525..89fb590d04a 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts @@ -35,9 +35,9 @@ export function grade_tasks(value: unknown) { const other = entity(o.state, "otherTask"); const first = observation(o.state.clarification); c.add("clarify-before-write", !!first && !!before && sameTask(before, first.tasks.find(row => row.id === before.id)) - && sameTask(other, first.tasks.find(row => row.id === other.id)) && /\?|\b(?:which|choose|select|clarify)\b/i.test(first.reply) + && sameTask(other, first.tasks.find(row => row.id === other.id)) && hasLink(first, "issues", before) && hasLink(first, "issues", other), - "The first real turn asks which of both linked tasks before changing either priority."); + "The first completed turn changes neither task and gives the person links to both candidates."); c.add("only-selected-task-changed", !!before && current?.priority === "high" && sameTask(before, current, ["priority"]) && sameTask(other, o.tasks.find(row => row.id === other.id)) && o.tasks.length === 2, "After the explicit Launch selection, only that task's requested priority changes."); @@ -91,7 +91,7 @@ function ambiguousTask(): ChatCatalogCase { await verifyDistinctTaskFixtures(ctx, [t, otherTask]); return { project: p, task: t, otherProject, otherTask }; }); - return { ...base, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-distinct-fixtures.ts", "chat-linked-question.ts"], expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000, + return { ...base, oracleVersion: "paperclip.chat-catalog.internal.ambiguous-task.v2", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-distinct-fixtures.ts", "chat-linked-question.ts"], expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000, async act(ctx, state) { const plain = await pollUntil({ label: "Task ambiguity clarification", deadlineAt: Date.now() + 150_000, load: () => observeInternal(ctx, state), accept: o => o.runs.length === 1 && o.runs.every(row => row.status === "succeeded") diff --git a/tests/runner-e2e/chat-linked-question-grade.test.ts b/tests/runner-e2e/chat-linked-question-grade.test.ts index 11a503d5e5d..230cd8f6842 100644 --- a/tests/runner-e2e/chat-linked-question-grade.test.ts +++ b/tests/runner-e2e/chat-linked-question-grade.test.ts @@ -29,3 +29,16 @@ it("preserves original no-early-write, exact person selection, mutation actor an expect(grade_tasks(actor).passed).toBe(false); expect(grade_tasks({ ...final, links: [] }).passed).toBe(false); }); +it("grades the clarification by saved state and candidate links, not by its wording", () => { + const input = valid(); + const first = { ...input.observation, reply: "There are two Landing page tasks: [Launch](/RUN/issues/RUN-1) and [Experiments](/RUN/issues/experiment-task). Tell me the one to raise to high.", + links: ["/RUN/issues/RUN-1", "/RUN/issues/experiment-task"] }; + const final = { ...structuredClone(input.observation), state: { ...input.observation.state, clarification: first }, + tasks: [{ ...input.observation.tasks[0]!, priority: "high" }, structuredClone(input.observation.tasks[1]!)], + links: ["/RUN/issues/RUN-1"], activity: [{ id: "update", action: "issue.updated", entityId: "launch-task", agentId: "ceo" }] }; + expect(grade_tasks(final).passed).toBe(true); + const oneCandidate = structuredClone(final); oneCandidate.state.clarification.links = ["/RUN/issues/RUN-1"]; + expect(grade_tasks(oneCandidate).passed).toBe(false); + const both = structuredClone(final); both.tasks[1]!.priority = "high"; + expect(grade_tasks(both).passed).toBe(false); +});