diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index e395ae84384..a602e150f1b 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5737,7 +5737,7 @@ "chat-catalog-domains/internal-distinct-fixtures.ts", "chat-linked-question.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.internal.ambiguous-task.v2", "coverageGaps": [], "expectedRunCount": 2, "expectedHumanMessageCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts index a12fe9ed525..89fb590d04a 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts @@ -35,9 +35,9 @@ export function grade_tasks(value: unknown) { const other = entity(o.state, "otherTask"); const first = observation(o.state.clarification); c.add("clarify-before-write", !!first && !!before && sameTask(before, first.tasks.find(row => row.id === before.id)) - && sameTask(other, first.tasks.find(row => row.id === other.id)) && /\?|\b(?:which|choose|select|clarify)\b/i.test(first.reply) + && sameTask(other, first.tasks.find(row => row.id === other.id)) && hasLink(first, "issues", before) && hasLink(first, "issues", other), - "The first real turn asks which of both linked tasks before changing either priority."); + "The first completed turn changes neither task and gives the person links to both candidates."); c.add("only-selected-task-changed", !!before && current?.priority === "high" && sameTask(before, current, ["priority"]) && sameTask(other, o.tasks.find(row => row.id === other.id)) && o.tasks.length === 2, "After the explicit Launch selection, only that task's requested priority changes."); @@ -91,7 +91,7 @@ function ambiguousTask(): ChatCatalogCase { await verifyDistinctTaskFixtures(ctx, [t, otherTask]); return { project: p, task: t, otherProject, otherTask }; }); - return { ...base, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-distinct-fixtures.ts", "chat-linked-question.ts"], expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000, + return { ...base, oracleVersion: "paperclip.chat-catalog.internal.ambiguous-task.v2", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-distinct-fixtures.ts", "chat-linked-question.ts"], expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000, async act(ctx, state) { const plain = await pollUntil({ label: "Task ambiguity clarification", deadlineAt: Date.now() + 150_000, load: () => observeInternal(ctx, state), accept: o => o.runs.length === 1 && o.runs.every(row => row.status === "succeeded") diff --git a/tests/runner-e2e/chat-linked-question-grade.test.ts b/tests/runner-e2e/chat-linked-question-grade.test.ts index 11a503d5e5d..230cd8f6842 100644 --- a/tests/runner-e2e/chat-linked-question-grade.test.ts +++ b/tests/runner-e2e/chat-linked-question-grade.test.ts @@ -29,3 +29,16 @@ it("preserves original no-early-write, exact person selection, mutation actor an expect(grade_tasks(actor).passed).toBe(false); expect(grade_tasks({ ...final, links: [] }).passed).toBe(false); }); +it("grades the clarification by saved state and candidate links, not by its wording", () => { + const input = valid(); + const first = { ...input.observation, reply: "There are two Landing page tasks: [Launch](/RUN/issues/RUN-1) and [Experiments](/RUN/issues/experiment-task). Tell me the one to raise to high.", + links: ["/RUN/issues/RUN-1", "/RUN/issues/experiment-task"] }; + const final = { ...structuredClone(input.observation), state: { ...input.observation.state, clarification: first }, + tasks: [{ ...input.observation.tasks[0]!, priority: "high" }, structuredClone(input.observation.tasks[1]!)], + links: ["/RUN/issues/RUN-1"], activity: [{ id: "update", action: "issue.updated", entityId: "launch-task", agentId: "ceo" }] }; + expect(grade_tasks(final).passed).toBe(true); + const oneCandidate = structuredClone(final); oneCandidate.state.clarification.links = ["/RUN/issues/RUN-1"]; + expect(grade_tasks(oneCandidate).passed).toBe(false); + const both = structuredClone(final); both.tasks[1]!.priority = "high"; + expect(grade_tasks(both).passed).toBe(false); +});