From 68a965452084e591fb4b2f7f775a9c8c363568a8 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 13:21:26 -0600 Subject: [PATCH] test(evals): grade ambiguous-yes disambiguation without a keyword list The reply path required one of "which/choose/select", so a valid "Do you want the launch poster or the welcome note?" failed. Now a reply must name both setup-chosen proposal titles and pose a question, or the evaluated run must have created a pending question naming both. Bumps the case oracle to paperclip.chat-catalog.identity.ambiguous-yes.v2. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../identity-questions-and-approvals.ts | 11 ++++++++--- .../runner-e2e/chat-catalog-domains/identity.test.ts | 12 ++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 7844effa6e8..e9eb5a4607e 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4118,7 +4118,7 @@ "chat-catalog-domains/identity-questions-and-approvals.ts", "chat-scenarios/questions-and-approvals.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.identity.ambiguous-yes.v2", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts index 7f7c6a6102c..1d28924014f 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts @@ -117,10 +117,14 @@ export function gradeQuestions(value: unknown) { && JSON.stringify(row.payload) === JSON.stringify(before.payload))), "The ambiguous message does not accept either current proposal."); c.add("neither-action-executed", o.tasks.length === 0 && o.projects.length === 0, "Neither candidate action has a durable side effect."); - c.add("specific-disambiguation", /\b(?:which|choose|select)\b/i.test(o.reply) && /welcome note/i.test(o.reply) && /launch poster/i.test(o.reply) + // The two setup-chosen proposal titles are the choices the person must see named; how the agent + // phrases its question is free. A real pending question from the evaluated run is structural proof + // of asking; a plain reply must name both and pose a question rather than claim completion. + const namesBoth = (text: string) => /welcome note/i.test(text) && /launch poster/i.test(text); + c.add("specific-disambiguation", namesBoth(o.reply) && o.reply.includes("?") || o.interactions.some(row => row.kind === "ask_user_questions" && row.status === "pending" && row.createdByAgentId === o.leadId - && /welcome note/i.test(JSON.stringify(row.payload)) && /launch poster/i.test(JSON.stringify(row.payload))), - "The reply or a real question names the two actual alternatives and requests a selection."); + && o.runs.some(run => run.id === row.sourceRunId && run.agentId === o.leadId) && namesBoth(JSON.stringify(row.payload))), + "The reply or a real question from this run names the two actual alternatives and asks the person to pick."); c.add("stale-and-expired-cannot-authorize", o.state.staleAttemptStatus === 409 && o.state.expiredAttemptStatus === 409 && o.state.staleErrorCode === "interaction_stale_target" && o.state.staleOldRevisionId !== o.state.staleCurrentRevisionId && typeof o.state.staleCurrentRevisionId === "string" && o.state.staleCurrentRevisionAfter === o.state.staleCurrentRevisionId && o.interactions.some(row => row.id === o.state.staleInteractionId && row.status === "expired" && row.result?.outcome === "stale_target" && !row.resolvedByAgentId), @@ -364,6 +368,7 @@ export const questionIdentityCases = [ } await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); return { proposals }; }, gradeQuestions, { + oracleVersion: "paperclip.chat-catalog.identity.ambiguous-yes.v2", async act(ctx, state) { await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); // Only after the canonical ambiguity response, exercise the additional diff --git a/tests/runner-e2e/chat-catalog-domains/identity.test.ts b/tests/runner-e2e/chat-catalog-domains/identity.test.ts index 54653ade43a..8e205ce75e4 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity.test.ts @@ -152,6 +152,18 @@ describe("identity catalog independent oracle calibration", () => { const o = valid("questions-and-approvals.ambiguous-yes"); o.state.staleCurrentRevisionAfter = "unexpected-revision"; expect(gradeQuestions(o).passed).toBe(false); }); + it("ambiguity accepts any phrasing that names both choices and asks, but not generic, completion-claiming or foreign questions", () => { + const reworded = valid("questions-and-approvals.ambiguous-yes"); reworded.reply = "Do you want the launch poster, or the welcome note?"; + expect(gradeQuestions(reworded).passed).toBe(true); + for (const reply of ["Which one do you mean?", "Done: the welcome note and launch poster are underway."]) { + const o = valid("questions-and-approvals.ambiguous-yes"); o.reply = reply; expect(gradeQuestions(o).passed).toBe(false); + } + const asked = valid("questions-and-approvals.ambiguous-yes"); asked.reply = ""; + asked.interactions.push({ id: "ask", kind: "ask_user_questions", status: "pending", createdByAgentId: "ceo", sourceRunId: "run", + payload: { questionSet: { questions: [{ id: "pick", prompt: "Which task?", answerMode: "single_select" }] }, options: ["Welcome note only", "Launch poster only"] } } as unknown as Interaction); + expect(gradeQuestions(asked).passed).toBe(true); + const foreign = structuredClone(asked); foreign.interactions.at(-1)!.sourceRunId = "other-run"; expect(gradeQuestions(foreign).passed).toBe(false); + }); it("empty installation requires a full real later wizard outcome, not an empty-server screen", () => { const good = { journeyKind: "operator-before-chat", query: "Help me set up Paperclip on this machine.", querySent: false, cliSha256: "a".repeat(64), cliVersion: "1.0.0", initialHomeEntries: 0, companiesBeforeWizard: 0,