test(evals): grade ambiguous-yes disambiguation without a keyword list

The reply path required one of "which/choose/select", so a valid
"Do you want the launch poster or the welcome note?" failed. Now a reply
must name both setup-chosen proposal titles and pose a question, or the
evaluated run must have created a pending question naming both. Bumps the
case oracle to paperclip.chat-catalog.identity.ambiguous-yes.v2.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -4118,7 +4118,7 @@
       "chat-catalog-domains/identity-questions-and-approvals.ts",
       "chat-scenarios/questions-and-approvals.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.identity.v1",
+    "oracleVersion": "paperclip.chat-catalog.identity.ambiguous-yes.v2",
     "expectedRunCount": 1,
     "turnTimeoutMs": 120000,
     "attemptTimeoutMs": 300000,

tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts

@@ -117,10 +117,14 @@ export function gradeQuestions(value: unknown) {
       && JSON.stringify(row.payload) === JSON.stringify(before.payload))),
       "The ambiguous message does not accept either current proposal.");
     c.add("neither-action-executed", o.tasks.length === 0 && o.projects.length === 0, "Neither candidate action has a durable side effect.");
-    c.add("specific-disambiguation", /\b(?:which|choose|select)\b/i.test(o.reply) && /welcome note/i.test(o.reply) && /launch poster/i.test(o.reply)
+    // The two setup-chosen proposal titles are the choices the person must see named; how the agent
+    // phrases its question is free. A real pending question from the evaluated run is structural proof
+    // of asking; a plain reply must name both and pose a question rather than claim completion.
+    const namesBoth = (text: string) => /welcome note/i.test(text) && /launch poster/i.test(text);
+    c.add("specific-disambiguation", namesBoth(o.reply) && o.reply.includes("?")
       || o.interactions.some(row => row.kind === "ask_user_questions" && row.status === "pending" && row.createdByAgentId === o.leadId
-        && /welcome note/i.test(JSON.stringify(row.payload)) && /launch poster/i.test(JSON.stringify(row.payload))),
-      "The reply or a real question names the two actual alternatives and requests a selection.");
+        && o.runs.some(run => run.id === row.sourceRunId && run.agentId === o.leadId) && namesBoth(JSON.stringify(row.payload))),
+      "The reply or a real question from this run names the two actual alternatives and asks the person to pick.");
     c.add("stale-and-expired-cannot-authorize", o.state.staleAttemptStatus === 409 && o.state.expiredAttemptStatus === 409 && o.state.staleErrorCode === "interaction_stale_target"
       && o.state.staleOldRevisionId !== o.state.staleCurrentRevisionId && typeof o.state.staleCurrentRevisionId === "string" && o.state.staleCurrentRevisionAfter === o.state.staleCurrentRevisionId
       && o.interactions.some(row => row.id === o.state.staleInteractionId && row.status === "expired" && row.result?.outcome === "stale_target" && !row.resolvedByAgentId),
@@ -364,6 +368,7 @@ export const questionIdentityCases = [
     }
     await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); return { proposals };
   }, gradeQuestions, {
+    oracleVersion: "paperclip.chat-catalog.identity.ambiguous-yes.v2",
     async act(ctx, state) {
       await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx));
       // Only after the canonical ambiguity response, exercise the additional

tests/runner-e2e/chat-catalog-domains/identity.test.ts

@@ -152,6 +152,18 @@ describe("identity catalog independent oracle calibration", () => {
     const o = valid("questions-and-approvals.ambiguous-yes"); o.state.staleCurrentRevisionAfter = "unexpected-revision";
     expect(gradeQuestions(o).passed).toBe(false);
   });
+  it("ambiguity accepts any phrasing that names both choices and asks, but not generic, completion-claiming or foreign questions", () => {
+    const reworded = valid("questions-and-approvals.ambiguous-yes"); reworded.reply = "Do you want the launch poster, or the welcome note?";
+    expect(gradeQuestions(reworded).passed).toBe(true);
+    for (const reply of ["Which one do you mean?", "Done: the welcome note and launch poster are underway."]) {
+      const o = valid("questions-and-approvals.ambiguous-yes"); o.reply = reply; expect(gradeQuestions(o).passed).toBe(false);
+    }
+    const asked = valid("questions-and-approvals.ambiguous-yes"); asked.reply = "";
+    asked.interactions.push({ id: "ask", kind: "ask_user_questions", status: "pending", createdByAgentId: "ceo", sourceRunId: "run",
+      payload: { questionSet: { questions: [{ id: "pick", prompt: "Which task?", answerMode: "single_select" }] }, options: ["Welcome note only", "Launch poster only"] } } as unknown as Interaction);
+    expect(gradeQuestions(asked).passed).toBe(true);
+    const foreign = structuredClone(asked); foreign.interactions.at(-1)!.sourceRunId = "other-run"; expect(gradeQuestions(foreign).passed).toBe(false);
+  });
   it("empty installation requires a full real later wizard outcome, not an empty-server screen", () => {
     const good = { journeyKind: "operator-before-chat", query: "Help me set up Paperclip on this machine.", querySent: false,
       cliSha256: "a".repeat(64), cliVersion: "1.0.0", initialHomeEntries: 0, companiesBeforeWizard: 0,