test(evals): observe visible multipart continuation summaries

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -4225,7 +4225,7 @@
       "chat-catalog-domains/identity-questions-and-approvals.ts",
       "chat-scenarios/questions-and-approvals.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.required-question.v2",
+    "oracleVersion": "paperclip.chat-catalog.required-question.v3",
     "expectedRunCount": 2,
     "turnTimeoutMs": 120000,
     "attemptTimeoutMs": 300000,

tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts

@@ -48,10 +48,30 @@ async function observeQuestion(ctx: ChatCatalogContext, state: Record<string, un
   const seen = new Set(state.beforeAnswerCommentIds as string[] | undefined);
   state.afterAnswerReply = (await ctx.input.api.get<Array<{ id: string; body: string; authorAgentId?: string }>>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`))
     .filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && !seen.has(row.id)).map(row => row.body).join("\n");
+  if (ctx.scenario.id === "questions-and-approvals.multi-part-question") {
+    state.visibleContinuationDescriptions = [];
+    for (const card of multipartContinuationCards(observed)) {
+      const description = card.payload.questionSet!.description!;
+      const row = ctx.input.page.locator(`[id=${JSON.stringify(`interaction-${card.id}`)}]`);
+      if (!await row.isVisible()) continue;
+      await row.click();
+      const context = ctx.input.page.getByTestId("task-chat-composer-takeover").getByRole("region", { name: "Question context", exact: true });
+      if (await context.getByText(description, { exact: true }).isVisible())
+        (state.visibleContinuationDescriptions as Array<{ id: string; description: string }>).push({ id: card.id, description });
+    }
+  }
   const brief = observed.tasks.find(row => `${row.title}\n${row.description}`.includes(String(state.answer)));
   state.taskBrief = brief ? await ctx.input.api.get(`/api/issues/${brief.id}`) : undefined;
   return observed;
 }
+function multipartContinuationCards(o: NonNullable<ReturnType<typeof identityObservation>>) {
+  const original = o.state.original as Interaction | undefined;
+  return o.interactions.filter(card => card.id !== original?.id && card.companyId === o.companyId && card.issueId === original?.issueId
+    && card.createdByAgentId === o.leadId && card.kind === "ask_user_questions" && card.status === "pending"
+    && typeof card.payload.questionSet?.description === "string" && o.runs.some(run => run.id === card.sourceRunId
+      && run.companyId === o.companyId && run.agentId === o.leadId && run.status === "succeeded"
+      && run.contextSnapshot?.interactionId === original?.id && run.contextSnapshot?.issueId === original?.issueId));
+}
 export function gradeQuestions(value: unknown) {
   const c = checks(); const o = identityObservation(value);
   c.add("complete-interaction-evidence", !!o, "Public interactions, persisted answers, run identities and result records are required.");
@@ -96,9 +116,12 @@ export function gradeQuestions(value: unknown) {
       && answers.some(row => row.questionId === "headline" && row.otherText === o.state.answer), "The submitted choice/choice/custom-text set preserves earlier pages.");
     c.add("no-default-consent", o.state.unsubmittedPending === true && o.state.validationObserved === true && o.state.retainedAudience === true,
       "Missing required input cannot submit, unfinished pages stay pending, and back navigation retains the first selection.");
+    const visible = o.state.visibleContinuationDescriptions as Array<{ id: string; description: string }> | undefined;
+    const summary = [String(o.state.afterAnswerReply), ...multipartContinuationCards(o).flatMap(card =>
+      visible?.some(row => row.id === card.id && row.description === card.payload.questionSet?.description) ? [card.payload.questionSet!.description!] : [])].join("\n");
     c.add("submitted-set-only-continuation", o.runs.filter(run => run.contextSnapshot?.interactionId === original?.id).length === 1
-      && /researchers/i.test(String(o.state.afterAnswerReply)) && /clear/i.test(String(o.state.afterAnswerReply)) && String(o.state.afterAnswerReply).includes(String(o.state.answer)),
-      "The submitted set causes one request-bound continuation whose actual later response accurately summarizes all three answers.");
+      && /researchers/i.test(summary) && /clear/i.test(summary) && summary.includes(String(o.state.answer)),
+      "The submitted set causes one request-bound continuation whose actual later visible chat response accurately summarizes all three answers.");
   } else {
     c.add("exact-free-text-answer", answers.length === 1 && answers[0]?.otherText === o.state.answer && !answers[0]?.optionIds?.length,
       "The exact human product name is attached once to the existing question.");
@@ -201,7 +224,7 @@ export const questionIdentityCases = [
     },
   }),
   identityDefinition("questions-and-approvals.multi-part-question", ctx => questionFixture(ctx, true), gradeQuestions, {
-    expectedRunCount: 2, observe: observeQuestion, oracleVersion: "paperclip.chat-catalog.required-question.v2",
+    expectedRunCount: 2, observe: observeQuestion, oracleVersion: "paperclip.chat-catalog.required-question.v3",
     async act(ctx, state) {
       const remaining = () => remainingIdentityObservationBudget(ctx);
       remaining();

tests/runner-e2e/chat-catalog-domains/identity-support.ts

@@ -7,7 +7,7 @@ export interface IdentityObservation extends InternalObservation {
 }
 export interface Interaction extends Entity {
   idempotencyKey?: string | null;
-  kind: string; status: string; payload: { prompt?: string; runtimeRequestId?: string; toolAction?: { connectionId?: string; actionRequestId?: string }; questionSet?: { submitLabel?: string; questions: Array<{ id: string; prompt: string; answerMode: string; required?: boolean; customAnswer?: { enabled: boolean; label?: string } }> } };
+  kind: string; status: string; payload: { prompt?: string; runtimeRequestId?: string; toolAction?: { connectionId?: string; actionRequestId?: string }; questionSet?: { description?: string; submitLabel?: string; questions: Array<{ id: string; prompt: string; answerMode: string; required?: boolean; customAnswer?: { enabled: boolean; label?: string } }> } };
   result?: { outcome?: string; commentId?: string; toolAction?: { status: string; errorCode?: string | null; errorMessage?: string | null }; answers?: Array<{ questionId: string; otherText?: string; optionIds?: string[] }> };
   resolvedByUserId?: string; resolvedByAgentId?: string; sourceRunId?: string;
 }

tests/runner-e2e/chat-catalog-domains/identity.test.ts

@@ -145,3 +145,33 @@ describe("identity catalog independent oracle calibration", () => {
     expect(gradeEmptyInstallation(undefined).passed).toBe(false);
   });
 });
+
+describe("multipart continuation on the visible question surface", () => {
+  function cardSummary() {
+    const o = valid("questions-and-approvals.multi-part-question");
+    Object.assign(o.state.original as Interaction, { issueId: "source", companyId: "company" });
+    Object.assign(o.runs[1]!, { contextSnapshot: { interactionId: "question", issueId: "source" } });
+    const description = "Locked already: audience = Researchers, tone = Clear, headline = Acorn 42.";
+    o.interactions.push({ id: "later", companyId: "company", issueId: "source", createdByAgentId: "ceo", sourceRunId: o.runs[1]!.id,
+      kind: "ask_user_questions", status: "pending", payload: { questionSet: { description, questions: [] } } });
+    o.state.afterAnswerReply = "";
+    o.state.visibleContinuationDescriptions = [{ id: "later", description }];
+    return o;
+  }
+  it("accepts the actual visible bound card summary without an extra comment", () => {
+    expect(gradeQuestions(cardSummary()).passed).toBe(true);
+  });
+  it.each(["hidden", "wrong-run", "wrong-issue", "wrong-agent", "historical", "missing-answer"])("rejects %s card evidence", mode => {
+    const o = cardSummary(); const card = o.interactions.at(-1)!;
+    if (mode === "hidden") o.state.visibleContinuationDescriptions = [];
+    if (mode === "wrong-run") card.sourceRunId = "unrelated";
+    if (mode === "wrong-issue") card.issueId = "other";
+    if (mode === "wrong-agent") card.createdByAgentId = "other";
+    if (mode === "historical") card.id = "question";
+    if (mode === "missing-answer") {
+      card.payload.questionSet!.description = "Researchers, Clear";
+      o.state.visibleContinuationDescriptions = [{ id: card.id, description: card.payload.questionSet!.description }];
+    }
+    expect(gradeQuestions(o).passed).toBe(false);
+  });
+});