From ee7283efb5044fde67df95666ed3cb3925fb2591 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 18:57:27 -0600 Subject: [PATCH] test(evals): preserve pending question during summary inspection Co-Authored-By: Paperclip --- .../identity-questions-and-approvals.ts | 36 +++++++++++++------ .../chat-catalog-domains/identity.test.ts | 7 ++-- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts index 3872cc08022..5855f843afd 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts @@ -53,19 +53,26 @@ async function observeQuestion(ctx: ChatCatalogContext, state: Record).push({ - id: card.id, rawDescription: description, visibleText: await context.innerText(), - }); + await expect(context).toBeVisible({ timeout: remainingIdentityObservationBudget(ctx) }); + (state.visibleContinuationDescriptions as Array<{ id: string; rawDescription: string; visibleText: string }>).push({ + id: card.id, rawDescription: description, visibleText: await context.innerText({ timeout: remainingIdentityObservationBudget(ctx) }), + }); } } const brief = observed.tasks.find(row => `${row.title}\n${row.description}`.includes(String(state.answer))); state.taskBrief = brief ? await ctx.input.api.get(`/api/issues/${brief.id}`) : undefined; return observed; } +export function multipartVisibleSummaries(value: unknown) { + const o = identityObservation(value); + if (!o) return []; + const visible = o.state.visibleContinuationDescriptions as Array<{ id: string; rawDescription: string; visibleText: string }> | undefined; + return multipartContinuationCards(o).flatMap(card => visible?.filter(row => row.id === card.id + && row.rawDescription === card.payload.questionSet?.description && typeof row.visibleText === "string" && row.visibleText.trim()) ?? []); +} function multipartContinuationCards(o: NonNullable>) { const original = o.state.original as Interaction | undefined; return o.interactions.filter(card => card.id !== original?.id && card.companyId === o.companyId && card.issueId === original?.issueId @@ -118,10 +125,7 @@ export function gradeQuestions(value: unknown) { && answers.some(row => row.questionId === "headline" && row.otherText === o.state.answer), "The submitted choice/choice/custom-text set preserves earlier pages."); c.add("no-default-consent", o.state.unsubmittedPending === true && o.state.validationObserved === true && o.state.retainedAudience === true, "Missing required input cannot submit, unfinished pages stay pending, and back navigation retains the first selection."); - const visible = o.state.visibleContinuationDescriptions as Array<{ id: string; rawDescription: string; visibleText: string }> | undefined; - const summary = [String(o.state.afterAnswerReply), ...multipartContinuationCards(o).flatMap(card => - visible?.filter(row => row.id === card.id && row.rawDescription === card.payload.questionSet?.description) - .map(row => row.visibleText) ?? [])].join("\n"); + const summary = [String(o.state.afterAnswerReply), ...multipartVisibleSummaries(o).map(row => row.visibleText)].join("\n"); c.add("submitted-set-only-continuation", o.runs.filter(run => run.contextSnapshot?.interactionId === original?.id).length === 1 && /researchers/i.test(summary) && /clear/i.test(summary) && summary.includes(String(o.state.answer)), "The submitted set causes one request-bound continuation whose actual later visible chat response accurately summarizes all three answers."); @@ -228,6 +232,18 @@ export const questionIdentityCases = [ }), identityDefinition("questions-and-approvals.multi-part-question", ctx => questionFixture(ctx, true), gradeQuestions, { expectedRunCount: 2, observe: observeQuestion, oracleVersion: "paperclip.chat-catalog.required-question.v3", + async verifyResult(ctx, _state, value) { + if (await ctx.input.page.getByTestId("task-chat-composer-input").isVisible()) return; + const o = identityObservation(value); + if (!o) throw new Error("Multipart final UI lacks its original observation"); + const current = await cards(ctx); + const summaries = multipartVisibleSummaries({ ...o, interactions: current }); + if (!summaries.length) throw new Error("Multipart final UI lacks a bound visible question summary"); + const context = ctx.input.page.getByTestId("task-chat-composer-takeover").getByRole("region", { name: "Question context", exact: true }); + await expect(context).toBeVisible({ timeout: remainingIdentityObservationBudget(ctx) }); + const text = await context.innerText({ timeout: remainingIdentityObservationBudget(ctx) }); + if (!summaries.some(row => row.visibleText === text)) throw new Error("Multipart final question context changed after observation"); + }, async act(ctx, state) { const remaining = () => remainingIdentityObservationBudget(ctx); remaining(); diff --git a/tests/runner-e2e/chat-catalog-domains/identity.test.ts b/tests/runner-e2e/chat-catalog-domains/identity.test.ts index 06ba0f680e9..f99f9b40ede 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { companyIdentityCases, gradeCompanyRename } from "./identity-companies.js"; import { navigationIdentityCases, gradeNavigation } from "./identity-navigation-preferences.js"; -import { questionIdentityCases, gradeQuestions } from "./identity-questions-and-approvals.js"; +import { questionIdentityCases, gradeQuestions, multipartVisibleSummaries } from "./identity-questions-and-approvals.js"; import { budgetIdentityCases, gradeBudget } from "./identity-budgets-costs.js"; import { accessIdentityCases } from "./identity-access.js"; import { toolReviewIdentityCases } from "./identity-tool-review.js"; @@ -159,7 +159,9 @@ describe("multipart continuation on the visible question surface", () => { return o; } it("accepts the actual visible bound card summary without an extra comment", () => { - expect(gradeQuestions(cardSummary()).passed).toBe(true); + const o = cardSummary(); + expect(gradeQuestions(o).passed).toBe(true); + expect(multipartVisibleSummaries(o)).toHaveLength(1); }); it("uses rendered Markdown text rather than its source markup", () => { const o = cardSummary(); const card = o.interactions.at(-1)!; @@ -183,5 +185,6 @@ describe("multipart continuation on the visible question surface", () => { o.state.visibleContinuationDescriptions = [{ id: card.id, rawDescription: card.payload.questionSet!.description, visibleText: card.payload.questionSet!.description }]; } expect(gradeQuestions(o).passed).toBe(false); + if (mode !== "missing-answer") expect(multipartVisibleSummaries(o)).toHaveLength(0); }); });