From ed4f01b15ad5a233f62dbc9b7bd001749c021809 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 18:48:29 -0600 Subject: [PATCH] test(evals): observe visible multipart continuation summaries Co-Authored-By: Paperclip --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../identity-questions-and-approvals.ts | 29 ++++++++++++++++-- .../chat-catalog-domains/identity-support.ts | 2 +- .../chat-catalog-domains/identity.test.ts | 30 +++++++++++++++++++ 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 578b1a239ee..8dc35a7581a 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4225,7 +4225,7 @@ "chat-catalog-domains/identity-questions-and-approvals.ts", "chat-scenarios/questions-and-approvals.yaml" ], - "oracleVersion": "paperclip.chat-catalog.required-question.v2", + "oracleVersion": "paperclip.chat-catalog.required-question.v3", "expectedRunCount": 2, "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts index 9fcfbdb009b..5b98f8e1b26 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts @@ -48,10 +48,30 @@ async function observeQuestion(ctx: ChatCatalogContext, state: Record>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`)) .filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && !seen.has(row.id)).map(row => row.body).join("\n"); + if (ctx.scenario.id === "questions-and-approvals.multi-part-question") { + state.visibleContinuationDescriptions = []; + for (const card of multipartContinuationCards(observed)) { + const description = card.payload.questionSet!.description!; + const row = ctx.input.page.locator(`[id=${JSON.stringify(`interaction-${card.id}`)}]`); + if (!await row.isVisible()) continue; + await row.click(); + const context = ctx.input.page.getByTestId("task-chat-composer-takeover").getByRole("region", { name: "Question context", exact: true }); + if (await context.getByText(description, { exact: true }).isVisible()) + (state.visibleContinuationDescriptions as Array<{ id: string; description: string }>).push({ id: card.id, description }); + } + } const brief = observed.tasks.find(row => `${row.title}\n${row.description}`.includes(String(state.answer))); state.taskBrief = brief ? await ctx.input.api.get(`/api/issues/${brief.id}`) : undefined; return observed; } +function multipartContinuationCards(o: NonNullable>) { + const original = o.state.original as Interaction | undefined; + return o.interactions.filter(card => card.id !== original?.id && card.companyId === o.companyId && card.issueId === original?.issueId + && card.createdByAgentId === o.leadId && card.kind === "ask_user_questions" && card.status === "pending" + && typeof card.payload.questionSet?.description === "string" && o.runs.some(run => run.id === card.sourceRunId + && run.companyId === o.companyId && run.agentId === o.leadId && run.status === "succeeded" + && run.contextSnapshot?.interactionId === original?.id && run.contextSnapshot?.issueId === original?.issueId)); +} export function gradeQuestions(value: unknown) { const c = checks(); const o = identityObservation(value); c.add("complete-interaction-evidence", !!o, "Public interactions, persisted answers, run identities and result records are required."); @@ -96,9 +116,12 @@ export function gradeQuestions(value: unknown) { && answers.some(row => row.questionId === "headline" && row.otherText === o.state.answer), "The submitted choice/choice/custom-text set preserves earlier pages."); c.add("no-default-consent", o.state.unsubmittedPending === true && o.state.validationObserved === true && o.state.retainedAudience === true, "Missing required input cannot submit, unfinished pages stay pending, and back navigation retains the first selection."); + const visible = o.state.visibleContinuationDescriptions as Array<{ id: string; description: string }> | undefined; + const summary = [String(o.state.afterAnswerReply), ...multipartContinuationCards(o).flatMap(card => + visible?.some(row => row.id === card.id && row.description === card.payload.questionSet?.description) ? [card.payload.questionSet!.description!] : [])].join("\n"); c.add("submitted-set-only-continuation", o.runs.filter(run => run.contextSnapshot?.interactionId === original?.id).length === 1 - && /researchers/i.test(String(o.state.afterAnswerReply)) && /clear/i.test(String(o.state.afterAnswerReply)) && String(o.state.afterAnswerReply).includes(String(o.state.answer)), - "The submitted set causes one request-bound continuation whose actual later response accurately summarizes all three answers."); + && /researchers/i.test(summary) && /clear/i.test(summary) && summary.includes(String(o.state.answer)), + "The submitted set causes one request-bound continuation whose actual later visible chat response accurately summarizes all three answers."); } else { c.add("exact-free-text-answer", answers.length === 1 && answers[0]?.otherText === o.state.answer && !answers[0]?.optionIds?.length, "The exact human product name is attached once to the existing question."); @@ -201,7 +224,7 @@ export const questionIdentityCases = [ }, }), identityDefinition("questions-and-approvals.multi-part-question", ctx => questionFixture(ctx, true), gradeQuestions, { - expectedRunCount: 2, observe: observeQuestion, oracleVersion: "paperclip.chat-catalog.required-question.v2", + expectedRunCount: 2, observe: observeQuestion, oracleVersion: "paperclip.chat-catalog.required-question.v3", async act(ctx, state) { const remaining = () => remainingIdentityObservationBudget(ctx); remaining(); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-support.ts b/tests/runner-e2e/chat-catalog-domains/identity-support.ts index 66035c62c41..ca4cc94c496 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-support.ts @@ -7,7 +7,7 @@ export interface IdentityObservation extends InternalObservation { } export interface Interaction extends Entity { idempotencyKey?: string | null; - kind: string; status: string; payload: { prompt?: string; runtimeRequestId?: string; toolAction?: { connectionId?: string; actionRequestId?: string }; questionSet?: { submitLabel?: string; questions: Array<{ id: string; prompt: string; answerMode: string; required?: boolean; customAnswer?: { enabled: boolean; label?: string } }> } }; + kind: string; status: string; payload: { prompt?: string; runtimeRequestId?: string; toolAction?: { connectionId?: string; actionRequestId?: string }; questionSet?: { description?: string; submitLabel?: string; questions: Array<{ id: string; prompt: string; answerMode: string; required?: boolean; customAnswer?: { enabled: boolean; label?: string } }> } }; result?: { outcome?: string; commentId?: string; toolAction?: { status: string; errorCode?: string | null; errorMessage?: string | null }; answers?: Array<{ questionId: string; otherText?: string; optionIds?: string[] }> }; resolvedByUserId?: string; resolvedByAgentId?: string; sourceRunId?: string; } diff --git a/tests/runner-e2e/chat-catalog-domains/identity.test.ts b/tests/runner-e2e/chat-catalog-domains/identity.test.ts index 8bd6972d470..afce91dedc7 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity.test.ts @@ -145,3 +145,33 @@ describe("identity catalog independent oracle calibration", () => { expect(gradeEmptyInstallation(undefined).passed).toBe(false); }); }); + +describe("multipart continuation on the visible question surface", () => { + function cardSummary() { + const o = valid("questions-and-approvals.multi-part-question"); + Object.assign(o.state.original as Interaction, { issueId: "source", companyId: "company" }); + Object.assign(o.runs[1]!, { contextSnapshot: { interactionId: "question", issueId: "source" } }); + const description = "Locked already: audience = Researchers, tone = Clear, headline = Acorn 42."; + o.interactions.push({ id: "later", companyId: "company", issueId: "source", createdByAgentId: "ceo", sourceRunId: o.runs[1]!.id, + kind: "ask_user_questions", status: "pending", payload: { questionSet: { description, questions: [] } } }); + o.state.afterAnswerReply = ""; + o.state.visibleContinuationDescriptions = [{ id: "later", description }]; + return o; + } + it("accepts the actual visible bound card summary without an extra comment", () => { + expect(gradeQuestions(cardSummary()).passed).toBe(true); + }); + it.each(["hidden", "wrong-run", "wrong-issue", "wrong-agent", "historical", "missing-answer"])("rejects %s card evidence", mode => { + const o = cardSummary(); const card = o.interactions.at(-1)!; + if (mode === "hidden") o.state.visibleContinuationDescriptions = []; + if (mode === "wrong-run") card.sourceRunId = "unrelated"; + if (mode === "wrong-issue") card.issueId = "other"; + if (mode === "wrong-agent") card.createdByAgentId = "other"; + if (mode === "historical") card.id = "question"; + if (mode === "missing-answer") { + card.payload.questionSet!.description = "Researchers, Clear"; + o.state.visibleContinuationDescriptions = [{ id: card.id, description: card.payload.questionSet!.description }]; + } + expect(gradeQuestions(o).passed).toBe(false); + }); +});