From 3d6d6d1bda9a34b9cedae4078fdf8f9d13b173b6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 21 Oct 2026 08:59:31 -0500 Subject: [PATCH] test: preserve complete question identity across an answer Co-Authored-By: Paperclip --- tests/runner-e2e/necessary-question.test.ts | 4 ++++ tests/runner-e2e/necessary-question.ts | 6 ++---- tests/runner-e2e/question-resume-scoring.ts | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/runner-e2e/necessary-question.test.ts b/tests/runner-e2e/necessary-question.test.ts index ded9fb8455a..47c9f48a2d9 100644 --- a/tests/runner-e2e/necessary-question.test.ts +++ b/tests/runner-e2e/necessary-question.test.ts @@ -49,6 +49,10 @@ describe("necessary free-text question journey", () => { ["unrelated rendered card", (points: Row[]) => { points[0].questionPresentation.interactionId = "foreign"; }], ["missing semantic receipt", (points: Row[]) => { points[0].runs[0].resultJson.semanticToolReceipts = {}; }], ["foreign question", (points: Row[]) => { points[0].interactions[0].companyId = "foreign"; }], + ["foreign resolved company", (points: Row[]) => { points[1].interactions[0].companyId = "foreign"; }], + ["changed creator", (points: Row[]) => { points[1].interactions[0].createdByAgentId = "foreign"; }], + ["changed idempotency key", (points: Row[]) => { points[1].interactions[0].idempotencyKey = "foreign"; }], + ["changed continuation policy", (points: Row[]) => { points[1].interactions[0].continuationPolicy = "none"; }], ["fabricated agent answer", (points: Row[]) => { points[1].interactions[0].resolvedByAgentId = "agent"; }], ["wrong answer", (points: Row[]) => { points[1].interactions[0].result.answers[0].otherText = "invented"; }], ["duplicate answer", (points: Row[]) => { points[1].interactions[0].result.answers.push(points[1].interactions[0].result.answers[0]); }], diff --git a/tests/runner-e2e/necessary-question.ts b/tests/runner-e2e/necessary-question.ts index a90ab1c8c10..98447a5a0f9 100644 --- a/tests/runner-e2e/necessary-question.ts +++ b/tests/runner-e2e/necessary-question.ts @@ -1,8 +1,7 @@ -import { isDeepStrictEqual } from "node:util"; import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import { continuationScenario, continuationTasks, necessaryQuestionContract } from "./continuation-cases.js"; -import { questionResumedOnce, validResumeQuestionForm } from "./question-resume-scoring.js"; +import { questionResumedOnce, sameQuestion, validResumeQuestionForm } from "./question-resume-scoring.js"; import type { ContinuationCheckpoint } from "./continuation-scoring.js"; import type { RunnerTaskFixture } from "./types.js"; @@ -31,8 +30,7 @@ export function gradeNecessaryQuestion(checkpoints: ContinuationCheckpoint[], an check("necessary-question.visible", initial?.questionPresentation?.interactionId === question?.id && initial?.questionPresentation?.promptVisible === true && initial?.questionPresentation?.contextVisible === true, "The browser must observe the actual question and context, not only its title."), - check("necessary-question.answer", finished.length === 1 && resolved?.id === question?.id && resolved?.sourceRunId === question?.sourceRunId && - isDeepStrictEqual(question?.payload, resolved?.payload) && resolved?.status === "answered" && + check("necessary-question.answer", finished.length === 1 && !!question && sameQuestion(question, resolved) && resolved?.status === "answered" && typeof resolved.resolvedByUserId === "string" && !!resolved.resolvedByUserId && !resolved.resolvedByAgentId && Array.isArray(response) && response.length === 1 && response[0]?.questionId === field?.id && response[0]?.otherText === answer && !response[0]?.optionIds?.length, diff --git a/tests/runner-e2e/question-resume-scoring.ts b/tests/runner-e2e/question-resume-scoring.ts index d806283f821..10e3f81e6bc 100644 --- a/tests/runner-e2e/question-resume-scoring.ts +++ b/tests/runner-e2e/question-resume-scoring.ts @@ -46,7 +46,7 @@ export function validResumeQuestionForm(checkpoint: ContinuationCheckpoint, i: R (fields.length === 1 || (path === "provider" && isSingleClaudeQuestion(fields))); } -function sameQuestion(a: Row, b?: Row) { +export function sameQuestion(a: Row, b?: Row) { return !!b && ["id", "kind", "sourceRunId", "issueId", "companyId", "createdByAgentId", "idempotencyKey", "continuationPolicy"] .every(k => a[k] === b[k]) && isDeepStrictEqual(a.payload, b.payload); }