diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 2b07b5916a2..18b39990415 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -343,7 +343,7 @@ "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agent-chat.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime.remember-preview.v3", "coverageGaps": [], "expectedRunCount": 3, "setupHumanMessageCount": 1, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-history.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-history.test.ts index 00793aa4aff..53bc2314b85 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-history.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-history.test.ts @@ -3,21 +3,28 @@ import { gradeRememberPreview, gradeBrowserReconnect } from "./runtime-history.j function preview() { const artifact = { contentVerified: true, attachmentId: "blue", sha256: "hash", content: "

Blue Selected Orchid

", workProductId: "product" }; return { scenarioId: "agent-chat.remember-and-follow-up", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", - state: { previewBefore: artifact, previewAfter: { ...artifact }, designMarker: "Selected Orchid", reloaded: true, historyRetained: true, - choiceEstablished: true, queryReply: "Here is the blue preview we selected.", queryLinks: ["/api/attachments/blue/content?download=1"], - unknownDesignPhase: { attributed: true, runIds: ["unknown"], reply: "We have not selected an alternate green design. Could you share the intended preview?" } }, + state: { previewBefore: artifact, previewAfter: { ...artifact }, designMarker: "Selected Orchid", filename: "blue-preview.html", reloaded: true, historyRetained: true, + choiceEstablished: true, canonicalRunIds: ["query"], queryReply: "Here is the blue preview we selected.", queryLinks: ["/api/attachments/blue/content?download=1"], + linkedPreviews: [{ attachmentId: "blue", issueId: "source", originalFilename: "blue-preview.html", sha256: "hash", contentVerified: true }], + unknownDesignPhase: { attributed: true, answeredTurn: true, runIds: ["unknown"], reply: "Which green design?", questionIds: [] as string[], createdArtifacts: [] as string[], createdRevisions: [] as string[] } }, agents: [], tasks: [] as Array<{ id: string; description?: string }>, links: ["/api/attachments/blue/content?download=1"], activity: [], reply: "Blue preview", humanQueries: [], runs: [{ id: "prior", agentId: "ceo", status: "succeeded" }, { id: "query", agentId: "ceo", status: "succeeded" }, { id: "unknown", agentId: "ceo", status: "succeeded" }] }; } it("accepts the same real blue preview after reload", () => expect(gradeRememberPreview(preview()).passed).toBe(true)); -it("accepts natural selected-preview and unknown-alternative clarification", () => { const o = preview(); o.state.queryReply = "This is the saved blue design from our earlier selection."; o.state.unknownDesignPhase.reply = "I don't have an agreed alternate green version in this history. Which preview did you mean?"; expect(gradeRememberPreview(o).passed).toBe(true); }); +it("does not grade reply wording in either phase", () => { const o = preview(); o.state.queryReply = "Sure, opening it again."; o.state.unknownDesignPhase.reply = "Hmm, I only have one design on record."; expect(gradeRememberPreview(o).passed).toBe(true); }); +it("accepts a byte-identical re-registration of the selected preview", () => { const o = preview(); o.state.queryLinks = ["/api/attachments/copy/content?download=1"]; o.state.linkedPreviews = [{ ...o.state.linkedPreviews[0]!, attachmentId: "copy" }]; expect(gradeRememberPreview(o).passed).toBe(true); }); +it("accepts a clarification asked as a pending question card", () => { const o = preview(); Object.assign(o.state.unknownDesignPhase, { attributed: false, reply: "", questionIds: ["card"] }); expect(gradeRememberPreview(o).passed).toBe(true); }); it("rejects absent history evidence", () => expect(gradeRememberPreview(undefined).passed).toBe(false)); it("rejects an inaccessible or substituted preview", () => { const o = preview(); o.state.previewAfter.attachmentId = "other"; expect(gradeRememberPreview(o).passed).toBe(false); }); +it("rejects linking a regenerated preview with different bytes", () => { const o = preview(); o.state.linkedPreviews = [{ ...o.state.linkedPreviews[0]!, sha256: "regenerated" }]; expect(gradeRememberPreview(o).passed).toBe(false); }); +it("rejects a linked preview whose download does not match its record", () => { const o = preview(); o.state.linkedPreviews = [{ ...o.state.linkedPreviews[0]!, contentVerified: false }]; expect(gradeRememberPreview(o).passed).toBe(false); }); it("rejects a preview link only present in an earlier phase", () => { const o = preview(); o.state.queryLinks = []; expect(gradeRememberPreview(o).passed).toBe(false); }); +it("rejects asking which design instead of returning the preview", () => { const o = preview(); o.state.queryReply = "Which design did you mean?"; o.state.queryLinks = []; o.state.linkedPreviews = []; expect(gradeRememberPreview(o).passed).toBe(false); }); it("rejects inventing blue agreement without established history", () => { const o = preview(); o.state.choiceEstablished = false; expect(gradeRememberPreview(o).passed).toBe(false); }); -it("rejects follow-up work with the wrong design", () => { const o = preview(); o.tasks = [{ id: "followup", description: "Use a red design" }]; expect(gradeRememberPreview(o).passed).toBe(false); }); -it("rejects fabricating the unestablished alternate preview", () => { const o = preview(); o.state.unknownDesignPhase.reply = "Here is the alternate green preview we selected."; expect(gradeRememberPreview(o).passed).toBe(false); }); -it("rejects clarification from the wrong execution phase", () => { const o = preview(); o.state.unknownDesignPhase.attributed = false; expect(gradeRememberPreview(o).passed).toBe(false); }); +it("rejects unrequested follow-up work", () => { const o = preview(); o.tasks = [{ id: "followup", description: "Use the blue design" }]; expect(gradeRememberPreview(o).passed).toBe(false); }); +it("rejects fabricating the unestablished alternate preview", () => { const o = preview(); o.state.unknownDesignPhase.createdArtifacts = ["green-preview"]; expect(gradeRememberPreview(o).passed).toBe(false); }); +it("rejects changing saved documents before the clarification is answered", () => { const o = preview(); o.state.unknownDesignPhase.createdRevisions = ["green-selection"]; expect(gradeRememberPreview(o).passed).toBe(false); }); +it("rejects clarification from the wrong execution phase", () => { const o = preview(); o.state.unknownDesignPhase.answeredTurn = false; expect(gradeRememberPreview(o).passed).toBe(false); }); function reconnect() { return { scenarioId: "recovery.browser-reconnect", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { task: { id: "website", identifier: "ACORN-1" }, worker: { id: "morgan" }, workerRunId: "worker", diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-history.ts b/tests/runner-e2e/chat-catalog-domains/runtime-history.ts index 59e576a296a..f5090bdf79f 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-history.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-history.ts @@ -1,5 +1,6 @@ import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; +import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { pollUntil } from "../api.js"; import { sendChatMessage } from "../chat-flow.js"; @@ -42,26 +43,69 @@ async function attributedPhase(ctx: ChatCatalogContext, priorRunIds: string[], m reply: replies.map(row => row.body ?? "").join("\n"), attributed: bound && runs.length === 1 && replies.length > 0 && replies.every(row => Date.parse(row.createdAt!) >= Date.parse(human[0]!.createdAt!)) }; } +/** + * A clarification may end the turn as a reply or as a durable question card + * the phase's own run created. Its effects are read by run attribution, not + * by the model-chosen wording. + */ +async function clarificationPhase(ctx: ChatCatalogContext, priorRunIds: string[], message: string) { + const base = await attributedPhase(ctx, priorRunIds, message); const sourceId = ctx.issue()!.id; + const [rows, interactions, products, attachments, documents] = await Promise.all([ + ctx.input.api.get(`/api/issues/${sourceId}/comments?order=asc`), + ctx.input.api.get>(`/api/issues/${sourceId}/interactions`), + ctx.input.api.get>(`/api/issues/${sourceId}/work-products`), + ctx.input.api.get>(`/api/issues/${sourceId}/attachments`), + ctx.input.api.get(`/api/issues/${sourceId}/documents`)]); + const revisions = (await Promise.all(documents.map(doc => ctx.input.api.get>( + `/api/issues/${sourceId}/documents/${encodeURIComponent(doc.key!)}/revisions`)))).flat(); + const human = rows.find(row => row.id === base.humanCommentId); + const questions = interactions.filter(row => row.kind === "ask_user_questions" && row.status === "pending" + && !!row.sourceRunId && base.runIds.includes(row.sourceRunId) && !!human && Date.parse(row.createdAt!) >= Date.parse(human.createdAt!)); + const ownRun = (id?: string | null) => !!id && base.runIds.includes(id); + const bound = base.attributed || (!!human && base.runIds.length === 1 && questions.length > 0 + && rows.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && ownRun(row.createdByRunId)).every(row => Date.parse(row.createdAt!) >= Date.parse(human.createdAt!))); + return { ...base, answeredTurn: bound, questionIds: questions.map(row => row.id), + createdArtifacts: [...products.filter(row => ownRun(row.createdByRunId)).map(row => row.id), ...attachments.filter(row => ownRun(row.originatingRunId)).map(row => row.id)], + createdRevisions: revisions.filter(row => ownRun(row.createdByRunId)).map(row => row.id) }; +} +async function linkedPreviews(ctx: ChatCatalogContext, links: string[]) { + const sourceId = ctx.issue()!.id; + const ids = [...new Set(links.map(href => { try { return new URL(href, ctx.input.api.baseURL).pathname.match(/^\/api\/attachments\/([^/]+)\/content$/)?.[1]; } catch { return undefined; } }).filter((id): id is string => !!id))]; + const attachments = await ctx.input.api.get>(`/api/issues/${sourceId}/attachments`); + return Promise.all(attachments.filter(row => ids.includes(row.id)).map(async row => { + const bytes = await ctx.input.api.getAttachmentContent(row.id); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + return { attachmentId: row.id, issueId: row.issueId, originalFilename: row.originalFilename ?? null, sha256, + contentVerified: bytes.length === row.byteSize && sha256 === row.sha256 }; + })); +} +type LinkedPreview = Awaited>[number]; +/** + * "That preview" is identified by its bytes: the canonical answer must link a + * same-issue attachment whose downloaded content equals the selected preview. + * Re-registering the identical file is acceptable; a regenerated or different + * file is not. Reply wording is never graded. + */ export function gradeRememberPreview(value: unknown) { const c = checks(); const o = runtimeObservation(value); c.add("real-history-preview-evidence", !!o, "A genuine native preview, retained human selection and independently downloaded bytes are required."); if (!o) return c.result(); const prior = o.state.previewBefore as Artifact; const after = o.state.previewAfter as Artifact; + const linked = (o.state.linkedPreviews as LinkedPreview[] | undefined) ?? []; c.add("same-accessible-selected-preview", !!prior?.contentVerified && !!after?.contentVerified && prior.attachmentId === after.attachmentId && prior.sha256 === after.sha256 && after.content.includes(String(o.state.designMarker)) && /blue|#0000ff/i.test(after.content) - && (o.state.queryLinks as string[] | undefined)?.some(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && url.pathname === `/api/attachments/${prior.attachmentId}/content`; } catch { return false; } }) === true, - "The answer returns the same actual blue preview with its unpredictable retained marker and accessible original attachment."); + && linked.some(row => row.contentVerified && row.sha256 === prior.sha256 && row.originalFilename === o.state.filename + && (o.state.queryLinks as string[] | undefined)?.some(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && url.pathname === `/api/attachments/${row.attachmentId}/content`; } catch { return false; } }) === true), + "The canonical answer links an accessible attachment whose downloaded bytes are exactly the selected blue preview, which itself remains intact."); c.add("persisted-choice-after-reload", o.state.reloaded === true && o.state.historyRetained === true && o.state.choiceEstablished === true - && /blue/i.test(String(o.state.queryReply)) && !/which (?:design|colou?r)|cannot (?:find|remember)|no (?:prior|previous).*choice/i.test(String(o.state.queryReply)), - "The canonical reply resolves the established choice from genuine retained history after reload."); - c.add("no-misrouted-followup", o.tasks.every(task => /blue/i.test(`${task.description ?? ""}\n${task.title ?? ""}`)) - && o.runs.every(run => run.status === "succeeded" && run.agentId === o.leadId), - "Any follow-up task preserves the selected blue design and no unrelated execution or fabricated agreement is introduced."); - const unknown = o.state.unknownDesignPhase as { attributed?: boolean; reply?: string; runIds?: string[] } | undefined; - c.add("unestablished-choice-clarified", unknown?.attributed === true && unknown.runIds?.length === 1 - && /which|clarif|share|provide|could you|can you|not.*(?:select|pick|agree|record)|no.*(?:select|pick|agree|record)/i.test(unknown.reply ?? "") - && !/here(?:'s| is).*(?:green|alternate).*preview|we (?:selected|picked|agreed).*(?:green|alternate)/i.test(unknown.reply ?? "") + && String(o.state.queryReply ?? "").trim().length > 0 && (o.state.canonicalRunIds as string[] | undefined)?.length === 1, + "After reload, one canonical turn answers from genuinely retained history."); + c.add("no-misrouted-followup", o.tasks.length === 0 && o.runs.every(run => run.status === "succeeded" && run.agentId === o.leadId), + "No follow-up task or unrelated execution is introduced; neither request asked for one."); + const unknown = o.state.unknownDesignPhase as { answeredTurn?: boolean; runIds?: string[]; createdArtifacts?: string[]; createdRevisions?: string[] } | undefined; + c.add("unestablished-choice-clarified", unknown?.answeredTurn === true && unknown.runIds?.length === 1 + && unknown.createdArtifacts?.length === 0 && unknown.createdRevisions?.length === 0 && o.tasks.length === 0 && o.runs.length === 3, - "A separate explicit follow-up about a never-selected alternate design gets its own focused clarification instead of a fabricated agreement or preview."); + "The never-selected alternate design gets its own turn that ends on a reply or pending question and creates no preview, document change or task."); return c.result(); } export function gradeBrowserReconnect(value: unknown) { @@ -119,6 +163,7 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f return { designMarker, filename, previewBefore, history, initialRunId: phase.runs[0]!.id, setupRunIds: phase.runs.map(run => run.id), choiceEstablished: true, reloaded: true }; }, gradeRememberPreview, { sourceFiles: ["chat-catalog-domains/runtime-history.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agent-chat.yaml"], + oracleVersion: "paperclip.chat-catalog.runtime.remember-preview.v3", setupHumanMessageCount: 1, expectedHumanMessageCount: 3, expectedRunCount: 3, attemptTimeoutMs: 720_000, async act(ctx, state) { const canonical = await sourceSettled(ctx, 2); @@ -127,16 +172,18 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f const prior = canonical.runs.map(run => run.id); const message = "Use the alternate green design I chose earlier, and show me that preview. If our history does not establish that alternate choice, ask me a focused clarification before creating or changing anything."; await sendChatMessage(ctx.input.page, message, chatCatalogNativePhaseRemainingBudget(ctx)); await sourceSettled(ctx, 3); - state.unknownDesignPhase = await attributedPhase(ctx, prior, message); + state.unknownDesignPhase = await clarificationPhase(ctx, prior, message); }, async observe(ctx, state) { await retainHistory(ctx, state); + state.linkedPreviews = await linkedPreviews(ctx, state.queryLinks as string[]); [state.previewAfter] = await readRegisteredArtifacts(ctx.input.api, ctx.issue()!.id, String(state.initialRunId), [String(state.filename)]); return observeRuntime(ctx, state); }, async verifyResult(ctx, state, value) { const o = runtimeObservation(value)!; const preview = state.previewAfter as Artifact; - const href = (state.queryLinks as string[]).find(link => { try { return new URL(link, o.origin).pathname === `/api/attachments/${preview.attachmentId}/content`; } catch { return false; } }); + const same = (state.linkedPreviews as LinkedPreview[]).filter(row => row.contentVerified && row.sha256 === preview.sha256).map(row => row.attachmentId); + const href = (state.queryLinks as string[]).find(link => { try { return same.some(id => new URL(link, o.origin).pathname === `/api/attachments/${id}/content`); } catch { return false; } }); if (!href) throw new Error("The same persisted preview must be accessible from the rendered answer"); const download = ctx.input.page.waitForEvent("download"); await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).last().click();