diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json
index 2b07b5916a2..18b39990415 100644
--- a/tests/runner-e2e/chat-catalog-descriptors.json
+++ b/tests/runner-e2e/chat-catalog-descriptors.json
@@ -343,7 +343,7 @@
"chat-catalog-domains/runtime-support.ts",
"chat-scenarios/agent-chat.yaml"
],
- "oracleVersion": "paperclip.chat-catalog.runtime.v2",
+ "oracleVersion": "paperclip.chat-catalog.runtime.remember-preview.v3",
"coverageGaps": [],
"expectedRunCount": 3,
"setupHumanMessageCount": 1,
diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-history.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-history.test.ts
index 00793aa4aff..53bc2314b85 100644
--- a/tests/runner-e2e/chat-catalog-domains/runtime-history.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/runtime-history.test.ts
@@ -3,21 +3,28 @@ import { gradeRememberPreview, gradeBrowserReconnect } from "./runtime-history.j
function preview() {
const artifact = { contentVerified: true, attachmentId: "blue", sha256: "hash", content: "
Blue Selected Orchid
", workProductId: "product" };
return { scenarioId: "agent-chat.remember-and-follow-up", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
- state: { previewBefore: artifact, previewAfter: { ...artifact }, designMarker: "Selected Orchid", reloaded: true, historyRetained: true,
- choiceEstablished: true, queryReply: "Here is the blue preview we selected.", queryLinks: ["/api/attachments/blue/content?download=1"],
- unknownDesignPhase: { attributed: true, runIds: ["unknown"], reply: "We have not selected an alternate green design. Could you share the intended preview?" } },
+ state: { previewBefore: artifact, previewAfter: { ...artifact }, designMarker: "Selected Orchid", filename: "blue-preview.html", reloaded: true, historyRetained: true,
+ choiceEstablished: true, canonicalRunIds: ["query"], queryReply: "Here is the blue preview we selected.", queryLinks: ["/api/attachments/blue/content?download=1"],
+ linkedPreviews: [{ attachmentId: "blue", issueId: "source", originalFilename: "blue-preview.html", sha256: "hash", contentVerified: true }],
+ unknownDesignPhase: { attributed: true, answeredTurn: true, runIds: ["unknown"], reply: "Which green design?", questionIds: [] as string[], createdArtifacts: [] as string[], createdRevisions: [] as string[] } },
agents: [], tasks: [] as Array<{ id: string; description?: string }>, links: ["/api/attachments/blue/content?download=1"], activity: [], reply: "Blue preview", humanQueries: [],
runs: [{ id: "prior", agentId: "ceo", status: "succeeded" }, { id: "query", agentId: "ceo", status: "succeeded" }, { id: "unknown", agentId: "ceo", status: "succeeded" }] };
}
it("accepts the same real blue preview after reload", () => expect(gradeRememberPreview(preview()).passed).toBe(true));
-it("accepts natural selected-preview and unknown-alternative clarification", () => { const o = preview(); o.state.queryReply = "This is the saved blue design from our earlier selection."; o.state.unknownDesignPhase.reply = "I don't have an agreed alternate green version in this history. Which preview did you mean?"; expect(gradeRememberPreview(o).passed).toBe(true); });
+it("does not grade reply wording in either phase", () => { const o = preview(); o.state.queryReply = "Sure, opening it again."; o.state.unknownDesignPhase.reply = "Hmm, I only have one design on record."; expect(gradeRememberPreview(o).passed).toBe(true); });
+it("accepts a byte-identical re-registration of the selected preview", () => { const o = preview(); o.state.queryLinks = ["/api/attachments/copy/content?download=1"]; o.state.linkedPreviews = [{ ...o.state.linkedPreviews[0]!, attachmentId: "copy" }]; expect(gradeRememberPreview(o).passed).toBe(true); });
+it("accepts a clarification asked as a pending question card", () => { const o = preview(); Object.assign(o.state.unknownDesignPhase, { attributed: false, reply: "", questionIds: ["card"] }); expect(gradeRememberPreview(o).passed).toBe(true); });
it("rejects absent history evidence", () => expect(gradeRememberPreview(undefined).passed).toBe(false));
it("rejects an inaccessible or substituted preview", () => { const o = preview(); o.state.previewAfter.attachmentId = "other"; expect(gradeRememberPreview(o).passed).toBe(false); });
+it("rejects linking a regenerated preview with different bytes", () => { const o = preview(); o.state.linkedPreviews = [{ ...o.state.linkedPreviews[0]!, sha256: "regenerated" }]; expect(gradeRememberPreview(o).passed).toBe(false); });
+it("rejects a linked preview whose download does not match its record", () => { const o = preview(); o.state.linkedPreviews = [{ ...o.state.linkedPreviews[0]!, contentVerified: false }]; expect(gradeRememberPreview(o).passed).toBe(false); });
it("rejects a preview link only present in an earlier phase", () => { const o = preview(); o.state.queryLinks = []; expect(gradeRememberPreview(o).passed).toBe(false); });
+it("rejects asking which design instead of returning the preview", () => { const o = preview(); o.state.queryReply = "Which design did you mean?"; o.state.queryLinks = []; o.state.linkedPreviews = []; expect(gradeRememberPreview(o).passed).toBe(false); });
it("rejects inventing blue agreement without established history", () => { const o = preview(); o.state.choiceEstablished = false; expect(gradeRememberPreview(o).passed).toBe(false); });
-it("rejects follow-up work with the wrong design", () => { const o = preview(); o.tasks = [{ id: "followup", description: "Use a red design" }]; expect(gradeRememberPreview(o).passed).toBe(false); });
-it("rejects fabricating the unestablished alternate preview", () => { const o = preview(); o.state.unknownDesignPhase.reply = "Here is the alternate green preview we selected."; expect(gradeRememberPreview(o).passed).toBe(false); });
-it("rejects clarification from the wrong execution phase", () => { const o = preview(); o.state.unknownDesignPhase.attributed = false; expect(gradeRememberPreview(o).passed).toBe(false); });
+it("rejects unrequested follow-up work", () => { const o = preview(); o.tasks = [{ id: "followup", description: "Use the blue design" }]; expect(gradeRememberPreview(o).passed).toBe(false); });
+it("rejects fabricating the unestablished alternate preview", () => { const o = preview(); o.state.unknownDesignPhase.createdArtifacts = ["green-preview"]; expect(gradeRememberPreview(o).passed).toBe(false); });
+it("rejects changing saved documents before the clarification is answered", () => { const o = preview(); o.state.unknownDesignPhase.createdRevisions = ["green-selection"]; expect(gradeRememberPreview(o).passed).toBe(false); });
+it("rejects clarification from the wrong execution phase", () => { const o = preview(); o.state.unknownDesignPhase.answeredTurn = false; expect(gradeRememberPreview(o).passed).toBe(false); });
function reconnect() {
return { scenarioId: "recovery.browser-reconnect", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
state: { task: { id: "website", identifier: "ACORN-1" }, worker: { id: "morgan" }, workerRunId: "worker",
diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-history.ts b/tests/runner-e2e/chat-catalog-domains/runtime-history.ts
index 59e576a296a..f5090bdf79f 100644
--- a/tests/runner-e2e/chat-catalog-domains/runtime-history.ts
+++ b/tests/runner-e2e/chat-catalog-domains/runtime-history.ts
@@ -1,5 +1,6 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
+import { createHash } from "node:crypto";
import { writeFile } from "node:fs/promises";
import { pollUntil } from "../api.js";
import { sendChatMessage } from "../chat-flow.js";
@@ -42,26 +43,69 @@ async function attributedPhase(ctx: ChatCatalogContext, priorRunIds: string[], m
reply: replies.map(row => row.body ?? "").join("\n"), attributed: bound && runs.length === 1 && replies.length > 0
&& replies.every(row => Date.parse(row.createdAt!) >= Date.parse(human[0]!.createdAt!)) };
}
+/**
+ * A clarification may end the turn as a reply or as a durable question card
+ * the phase's own run created. Its effects are read by run attribution, not
+ * by the model-chosen wording.
+ */
+async function clarificationPhase(ctx: ChatCatalogContext, priorRunIds: string[], message: string) {
+ const base = await attributedPhase(ctx, priorRunIds, message); const sourceId = ctx.issue()!.id;
+ const [rows, interactions, products, attachments, documents] = await Promise.all([
+ ctx.input.api.get(`/api/issues/${sourceId}/comments?order=asc`),
+ ctx.input.api.get>(`/api/issues/${sourceId}/interactions`),
+ ctx.input.api.get>(`/api/issues/${sourceId}/work-products`),
+ ctx.input.api.get>(`/api/issues/${sourceId}/attachments`),
+ ctx.input.api.get(`/api/issues/${sourceId}/documents`)]);
+ const revisions = (await Promise.all(documents.map(doc => ctx.input.api.get>(
+ `/api/issues/${sourceId}/documents/${encodeURIComponent(doc.key!)}/revisions`)))).flat();
+ const human = rows.find(row => row.id === base.humanCommentId);
+ const questions = interactions.filter(row => row.kind === "ask_user_questions" && row.status === "pending"
+ && !!row.sourceRunId && base.runIds.includes(row.sourceRunId) && !!human && Date.parse(row.createdAt!) >= Date.parse(human.createdAt!));
+ const ownRun = (id?: string | null) => !!id && base.runIds.includes(id);
+ const bound = base.attributed || (!!human && base.runIds.length === 1 && questions.length > 0
+ && rows.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && ownRun(row.createdByRunId)).every(row => Date.parse(row.createdAt!) >= Date.parse(human.createdAt!)));
+ return { ...base, answeredTurn: bound, questionIds: questions.map(row => row.id),
+ createdArtifacts: [...products.filter(row => ownRun(row.createdByRunId)).map(row => row.id), ...attachments.filter(row => ownRun(row.originatingRunId)).map(row => row.id)],
+ createdRevisions: revisions.filter(row => ownRun(row.createdByRunId)).map(row => row.id) };
+}
+async function linkedPreviews(ctx: ChatCatalogContext, links: string[]) {
+ const sourceId = ctx.issue()!.id;
+ const ids = [...new Set(links.map(href => { try { return new URL(href, ctx.input.api.baseURL).pathname.match(/^\/api\/attachments\/([^/]+)\/content$/)?.[1]; } catch { return undefined; } }).filter((id): id is string => !!id))];
+ const attachments = await ctx.input.api.get>(`/api/issues/${sourceId}/attachments`);
+ return Promise.all(attachments.filter(row => ids.includes(row.id)).map(async row => {
+ const bytes = await ctx.input.api.getAttachmentContent(row.id);
+ const sha256 = createHash("sha256").update(bytes).digest("hex");
+ return { attachmentId: row.id, issueId: row.issueId, originalFilename: row.originalFilename ?? null, sha256,
+ contentVerified: bytes.length === row.byteSize && sha256 === row.sha256 };
+ }));
+}
+type LinkedPreview = Awaited>[number];
+/**
+ * "That preview" is identified by its bytes: the canonical answer must link a
+ * same-issue attachment whose downloaded content equals the selected preview.
+ * Re-registering the identical file is acceptable; a regenerated or different
+ * file is not. Reply wording is never graded.
+ */
export function gradeRememberPreview(value: unknown) {
const c = checks(); const o = runtimeObservation(value);
c.add("real-history-preview-evidence", !!o, "A genuine native preview, retained human selection and independently downloaded bytes are required."); if (!o) return c.result();
const prior = o.state.previewBefore as Artifact; const after = o.state.previewAfter as Artifact;
+ const linked = (o.state.linkedPreviews as LinkedPreview[] | undefined) ?? [];
c.add("same-accessible-selected-preview", !!prior?.contentVerified && !!after?.contentVerified && prior.attachmentId === after.attachmentId
&& prior.sha256 === after.sha256 && after.content.includes(String(o.state.designMarker)) && /blue|#0000ff/i.test(after.content)
- && (o.state.queryLinks as string[] | undefined)?.some(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && url.pathname === `/api/attachments/${prior.attachmentId}/content`; } catch { return false; } }) === true,
- "The answer returns the same actual blue preview with its unpredictable retained marker and accessible original attachment.");
+ && linked.some(row => row.contentVerified && row.sha256 === prior.sha256 && row.originalFilename === o.state.filename
+ && (o.state.queryLinks as string[] | undefined)?.some(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && url.pathname === `/api/attachments/${row.attachmentId}/content`; } catch { return false; } }) === true),
+ "The canonical answer links an accessible attachment whose downloaded bytes are exactly the selected blue preview, which itself remains intact.");
c.add("persisted-choice-after-reload", o.state.reloaded === true && o.state.historyRetained === true && o.state.choiceEstablished === true
- && /blue/i.test(String(o.state.queryReply)) && !/which (?:design|colou?r)|cannot (?:find|remember)|no (?:prior|previous).*choice/i.test(String(o.state.queryReply)),
- "The canonical reply resolves the established choice from genuine retained history after reload.");
- c.add("no-misrouted-followup", o.tasks.every(task => /blue/i.test(`${task.description ?? ""}\n${task.title ?? ""}`))
- && o.runs.every(run => run.status === "succeeded" && run.agentId === o.leadId),
- "Any follow-up task preserves the selected blue design and no unrelated execution or fabricated agreement is introduced.");
- const unknown = o.state.unknownDesignPhase as { attributed?: boolean; reply?: string; runIds?: string[] } | undefined;
- c.add("unestablished-choice-clarified", unknown?.attributed === true && unknown.runIds?.length === 1
- && /which|clarif|share|provide|could you|can you|not.*(?:select|pick|agree|record)|no.*(?:select|pick|agree|record)/i.test(unknown.reply ?? "")
- && !/here(?:'s| is).*(?:green|alternate).*preview|we (?:selected|picked|agreed).*(?:green|alternate)/i.test(unknown.reply ?? "")
+ && String(o.state.queryReply ?? "").trim().length > 0 && (o.state.canonicalRunIds as string[] | undefined)?.length === 1,
+ "After reload, one canonical turn answers from genuinely retained history.");
+ c.add("no-misrouted-followup", o.tasks.length === 0 && o.runs.every(run => run.status === "succeeded" && run.agentId === o.leadId),
+ "No follow-up task or unrelated execution is introduced; neither request asked for one.");
+ const unknown = o.state.unknownDesignPhase as { answeredTurn?: boolean; runIds?: string[]; createdArtifacts?: string[]; createdRevisions?: string[] } | undefined;
+ c.add("unestablished-choice-clarified", unknown?.answeredTurn === true && unknown.runIds?.length === 1
+ && unknown.createdArtifacts?.length === 0 && unknown.createdRevisions?.length === 0
&& o.tasks.length === 0 && o.runs.length === 3,
- "A separate explicit follow-up about a never-selected alternate design gets its own focused clarification instead of a fabricated agreement or preview.");
+ "The never-selected alternate design gets its own turn that ends on a reply or pending question and creates no preview, document change or task.");
return c.result();
}
export function gradeBrowserReconnect(value: unknown) {
@@ -119,6 +163,7 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f
return { designMarker, filename, previewBefore, history, initialRunId: phase.runs[0]!.id, setupRunIds: phase.runs.map(run => run.id), choiceEstablished: true, reloaded: true };
}, gradeRememberPreview, {
sourceFiles: ["chat-catalog-domains/runtime-history.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agent-chat.yaml"],
+ oracleVersion: "paperclip.chat-catalog.runtime.remember-preview.v3",
setupHumanMessageCount: 1, expectedHumanMessageCount: 3, expectedRunCount: 3, attemptTimeoutMs: 720_000,
async act(ctx, state) {
const canonical = await sourceSettled(ctx, 2);
@@ -127,16 +172,18 @@ export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-f
const prior = canonical.runs.map(run => run.id);
const message = "Use the alternate green design I chose earlier, and show me that preview. If our history does not establish that alternate choice, ask me a focused clarification before creating or changing anything.";
await sendChatMessage(ctx.input.page, message, chatCatalogNativePhaseRemainingBudget(ctx)); await sourceSettled(ctx, 3);
- state.unknownDesignPhase = await attributedPhase(ctx, prior, message);
+ state.unknownDesignPhase = await clarificationPhase(ctx, prior, message);
},
async observe(ctx, state) {
await retainHistory(ctx, state);
+ state.linkedPreviews = await linkedPreviews(ctx, state.queryLinks as string[]);
[state.previewAfter] = await readRegisteredArtifacts(ctx.input.api, ctx.issue()!.id, String(state.initialRunId), [String(state.filename)]);
return observeRuntime(ctx, state);
},
async verifyResult(ctx, state, value) {
const o = runtimeObservation(value)!; const preview = state.previewAfter as Artifact;
- const href = (state.queryLinks as string[]).find(link => { try { return new URL(link, o.origin).pathname === `/api/attachments/${preview.attachmentId}/content`; } catch { return false; } });
+ const same = (state.linkedPreviews as LinkedPreview[]).filter(row => row.contentVerified && row.sha256 === preview.sha256).map(row => row.attachmentId);
+ const href = (state.queryLinks as string[]).find(link => { try { return same.some(id => new URL(link, o.origin).pathname === `/api/attachments/${id}/content`); } catch { return false; } });
if (!href) throw new Error("The same persisted preview must be accessible from the rendered answer");
const download = ctx.input.page.waitForEvent("download");
await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).last().click();