fix(evals): grade website revision by its saved versions, not reply wording

The revise-delivery grader rejected a correct revision for four eval reasons:
the fixture footer marker ended in the nonce's attempt suffix (-1), which the
agent reasonably bumped as a version label on website-v1.html; it required a
link to the source task the person had attached; it required the revision on
the source task although the chat run's register_deliverable files it on the
requesting conversation; and it graded revision context with keyword regexes.

Hash the marker, accept the source task or the conversation, and grade
revision context structurally: a separately titled registered version plus a
returned route to the retained original.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -2872,7 +2872,7 @@
       "chat-catalog-domains/internal-website-delivery.ts",
       "registered-artifact.ts"
     ],
-    "oracleVersion": "paperclip.chat-catalog.internal.v1",
+    "oracleVersion": "paperclip.chat-catalog.internal.website-revision.v2",
     "coverageGaps": [],
     "expectedRunCount": 2,
     "minimumRunCount": 2,

tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts

@@ -14,11 +14,11 @@ function valid(revising = false) {
   const prior = "<html><body><h1>Bring your research team notes together in one accessible place</h1><p>Research teams stay organized.</p><a href='https://example.test/signup'>Signup</a><footer>ORCHID</footer></body></html>";
   const body = revising ? prior.replace("Bring your research team notes together in one accessible place", "Notes, together") : prior;
   const attachment = (id: string, bytes: string, run: string) => ({ id, issueId: "task", sha256: hash(bytes), originatingRunId: run, contentPath: `/api/attachments/${id}/content` });
-  const product = (id: string, file: string, run: string) => ({ id, issueId: "task", type: "artifact", provider: "paperclip", status: "active", createdByRunId: run, metadata: { attachmentId: file } });
+  const product = (id: string, file: string, run: string, issueId = "task") => ({ id, issueId, title: file === "original" ? "website-v1.html" : "website-v2.html", type: "artifact", provider: "paperclip", status: "active", createdByRunId: run, metadata: { attachmentId: file } });
   const original = attachment("original", prior, "build"); const newFile = attachment("new", body, "run");
   return { scenarioId: revising ? "documents-artifacts.revise-delivery" : "documents-artifacts.produce-download", origin: "https://fixture.test", companyId: "company", leadId: "ceo",
     state: { task: structuredClone(task), worker: { id: "morgan" }, priorRunId: "build", producedBody: prior, producedHash: hash(prior), bodyFact: "Research teams stay organized.", signupHref: "https://example.test/signup", marker: "ORCHID", originalAttachment: original, originalProductId: "old-product" },
-    tasks: [task], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], humanQueries: [], links: ["/RUN/issues/task", "/api/attachments/new/content?download=1"],
+    tasks: [task], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], humanQueries: [], links: ["/RUN/issues/task", "/api/attachments/new/content?download=1", ...(revising ? ["/api/attachments/original/content"] : [])], chatIssueId: "chat",
     reply: "The updated shorter version is downloadable; the original remains available.", runs: [{ id: "run", companyId: "company", agentId: "ceo", status: "succeeded" }, { id: "build", companyId: "company", agentId: "morgan", status: "succeeded" }],
     attachments: [newFile, ...(revising ? [original] : [])], products: [product("new-product", "new", "run"), ...(revising ? [product("old-product", "original", "build")] : [])], downloaded: [{ id: "new", sha256: hash(body), body }, ...(revising ? [{ id: "original", sha256: hash(prior), body: prior }] : [])] };
 }
@@ -28,6 +28,14 @@ describe("registered website delivery calibration", () => {
   it("rejects stale or mismatched downloaded bytes", () => { const o = valid(); o.downloaded[0]!.body = "<html>stale</html>"; expect(gradeWebsiteDelivery(o).passed).toBe(false); });
   it("rejects wrong immutable origin or work-product source", () => { const o = valid(); o.attachments[0]!.originatingRunId = "build"; expect(gradeWebsiteDelivery(o).passed).toBe(false); const source = valid(true); source.products[0]!.issueId = "another-task"; expect(gradeWebsiteDelivery(source).passed).toBe(false); });
   it("rejects overwriting the only old copy", () => { const o = valid(true); o.downloaded = o.downloaded.filter(row => row.id !== "original"); expect(gradeWebsiteDelivery(o).passed).toBe(false); });
+  it("accepts the revision registered on the requesting conversation, whatever the reply wording", () => {
+    const o = valid(true); const file = o.attachments.find(row => row.id === "new")!; file.issueId = "chat"; o.products.find(row => row.id === "new-product")!.issueId = "chat";
+    o.reply = "Done. Shorter headline is in the new file; the first one is untouched."; expect(gradeWebsiteDelivery(o).passed).toBe(true);
+    o.reply = "Here you go."; expect(gradeWebsiteDelivery(o).passed).toBe(true);
+  });
+  it("rejects a revision registered on an unrelated task", () => { const o = valid(true); o.attachments.find(row => row.id === "new")!.issueId = "other"; o.products.find(row => row.id === "new-product")!.issueId = "other"; expect(gradeWebsiteDelivery(o).passed).toBe(false); });
+  it("rejects a revision with no returned route to the retained original", () => { const o = valid(true); o.links = ["/api/attachments/new/content?download=1"]; expect(gradeWebsiteDelivery(o).passed).toBe(false); o.links.push("/api/attachments/original/content"); expect(gradeWebsiteDelivery(o).passed).toBe(true); });
+  it("rejects new bytes filed under the original version's title", () => { const o = valid(true); o.products.find(row => row.id === "new-product")!.title = "website-v1.html"; expect(gradeWebsiteDelivery(o).passed).toBe(false); });
   it("rejects a headline that was not shortened", () => { const o = valid(true); o.downloaded[0]!.body = o.state.producedBody; o.downloaded[0]!.sha256 = hash(o.downloaded[0]!.body); o.attachments[0]!.sha256 = o.downloaded[0]!.sha256; expect(gradeWebsiteDelivery(o).passed).toBe(false); });
   it("rejects a shorter headline with unrelated body edits", () => { const o = valid(true); o.downloaded[0]!.body = o.downloaded[0]!.body.replace("Research teams stay organized.", "Ads for sale."); o.downloaded[0]!.sha256 = hash(o.downloaded[0]!.body); o.attachments[0]!.sha256 = o.downloaded[0]!.sha256; expect(gradeWebsiteDelivery(o).passed).toBe(false); });
 });

tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts

@@ -12,7 +12,7 @@ import { checks, entity, hasLink, observation, observeInternal, project, task, t
 import { selectDocumentSource } from "./internal-document-context.js";
 import { factualContextQueryAttachment } from "../chat-query-attachments.js";
 interface Downloaded { id: string; sha256: string; body: string }
-interface DeliveryObservation extends InternalObservation { attachments: IssueAttachment[]; products: IssueWorkProduct[]; downloaded: Downloaded[] }
+interface DeliveryObservation extends InternalObservation { attachments: IssueAttachment[]; products: IssueWorkProduct[]; downloaded: Downloaded[]; chatIssueId?: string }
 const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex");
 const headline = (body: string) => /<h1\b[^>]*>([\s\S]*?)<\/h1>/i.exec(body)?.[1]?.replace(/<[^>]+>/g, "").trim() ?? "";
 export function websitePrerequisiteSettled(value: { task: Entity; runs: { agentId: string; status: string }[] }, workerId: string, revising: boolean) {
@@ -23,6 +23,9 @@ function downloadHref(o: InternalObservation, attachment: IssueAttachment) {
   const paths = [attachment.contentPath, attachment.openPath, attachment.downloadPath].filter((path): path is string => !!path);
   return o.links.find(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && paths.some(path => new URL(path, o.origin).pathname === url.pathname); } catch { return false; } });
 }
+function linksAttachment(o: InternalObservation, attachment: IssueAttachment | undefined) {
+  return !!attachment && !!downloadHref(o, attachment);
+}
 function deliveredFiles(o: DeliveryObservation) {
   return o.attachments.filter(file => o.products.some(product => product.type === "artifact" && product.provider === "paperclip" && product.issueId === file.issueId
     && product.metadata?.attachmentId === file.id && ["active", "approved", "ready_for_review"].includes(product.status)
@@ -35,7 +38,9 @@ export function gradeWebsiteDelivery(value: unknown) {
   const source = o.state.task ? entity(o.state, "task") : undefined; const prior = o.state.producedBody as string; const originalHash = o.state.producedHash;
   const files = deliveredFiles(o); const revising = o.scenarioId === "documents-artifacts.revise-delivery";
   c.add("actual-prior-work-and-source-retained", typeof prior === "string" && hash(prior) === originalHash && /<html\b/i.test(prior)
-    && !!source && o.tasks.some(row => row.id === source.id && row.companyId === o.companyId && row.projectId === source.projectId) && hasLink(o, "issues", entity(o.state, "task"))
+    && !!source && o.tasks.some(row => row.id === source.id && row.companyId === o.companyId && row.projectId === source.projectId)
+    // The revision request attaches the source task, so its link is not something the person still needs.
+    && (revising || hasLink(o, "issues", entity(o.state, "task")))
     && o.runs.some(run => run.id === o.state.priorRunId && run.agentId === entity(o.state, "worker").id && run.status === "succeeded"),
     "The request refers to actual earlier worker-produced website bytes and preserves its source task and execution attribution.");
   const correct = files.filter(file => o.downloaded.some(bytes => bytes.id === file.id && bytes.sha256 === file.sha256 && hash(bytes.body) === file.sha256
@@ -47,9 +52,17 @@ export function gradeWebsiteDelivery(value: unknown) {
     c.add("original-version-still-retrievable", !!original && o.attachments.some(file => file.id === original.id && file.sha256 === originalHash && file.issueId === source?.id)
       && o.downloaded.some(bytes => bytes.id === original.id && bytes.sha256 === originalHash && bytes.body === prior)
       && o.products.some(product => product.id === o.state.originalProductId && product.metadata?.attachmentId === original.id && product.issueId === source?.id)
-      && correct[0]?.issueId === source?.id, "The original attachment and work product remain retrievable, while the shorter-headline revision belongs to the same source work.");
-    c.add("clear-revision-context", /\b(?:new|updated|revised|shorter)\b/i.test(o.reply) && /\b(?:old|original|previous|v1|version one|first version)\b/i.test(o.reply)
-      && /\b(?:kept|retained|available|preserved|still|unchanged)\b/i.test(o.reply), "Chat distinguishes the new revision from the retained original instead of relabeling stale bytes.");
+      // The chat run's own register_deliverable files the revision on this conversation; the source task is equally valid.
+      && !!correct[0] && [source?.id, o.chatIssueId].filter(Boolean).includes(correct[0].issueId),
+      "The original attachment and work product remain retrievable, while the shorter-headline revision is registered on the source work or the conversation that requested it.");
+    const revisedProduct = correct[0] && o.products.find(product => product.metadata?.attachmentId === correct[0]!.id);
+    const originalProduct = o.products.find(product => product.id === o.state.originalProductId);
+    // Revision context is structural: the new version is a separately titled record and the reply
+    // returns a route to the retained original (its download or the source task) beside the new download.
+    c.add("clear-revision-context", !!revisedProduct && !!originalProduct && revisedProduct.id !== originalProduct.id && !!String(revisedProduct.title ?? "").trim()
+      && String(revisedProduct.title ?? "").trim() !== String(originalProduct.title ?? "").trim()
+      && (linksAttachment(o, original) || (!!source && hasLink(o, "issues", source))),
+      "The new revision is distinguishable from the retained original, and the reply links both the new download and a route to the original.");
   }
   return c.result();
 }
@@ -59,7 +72,9 @@ function websiteDelivery(revising: boolean): ChatCatalogCase {
     const root = `/api/companies/${ctx.input.fixtures.company.id}`;
     for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }])
       await ctx.input.api.post(`${root}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
-    const filename = revising ? "website-v1.html" : "website.html"; const marker = `ACORN-WEBSITE-${ctx.input.nonce}`;
+    const filename = revising ? "website-v1.html" : "website.html"; // The nonce ends in an attempt number ("-1"), which reads as a version label on website-v1.html; hash it so
+    // preserving unrelated content stays an unambiguous requirement for the revision.
+    const marker = `ACORN-REF-${hash(String(ctx.input.nonce)).slice(0, 12)}`;
     const bodyFact = "Acorn Notes keeps quiet research teams organized."; const signupHref = "https://example.test/acorn-signup";
     const producedBody = `<!doctype html><html><head><meta charset="utf-8"><title>Acorn Notes</title></head><body><h1>Bring all of your research team notes together in one calm and accessible place</h1><p>${bodyFact}</p><a href="${signupHref}">Join the list</a><footer>${marker}</footer></body></html>\n`;
     const t = await task(ctx, "Website task", p.id, { status: "todo", assigneeAgentId: worker.id,
@@ -87,14 +102,14 @@ function websiteDelivery(revising: boolean): ChatCatalogCase {
     const state = { project: p, task: handedOff, worker, priorRunId, filename, producedBody: actual, producedHash: hash(actual), marker, bodyFact, signupHref, originalAttachment, originalProductId, selectedSource, factualContextAttachment: selectedSource.factualContextAttachment };
     await ctx.input.evidence("chat-catalog-actual-prior-website.json", state); return state;
   }, gradeWebsiteDelivery);
-  return { ...base, authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-source.md"), expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 600_000,
+  return { ...base, ...(revising ? { oracleVersion: "paperclip.chat-catalog.internal.website-revision.v2" } : {}), authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-source.md"), expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 600_000,
     sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-website-delivery.ts", "registered-artifact.ts"],
     async observe(ctx, state): Promise<DeliveryObservation> {
       const o = await observeInternal(ctx, state); const ids = [entity(state, "task").id, ...(ctx.issue() ? [ctx.issue()!.id] : [])];
       const rows = await Promise.all(ids.map(async id => ({ attachments: await ctx.input.api.get<IssueAttachment[]>(`/api/issues/${id}/attachments`), products: await ctx.input.api.get<IssueWorkProduct[]>(`/api/issues/${id}/work-products`) })));
       const attachments = rows.flatMap(row => row.attachments); const products = rows.flatMap(row => row.products); const downloaded: Downloaded[] = [];
       for (const file of attachments.filter(row => /\.html?$/i.test(row.originalFilename ?? "")).slice(0, 10)) { const bytes = await ctx.input.api.getAttachmentContent(file.id); if (bytes.length > 262144) throw new Error("Website artifact exceeds fixture bounds"); downloaded.push({ id: file.id, sha256: hash(bytes), body: bytes.toString("utf8") }); }
-      return { ...o, attachments, products, downloaded };
+      return { ...o, attachments, products, downloaded, ...(ctx.issue() ? { chatIssueId: ctx.issue()!.id } : {}) };
     },
     async verifyResult(ctx, state, value) {
       const o = value as DeliveryObservation; await base.verifyResult(ctx, state, value);
@@ -110,7 +125,13 @@ function websiteDelivery(revising: boolean): ChatCatalogCase {
       } finally { await page.close(); }
       if (revising) { const source = ctx.input.page.url(); try {
         await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/artifacts?groupIssueId=${entity(state, "task").id}`, o.origin).href, { waitUntil: "domcontentloaded" });
-        for (const product of o.products.filter(row => row.issueId === entity(state, "task").id && row.metadata?.attachmentId === file.id)) await expect(ctx.input.page.getByText(product.title, { exact: true }).first()).toBeVisible();
+        const originalTitle = o.products.find(row => row.id === state.originalProductId)?.title;
+        if (!originalTitle) throw new Error("The original work product is missing from the observation");
+        await expect(ctx.input.page.getByText(originalTitle, { exact: true }).first()).toBeVisible();
+        const revised = o.products.find(row => row.metadata?.attachmentId === file.id);
+        if (!revised) throw new Error("The revised work product is missing from the observation");
+        if (revised.issueId !== entity(state, "task").id) await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/artifacts?groupIssueId=${revised.issueId}`, o.origin).href, { waitUntil: "domcontentloaded" });
+        await expect(ctx.input.page.getByText(revised.title, { exact: true }).first()).toBeVisible();
         await ctx.input.evidence("chat-catalog-website-version-library-opened.json", { taskId: entity(state, "task").id, url: ctx.input.page.url(), originalProductId: state.originalProductId });
       } finally { await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } }
     },

tests/runner-e2e/chat-catalog-domains/internal.test.ts

@@ -53,9 +53,9 @@ function valid(scenario: string): InternalObservation {
       const original = { id: "original", issueId: "task", sha256: sha(prior), originatingRunId: "build", contentPath: "/api/attachments/original/content" };
       const file = { id: "new", issueId: "task", sha256: sha(body), originatingRunId: "run", contentPath: "/api/attachments/new/content" };
       Object.assign(o.state, { worker: { id: "morgan" }, priorRunId: "build", producedBody: prior, producedHash: sha(prior), bodyFact: "Research teams stay organized.", signupHref: "https://example.test/signup", marker: "ORCHID", originalAttachment: original, originalProductId: "old-product" });
-      const product = (id: string, attachmentId: string, run: string) => ({ id, issueId: "task", type: "artifact", provider: "paperclip", status: "active", createdByRunId: run, metadata: { attachmentId } });
+      const product = (id: string, attachmentId: string, run: string) => ({ id, issueId: "task", title: attachmentId === "original" ? "website-v1.html" : "website-v2.html", type: "artifact", provider: "paperclip", status: "active", createdByRunId: run, metadata: { attachmentId } });
       Object.assign(o, { attachments: [file, ...(revising ? [original] : [])], products: [product("new-product", "new", "run"), ...(revising ? [product("old-product", "original", "build")] : [])], downloaded: [{ id: "new", sha256: sha(body), body }, ...(revising ? [{ id: "original", sha256: sha(prior), body: prior }] : [])] });
-      o.runs.push({ id: "build", companyId: "company", agentId: "morgan", status: "succeeded" }); o.links.push("/api/attachments/new/content?download=1");
+      o.runs.push({ id: "build", companyId: "company", agentId: "morgan", status: "succeeded" }); o.links.push("/api/attachments/new/content?download=1", ...(revising ? ["/api/attachments/original/content"] : []));
       o.reply = "The updated shorter version is available; the original remains available."; break;
     }
     case "tasks.assignment-conflict": {
@@ -363,7 +363,8 @@ const corrupt: Record<string, (o: InternalObservation) => void> = {
   "inbox-search.notification-audience": o => { Object.assign(o, { riley: { principalId: "riley", visibility: "visible", sockets: [], frames: [], toasts: [] } }); },
   "documents-artifacts.find-artifact": o => { o.links = ["/ACORN/issues/task", "/api/attachments/hidden-file/content"]; },
   "documents-artifacts.produce-download": o => { o.links = ["/ACORN/issues/task", "/tmp/website.html"]; },
-  "documents-artifacts.revise-delivery": o => { o.reply = "Here is the old version again."; },
+  // Relabeling: the new bytes are filed under the original version's title, so the two versions are indistinguishable.
+  "documents-artifacts.revise-delivery": o => { (o as unknown as { products: { metadata: { attachmentId: string }; title: string }[] }).products.find(row => row.metadata.attachmentId === "new")!.title = "website-v1.html"; },
   "tasks.assignment-conflict": o => { o.tasks[0]!.assigneeAgentId = "riley"; },
   "activity.find-runs": o => { o.runs = o.runs.filter(run => run.id !== "interrupted"); },
   // The grader does not judge claim wording; the failure that matters is relaying another agent's feedback text.

tests/runner-e2e/chat-scenarios/documents-artifacts.yaml

@@ -63,8 +63,9 @@ scenarios:
     query: >-
       Shorten the headline and give me a new version, but keep the old one available.
     expected_outcome: >-
-      The updated deliverable is linked to the same work with clear revision context, and the
-      original remains retrievable. The library opens the intended stack or work product. The
+      The updated deliverable is a new, separately titled registered version, filed on the
+      source work or in the conversation that requested it, and the reply links it beside a route
+      to the original, which remains retrievable. The library opens both versions' stacks. The
       agent does not overwrite the only retained copy or label stale bytes as the new revision.
   - id: documents-artifacts.find-artifact
     capabilities: