From a3f29839fd4101e53640d52ed05af5c166440466 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 20:12:46 -0600 Subject: [PATCH] fix(evals): grade website revision by its saved versions, not reply wording The revise-delivery grader rejected a correct revision for four eval reasons: the fixture footer marker ended in the nonce's attempt suffix (-1), which the agent reasonably bumped as a version label on website-v1.html; it required a link to the source task the person had attached; it required the revision on the source task although the chat run's register_deliverable files it on the requesting conversation; and it graded revision context with keyword regexes. Hash the marker, accept the source task or the conversation, and grade revision context structurally: a separately titled registered version plus a returned route to the retained original. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../internal-website-delivery.test.ts | 12 +++++- .../internal-website-delivery.ts | 39 ++++++++++++++----- .../chat-catalog-domains/internal.test.ts | 7 ++-- .../chat-scenarios/documents-artifacts.yaml | 5 ++- 5 files changed, 48 insertions(+), 17 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 01c1a72a3d8..46094a31977 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -2872,7 +2872,7 @@ "chat-catalog-domains/internal-website-delivery.ts", "registered-artifact.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.internal.website-revision.v2", "coverageGaps": [], "expectedRunCount": 2, "minimumRunCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts index e2f89893dd2..e7672e4a90b 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts @@ -14,11 +14,11 @@ function valid(revising = false) { const prior = "

Bring your research team notes together in one accessible place

Research teams stay organized.

Signup
ORCHID
"; const body = revising ? prior.replace("Bring your research team notes together in one accessible place", "Notes, together") : prior; const attachment = (id: string, bytes: string, run: string) => ({ id, issueId: "task", sha256: hash(bytes), originatingRunId: run, contentPath: `/api/attachments/${id}/content` }); - const product = (id: string, file: string, run: string) => ({ id, issueId: "task", type: "artifact", provider: "paperclip", status: "active", createdByRunId: run, metadata: { attachmentId: file } }); + const product = (id: string, file: string, run: string, issueId = "task") => ({ id, issueId, title: file === "original" ? "website-v1.html" : "website-v2.html", type: "artifact", provider: "paperclip", status: "active", createdByRunId: run, metadata: { attachmentId: file } }); const original = attachment("original", prior, "build"); const newFile = attachment("new", body, "run"); return { scenarioId: revising ? "documents-artifacts.revise-delivery" : "documents-artifacts.produce-download", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(task), worker: { id: "morgan" }, priorRunId: "build", producedBody: prior, producedHash: hash(prior), bodyFact: "Research teams stay organized.", signupHref: "https://example.test/signup", marker: "ORCHID", originalAttachment: original, originalProductId: "old-product" }, - tasks: [task], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], humanQueries: [], links: ["/RUN/issues/task", "/api/attachments/new/content?download=1"], + tasks: [task], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], humanQueries: [], links: ["/RUN/issues/task", "/api/attachments/new/content?download=1", ...(revising ? ["/api/attachments/original/content"] : [])], chatIssueId: "chat", reply: "The updated shorter version is downloadable; the original remains available.", runs: [{ id: "run", companyId: "company", agentId: "ceo", status: "succeeded" }, { id: "build", companyId: "company", agentId: "morgan", status: "succeeded" }], attachments: [newFile, ...(revising ? [original] : [])], products: [product("new-product", "new", "run"), ...(revising ? [product("old-product", "original", "build")] : [])], downloaded: [{ id: "new", sha256: hash(body), body }, ...(revising ? [{ id: "original", sha256: hash(prior), body: prior }] : [])] }; } @@ -28,6 +28,14 @@ describe("registered website delivery calibration", () => { it("rejects stale or mismatched downloaded bytes", () => { const o = valid(); o.downloaded[0]!.body = "stale"; expect(gradeWebsiteDelivery(o).passed).toBe(false); }); it("rejects wrong immutable origin or work-product source", () => { const o = valid(); o.attachments[0]!.originatingRunId = "build"; expect(gradeWebsiteDelivery(o).passed).toBe(false); const source = valid(true); source.products[0]!.issueId = "another-task"; expect(gradeWebsiteDelivery(source).passed).toBe(false); }); it("rejects overwriting the only old copy", () => { const o = valid(true); o.downloaded = o.downloaded.filter(row => row.id !== "original"); expect(gradeWebsiteDelivery(o).passed).toBe(false); }); + it("accepts the revision registered on the requesting conversation, whatever the reply wording", () => { + const o = valid(true); const file = o.attachments.find(row => row.id === "new")!; file.issueId = "chat"; o.products.find(row => row.id === "new-product")!.issueId = "chat"; + o.reply = "Done. Shorter headline is in the new file; the first one is untouched."; expect(gradeWebsiteDelivery(o).passed).toBe(true); + o.reply = "Here you go."; expect(gradeWebsiteDelivery(o).passed).toBe(true); + }); + it("rejects a revision registered on an unrelated task", () => { const o = valid(true); o.attachments.find(row => row.id === "new")!.issueId = "other"; o.products.find(row => row.id === "new-product")!.issueId = "other"; expect(gradeWebsiteDelivery(o).passed).toBe(false); }); + it("rejects a revision with no returned route to the retained original", () => { const o = valid(true); o.links = ["/api/attachments/new/content?download=1"]; expect(gradeWebsiteDelivery(o).passed).toBe(false); o.links.push("/api/attachments/original/content"); expect(gradeWebsiteDelivery(o).passed).toBe(true); }); + it("rejects new bytes filed under the original version's title", () => { const o = valid(true); o.products.find(row => row.id === "new-product")!.title = "website-v1.html"; expect(gradeWebsiteDelivery(o).passed).toBe(false); }); it("rejects a headline that was not shortened", () => { const o = valid(true); o.downloaded[0]!.body = o.state.producedBody; o.downloaded[0]!.sha256 = hash(o.downloaded[0]!.body); o.attachments[0]!.sha256 = o.downloaded[0]!.sha256; expect(gradeWebsiteDelivery(o).passed).toBe(false); }); it("rejects a shorter headline with unrelated body edits", () => { const o = valid(true); o.downloaded[0]!.body = o.downloaded[0]!.body.replace("Research teams stay organized.", "Ads for sale."); o.downloaded[0]!.sha256 = hash(o.downloaded[0]!.body); o.attachments[0]!.sha256 = o.downloaded[0]!.sha256; expect(gradeWebsiteDelivery(o).passed).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts index 0f5b471af60..18edbd65af0 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts @@ -12,7 +12,7 @@ import { checks, entity, hasLink, observation, observeInternal, project, task, t import { selectDocumentSource } from "./internal-document-context.js"; import { factualContextQueryAttachment } from "../chat-query-attachments.js"; interface Downloaded { id: string; sha256: string; body: string } -interface DeliveryObservation extends InternalObservation { attachments: IssueAttachment[]; products: IssueWorkProduct[]; downloaded: Downloaded[] } +interface DeliveryObservation extends InternalObservation { attachments: IssueAttachment[]; products: IssueWorkProduct[]; downloaded: Downloaded[]; chatIssueId?: string } const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex"); const headline = (body: string) => /]*>([\s\S]*?)<\/h1>/i.exec(body)?.[1]?.replace(/<[^>]+>/g, "").trim() ?? ""; export function websitePrerequisiteSettled(value: { task: Entity; runs: { agentId: string; status: string }[] }, workerId: string, revising: boolean) { @@ -23,6 +23,9 @@ function downloadHref(o: InternalObservation, attachment: IssueAttachment) { const paths = [attachment.contentPath, attachment.openPath, attachment.downloadPath].filter((path): path is string => !!path); return o.links.find(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && paths.some(path => new URL(path, o.origin).pathname === url.pathname); } catch { return false; } }); } +function linksAttachment(o: InternalObservation, attachment: IssueAttachment | undefined) { + return !!attachment && !!downloadHref(o, attachment); +} function deliveredFiles(o: DeliveryObservation) { return o.attachments.filter(file => o.products.some(product => product.type === "artifact" && product.provider === "paperclip" && product.issueId === file.issueId && product.metadata?.attachmentId === file.id && ["active", "approved", "ready_for_review"].includes(product.status) @@ -35,7 +38,9 @@ export function gradeWebsiteDelivery(value: unknown) { const source = o.state.task ? entity(o.state, "task") : undefined; const prior = o.state.producedBody as string; const originalHash = o.state.producedHash; const files = deliveredFiles(o); const revising = o.scenarioId === "documents-artifacts.revise-delivery"; c.add("actual-prior-work-and-source-retained", typeof prior === "string" && hash(prior) === originalHash && / row.id === source.id && row.companyId === o.companyId && row.projectId === source.projectId) && hasLink(o, "issues", entity(o.state, "task")) + && !!source && o.tasks.some(row => row.id === source.id && row.companyId === o.companyId && row.projectId === source.projectId) + // The revision request attaches the source task, so its link is not something the person still needs. + && (revising || hasLink(o, "issues", entity(o.state, "task"))) && o.runs.some(run => run.id === o.state.priorRunId && run.agentId === entity(o.state, "worker").id && run.status === "succeeded"), "The request refers to actual earlier worker-produced website bytes and preserves its source task and execution attribution."); const correct = files.filter(file => o.downloaded.some(bytes => bytes.id === file.id && bytes.sha256 === file.sha256 && hash(bytes.body) === file.sha256 @@ -47,9 +52,17 @@ export function gradeWebsiteDelivery(value: unknown) { c.add("original-version-still-retrievable", !!original && o.attachments.some(file => file.id === original.id && file.sha256 === originalHash && file.issueId === source?.id) && o.downloaded.some(bytes => bytes.id === original.id && bytes.sha256 === originalHash && bytes.body === prior) && o.products.some(product => product.id === o.state.originalProductId && product.metadata?.attachmentId === original.id && product.issueId === source?.id) - && correct[0]?.issueId === source?.id, "The original attachment and work product remain retrievable, while the shorter-headline revision belongs to the same source work."); - c.add("clear-revision-context", /\b(?:new|updated|revised|shorter)\b/i.test(o.reply) && /\b(?:old|original|previous|v1|version one|first version)\b/i.test(o.reply) - && /\b(?:kept|retained|available|preserved|still|unchanged)\b/i.test(o.reply), "Chat distinguishes the new revision from the retained original instead of relabeling stale bytes."); + // The chat run's own register_deliverable files the revision on this conversation; the source task is equally valid. + && !!correct[0] && [source?.id, o.chatIssueId].filter(Boolean).includes(correct[0].issueId), + "The original attachment and work product remain retrievable, while the shorter-headline revision is registered on the source work or the conversation that requested it."); + const revisedProduct = correct[0] && o.products.find(product => product.metadata?.attachmentId === correct[0]!.id); + const originalProduct = o.products.find(product => product.id === o.state.originalProductId); + // Revision context is structural: the new version is a separately titled record and the reply + // returns a route to the retained original (its download or the source task) beside the new download. + c.add("clear-revision-context", !!revisedProduct && !!originalProduct && revisedProduct.id !== originalProduct.id && !!String(revisedProduct.title ?? "").trim() + && String(revisedProduct.title ?? "").trim() !== String(originalProduct.title ?? "").trim() + && (linksAttachment(o, original) || (!!source && hasLink(o, "issues", source))), + "The new revision is distinguishable from the retained original, and the reply links both the new download and a route to the original."); } return c.result(); } @@ -59,7 +72,9 @@ function websiteDelivery(revising: boolean): ChatCatalogCase { const root = `/api/companies/${ctx.input.fixtures.company.id}`; for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }]) await ctx.input.api.post(`${root}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); - const filename = revising ? "website-v1.html" : "website.html"; const marker = `ACORN-WEBSITE-${ctx.input.nonce}`; + const filename = revising ? "website-v1.html" : "website.html"; // The nonce ends in an attempt number ("-1"), which reads as a version label on website-v1.html; hash it so + // preserving unrelated content stays an unambiguous requirement for the revision. + const marker = `ACORN-REF-${hash(String(ctx.input.nonce)).slice(0, 12)}`; const bodyFact = "Acorn Notes keeps quiet research teams organized."; const signupHref = "https://example.test/acorn-signup"; const producedBody = `Acorn Notes

Bring all of your research team notes together in one calm and accessible place

${bodyFact}

Join the list
${marker}
\n`; const t = await task(ctx, "Website task", p.id, { status: "todo", assigneeAgentId: worker.id, @@ -87,14 +102,14 @@ function websiteDelivery(revising: boolean): ChatCatalogCase { const state = { project: p, task: handedOff, worker, priorRunId, filename, producedBody: actual, producedHash: hash(actual), marker, bodyFact, signupHref, originalAttachment, originalProductId, selectedSource, factualContextAttachment: selectedSource.factualContextAttachment }; await ctx.input.evidence("chat-catalog-actual-prior-website.json", state); return state; }, gradeWebsiteDelivery); - return { ...base, authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-source.md"), expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 600_000, + return { ...base, ...(revising ? { oracleVersion: "paperclip.chat-catalog.internal.website-revision.v2" } : {}), authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-source.md"), expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 600_000, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-website-delivery.ts", "registered-artifact.ts"], async observe(ctx, state): Promise { const o = await observeInternal(ctx, state); const ids = [entity(state, "task").id, ...(ctx.issue() ? [ctx.issue()!.id] : [])]; const rows = await Promise.all(ids.map(async id => ({ attachments: await ctx.input.api.get(`/api/issues/${id}/attachments`), products: await ctx.input.api.get(`/api/issues/${id}/work-products`) }))); const attachments = rows.flatMap(row => row.attachments); const products = rows.flatMap(row => row.products); const downloaded: Downloaded[] = []; for (const file of attachments.filter(row => /\.html?$/i.test(row.originalFilename ?? "")).slice(0, 10)) { const bytes = await ctx.input.api.getAttachmentContent(file.id); if (bytes.length > 262144) throw new Error("Website artifact exceeds fixture bounds"); downloaded.push({ id: file.id, sha256: hash(bytes), body: bytes.toString("utf8") }); } - return { ...o, attachments, products, downloaded }; + return { ...o, attachments, products, downloaded, ...(ctx.issue() ? { chatIssueId: ctx.issue()!.id } : {}) }; }, async verifyResult(ctx, state, value) { const o = value as DeliveryObservation; await base.verifyResult(ctx, state, value); @@ -110,7 +125,13 @@ function websiteDelivery(revising: boolean): ChatCatalogCase { } finally { await page.close(); } if (revising) { const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/artifacts?groupIssueId=${entity(state, "task").id}`, o.origin).href, { waitUntil: "domcontentloaded" }); - for (const product of o.products.filter(row => row.issueId === entity(state, "task").id && row.metadata?.attachmentId === file.id)) await expect(ctx.input.page.getByText(product.title, { exact: true }).first()).toBeVisible(); + const originalTitle = o.products.find(row => row.id === state.originalProductId)?.title; + if (!originalTitle) throw new Error("The original work product is missing from the observation"); + await expect(ctx.input.page.getByText(originalTitle, { exact: true }).first()).toBeVisible(); + const revised = o.products.find(row => row.metadata?.attachmentId === file.id); + if (!revised) throw new Error("The revised work product is missing from the observation"); + if (revised.issueId !== entity(state, "task").id) await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/artifacts?groupIssueId=${revised.issueId}`, o.origin).href, { waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByText(revised.title, { exact: true }).first()).toBeVisible(); await ctx.input.evidence("chat-catalog-website-version-library-opened.json", { taskId: entity(state, "task").id, url: ctx.input.page.url(), originalProductId: state.originalProductId }); } finally { await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } } }, diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index ba5af3b469a..a4706fae07b 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -53,9 +53,9 @@ function valid(scenario: string): InternalObservation { const original = { id: "original", issueId: "task", sha256: sha(prior), originatingRunId: "build", contentPath: "/api/attachments/original/content" }; const file = { id: "new", issueId: "task", sha256: sha(body), originatingRunId: "run", contentPath: "/api/attachments/new/content" }; Object.assign(o.state, { worker: { id: "morgan" }, priorRunId: "build", producedBody: prior, producedHash: sha(prior), bodyFact: "Research teams stay organized.", signupHref: "https://example.test/signup", marker: "ORCHID", originalAttachment: original, originalProductId: "old-product" }); - const product = (id: string, attachmentId: string, run: string) => ({ id, issueId: "task", type: "artifact", provider: "paperclip", status: "active", createdByRunId: run, metadata: { attachmentId } }); + const product = (id: string, attachmentId: string, run: string) => ({ id, issueId: "task", title: attachmentId === "original" ? "website-v1.html" : "website-v2.html", type: "artifact", provider: "paperclip", status: "active", createdByRunId: run, metadata: { attachmentId } }); Object.assign(o, { attachments: [file, ...(revising ? [original] : [])], products: [product("new-product", "new", "run"), ...(revising ? [product("old-product", "original", "build")] : [])], downloaded: [{ id: "new", sha256: sha(body), body }, ...(revising ? [{ id: "original", sha256: sha(prior), body: prior }] : [])] }); - o.runs.push({ id: "build", companyId: "company", agentId: "morgan", status: "succeeded" }); o.links.push("/api/attachments/new/content?download=1"); + o.runs.push({ id: "build", companyId: "company", agentId: "morgan", status: "succeeded" }); o.links.push("/api/attachments/new/content?download=1", ...(revising ? ["/api/attachments/original/content"] : [])); o.reply = "The updated shorter version is available; the original remains available."; break; } case "tasks.assignment-conflict": { @@ -363,7 +363,8 @@ const corrupt: Record void> = { "inbox-search.notification-audience": o => { Object.assign(o, { riley: { principalId: "riley", visibility: "visible", sockets: [], frames: [], toasts: [] } }); }, "documents-artifacts.find-artifact": o => { o.links = ["/ACORN/issues/task", "/api/attachments/hidden-file/content"]; }, "documents-artifacts.produce-download": o => { o.links = ["/ACORN/issues/task", "/tmp/website.html"]; }, - "documents-artifacts.revise-delivery": o => { o.reply = "Here is the old version again."; }, + // Relabeling: the new bytes are filed under the original version's title, so the two versions are indistinguishable. + "documents-artifacts.revise-delivery": o => { (o as unknown as { products: { metadata: { attachmentId: string }; title: string }[] }).products.find(row => row.metadata.attachmentId === "new")!.title = "website-v1.html"; }, "tasks.assignment-conflict": o => { o.tasks[0]!.assigneeAgentId = "riley"; }, "activity.find-runs": o => { o.runs = o.runs.filter(run => run.id !== "interrupted"); }, // The grader does not judge claim wording; the failure that matters is relaying another agent's feedback text. diff --git a/tests/runner-e2e/chat-scenarios/documents-artifacts.yaml b/tests/runner-e2e/chat-scenarios/documents-artifacts.yaml index 5eba861e55a..97690df7bbb 100644 --- a/tests/runner-e2e/chat-scenarios/documents-artifacts.yaml +++ b/tests/runner-e2e/chat-scenarios/documents-artifacts.yaml @@ -63,8 +63,9 @@ scenarios: query: >- Shorten the headline and give me a new version, but keep the old one available. expected_outcome: >- - The updated deliverable is linked to the same work with clear revision context, and the - original remains retrievable. The library opens the intended stack or work product. The + The updated deliverable is a new, separately titled registered version, filed on the + source work or in the conversation that requested it, and the reply links it beside a route + to the original, which remains retrievable. The library opens both versions' stacks. The agent does not overwrite the only retained copy or label stale bytes as the new revision. - id: documents-artifacts.find-artifact capabilities: