diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json
index 01c1a72a3d8..46094a31977 100644
--- a/tests/runner-e2e/chat-catalog-descriptors.json
+++ b/tests/runner-e2e/chat-catalog-descriptors.json
@@ -2872,7 +2872,7 @@
"chat-catalog-domains/internal-website-delivery.ts",
"registered-artifact.ts"
],
- "oracleVersion": "paperclip.chat-catalog.internal.v1",
+ "oracleVersion": "paperclip.chat-catalog.internal.website-revision.v2",
"coverageGaps": [],
"expectedRunCount": 2,
"minimumRunCount": 2,
diff --git a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts
index e2f89893dd2..e7672e4a90b 100644
--- a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts
@@ -14,11 +14,11 @@ function valid(revising = false) {
const prior = "
Bring your research team notes together in one accessible place
Research teams stay organized.
Signup";
const body = revising ? prior.replace("Bring your research team notes together in one accessible place", "Notes, together") : prior;
const attachment = (id: string, bytes: string, run: string) => ({ id, issueId: "task", sha256: hash(bytes), originatingRunId: run, contentPath: `/api/attachments/${id}/content` });
- const product = (id: string, file: string, run: string) => ({ id, issueId: "task", type: "artifact", provider: "paperclip", status: "active", createdByRunId: run, metadata: { attachmentId: file } });
+ const product = (id: string, file: string, run: string, issueId = "task") => ({ id, issueId, title: file === "original" ? "website-v1.html" : "website-v2.html", type: "artifact", provider: "paperclip", status: "active", createdByRunId: run, metadata: { attachmentId: file } });
const original = attachment("original", prior, "build"); const newFile = attachment("new", body, "run");
return { scenarioId: revising ? "documents-artifacts.revise-delivery" : "documents-artifacts.produce-download", origin: "https://fixture.test", companyId: "company", leadId: "ceo",
state: { task: structuredClone(task), worker: { id: "morgan" }, priorRunId: "build", producedBody: prior, producedHash: hash(prior), bodyFact: "Research teams stay organized.", signupHref: "https://example.test/signup", marker: "ORCHID", originalAttachment: original, originalProductId: "old-product" },
- tasks: [task], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], humanQueries: [], links: ["/RUN/issues/task", "/api/attachments/new/content?download=1"],
+ tasks: [task], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], humanQueries: [], links: ["/RUN/issues/task", "/api/attachments/new/content?download=1", ...(revising ? ["/api/attachments/original/content"] : [])], chatIssueId: "chat",
reply: "The updated shorter version is downloadable; the original remains available.", runs: [{ id: "run", companyId: "company", agentId: "ceo", status: "succeeded" }, { id: "build", companyId: "company", agentId: "morgan", status: "succeeded" }],
attachments: [newFile, ...(revising ? [original] : [])], products: [product("new-product", "new", "run"), ...(revising ? [product("old-product", "original", "build")] : [])], downloaded: [{ id: "new", sha256: hash(body), body }, ...(revising ? [{ id: "original", sha256: hash(prior), body: prior }] : [])] };
}
@@ -28,6 +28,14 @@ describe("registered website delivery calibration", () => {
it("rejects stale or mismatched downloaded bytes", () => { const o = valid(); o.downloaded[0]!.body = "stale"; expect(gradeWebsiteDelivery(o).passed).toBe(false); });
it("rejects wrong immutable origin or work-product source", () => { const o = valid(); o.attachments[0]!.originatingRunId = "build"; expect(gradeWebsiteDelivery(o).passed).toBe(false); const source = valid(true); source.products[0]!.issueId = "another-task"; expect(gradeWebsiteDelivery(source).passed).toBe(false); });
it("rejects overwriting the only old copy", () => { const o = valid(true); o.downloaded = o.downloaded.filter(row => row.id !== "original"); expect(gradeWebsiteDelivery(o).passed).toBe(false); });
+ it("accepts the revision registered on the requesting conversation, whatever the reply wording", () => {
+ const o = valid(true); const file = o.attachments.find(row => row.id === "new")!; file.issueId = "chat"; o.products.find(row => row.id === "new-product")!.issueId = "chat";
+ o.reply = "Done. Shorter headline is in the new file; the first one is untouched."; expect(gradeWebsiteDelivery(o).passed).toBe(true);
+ o.reply = "Here you go."; expect(gradeWebsiteDelivery(o).passed).toBe(true);
+ });
+ it("rejects a revision registered on an unrelated task", () => { const o = valid(true); o.attachments.find(row => row.id === "new")!.issueId = "other"; o.products.find(row => row.id === "new-product")!.issueId = "other"; expect(gradeWebsiteDelivery(o).passed).toBe(false); });
+ it("rejects a revision with no returned route to the retained original", () => { const o = valid(true); o.links = ["/api/attachments/new/content?download=1"]; expect(gradeWebsiteDelivery(o).passed).toBe(false); o.links.push("/api/attachments/original/content"); expect(gradeWebsiteDelivery(o).passed).toBe(true); });
+ it("rejects new bytes filed under the original version's title", () => { const o = valid(true); o.products.find(row => row.id === "new-product")!.title = "website-v1.html"; expect(gradeWebsiteDelivery(o).passed).toBe(false); });
it("rejects a headline that was not shortened", () => { const o = valid(true); o.downloaded[0]!.body = o.state.producedBody; o.downloaded[0]!.sha256 = hash(o.downloaded[0]!.body); o.attachments[0]!.sha256 = o.downloaded[0]!.sha256; expect(gradeWebsiteDelivery(o).passed).toBe(false); });
it("rejects a shorter headline with unrelated body edits", () => { const o = valid(true); o.downloaded[0]!.body = o.downloaded[0]!.body.replace("Research teams stay organized.", "Ads for sale."); o.downloaded[0]!.sha256 = hash(o.downloaded[0]!.body); o.attachments[0]!.sha256 = o.downloaded[0]!.sha256; expect(gradeWebsiteDelivery(o).passed).toBe(false); });
});
diff --git a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts
index 0f5b471af60..18edbd65af0 100644
--- a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts
+++ b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts
@@ -12,7 +12,7 @@ import { checks, entity, hasLink, observation, observeInternal, project, task, t
import { selectDocumentSource } from "./internal-document-context.js";
import { factualContextQueryAttachment } from "../chat-query-attachments.js";
interface Downloaded { id: string; sha256: string; body: string }
-interface DeliveryObservation extends InternalObservation { attachments: IssueAttachment[]; products: IssueWorkProduct[]; downloaded: Downloaded[] }
+interface DeliveryObservation extends InternalObservation { attachments: IssueAttachment[]; products: IssueWorkProduct[]; downloaded: Downloaded[]; chatIssueId?: string }
const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex");
const headline = (body: string) => /]*>([\s\S]*?)<\/h1>/i.exec(body)?.[1]?.replace(/<[^>]+>/g, "").trim() ?? "";
export function websitePrerequisiteSettled(value: { task: Entity; runs: { agentId: string; status: string }[] }, workerId: string, revising: boolean) {
@@ -23,6 +23,9 @@ function downloadHref(o: InternalObservation, attachment: IssueAttachment) {
const paths = [attachment.contentPath, attachment.openPath, attachment.downloadPath].filter((path): path is string => !!path);
return o.links.find(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && paths.some(path => new URL(path, o.origin).pathname === url.pathname); } catch { return false; } });
}
+function linksAttachment(o: InternalObservation, attachment: IssueAttachment | undefined) {
+ return !!attachment && !!downloadHref(o, attachment);
+}
function deliveredFiles(o: DeliveryObservation) {
return o.attachments.filter(file => o.products.some(product => product.type === "artifact" && product.provider === "paperclip" && product.issueId === file.issueId
&& product.metadata?.attachmentId === file.id && ["active", "approved", "ready_for_review"].includes(product.status)
@@ -35,7 +38,9 @@ export function gradeWebsiteDelivery(value: unknown) {
const source = o.state.task ? entity(o.state, "task") : undefined; const prior = o.state.producedBody as string; const originalHash = o.state.producedHash;
const files = deliveredFiles(o); const revising = o.scenarioId === "documents-artifacts.revise-delivery";
c.add("actual-prior-work-and-source-retained", typeof prior === "string" && hash(prior) === originalHash && / row.id === source.id && row.companyId === o.companyId && row.projectId === source.projectId) && hasLink(o, "issues", entity(o.state, "task"))
+ && !!source && o.tasks.some(row => row.id === source.id && row.companyId === o.companyId && row.projectId === source.projectId)
+ // The revision request attaches the source task, so its link is not something the person still needs.
+ && (revising || hasLink(o, "issues", entity(o.state, "task")))
&& o.runs.some(run => run.id === o.state.priorRunId && run.agentId === entity(o.state, "worker").id && run.status === "succeeded"),
"The request refers to actual earlier worker-produced website bytes and preserves its source task and execution attribution.");
const correct = files.filter(file => o.downloaded.some(bytes => bytes.id === file.id && bytes.sha256 === file.sha256 && hash(bytes.body) === file.sha256
@@ -47,9 +52,17 @@ export function gradeWebsiteDelivery(value: unknown) {
c.add("original-version-still-retrievable", !!original && o.attachments.some(file => file.id === original.id && file.sha256 === originalHash && file.issueId === source?.id)
&& o.downloaded.some(bytes => bytes.id === original.id && bytes.sha256 === originalHash && bytes.body === prior)
&& o.products.some(product => product.id === o.state.originalProductId && product.metadata?.attachmentId === original.id && product.issueId === source?.id)
- && correct[0]?.issueId === source?.id, "The original attachment and work product remain retrievable, while the shorter-headline revision belongs to the same source work.");
- c.add("clear-revision-context", /\b(?:new|updated|revised|shorter)\b/i.test(o.reply) && /\b(?:old|original|previous|v1|version one|first version)\b/i.test(o.reply)
- && /\b(?:kept|retained|available|preserved|still|unchanged)\b/i.test(o.reply), "Chat distinguishes the new revision from the retained original instead of relabeling stale bytes.");
+ // The chat run's own register_deliverable files the revision on this conversation; the source task is equally valid.
+ && !!correct[0] && [source?.id, o.chatIssueId].filter(Boolean).includes(correct[0].issueId),
+ "The original attachment and work product remain retrievable, while the shorter-headline revision is registered on the source work or the conversation that requested it.");
+ const revisedProduct = correct[0] && o.products.find(product => product.metadata?.attachmentId === correct[0]!.id);
+ const originalProduct = o.products.find(product => product.id === o.state.originalProductId);
+ // Revision context is structural: the new version is a separately titled record and the reply
+ // returns a route to the retained original (its download or the source task) beside the new download.
+ c.add("clear-revision-context", !!revisedProduct && !!originalProduct && revisedProduct.id !== originalProduct.id && !!String(revisedProduct.title ?? "").trim()
+ && String(revisedProduct.title ?? "").trim() !== String(originalProduct.title ?? "").trim()
+ && (linksAttachment(o, original) || (!!source && hasLink(o, "issues", source))),
+ "The new revision is distinguishable from the retained original, and the reply links both the new download and a route to the original.");
}
return c.result();
}
@@ -59,7 +72,9 @@ function websiteDelivery(revising: boolean): ChatCatalogCase {
const root = `/api/companies/${ctx.input.fixtures.company.id}`;
for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }])
await ctx.input.api.post(`${root}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
- const filename = revising ? "website-v1.html" : "website.html"; const marker = `ACORN-WEBSITE-${ctx.input.nonce}`;
+ const filename = revising ? "website-v1.html" : "website.html"; // The nonce ends in an attempt number ("-1"), which reads as a version label on website-v1.html; hash it so
+ // preserving unrelated content stays an unambiguous requirement for the revision.
+ const marker = `ACORN-REF-${hash(String(ctx.input.nonce)).slice(0, 12)}`;
const bodyFact = "Acorn Notes keeps quiet research teams organized."; const signupHref = "https://example.test/acorn-signup";
const producedBody = `Acorn NotesBring all of your research team notes together in one calm and accessible place
${bodyFact}
Join the list\n`;
const t = await task(ctx, "Website task", p.id, { status: "todo", assigneeAgentId: worker.id,
@@ -87,14 +102,14 @@ function websiteDelivery(revising: boolean): ChatCatalogCase {
const state = { project: p, task: handedOff, worker, priorRunId, filename, producedBody: actual, producedHash: hash(actual), marker, bodyFact, signupHref, originalAttachment, originalProductId, selectedSource, factualContextAttachment: selectedSource.factualContextAttachment };
await ctx.input.evidence("chat-catalog-actual-prior-website.json", state); return state;
}, gradeWebsiteDelivery);
- return { ...base, authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-source.md"), expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 600_000,
+ return { ...base, ...(revising ? { oracleVersion: "paperclip.chat-catalog.internal.website-revision.v2" } : {}), authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-source.md"), expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 600_000,
sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-website-delivery.ts", "registered-artifact.ts"],
async observe(ctx, state): Promise {
const o = await observeInternal(ctx, state); const ids = [entity(state, "task").id, ...(ctx.issue() ? [ctx.issue()!.id] : [])];
const rows = await Promise.all(ids.map(async id => ({ attachments: await ctx.input.api.get(`/api/issues/${id}/attachments`), products: await ctx.input.api.get(`/api/issues/${id}/work-products`) })));
const attachments = rows.flatMap(row => row.attachments); const products = rows.flatMap(row => row.products); const downloaded: Downloaded[] = [];
for (const file of attachments.filter(row => /\.html?$/i.test(row.originalFilename ?? "")).slice(0, 10)) { const bytes = await ctx.input.api.getAttachmentContent(file.id); if (bytes.length > 262144) throw new Error("Website artifact exceeds fixture bounds"); downloaded.push({ id: file.id, sha256: hash(bytes), body: bytes.toString("utf8") }); }
- return { ...o, attachments, products, downloaded };
+ return { ...o, attachments, products, downloaded, ...(ctx.issue() ? { chatIssueId: ctx.issue()!.id } : {}) };
},
async verifyResult(ctx, state, value) {
const o = value as DeliveryObservation; await base.verifyResult(ctx, state, value);
@@ -110,7 +125,13 @@ function websiteDelivery(revising: boolean): ChatCatalogCase {
} finally { await page.close(); }
if (revising) { const source = ctx.input.page.url(); try {
await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/artifacts?groupIssueId=${entity(state, "task").id}`, o.origin).href, { waitUntil: "domcontentloaded" });
- for (const product of o.products.filter(row => row.issueId === entity(state, "task").id && row.metadata?.attachmentId === file.id)) await expect(ctx.input.page.getByText(product.title, { exact: true }).first()).toBeVisible();
+ const originalTitle = o.products.find(row => row.id === state.originalProductId)?.title;
+ if (!originalTitle) throw new Error("The original work product is missing from the observation");
+ await expect(ctx.input.page.getByText(originalTitle, { exact: true }).first()).toBeVisible();
+ const revised = o.products.find(row => row.metadata?.attachmentId === file.id);
+ if (!revised) throw new Error("The revised work product is missing from the observation");
+ if (revised.issueId !== entity(state, "task").id) await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/artifacts?groupIssueId=${revised.issueId}`, o.origin).href, { waitUntil: "domcontentloaded" });
+ await expect(ctx.input.page.getByText(revised.title, { exact: true }).first()).toBeVisible();
await ctx.input.evidence("chat-catalog-website-version-library-opened.json", { taskId: entity(state, "task").id, url: ctx.input.page.url(), originalProductId: state.originalProductId });
} finally { await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } }
},
diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts
index ba5af3b469a..a4706fae07b 100644
--- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts
@@ -53,9 +53,9 @@ function valid(scenario: string): InternalObservation {
const original = { id: "original", issueId: "task", sha256: sha(prior), originatingRunId: "build", contentPath: "/api/attachments/original/content" };
const file = { id: "new", issueId: "task", sha256: sha(body), originatingRunId: "run", contentPath: "/api/attachments/new/content" };
Object.assign(o.state, { worker: { id: "morgan" }, priorRunId: "build", producedBody: prior, producedHash: sha(prior), bodyFact: "Research teams stay organized.", signupHref: "https://example.test/signup", marker: "ORCHID", originalAttachment: original, originalProductId: "old-product" });
- const product = (id: string, attachmentId: string, run: string) => ({ id, issueId: "task", type: "artifact", provider: "paperclip", status: "active", createdByRunId: run, metadata: { attachmentId } });
+ const product = (id: string, attachmentId: string, run: string) => ({ id, issueId: "task", title: attachmentId === "original" ? "website-v1.html" : "website-v2.html", type: "artifact", provider: "paperclip", status: "active", createdByRunId: run, metadata: { attachmentId } });
Object.assign(o, { attachments: [file, ...(revising ? [original] : [])], products: [product("new-product", "new", "run"), ...(revising ? [product("old-product", "original", "build")] : [])], downloaded: [{ id: "new", sha256: sha(body), body }, ...(revising ? [{ id: "original", sha256: sha(prior), body: prior }] : [])] });
- o.runs.push({ id: "build", companyId: "company", agentId: "morgan", status: "succeeded" }); o.links.push("/api/attachments/new/content?download=1");
+ o.runs.push({ id: "build", companyId: "company", agentId: "morgan", status: "succeeded" }); o.links.push("/api/attachments/new/content?download=1", ...(revising ? ["/api/attachments/original/content"] : []));
o.reply = "The updated shorter version is available; the original remains available."; break;
}
case "tasks.assignment-conflict": {
@@ -363,7 +363,8 @@ const corrupt: Record void> = {
"inbox-search.notification-audience": o => { Object.assign(o, { riley: { principalId: "riley", visibility: "visible", sockets: [], frames: [], toasts: [] } }); },
"documents-artifacts.find-artifact": o => { o.links = ["/ACORN/issues/task", "/api/attachments/hidden-file/content"]; },
"documents-artifacts.produce-download": o => { o.links = ["/ACORN/issues/task", "/tmp/website.html"]; },
- "documents-artifacts.revise-delivery": o => { o.reply = "Here is the old version again."; },
+ // Relabeling: the new bytes are filed under the original version's title, so the two versions are indistinguishable.
+ "documents-artifacts.revise-delivery": o => { (o as unknown as { products: { metadata: { attachmentId: string }; title: string }[] }).products.find(row => row.metadata.attachmentId === "new")!.title = "website-v1.html"; },
"tasks.assignment-conflict": o => { o.tasks[0]!.assigneeAgentId = "riley"; },
"activity.find-runs": o => { o.runs = o.runs.filter(run => run.id !== "interrupted"); },
// The grader does not judge claim wording; the failure that matters is relaying another agent's feedback text.
diff --git a/tests/runner-e2e/chat-scenarios/documents-artifacts.yaml b/tests/runner-e2e/chat-scenarios/documents-artifacts.yaml
index 5eba861e55a..97690df7bbb 100644
--- a/tests/runner-e2e/chat-scenarios/documents-artifacts.yaml
+++ b/tests/runner-e2e/chat-scenarios/documents-artifacts.yaml
@@ -63,8 +63,9 @@ scenarios:
query: >-
Shorten the headline and give me a new version, but keep the old one available.
expected_outcome: >-
- The updated deliverable is linked to the same work with clear revision context, and the
- original remains retrievable. The library opens the intended stack or work product. The
+ The updated deliverable is a new, separately titled registered version, filed on the
+ source work or in the conversation that requested it, and the reply links it beside a route
+ to the original, which remains retrievable. The library opens both versions' stacks. The
agent does not overwrite the only retained copy or label stale bytes as the new revision.
- id: documents-artifacts.find-artifact
capabilities: