From d4e8906dc36af80bff50cf5585954f0780b3ed27 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 15:41:19 -0500 Subject: [PATCH] test(evals): verify rendered references inside saved task comments --- .../runner-e2e/chat-catalog-descriptors.json | 5 +- .../internal-comment-reference.test.ts | 32 ++++++- .../internal-comment-reference.ts | 85 +++++++++++++++++++ .../chat-catalog-domains/internal-tasks.ts | 27 +----- .../chat-catalog-domains/internal.test.ts | 3 +- 5 files changed, 126 insertions(+), 26 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/internal-comment-reference.ts diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 8530516651b..53fc610d779 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5712,7 +5712,10 @@ "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-tasks.ts", - "chat-scenarios/tasks.yaml" + "chat-scenarios/tasks.yaml", + "chat-catalog-domains/internal-comment-reference.ts", + "../../ui/src/lib/issue-reference.ts", + "../../ui/src/components/MarkdownBody.tsx" ], "oracleVersion": "paperclip.chat-catalog.internal.v1", "coverageGaps": [], diff --git a/tests/runner-e2e/chat-catalog-domains/internal-comment-reference.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-comment-reference.test.ts index 008dcedd4ac..220b7dd2487 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-comment-reference.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-comment-reference.test.ts @@ -1,9 +1,39 @@ +import { createHash } from "node:crypto"; import { expect, it } from "vitest"; +import { savedDesignInstruction, validRenderedCommentReference } from "./internal-comment-reference.js"; import { grade_tasks } from "./internal-tasks.js"; import type { InternalObservation } from "./internal-support.js"; +function valid(body = "Follow the approved design task RUN-1 as the source of truth for this landing page build."): InternalObservation { + const task = { id: "landing", companyId: "company", title: "Landing page", status: "backlog" }; + const design = { id: "design", companyId: "company", identifier: "RUN-1", title: "Approved design", status: "done" }; + const comment = { id: "note", issueId: task.id, body, authorAgentId: "ceo", createdByRunId: "run" }; + return { scenarioId: "tasks.comment-with-reference", origin: "http://127.0.0.1:3100", companyId: "company", leadId: "ceo", state: { task, design, + savedCommentReference: { commentId: comment.id, taskId: task.id, designId: design.id, bodySha256: createHash("sha256").update(body).digest("hex"), href: "/RUN/issues/RUN-1", visible: true, destinationMatched: true, publicReadMatched: true } }, + tasks: [task, design], comments: [comment], projects: [], goals: [], activity: [], documents: [], revisions: [], reply: "Saved the note on the landing page task.", links: ["/RUN/issues/landing"], humanQueries: ["Add a note."], runs: [{ id: "run", agentId: "ceo", companyId: "company", status: "succeeded" }] }; +} +it("accepts the retained natural bare-identifier note and explicit Markdown links with actual comment-scoped proof", () => { + for (const body of [valid().comments[0]!.body!, "Use [the approved design](/RUN/issues/RUN-1) when implementing the landing page.", "Build according to `RUN-1`.", "Follow [the design](http://127.0.0.1:3100/RUN/issues/RUN-1)."]) expect(grade_tasks(valid(body)).passed).toBe(true); +}); +it("does not accept wrong/foreign/hidden or negated design instructions", () => { + for (const body of ["Follow RUN-2.", "Follow [RUN-1](https://foreign.invalid/issues/RUN-1).", "Follow [RUN-1](/FOREIGN/issues/RUN-1).", "Do not follow RUN-1.", "Ignore the approved design RUN-1.", "Follow RUN-1. Ignore that design.", "```\nFollow RUN-1\n```", "", "RUN-1 is the design, but this note gives no instruction."]) { + const o = valid(body); expect(savedDesignInstruction(o, body, o.state.design as typeof o.tasks[0]), body).toBe(false); expect(grade_tasks(o).passed).toBe(false); + } +}); +it("requires exact saved-comment, body, task, destination and readable-resource proof instead of aggregate links", () => { + for (const change of [{ commentId: "other-note" }, { taskId: "other-task" }, { designId: "other-design" }, { bodySha256: "stale" }, { href: "https://foreign.invalid/issues/RUN-1" }, { visible: false }, { destinationMatched: false }, { publicReadMatched: false }]) { + const o = valid(); Object.assign(o.state.savedCommentReference!, change); o.links.push("/RUN/issues/RUN-1"); expect(grade_tasks(o).passed).toBe(false); + } + const o = valid(); delete o.state.savedCommentReference; expect(validRenderedCommentReference(o, o.comments[0]!, o.tasks[1]!)).toBe(false); +}); +it("retains exactly one real agent/run-attributed comment", () => { + for (const change of [{ authorAgentId: "human" }, { createdByRunId: "unrelated" }, { issueId: "foreign-task" }]) { const o = valid(); Object.assign(o.comments[0]!, change); expect(grade_tasks(o).passed).toBe(false); } + const o = valid(); o.comments.push({ ...o.comments[0]!, id: "duplicate" }); expect(grade_tasks(o).passed).toBe(false); +}); + +// Retain the original affirmative/negative instruction calibration matrix. function observation(body: string): InternalObservation { const task = { id: "task", companyId: "company", title: "Landing page", status: "backlog" }, design = { id: "design", identifier: "ACO-2", companyId: "company", title: "Approved design", status: "done" }; - return { scenarioId: "tasks.comment-with-reference", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(task), design: structuredClone(design) }, tasks: [task, design], projects: [], goals: [], documents: [], revisions: [], activity: [], reply: "Note added.", humanQueries: [], links: ["/ACO/issues/task"], comments: [{ id: "note", authorAgentId: "ceo", createdByRunId: "run", body }], runs: [{ id: "run", companyId: "company", agentId: "ceo", status: "succeeded" }] }; + return { scenarioId: "tasks.comment-with-reference", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(task), design: structuredClone(design), savedCommentReference: { commentId: "note", taskId: "task", designId: "design", bodySha256: createHash("sha256").update(body).digest("hex"), href: "/ACO/issues/design", visible: true, destinationMatched: true, publicReadMatched: true } }, tasks: [task, design], projects: [], goals: [], documents: [], revisions: [], activity: [], reply: "Note added.", humanQueries: [], links: ["/ACO/issues/task"], comments: [{ id: "note", issueId: "task", authorAgentId: "ceo", createdByRunId: "run", body }], runs: [{ id: "run", companyId: "company", agentId: "ceo", status: "succeeded" }] }; } for (const body of ["Follow [the approved design](/ACO/issues/design).", "Please use [this design](https://fixture.test/ACO/issues/ACO-2) for implementation.", "Adhere to [the approved design](/ACO/issues/design?tab=documents).", "Implement according to [the design](/ACO/issues/design).", "Build in line with [the approved design](/ACO/issues/design).", "Follow [the approved design](/ACO/issues/design) and avoid unrelated layout changes.", "Do not forget to follow [the approved design](/ACO/issues/design)."]) it(`accepts affirmative actual design instruction: ${body}`, () => expect(grade_tasks(observation(body)).passed).toBe(true)); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-comment-reference.ts b/tests/runner-e2e/chat-catalog-domains/internal-comment-reference.ts new file mode 100644 index 00000000000..097a45a97e2 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-comment-reference.ts @@ -0,0 +1,85 @@ +import { createHash } from "node:crypto"; +import { expect } from "@playwright/test"; +import { parseIssueReferenceFromHref } from "../../../ui/src/lib/issue-reference.js"; +import type { ChatCatalogCase } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { entity, hasLink, observeInternal, type Entity, type InternalObservation } from "./internal-support.js"; + +const digest = (body: string) => createHash("sha256").update(body).digest("hex"); +export function savedDesignInstruction(o: InternalObservation, body: string, design: Entity) { + const valid: string[] = []; + const reference = (href: string) => { + let local = href; + try { + if (/^https?:/i.test(href)) { + const url = new URL(href); if (url.origin !== new URL(o.origin).origin) return "{{OTHERREF}}"; + local = `${url.pathname}${url.search}${url.hash}`; + } + const segments = local.split(/[?#]/, 1)[0]!.split("/").filter(Boolean), issueAt = segments.indexOf("issues"); + if (design.identifier && issueAt > 0 && segments[issueAt - 1] !== design.identifier.split("-")[0]) return "{{OTHERREF}}"; + } catch { return "{{OTHERREF}}"; } + const parsed = parseIssueReferenceFromHref(local, design.identifier ? new Set([design.identifier.split("-")[0]!]) : undefined); + if (!parsed || !hasLink({ ...o, links: [parsed.href] }, "issues", design)) return "{{OTHERREF}}"; + const token = `{{DESIGNREF${valid.length}}}`; valid.push(token); return token; + }; + const visibleSource = body.replace(/^ {0,3}(`{3,}|~{3,})[^\n]*\n[\s\S]*?^ {0,3}\1[^\n]*$/gm, "") + .replace(//g, "").replace(/<[^>]*>[\s\S]*?<\/[^>]*>/g, ""); + // Rendered-comment proof below is mandatory; lexical recognition alone never + // proves that a Markdown or bare identifier became a visible, usable link. + const text = visibleSource.replace(/\[[^\]]+\]\(]+)>?(?:\s+["'][^"']*["'])?\)/g, (_link, href: string) => reference(href)) + .replace(/\b[A-Z][A-Z0-9]*-\d+\b/gi, identifier => reference(identifier)); + if (!valid.length) return false; + const directive = /\b(?:follow|use|adhere\s+to|implement\s+(?:according\s+to|in\s+line\s+with)|build\s+(?:according\s+to|in\s+line\s+with))\b/i; + const negative = (prefix: string) => { + const t = prefix.replace(/\b(?:do not|don't) forget to\b/gi, "remember to"); + return /\b(?:do not|don't|must not|should not|never|not to|not)\b[^{}.;!\n]{0,60}\b(?:follow|use|adhere|implement|build)\b[^{}.;!\n]{0,80}$/i.test(t) + || /\b(?:avoid|ignore|disregard|stop)\b[^{}.;!\n]{0,80}$/i.test(t); + }; + const clauses = text.split(/[.;!\n]+/).filter(clause => valid.some(token => clause.includes(token))); + if (clauses.some(clause => valid.some(token => { const at = clause.indexOf(token); return at >= 0 && negative(clause.slice(0, at)); }))) return false; + if (/\b(?:(?:do not|don't|never|must not|should not)\s+(?:follow|use|adhere\s+to)|ignore|disregard)\s+(?:it|this|that|the\s+(?:approved\s+)?design)\b/i.test(text)) return false; + return clauses.some(clause => directive.test(clause) && valid.some(token => { + const instruction = directive.exec(clause); return !!instruction && clause.indexOf(token) > instruction.index; + })); +} +interface ReferenceProof { commentId: string; taskId: string; designId: string; bodySha256: string; href: string; visible: boolean; destinationMatched: boolean; publicReadMatched: boolean } +export function validRenderedCommentReference(o: InternalObservation, comment: Entity, design: Entity) { + const proof = o.state.savedCommentReference as ReferenceProof | undefined; + return !!proof && proof.commentId === comment.id && comment.issueId === proof.taskId && proof.taskId === entity(o.state, "task").id && proof.designId === design.id + && proof.bodySha256 === digest(comment.body ?? "") && proof.visible === true && proof.destinationMatched === true && proof.publicReadMatched === true + && hasLink({ ...o, links: [proof.href] }, "issues", design); +} +export function withRenderedCommentReference(base: ChatCatalogCase): ChatCatalogCase { + return { ...base, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-comment-reference.ts", "../../ui/src/lib/issue-reference.ts", "../../ui/src/components/MarkdownBody.tsx"], + async act(ctx, state) { + await pollUntil({ label: "Saved comment source turn", deadlineAt: Date.now() + 150000, + load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); }, + accept: runs => runs.length === 1 && runs[0]!.status === "succeeded" && ctx.issue()?.conversationState === "waiting", + reject: runs => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Source run failed" : undefined }); + const o = await observeInternal(ctx, state), target = entity(state, "task"), design = entity(state, "design"); + const added = o.comments.filter(row => row.authorAgentId === o.leadId && row.createdByRunId && o.runs.some(run => run.id === row.createdByRunId)); + if (added.length !== 1 || !savedDesignInstruction(o, added[0]!.body ?? "", design)) return; + const taskHref = o.links.find(href => hasLink({ ...o, links: [href] }, "issues", target)); + if (!taskHref) return; + const page = ctx.input.page, source = page.url(); + try { + const link = page.locator(`[id^="comment-"] a[href=${JSON.stringify(taskHref)}]`).first(); + await expect(link).toBeVisible(); await link.click(); + await page.waitForURL(url => hasLink({ ...o, links: [url.href] }, "issues", target)); + await expect(page.getByRole("heading", { name: target.title!, exact: true }).first()).toBeVisible(); + const comment = page.locator(`[id=${JSON.stringify(`comment-${added[0]!.id}`)}]`); + await expect(comment).toBeVisible(); + const hrefs = await comment.locator("a[href]").evaluateAll(elements => elements.map(element => element.getAttribute("href")!)); + const href = hrefs.find(value => hasLink({ ...o, links: [value] }, "issues", design)); + if (!href) return; + const designLink = comment.locator(`a[href=${JSON.stringify(href)}]`).first(); await expect(designLink).toBeVisible(); await designLink.click(); + await page.waitForURL(url => hasLink({ ...o, links: [url.href] }, "issues", design)); + await expect(page.getByRole("heading", { name: design.title!, exact: true }).first()).toBeVisible(); + const actual = await ctx.input.api.get(`/api/issues/${design.id}`); + state.savedCommentReference = { commentId: added[0]!.id, taskId: target.id, designId: design.id, bodySha256: digest(added[0]!.body ?? ""), href, + visible: true, destinationMatched: true, publicReadMatched: actual.id === design.id && actual.companyId === o.companyId } satisfies ReferenceProof; + await ctx.input.evidence("chat-saved-comment-reference.json", state.savedCommentReference); + } finally { await page.goto(source, { waitUntil: "domcontentloaded" }); await expect(page.getByTestId("task-chat-composer-input")).toBeVisible(); } + }, + }; +} diff --git a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts index d2c443f95a6..764ddc4eb89 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts @@ -1,3 +1,4 @@ +import { savedDesignInstruction, withRenderedCommentReference, validRenderedCommentReference } from "./internal-comment-reference.js"; import { withEditOwnership } from "./internal-edit-ownership.js"; import { verifyDistinctTaskFixtures } from "./internal-distinct-fixtures.js"; import { delegatedTaskCases } from "./internal-task-creation.js"; @@ -10,26 +11,6 @@ import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { project, task, goal, putDocument, entity, observation, checks, hasLink, sameTask, type Entity, type InternalObservation } from "./internal-support.js"; import { makeDefinition, retryDefinition, fixtureWorker } from "./internal-definition.js"; -function savedDesignInstruction(o: InternalObservation, body: string, design: Entity) { - const targets: string[] = []; - const prose = body.replace(/\[[^\]]+\]\(]+)>?(?:\s+["'][^"']*["'])?\)/g, (_link, href: string) => { - const token = `{{DESIGNREF${targets.length}}}`; targets.push(href); return token; - }); - const valid = targets.map((href, index) => hasLink({ ...o, links: [href] }, "issues", design) ? `{{DESIGNREF${index}}}` : null).filter((x): x is string => !!x); - if (!valid.length) return false; - const directive = /\b(?:follow|use|adhere\s+to|implement\s+(?:according\s+to|in\s+line\s+with)|build\s+(?:according\s+to|in\s+line\s+with))\b/i; - const governsNegativeReference = (prefix: string) => { - const text = prefix.replace(/\b(?:do not|don't) forget to\b/gi, "remember to"); - return /\b(?:do not|don't|must not|should not|never|not to|not)\b[^{}.;!\n]{0,60}\b(?:follow|use|adhere|implement|build)\b[^{}.;!\n]{0,80}$/i.test(text) - || /\b(?:avoid|ignore|disregard|stop)\b[^{}.;!\n]{0,80}$/i.test(text); - }; - const clauses = prose.split(/[.;!\n]+/).filter(clause => valid.some(token => clause.includes(token))); - if (clauses.some(clause => valid.some(token => { const at = clause.indexOf(token); return at >= 0 && governsNegativeReference(clause.slice(0, at)); }))) return false; - if (/\b(?:(?:do not|don't|never|must not|should not)\s+(?:follow|use|adhere\s+to)|ignore|disregard)\s+(?:it|this|that|the\s+(?:approved\s+)?design)\b/i.test(prose)) return false; - return clauses.some(clause => directive.test(clause) && valid.some(token => { - const instruction = directive.exec(clause); return !!instruction && clause.indexOf(token) > instruction.index; - })); -} export function grade_tasks(value: unknown) { const c = checks(); const o = observation(value); c.add("complete-independent-observation", !!o, "Missing API, document, audit or rendered-link evidence cannot pass."); @@ -67,7 +48,7 @@ export function grade_tasks(value: unknown) { case "tasks.comment-with-reference": { const design = entity(o.state, "design"); const added = o.comments.filter(row => row.authorAgentId === o.leadId && row.createdByRunId && o.runs.some(run => run.id === row.createdByRunId)); - c.add("one-attributed-note", added.length === 1 && savedDesignInstruction(o, added[0]?.body ?? "", design), "One CEO-authored note retains a usable reference to the actual approved design task."); + c.add("one-attributed-note", added.length === 1 && savedDesignInstruction(o, added[0]?.body ?? "", design) && validRenderedCommentReference(o, added[0]!, design), "One CEO-authored note retains a usable reference to the actual approved design task."); c.add("records-preserved", !!before && sameTask(before, current) && sameTask(design, o.tasks.find(row => row.id === design.id)) && o.tasks.length === 2, "The note neither changes either task nor creates another task."); c.add("task-link", linkedTask(), "The original conversation links back to the task containing the note."); @@ -92,11 +73,11 @@ export const tasksCases: readonly ChatCatalogCase[] = [ const t = await task(ctx, "Landing page", p.id, { description: "Build the Launch landing page; this work is saved for later." }); return { project: p, worker, task: t }; })), - define("tasks.comment-with-reference", async ctx => { + withRenderedCommentReference(define("tasks.comment-with-reference", async ctx => { const p = await project(ctx); const design = await task(ctx, "Approved design", p.id, { status: "done", description: "Approved design: semantic HTML, clear contrast, one primary signup action." }); const t = await task(ctx, "Landing page", p.id, { description: "Build the launch website." }); return { project: p, design, task: t }; - }), + })), ]; diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index cff02cd765c..a4873f44471 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -241,7 +241,8 @@ function valid(scenario: string): InternalObservation { } case "tasks.comment-with-reference": { const design = { id: "design", title: "Approved design", status: "done" }; o.state.design = design; o.tasks.push(design); - o.comments = [{ id: "comment", authorAgentId: "ceo", createdByRunId: "run", body: "Follow the [approved design](/ACORN/issues/design)." }]; break; + o.comments = [{ id: "comment", issueId: "task", authorAgentId: "ceo", createdByRunId: "run", body: "Follow the [approved design](/ACORN/issues/design)." }]; + o.state.savedCommentReference = { commentId: "comment", taskId: "task", designId: "design", bodySha256: createHash("sha256").update(o.comments[0]!.body!).digest("hex"), href: "/ACORN/issues/design", visible: true, destinationMatched: true, publicReadMatched: true }; break; } case "goals.create-goal-tree": { o.state = { firstIds: ["growth", "team"], retryQuery: "query" }; o.humanQueries = ["query", "query"]; o.tasks = []; o.projects = [];