Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-05 03:51:54 +01:00
Code Issues Packages Projects Releases Wiki Activity

test(evals): verify rendered references inside saved task comments

Browse Source
This commit is contained in:
Dotta committed 2026-10-27 16:10:58 -05:00
1 parent d6f00a8136
commit d4e8906dc3
5 files changed
+126 -26
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/chat-catalog-descriptors.json
+4 -1
View File
@@ -5712,7 +5712,10 @@
"chat-catalog-domains/internal-definition.ts",
"chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/internal-tasks.ts",
"chat-scenarios/tasks.yaml"
"chat-scenarios/tasks.yaml",
"chat-catalog-domains/internal-comment-reference.ts",
"../../ui/src/lib/issue-reference.ts",
"../../ui/src/components/MarkdownBody.tsx"
],
"oracleVersion": "paperclip.chat-catalog.internal.v1",
"coverageGaps": [],
tests/runner-e2e/chat-catalog-domains/internal-comment-reference.test.ts
+31 -1
View File
@@ -1,9 +1,39 @@
import { createHash } from "node:crypto";
import { expect, it } from "vitest";
import { savedDesignInstruction, validRenderedCommentReference } from "./internal-comment-reference.js";
import { grade_tasks } from "./internal-tasks.js";
import type { InternalObservation } from "./internal-support.js";
function valid(body = "Follow the approved design task RUN-1 as the source of truth for this landing page build."): InternalObservation {
const task = { id: "landing", companyId: "company", title: "Landing page", status: "backlog" };
const design = { id: "design", companyId: "company", identifier: "RUN-1", title: "Approved design", status: "done" };
const comment = { id: "note", issueId: task.id, body, authorAgentId: "ceo", createdByRunId: "run" };
return { scenarioId: "tasks.comment-with-reference", origin: "http://127.0.0.1:3100", companyId: "company", leadId: "ceo", state: { task, design,
savedCommentReference: { commentId: comment.id, taskId: task.id, designId: design.id, bodySha256: createHash("sha256").update(body).digest("hex"), href: "/RUN/issues/RUN-1", visible: true, destinationMatched: true, publicReadMatched: true } },
tasks: [task, design], comments: [comment], projects: [], goals: [], activity: [], documents: [], revisions: [], reply: "Saved the note on the landing page task.", links: ["/RUN/issues/landing"], humanQueries: ["Add a note."], runs: [{ id: "run", agentId: "ceo", companyId: "company", status: "succeeded" }] };
}
it("accepts the retained natural bare-identifier note and explicit Markdown links with actual comment-scoped proof", () => {
for (const body of [valid().comments[0]!.body!, "Use [the approved design](/RUN/issues/RUN-1) when implementing the landing page.", "Build according to `RUN-1`.", "Follow [the design](http://127.0.0.1:3100/RUN/issues/RUN-1)."]) expect(grade_tasks(valid(body)).passed).toBe(true);
});
it("does not accept wrong/foreign/hidden or negated design instructions", () => {
for (const body of ["Follow RUN-2.", "Follow [RUN-1](https://foreign.invalid/issues/RUN-1).", "Follow [RUN-1](/FOREIGN/issues/RUN-1).", "Do not follow RUN-1.", "Ignore the approved design RUN-1.", "Follow RUN-1. Ignore that design.", "```\nFollow RUN-1\n```", "<!-- Follow RUN-1 -->", "RUN-1 is the design, but this note gives no instruction."]) {
const o = valid(body); expect(savedDesignInstruction(o, body, o.state.design as typeof o.tasks[0]), body).toBe(false); expect(grade_tasks(o).passed).toBe(false);
}
});
it("requires exact saved-comment, body, task, destination and readable-resource proof instead of aggregate links", () => {
for (const change of [{ commentId: "other-note" }, { taskId: "other-task" }, { designId: "other-design" }, { bodySha256: "stale" }, { href: "https://foreign.invalid/issues/RUN-1" }, { visible: false }, { destinationMatched: false }, { publicReadMatched: false }]) {
const o = valid(); Object.assign(o.state.savedCommentReference!, change); o.links.push("/RUN/issues/RUN-1"); expect(grade_tasks(o).passed).toBe(false);
}
const o = valid(); delete o.state.savedCommentReference; expect(validRenderedCommentReference(o, o.comments[0]!, o.tasks[1]!)).toBe(false);
});
it("retains exactly one real agent/run-attributed comment", () => {
for (const change of [{ authorAgentId: "human" }, { createdByRunId: "unrelated" }, { issueId: "foreign-task" }]) { const o = valid(); Object.assign(o.comments[0]!, change); expect(grade_tasks(o).passed).toBe(false); }
const o = valid(); o.comments.push({ ...o.comments[0]!, id: "duplicate" }); expect(grade_tasks(o).passed).toBe(false);
});
// Retain the original affirmative/negative instruction calibration matrix.
function observation(body: string): InternalObservation {
const task = { id: "task", companyId: "company", title: "Landing page", status: "backlog" }, design = { id: "design", identifier: "ACO-2", companyId: "company", title: "Approved design", status: "done" };
return { scenarioId: "tasks.comment-with-reference", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(task), design: structuredClone(design) }, tasks: [task, design], projects: [], goals: [], documents: [], revisions: [], activity: [], reply: "Note added.", humanQueries: [], links: ["/ACO/issues/task"], comments: [{ id: "note", authorAgentId: "ceo", createdByRunId: "run", body }], runs: [{ id: "run", companyId: "company", agentId: "ceo", status: "succeeded" }] };
return { scenarioId: "tasks.comment-with-reference", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(task), design: structuredClone(design), savedCommentReference: { commentId: "note", taskId: "task", designId: "design", bodySha256: createHash("sha256").update(body).digest("hex"), href: "/ACO/issues/design", visible: true, destinationMatched: true, publicReadMatched: true } }, tasks: [task, design], projects: [], goals: [], documents: [], revisions: [], activity: [], reply: "Note added.", humanQueries: [], links: ["/ACO/issues/task"], comments: [{ id: "note", issueId: "task", authorAgentId: "ceo", createdByRunId: "run", body }], runs: [{ id: "run", companyId: "company", agentId: "ceo", status: "succeeded" }] };
}
for (const body of ["Follow [the approved design](/ACO/issues/design).", "Please use [this design](https://fixture.test/ACO/issues/ACO-2) for implementation.", "Adhere to [the approved design](/ACO/issues/design?tab=documents).", "Implement according to [the design](/ACO/issues/design).", "Build in line with [the approved design](/ACO/issues/design).", "Follow [the approved design](/ACO/issues/design) and avoid unrelated layout changes.", "Do not forget to follow [the approved design](/ACO/issues/design)."])
it(`accepts affirmative actual design instruction: ${body}`, () => expect(grade_tasks(observation(body)).passed).toBe(true));
tests/runner-e2e/chat-catalog-domains/internal-comment-reference.ts
+85
View File
@@ -0,0 +1,85 @@
import { createHash } from "node:crypto";
import { expect } from "@playwright/test";
import { parseIssueReferenceFromHref } from "../../../ui/src/lib/issue-reference.js";
import type { ChatCatalogCase } from "../chat-catalog-contract.js";
import { pollUntil } from "../api.js";
import { entity, hasLink, observeInternal, type Entity, type InternalObservation } from "./internal-support.js";
const digest = (body: string) => createHash("sha256").update(body).digest("hex");
export function savedDesignInstruction(o: InternalObservation, body: string, design: Entity) {
const valid: string[] = [];
const reference = (href: string) => {
let local = href;
try {
if (/^https?:/i.test(href)) {
const url = new URL(href); if (url.origin !== new URL(o.origin).origin) return "{{OTHERREF}}";
local = `${url.pathname}${url.search}${url.hash}`;
}
const segments = local.split(/[?#]/, 1)[0]!.split("/").filter(Boolean), issueAt = segments.indexOf("issues");
if (design.identifier && issueAt > 0 && segments[issueAt - 1] !== design.identifier.split("-")[0]) return "{{OTHERREF}}";
} catch { return "{{OTHERREF}}"; }
const parsed = parseIssueReferenceFromHref(local, design.identifier ? new Set([design.identifier.split("-")[0]!]) : undefined);
if (!parsed || !hasLink({ ...o, links: [parsed.href] }, "issues", design)) return "{{OTHERREF}}";
const token = `{{DESIGNREF${valid.length}}}`; valid.push(token); return token;
};
const visibleSource = body.replace(/^ {0,3}(`{3,}|~{3,})[^\n]*\n[\s\S]*?^ {0,3}\1[^\n]*$/gm, "")
.replace(/<!--[\s\S]*?-->/g, "").replace(/<[^>]*>[\s\S]*?<\/[^>]*>/g, "");
// Rendered-comment proof below is mandatory; lexical recognition alone never
// proves that a Markdown or bare identifier became a visible, usable link.
const text = visibleSource.replace(/\[[^\]]+\]\(<?([^\s)>]+)>?(?:\s+["'][^"']*["'])?\)/g, (_link, href: string) => reference(href))
.replace(/\b[A-Z][A-Z0-9]*-\d+\b/gi, identifier => reference(identifier));
if (!valid.length) return false;
const directive = /\b(?:follow|use|adhere\s+to|implement\s+(?:according\s+to|in\s+line\s+with)|build\s+(?:according\s+to|in\s+line\s+with))\b/i;
const negative = (prefix: string) => {
const t = prefix.replace(/\b(?:do not|don't) forget to\b/gi, "remember to");
return /\b(?:do not|don't|must not|should not|never|not to|not)\b[^{}.;!\n]{0,60}\b(?:follow|use|adhere|implement|build)\b[^{}.;!\n]{0,80}$/i.test(t)
|| /\b(?:avoid|ignore|disregard|stop)\b[^{}.;!\n]{0,80}$/i.test(t);
};
const clauses = text.split(/[.;!\n]+/).filter(clause => valid.some(token => clause.includes(token)));
if (clauses.some(clause => valid.some(token => { const at = clause.indexOf(token); return at >= 0 && negative(clause.slice(0, at)); }))) return false;
if (/\b(?:(?:do not|don't|never|must not|should not)\s+(?:follow|use|adhere\s+to)|ignore|disregard)\s+(?:it|this|that|the\s+(?:approved\s+)?design)\b/i.test(text)) return false;
return clauses.some(clause => directive.test(clause) && valid.some(token => {
const instruction = directive.exec(clause); return !!instruction && clause.indexOf(token) > instruction.index;
}));
}
interface ReferenceProof { commentId: string; taskId: string; designId: string; bodySha256: string; href: string; visible: boolean; destinationMatched: boolean; publicReadMatched: boolean }
export function validRenderedCommentReference(o: InternalObservation, comment: Entity, design: Entity) {
const proof = o.state.savedCommentReference as ReferenceProof | undefined;
return !!proof && proof.commentId === comment.id && comment.issueId === proof.taskId && proof.taskId === entity(o.state, "task").id && proof.designId === design.id
&& proof.bodySha256 === digest(comment.body ?? "") && proof.visible === true && proof.destinationMatched === true && proof.publicReadMatched === true
&& hasLink({ ...o, links: [proof.href] }, "issues", design);
}
export function withRenderedCommentReference(base: ChatCatalogCase): ChatCatalogCase {
return { ...base, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-comment-reference.ts", "../../ui/src/lib/issue-reference.ts", "../../ui/src/components/MarkdownBody.tsx"],
async act(ctx, state) {
await pollUntil({ label: "Saved comment source turn", deadlineAt: Date.now() + 150000,
load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); },
accept: runs => runs.length === 1 && runs[0]!.status === "succeeded" && ctx.issue()?.conversationState === "waiting",
reject: runs => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Source run failed" : undefined });
const o = await observeInternal(ctx, state), target = entity(state, "task"), design = entity(state, "design");
const added = o.comments.filter(row => row.authorAgentId === o.leadId && row.createdByRunId && o.runs.some(run => run.id === row.createdByRunId));
if (added.length !== 1 || !savedDesignInstruction(o, added[0]!.body ?? "", design)) return;
const taskHref = o.links.find(href => hasLink({ ...o, links: [href] }, "issues", target));
if (!taskHref) return;
const page = ctx.input.page, source = page.url();
try {
const link = page.locator(`[id^="comment-"] a[href=${JSON.stringify(taskHref)}]`).first();
await expect(link).toBeVisible(); await link.click();
await page.waitForURL(url => hasLink({ ...o, links: [url.href] }, "issues", target));
await expect(page.getByRole("heading", { name: target.title!, exact: true }).first()).toBeVisible();
const comment = page.locator(`[id=${JSON.stringify(`comment-${added[0]!.id}`)}]`);
await expect(comment).toBeVisible();
const hrefs = await comment.locator("a[href]").evaluateAll(elements => elements.map(element => element.getAttribute("href")!));
const href = hrefs.find(value => hasLink({ ...o, links: [value] }, "issues", design));
if (!href) return;
const designLink = comment.locator(`a[href=${JSON.stringify(href)}]`).first(); await expect(designLink).toBeVisible(); await designLink.click();
await page.waitForURL(url => hasLink({ ...o, links: [url.href] }, "issues", design));
await expect(page.getByRole("heading", { name: design.title!, exact: true }).first()).toBeVisible();
const actual = await ctx.input.api.get<Entity>(`/api/issues/${design.id}`);
state.savedCommentReference = { commentId: added[0]!.id, taskId: target.id, designId: design.id, bodySha256: digest(added[0]!.body ?? ""), href,
visible: true, destinationMatched: true, publicReadMatched: actual.id === design.id && actual.companyId === o.companyId } satisfies ReferenceProof;
await ctx.input.evidence("chat-saved-comment-reference.json", state.savedCommentReference);
} finally { await page.goto(source, { waitUntil: "domcontentloaded" }); await expect(page.getByTestId("task-chat-composer-input")).toBeVisible(); }
},
};
}
tests/runner-e2e/chat-catalog-domains/internal-tasks.ts
+4 -23
View File
@@ -1,3 +1,4 @@
import { savedDesignInstruction, withRenderedCommentReference, validRenderedCommentReference } from "./internal-comment-reference.js";
import { withEditOwnership } from "./internal-edit-ownership.js";
import { verifyDistinctTaskFixtures } from "./internal-distinct-fixtures.js";
import { delegatedTaskCases } from "./internal-task-creation.js";
@@ -10,26 +11,6 @@ import type { ChatCatalogCase } from "../chat-catalog-contract.js";
import { project, task, goal, putDocument, entity, observation, checks, hasLink, sameTask, type Entity, type InternalObservation } from "./internal-support.js";
import { makeDefinition, retryDefinition, fixtureWorker } from "./internal-definition.js";
function savedDesignInstruction(o: InternalObservation, body: string, design: Entity) {
const targets: string[] = [];
const prose = body.replace(/\[[^\]]+\]\(<?([^\s)>]+)>?(?:\s+["'][^"']*["'])?\)/g, (_link, href: string) => {
const token = `{{DESIGNREF${targets.length}}}`; targets.push(href); return token;
});
const valid = targets.map((href, index) => hasLink({ ...o, links: [href] }, "issues", design) ? `{{DESIGNREF${index}}}` : null).filter((x): x is string => !!x);
if (!valid.length) return false;
const directive = /\b(?:follow|use|adhere\s+to|implement\s+(?:according\s+to|in\s+line\s+with)|build\s+(?:according\s+to|in\s+line\s+with))\b/i;
const governsNegativeReference = (prefix: string) => {
const text = prefix.replace(/\b(?:do not|don't) forget to\b/gi, "remember to");
return /\b(?:do not|don't|must not|should not|never|not to|not)\b[^{}.;!\n]{0,60}\b(?:follow|use|adhere|implement|build)\b[^{}.;!\n]{0,80}$/i.test(text)
|| /\b(?:avoid|ignore|disregard|stop)\b[^{}.;!\n]{0,80}$/i.test(text);
};
const clauses = prose.split(/[.;!\n]+/).filter(clause => valid.some(token => clause.includes(token)));
if (clauses.some(clause => valid.some(token => { const at = clause.indexOf(token); return at >= 0 && governsNegativeReference(clause.slice(0, at)); }))) return false;
if (/\b(?:(?:do not|don't|never|must not|should not)\s+(?:follow|use|adhere\s+to)|ignore|disregard)\s+(?:it|this|that|the\s+(?:approved\s+)?design)\b/i.test(prose)) return false;
return clauses.some(clause => directive.test(clause) && valid.some(token => {
const instruction = directive.exec(clause); return !!instruction && clause.indexOf(token) > instruction.index;
}));
}
export function grade_tasks(value: unknown) {
const c = checks(); const o = observation(value);
c.add("complete-independent-observation", !!o, "Missing API, document, audit or rendered-link evidence cannot pass.");
@@ -67,7 +48,7 @@ export function grade_tasks(value: unknown) {
case "tasks.comment-with-reference": {
const design = entity(o.state, "design");
const added = o.comments.filter(row => row.authorAgentId === o.leadId && row.createdByRunId && o.runs.some(run => run.id === row.createdByRunId));
c.add("one-attributed-note", added.length === 1 && savedDesignInstruction(o, added[0]?.body ?? "", design), "One CEO-authored note retains a usable reference to the actual approved design task.");
c.add("one-attributed-note", added.length === 1 && savedDesignInstruction(o, added[0]?.body ?? "", design) && validRenderedCommentReference(o, added[0]!, design), "One CEO-authored note retains a usable reference to the actual approved design task.");
c.add("records-preserved", !!before && sameTask(before, current) && sameTask(design, o.tasks.find(row => row.id === design.id)) && o.tasks.length === 2,
"The note neither changes either task nor creates another task.");
c.add("task-link", linkedTask(), "The original conversation links back to the task containing the note.");
@@ -92,11 +73,11 @@ export const tasksCases: readonly ChatCatalogCase[] = [
const t = await task(ctx, "Landing page", p.id, { description: "Build the Launch landing page; this work is saved for later." });
return { project: p, worker, task: t };
})),
define("tasks.comment-with-reference", async ctx => {
withRenderedCommentReference(define("tasks.comment-with-reference", async ctx => {
const p = await project(ctx); const design = await task(ctx, "Approved design", p.id, { status: "done", description: "Approved design: semantic HTML, clear contrast, one primary signup action." });
const t = await task(ctx, "Landing page", p.id, { description: "Build the launch website." });
return { project: p, design, task: t };
}),
})),
];
tests/runner-e2e/chat-catalog-domains/internal.test.ts
+2 -1
View File
@@ -241,7 +241,8 @@ function valid(scenario: string): InternalObservation {
}
case "tasks.comment-with-reference": {
const design = { id: "design", title: "Approved design", status: "done" }; o.state.design = design; o.tasks.push(design);
o.comments = [{ id: "comment", authorAgentId: "ceo", createdByRunId: "run", body: "Follow the [approved design](/ACORN/issues/design)." }]; break;
o.comments = [{ id: "comment", issueId: "task", authorAgentId: "ceo", createdByRunId: "run", body: "Follow the [approved design](/ACORN/issues/design)." }];
o.state.savedCommentReference = { commentId: "comment", taskId: "task", designId: "design", bodySha256: createHash("sha256").update(o.comments[0]!.body!).digest("hex"), href: "/ACORN/issues/design", visible: true, destinationMatched: true, publicReadMatched: true }; break;
}
case "goals.create-goal-tree": {
o.state = { firstIds: ["growth", "team"], retryQuery: "query" }; o.humanQueries = ["query", "query"]; o.tasks = []; o.projects = [];