diff --git a/tests/runner-e2e/chat-catalog-domains/internal-comment-reference.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-comment-reference.test.ts new file mode 100644 index 00000000000..02d57eb01af --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-comment-reference.test.ts @@ -0,0 +1,12 @@ +import { expect, it } from "vitest"; +import { grade_tasks } from "./internal-tasks.js"; +import type { InternalObservation } from "./internal-support.js"; +function observation(body: string): InternalObservation { + const task = { id: "task", companyId: "company", title: "Landing page", status: "backlog" }, design = { id: "design", identifier: "ACO-2", companyId: "company", title: "Approved design", status: "done" }; + return { scenarioId: "tasks.comment-with-reference", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(task), design: structuredClone(design) }, tasks: [task, design], projects: [], goals: [], documents: [], revisions: [], activity: [], reply: "Note added.", humanQueries: [], links: ["/ACO/issues/task"], comments: [{ id: "note", authorAgentId: "ceo", createdByRunId: "run", body }], runs: [{ id: "run", status: "succeeded" }] }; +} +for (const body of ["Follow [the approved design](/ACO/issues/design).", "Please use [this design](https://fixture.test/ACO/issues/ACO-2) for implementation.", "Adhere to [the approved design](/ACO/issues/design?tab=documents).", "Implement according to [the design](/ACO/issues/design).", "Build in line with [the approved design](/ACO/issues/design)."]) + it(`accepts affirmative actual design instruction: ${body}`, () => expect(grade_tasks(observation(body)).passed).toBe(true)); +for (const body of ["Do not follow [the design](/ACO/issues/design).", "Never use [the design](/ACO/issues/design).", "Follow [the wrong task](/ACO/issues/other); the intended design ID is design.", "Follow [the design](https://foreign.test/ACO/issues/design).", "Follow [the design](/ACO/issues/design), but do not follow it.", "Here is [the design](/ACO/issues/design).", "Follow [another task](/ACO/issues/other). Reference [the design](/ACO/issues/design)."]) + it(`rejects negated, unrelated or merely mentioned reference: ${body}`, () => expect(grade_tasks(observation(body)).passed).toBe(false)); +it("preserves exact note authorship and duplicate rejection", () => { const o = observation("Follow [the design](/ACO/issues/design)."); o.comments[0]!.authorAgentId = "other"; expect(grade_tasks(o).passed).toBe(false); o.comments[0]!.authorAgentId = "ceo"; o.comments.push(structuredClone(o.comments[0]!)); expect(grade_tasks(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts index e625b449e63..d27b86ad1b3 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts @@ -6,9 +6,25 @@ import { taskAttachmentCases } from "./internal-task-attachment.js"; import { taskCompletionCases } from "./internal-task-completion.js"; import { privateDelegationCases } from "./internal-private-delegation.js"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; -import { project, task, goal, putDocument, entity, observation, checks, hasLink, sameTask, type Entity } from "./internal-support.js"; +import { project, task, goal, putDocument, entity, observation, checks, hasLink, sameTask, type Entity, type InternalObservation } from "./internal-support.js"; import { makeDefinition, retryDefinition, fixtureWorker } from "./internal-definition.js"; +function savedDesignInstruction(o: InternalObservation, body: string, design: Entity) { + const targets: string[] = []; + const prose = body.replace(/\[[^\]]+\]\(]+)>?(?:\s+["'][^"']*["'])?\)/g, (_link, href: string) => { + const token = `{{DESIGNREF${targets.length}}}`; targets.push(href); return token; + }); + const valid = targets.map((href, index) => hasLink({ ...o, links: [href] }, "issues", design) ? `{{DESIGNREF${index}}}` : null).filter((x): x is string => !!x); + if (!valid.length) return false; + const directive = /\b(?:follow|use|adhere\s+to|implement\s+(?:according\s+to|in\s+line\s+with)|build\s+(?:according\s+to|in\s+line\s+with))\b/i; + const negative = /\b(?:do not|don't|must not|should not|never|not to|avoid|ignore|disregard|stop)\b/i; + const clauses = prose.split(/[.;!\n]+/).filter(clause => valid.some(token => clause.includes(token))); + if (clauses.some(clause => negative.test(clause))) return false; + if (/\b(?:do not|don't|never|must not|should not)\s+(?:follow|use|adhere\s+to)\s+(?:it|this|that|the\s+(?:approved\s+)?design)\b/i.test(prose)) return false; + return clauses.some(clause => directive.test(clause) && valid.some(token => { + const instruction = directive.exec(clause); return !!instruction && clause.indexOf(token) > instruction.index; + })); +} export function grade_tasks(value: unknown) { const c = checks(); const o = observation(value); c.add("complete-independent-observation", !!o, "Missing API, document, audit or rendered-link evidence cannot pass."); @@ -46,9 +62,7 @@ export function grade_tasks(value: unknown) { case "tasks.comment-with-reference": { const design = entity(o.state, "design"); const added = o.comments.filter(row => row.authorAgentId === o.leadId && row.createdByRunId && o.runs.some(run => run.id === row.createdByRunId)); - c.add("one-attributed-note", added.length === 1 && /\bfollow\b/i.test(added[0]?.body ?? "") - && [design.id, design.identifier].filter(Boolean).some(id => (added[0]?.body ?? "").includes(id!)) - && /\[[^\]]+\]\([^)]*\/issues\//.test(added[0]?.body ?? ""), "One CEO-authored note retains a usable reference to the actual approved design task."); + c.add("one-attributed-note", added.length === 1 && savedDesignInstruction(o, added[0]?.body ?? "", design), "One CEO-authored note retains a usable reference to the actual approved design task."); c.add("records-preserved", !!before && sameTask(before, current) && sameTask(design, o.tasks.find(row => row.id === design.id)) && o.tasks.length === 2, "The note neither changes either task nor creates another task."); c.add("task-link", linkedTask(), "The original conversation links back to the task containing the note.");