From 913ffeec7c1a82b53dc295a42d433796fa3fa50c Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 21:42:15 -0500 Subject: [PATCH] test(evals): accept substantive single-page plan paraphrases Co-Authored-By: Paperclip --- .../internal-document-plan-oracle.test.ts | 40 +++++++++++++++++++ .../internal-documents-artifacts.ts | 28 +++++++++++-- 2 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/internal-document-plan-oracle.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/internal-document-plan-oracle.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-document-plan-oracle.test.ts new file mode 100644 index 00000000000..aacd882bee0 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-document-plan-oracle.test.ts @@ -0,0 +1,40 @@ +import { expect, it } from "vitest"; +import { grade_documents_artifacts, meaningfulOrchidPlan } from "./internal-documents-artifacts.js"; + +it.each([ + "Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout.", + "Implement Orchid in a single responsive page with accessible signup and do not add unrelated registration flows. Verify keyboard navigation and mobile layout.", + "Develop one accessible web page for Orchid registration, using responsive breakpoints. Test keyboard navigation and mobile presentation.", + "Build an accessible single-page Orchid signup website with responsive layout. Verify keyboard navigation and mobile layout; never sacrifice accuracy for brevity.", + "Implement a single responsive page for Orchid with accessible signup. Do not build multiple pages. Verify keyboard navigation and mobile layout.", +])("accepts substantive plan requirements despite word order and unrelated negation: %s", body => { + expect(meaningfulOrchidPlan(body)).toBe(true); +}); +it.each([ + "Implement Birch in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout.", + "Implement Orchid in a single responsive page with accessible signup. Verify mobile layout.", + "Implement Orchid in a single page with accessible signup. Verify keyboard navigation and mobile layout.", + "Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout. Build multiple pages instead.", + "Do not implement a single responsive page for Orchid with accessible signup. Develop multiple pages. Verify keyboard navigation and mobile layout.", + "Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout. Never ever include mobile support.", + "Implement Orchid in a single responsive page without signup. Verify keyboard navigation, accessibility and mobile layout.", + "Implement Orchid in a single responsive page with accessible signup. No need to test keyboard or mobile layout.", +])("rejects missing, foreign or contradictory requirements: %s", body => { + expect(meaningfulOrchidPlan(body)).toBe(false); +}); + +it("retains real task, saved revision and run-attributed creation requirements", () => { + const task = { id: "task", companyId: "company", title: "Website task", status: "backlog", projectId: "project" }; + const body = "Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout."; + const o = { scenarioId: "documents-artifacts.write-plan", companyId: "company", origin: "https://fixture.test", leadId: "ceo", + state: { task }, tasks: [task], projects: [], goals: [], comments: [], + documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], + revisions: [{ id: "revision", body, createdByAgentId: "ceo", createdByRunId: "run" }], + activity: [{ id: "audit", action: "issue.document_created", entityId: task.id, agentId: "ceo" }], + reply: "Saved the implementation plan.", links: ["/RUN/issues/task"], humanQueries: [], runs: [{ id: "run", status: "succeeded" }] }; + expect(grade_documents_artifacts(o).passed).toBe(true); + expect(grade_documents_artifacts({ ...o, revisions: [{ ...o.revisions[0], createdByAgentId: "other" }] }).passed).toBe(false); + expect(grade_documents_artifacts({ ...o, documents: [] }).passed).toBe(false); + expect(grade_documents_artifacts({ ...o, tasks: [{ ...task, status: "done" }] }).passed).toBe(false); + expect(grade_documents_artifacts({ ...o, activity: [] }).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts index fd30ff04666..d9d00a5c7b6 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts @@ -124,8 +124,28 @@ function resolveFeedback(): ChatCatalogCase { import { observeInternal } from "./internal-support.js"; export function meaningfulOrchidPlan(body: string): boolean { - return /\b(?:implement|build|develop)\b/i.test(body) && /\b(?:test|verify|verification|validation)\b/i.test(body) - && /\bOrchid\b/i.test(body) && /\b(?:single|one)[- ]page\b/i.test(body) - && /\b(?:signup|sign.up|registration)\b/i.test(body) && /\b(?:accessible|accessibility|labels?|semantic)\b/i.test(body) - && /\b(?:responsive|breakpoints?)\b/i.test(body) && /\bkeyboard\b/i.test(body) && /\bmobile\b/i.test(body); + const clauses = body.replace(/[*_`]/g, "").split(/[.!?;\n]|\b(?:but|however|instead)\b|\band\s+(?=(?:do not|don't|never|skip|omit|exclude|no need))/i).filter(Boolean); + const negativeDirective = /\b(?:do not|don't|never)(?:\s+(?:ever|actually|really|even))?\s+(?:build|implement|develop|include|add|provide|support|verify|test|check|make|use|preserve)\b|\b(?:skip|omit|exclude)\b|\bno need to\s+(?:build|implement|develop|include|add|provide|support|verify|test|check|make|use|preserve)\b/i; + const requirements = [ + /\b(?:implement|build|develop)\b/i, + /\b(?:test|verify|verification|validation)\b/i, + /\bOrchid\b/i, + /\b(?:single|one)(?:[- ]+[a-z]+){0,3}[- ]+page\b/i, + /\b(?:signup|sign.up|registration)\b/i, + /\b(?:accessible|accessibility|labels?|semantic)\b/i, + /\b(?:responsive|breakpoints?)\b/i, + /\bkeyboard\b/i, + /\bmobile\b/i, + ]; + if (!requirements.every(fact => clauses.some(clause => fact.test(clause) && !negativeDirective.test(clause)))) return false; + const features = requirements.slice(3); + if (clauses.some(clause => { + const directive = negativeDirective.exec(clause); + if (!directive) return false; + const target = clause.slice(directive.index + directive[0].length).trim(); + return features.some(fact => new RegExp(`^(?:(?:a|an|the|any|our|this)\\s+)?${fact.source}`, "i").test(target)); + })) return false; + if (/\bwithout\s+(?:(?:any|the)\s+)?(?:signup|registration|accessibility|labels?|responsive(?:ness)?|keyboard|mobile)\b/i.test(body)) return false; + return !clauses.some(clause => /\b(?:multi[- ]page|multiple\s+pages|separate\s+pages)\b/i.test(clause) + && !negativeDirective.test(clause) && !/\b(?:not|no|without|avoid)\s+(?:a\s+)?(?:multi[- ]page|multiple\s+pages|separate\s+pages)\b/i.test(clause)); }