test(evals): accept substantive single-page plan paraphrases

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/internal-document-plan-oracle.test.ts

@@ -0,0 +1,40 @@
+import { expect, it } from "vitest";
+import { grade_documents_artifacts, meaningfulOrchidPlan } from "./internal-documents-artifacts.js";
+
+it.each([
+  "Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout.",
+  "Implement Orchid in a single responsive page with accessible signup and do not add unrelated registration flows. Verify keyboard navigation and mobile layout.",
+  "Develop one accessible web page for Orchid registration, using responsive breakpoints. Test keyboard navigation and mobile presentation.",
+  "Build an accessible single-page Orchid signup website with responsive layout. Verify keyboard navigation and mobile layout; never sacrifice accuracy for brevity.",
+  "Implement a single responsive page for Orchid with accessible signup. Do not build multiple pages. Verify keyboard navigation and mobile layout.",
+])("accepts substantive plan requirements despite word order and unrelated negation: %s", body => {
+  expect(meaningfulOrchidPlan(body)).toBe(true);
+});
+it.each([
+  "Implement Birch in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout.",
+  "Implement Orchid in a single responsive page with accessible signup. Verify mobile layout.",
+  "Implement Orchid in a single page with accessible signup. Verify keyboard navigation and mobile layout.",
+  "Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout. Build multiple pages instead.",
+  "Do not implement a single responsive page for Orchid with accessible signup. Develop multiple pages. Verify keyboard navigation and mobile layout.",
+  "Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout. Never ever include mobile support.",
+  "Implement Orchid in a single responsive page without signup. Verify keyboard navigation, accessibility and mobile layout.",
+  "Implement Orchid in a single responsive page with accessible signup. No need to test keyboard or mobile layout.",
+])("rejects missing, foreign or contradictory requirements: %s", body => {
+  expect(meaningfulOrchidPlan(body)).toBe(false);
+});
+
+it("retains real task, saved revision and run-attributed creation requirements", () => {
+  const task = { id: "task", companyId: "company", title: "Website task", status: "backlog", projectId: "project" };
+  const body = "Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout.";
+  const o = { scenarioId: "documents-artifacts.write-plan", companyId: "company", origin: "https://fixture.test", leadId: "ceo",
+    state: { task }, tasks: [task], projects: [], goals: [], comments: [],
+    documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }],
+    revisions: [{ id: "revision", body, createdByAgentId: "ceo", createdByRunId: "run" }],
+    activity: [{ id: "audit", action: "issue.document_created", entityId: task.id, agentId: "ceo" }],
+    reply: "Saved the implementation plan.", links: ["/RUN/issues/task"], humanQueries: [], runs: [{ id: "run", status: "succeeded" }] };
+  expect(grade_documents_artifacts(o).passed).toBe(true);
+  expect(grade_documents_artifacts({ ...o, revisions: [{ ...o.revisions[0], createdByAgentId: "other" }] }).passed).toBe(false);
+  expect(grade_documents_artifacts({ ...o, documents: [] }).passed).toBe(false);
+  expect(grade_documents_artifacts({ ...o, tasks: [{ ...task, status: "done" }] }).passed).toBe(false);
+  expect(grade_documents_artifacts({ ...o, activity: [] }).passed).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts

@@ -124,8 +124,28 @@ function resolveFeedback(): ChatCatalogCase {
 import { observeInternal } from "./internal-support.js";
 
 export function meaningfulOrchidPlan(body: string): boolean {
-  return /\b(?:implement|build|develop)\b/i.test(body) && /\b(?:test|verify|verification|validation)\b/i.test(body)
-    && /\bOrchid\b/i.test(body) && /\b(?:single|one)[- ]page\b/i.test(body)
-    && /\b(?:signup|sign.up|registration)\b/i.test(body) && /\b(?:accessible|accessibility|labels?|semantic)\b/i.test(body)
-    && /\b(?:responsive|breakpoints?)\b/i.test(body) && /\bkeyboard\b/i.test(body) && /\bmobile\b/i.test(body);
+  const clauses = body.replace(/[*_`]/g, "").split(/[.!?;\n]|\b(?:but|however|instead)\b|\band\s+(?=(?:do not|don't|never|skip|omit|exclude|no need))/i).filter(Boolean);
+  const negativeDirective = /\b(?:do not|don't|never)(?:\s+(?:ever|actually|really|even))?\s+(?:build|implement|develop|include|add|provide|support|verify|test|check|make|use|preserve)\b|\b(?:skip|omit|exclude)\b|\bno need to\s+(?:build|implement|develop|include|add|provide|support|verify|test|check|make|use|preserve)\b/i;
+  const requirements = [
+    /\b(?:implement|build|develop)\b/i,
+    /\b(?:test|verify|verification|validation)\b/i,
+    /\bOrchid\b/i,
+    /\b(?:single|one)(?:[- ]+[a-z]+){0,3}[- ]+page\b/i,
+    /\b(?:signup|sign.up|registration)\b/i,
+    /\b(?:accessible|accessibility|labels?|semantic)\b/i,
+    /\b(?:responsive|breakpoints?)\b/i,
+    /\bkeyboard\b/i,
+    /\bmobile\b/i,
+  ];
+  if (!requirements.every(fact => clauses.some(clause => fact.test(clause) && !negativeDirective.test(clause)))) return false;
+  const features = requirements.slice(3);
+  if (clauses.some(clause => {
+    const directive = negativeDirective.exec(clause);
+    if (!directive) return false;
+    const target = clause.slice(directive.index + directive[0].length).trim();
+    return features.some(fact => new RegExp(`^(?:(?:a|an|the|any|our|this)\\s+)?${fact.source}`, "i").test(target));
+  })) return false;
+  if (/\bwithout\s+(?:(?:any|the)\s+)?(?:signup|registration|accessibility|labels?|responsive(?:ness)?|keyboard|mobile)\b/i.test(body)) return false;
+  return !clauses.some(clause => /\b(?:multi[- ]page|multiple\s+pages|separate\s+pages)\b/i.test(clause)
+    && !negativeDirective.test(clause) && !/\b(?:not|no|without|avoid)\s+(?:a\s+)?(?:multi[- ]page|multiple\s+pages|separate\s+pages)\b/i.test(clause));
 }