From 96dcd154491aa9bc085a65bbbf242bdefc0a983e Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 21:46:44 -0500 Subject: [PATCH] test(evals): reject modal contradictions of required plan features Co-Authored-By: Paperclip --- .../chat-catalog-domains/internal-document-plan-oracle.test.ts | 2 ++ .../chat-catalog-domains/internal-documents-artifacts.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-catalog-domains/internal-document-plan-oracle.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-document-plan-oracle.test.ts index aacd882bee0..eebecbcef1f 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-document-plan-oracle.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-document-plan-oracle.test.ts @@ -4,6 +4,7 @@ import { grade_documents_artifacts, meaningfulOrchidPlan } from "./internal-docu it.each([ "Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout.", "Implement Orchid in a single responsive page with accessible signup and do not add unrelated registration flows. Verify keyboard navigation and mobile layout.", + "Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout. The team must not sacrifice accuracy. A previous unrelated site did not support mobile devices.", "Develop one accessible web page for Orchid registration, using responsive breakpoints. Test keyboard navigation and mobile presentation.", "Build an accessible single-page Orchid signup website with responsive layout. Verify keyboard navigation and mobile layout; never sacrifice accuracy for brevity.", "Implement a single responsive page for Orchid with accessible signup. Do not build multiple pages. Verify keyboard navigation and mobile layout.", @@ -17,6 +18,7 @@ it.each([ "Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout. Build multiple pages instead.", "Do not implement a single responsive page for Orchid with accessible signup. Develop multiple pages. Verify keyboard navigation and mobile layout.", "Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout. Never ever include mobile support.", + "Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout. The website must not support mobile devices.", "Implement Orchid in a single responsive page without signup. Verify keyboard navigation, accessibility and mobile layout.", "Implement Orchid in a single responsive page with accessible signup. No need to test keyboard or mobile layout.", ])("rejects missing, foreign or contradictory requirements: %s", body => { diff --git a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts index d9d00a5c7b6..0b163da408b 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts @@ -125,7 +125,7 @@ import { observeInternal } from "./internal-support.js"; export function meaningfulOrchidPlan(body: string): boolean { const clauses = body.replace(/[*_`]/g, "").split(/[.!?;\n]|\b(?:but|however|instead)\b|\band\s+(?=(?:do not|don't|never|skip|omit|exclude|no need))/i).filter(Boolean); - const negativeDirective = /\b(?:do not|don't|never)(?:\s+(?:ever|actually|really|even))?\s+(?:build|implement|develop|include|add|provide|support|verify|test|check|make|use|preserve)\b|\b(?:skip|omit|exclude)\b|\bno need to\s+(?:build|implement|develop|include|add|provide|support|verify|test|check|make|use|preserve)\b/i; + const negativeDirective = /\b(?:do not|don't|never|(?:must|should|will|can|shall)\s+not)(?:\s+(?:ever|actually|really|even))?\s+(?:build|implement|develop|include|add|provide|support|verify|test|check|make|use|preserve)\b|\b(?:skip|omit|exclude)\b|\bno need to\s+(?:build|implement|develop|include|add|provide|support|verify|test|check|make|use|preserve)\b/i; const requirements = [ /\b(?:implement|build|develop)\b/i, /\b(?:test|verify|verification|validation)\b/i,