Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-05 03:51:54 +01:00
Code Issues Packages Projects Releases Wiki Activity

test(evals): reject modal contradictions of required plan features

Browse Source 
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-27 21:56:43 -05:00
1 parent 021c155682
commit 96dcd15449
2 files changed
+3 -1
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/chat-catalog-domains/internal-document-plan-oracle.test.ts
+2
View File
@@ -4,6 +4,7 @@ import { grade_documents_artifacts, meaningfulOrchidPlan } from "./internal-docu
it.each([
"Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout.",
"Implement Orchid in a single responsive page with accessible signup and do not add unrelated registration flows. Verify keyboard navigation and mobile layout.",
"Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout. The team must not sacrifice accuracy. A previous unrelated site did not support mobile devices.",
"Develop one accessible web page for Orchid registration, using responsive breakpoints. Test keyboard navigation and mobile presentation.",
"Build an accessible single-page Orchid signup website with responsive layout. Verify keyboard navigation and mobile layout; never sacrifice accuracy for brevity.",
"Implement a single responsive page for Orchid with accessible signup. Do not build multiple pages. Verify keyboard navigation and mobile layout.",
@@ -17,6 +18,7 @@ it.each([
"Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout. Build multiple pages instead.",
"Do not implement a single responsive page for Orchid with accessible signup. Develop multiple pages. Verify keyboard navigation and mobile layout.",
"Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout. Never ever include mobile support.",
"Implement Orchid in a single responsive page with accessible signup. Verify keyboard navigation and mobile layout. The website must not support mobile devices.",
"Implement Orchid in a single responsive page without signup. Verify keyboard navigation, accessibility and mobile layout.",
"Implement Orchid in a single responsive page with accessible signup. No need to test keyboard or mobile layout.",
])("rejects missing, foreign or contradictory requirements: %s", body => {
tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts
+1 -1
View File
@@ -125,7 +125,7 @@ import { observeInternal } from "./internal-support.js";
export function meaningfulOrchidPlan(body: string): boolean {
const clauses = body.replace(/[*_`]/g, "").split(/[.!?;\n]|\b(?:but|however|instead)\b|\band\s+(?=(?:do not|don't|never|skip|omit|exclude|no need))/i).filter(Boolean);
const negativeDirective = /\b(?:do not|don't|never)(?:\s+(?:ever|actually|really|even))?\s+(?:build|implement|develop|include|add|provide|support|verify|test|check|make|use|preserve)\b|\b(?:skip|omit|exclude)\b|\bno need to\s+(?:build|implement|develop|include|add|provide|support|verify|test|check|make|use|preserve)\b/i;
const negativeDirective = /\b(?:do not|don't|never|(?:must|should|will|can|shall)\s+not)(?:\s+(?:ever|actually|really|even))?\s+(?:build|implement|develop|include|add|provide|support|verify|test|check|make|use|preserve)\b|\b(?:skip|omit|exclude)\b|\bno need to\s+(?:build|implement|develop|include|add|provide|support|verify|test|check|make|use|preserve)\b/i;
const requirements = [
/\b(?:implement|build|develop)\b/i,
/\b(?:test|verify|verification|validation)\b/i,