From 81bba1dd0d4f77448bf4c5e360f3c78760e34ce2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 22:58:35 -0500 Subject: [PATCH] fix(evals): scope design approval claims to contrast clauses --- .../internal-design-search.test.ts | 9 +++++++++ .../internal-design-search.ts | 16 +++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/internal-design-search.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-design-search.test.ts index 4d0bab3a67f..ac874d56d2b 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-design-search.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-design-search.test.ts @@ -25,7 +25,12 @@ describe("approved design search calibration", () => { }); it.each([ "Version 1 with amber navigation is signed off. Version 2 remains pending review.", + "Revision 1 is approved, whereas revision 2 remains an unapproved draft.", + "Revision 1 is approved, but revision 2 is not yet approved.", + "Revision 1 is approved, while revision 2 is an unapproved draft.", "The approved design is v1 with amber navigation. V2 is not yet approved.", + "Revision 1 was approved. Revision 2 is not officially approved and is an unapproved draft.", + "Revision 1 was approved. Revision 2 has never been approved and is an unapproved draft.", "Revision 1 is approved. RUN-2 revision 2 has not been approved and is an unapproved draft.", ])("accepts truthful current-version disclosure: %s", reply => { const o = valid(); o.reply = reply; expect(gradeDesignSearch(o).passed).toBe(true); }); it.each([ @@ -36,6 +41,10 @@ describe("approved design search calibration", () => { "Revision 1 is approved. Revision 1 is an unapproved draft.", "Revision 1 is approved. The newer draft is no longer unapproved.", "Revision 1 is approved. Revision 2 is an unapproved draft. Revision 2 is approved.", + "Revision 1 is approved. Revision 2 is an unapproved draft. Revision 2 approved.", + "Revision 1 is approved. Revision 2 is an unapproved draft. Also approved: revision 2.", + "Revision 1 is approved. Revision 2 is an unapproved draft. We did not choose a filename, but revision 2 is approved.", + "Revision 1 is approved. Revision 2 is an unapproved draft. V2 has been signed off.", ])("rejects unrelated, wrong-version, negated or contradictory disclosure: %s", reply => { const o = valid(); o.reply = reply; expect(gradeDesignSearch(o).passed).toBe(false); }); it("retains exact linked target and historical approval checks with explicit revision ordering", () => { const o = valid(); o.reply = "Revision 1 is approved. Revision 2 is an unapproved draft."; o.links = ["/RUN/issues/other"]; expect(gradeDesignSearch(o).passed).toBe(false); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-design-search.ts b/tests/runner-e2e/chat-catalog-domains/internal-design-search.ts index 1c1e6153b8e..e61bf6743cb 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-design-search.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-design-search.ts @@ -7,8 +7,18 @@ import { checks, entity, hasLink, observation, observeInternal, project, putDocu interface DesignObservation extends InternalObservation { approvedRevision?: Entity; approvalComments: Entity[]; olderApprovalComments: Entity[]; privateReadStatus: number; agentSearchStatus: number; agentSearchResults: unknown[]; } +function designClauses(reply: string) { return reply.split(/[.\n;]|\b(?:whereas|while|but)\b/i); } +function contradictsDraftApproval(reply: string, target: Entity) { + return designClauses(reply).some(part => { + const taskReferences = part.match(/\b[A-Z][A-Z0-9]*-\d+\b/g) ?? []; + if (taskReferences.some(id => id !== target.identifier)) return false; + if (!/\b(?:revision\s*2|version\s*2|v2|violet\s+navigation)\b/i.test(part)) return false; + return [...part.matchAll(/\b(?:approved|signed[ -]off)\b/gi)].some(match => + !/(?:\b(?:not|never)|\bno longer|\b(?:hasn't|haven't|wasn't|weren't|isn't|aren't))(?:\s+(?:yet|been|ever|officially|currently|formally))*\s*$/i.test(part.slice(0, match.index))); + }); +} function describesUnapprovedCurrentDraft(reply: string, target: Entity) { - return reply.split(/[.\n;]/).some(part => { + return designClauses(reply).some(part => { const taskReferences = part.match(/\b[A-Z][A-Z0-9]*-\d+\b/g) ?? []; if (taskReferences.length && (!target.identifier || taskReferences.some(id => id !== target.identifier))) return false; const versions = [...part.matchAll(/\b(?:revision|version)\s*(\d+)\b|\bv(\d+)\b/gi)].map(match => Number(match[1] ?? match[2])); @@ -31,11 +41,11 @@ export function gradeDesignSearch(value: unknown) { && o.approvalComments.some(row => row.id === (o.state.approvalComment as Entity | undefined)?.id && row.authorUserId === o.state.ownerId && row.body === (o.state.approvalComment as Entity).body), "The exact historical revision and real human approval comment remain saved rather than inferring approval from latest edit time."); c.add("approved-design-distinguished-from-newer-draft", !!target && hasLink(o, "issues", target) - && o.reply.split(/[.\n]/).some(sentence => /\b(?:approved|signed[ -]off)\b/i.test(sentence) + && designClauses(o.reply).some(sentence => /\b(?:approved|signed[ -]off)\b/i.test(sentence) && /\b(?:revision\s*1|version\s*1|v1|amber\s+navigation)/i.test(sentence) && !/\b(?:not|never|no longer)\s+(?:yet\s+)?(?:approved|signed[ -]off)\b|\bunapproved\b/i.test(sentence)) && describesUnapprovedCurrentDraft(o.reply, target) - && !/\b(?:violet\s+navigation|revision\s*2|version\s*2|v2)\b[^.\n]*(?:is|was|has been)\s+(?:the\s+)?(?:latest\s+)?approved/i.test(o.reply), + && !contradictsDraftApproval(o.reply, target), "The answer links the actual approved design and explains that the later revision is still a draft."); c.add("read-only-design-search", !!originals && originals.every(row => sameTask(row, o.tasks.find(now => now.id === row.id))) && current?.latestRevisionId === o.state.draftRevisionId && current?.body === o.state.draftBody