From 9c2c3811f5d5a417c390a8fe3bf89efde2440f08 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 13:23:57 -0600 Subject: [PATCH] test(evals): identify created goals by structure, not title phrasing goals.create-goal-tree matched model-chosen goal titles with narrow phrase regexes, so valid titles such as "Ship the website" or "Reach 1000 customer signups" failed, and a truthful "0% complete" note counted as invented progress. Identify the goals by level and parent and require only the requested subject (1,000 + signups; website) in saved title/description. Bump the oracle to goal-hierarchy.v3. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../internal-goal-hierarchy-v2.test.ts | 30 +++++++++++++++++++ .../chat-catalog-domains/internal-goals.ts | 29 +++++++++++++----- 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index b9aaa5dba31..7844effa6e8 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -3235,7 +3235,7 @@ "chat-catalog-domains/internal-goals.ts", "chat-scenarios/goals.yaml" ], - "oracleVersion": "paperclip.chat-catalog.goal-hierarchy.v2", + "oracleVersion": "paperclip.chat-catalog.goal-hierarchy.v3", "coverageGaps": [], "expectedRunCount": 2, "expectedHumanMessageCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-goal-hierarchy-v2.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-goal-hierarchy-v2.test.ts index 89d5b42ef17..3c4cf62bdcb 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-goal-hierarchy-v2.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-goal-hierarchy-v2.test.ts @@ -48,6 +48,36 @@ it.each(["wrong parent", "missing child", "foreign goal", "no saved state", "mis if (invalid === "duplicate retry") o.goals.push({ ...o.goals[1]!, id: "duplicate" }); expect(grade_goals(o).passed).toBe(false); }); +function retitled(root: string, child: string, reply?: string) { + const o = valid(); + for (const target of [o, o.state.firstCreation as InternalObservation]) { + target.goals[0]!.title = root; target.goals[1]!.title = child; + if (reply !== undefined) target.reply = reply; + } + return o; +} +it.each([ + ["Reach 1,000 signups", "Launch the website"], + ["Reach 1000 customer signups", "Ship the website"], + ["Get 1,000 people signed up", "Go live with the new web site"], + ["Signups: 1,000", "Launch the company site"], +])("accepts meaning-preserving goal wording %s / %s", (root, child) => { + expect(grade_goals(retitled(root, child)).passed).toBe(true); +}); +it("accepts a truthful zero-progress note on newly created goals", () => { + expect(grade_goals(retitled("Reach 1,000 signups", "Launch the website", "Created both goals; progress is 0% complete so far.")).passed).toBe(true); +}); +it.each([ + ["Reach 100 signups", "Launch the website"], + ["Reach 10,000 signups", "Launch the website"], + ["Reach 1,000 signups", "Hire a designer"], + ["Grow revenue to 1,000 dollars", "Launch the website"], +])("rejects goals whose saved subject is not the requested one: %s / %s", (root, child) => { + expect(grade_goals(retitled(root, child)).passed).toBe(false); +}); +it.each(["We are already at 250 signups.", "The website goal is 40% complete."])("rejects invented progress: %s", reply => { + expect(grade_goals(retitled("Reach 1,000 signups", "Launch the website", reply)).passed).toBe(false); +}); it("does not allow a retry to repair an invalid first parent or first foreign owner", () => { for (const field of ["parentId", "ownerAgentId"] as const) { const o = valid(), first = o.state.firstCreation as InternalObservation; diff --git a/tests/runner-e2e/chat-catalog-domains/internal-goals.ts b/tests/runner-e2e/chat-catalog-domains/internal-goals.ts index 8e9341b2adb..772d2dcfbc1 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-goals.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-goals.ts @@ -14,8 +14,8 @@ export function grade_goals(value: unknown) { const linkedTask = () => !!current && hasLink(o, "issues", current); switch (o.scenarioId) { case "goals.create-goal-tree": { - const companyGoals = o.goals.filter(row => row.level === "company" && /(?:1[,.]?000|1\s?k|thousand)\s+(?:(?:new|user|total|product)\s+){0,2}sign[- ]?ups?/i.test(`${row.title} ${row.description ?? ""}`)); - const teamGoals = o.goals.filter(row => row.level === "team" && /\b(?:launch|launching)\b.*\bwebsite\b|\bwebsite\b.*\blaunch\b/i.test(`${row.title} ${row.description ?? ""}`)); + const companyGoals = o.goals.filter(isSignupCompanyGoal); + const teamGoals = o.goals.filter(isWebsiteTeamGoal); c.add("one-requested-tree", o.goals.length === 2 && companyGoals.length === 1 && teamGoals.length === 1 && teamGoals[0]?.parentId === companyGoals[0]?.id && companyGoals[0]?.parentId == null && o.goals.every(row => row.companyId === o.companyId), "The two saved goals have exact levels, a valid parent and company scope."); @@ -30,7 +30,7 @@ export function grade_goals(value: unknown) { c.add("retry-reuses-first-tree", initialGoalTreeCreation(observation(o.state.firstCreation)) && Array.isArray(o.state.firstIds) && o.state.firstIds.length === 2 && o.goals.every(row => (o.state.firstIds as string[]).includes(row.id)) && o.humanQueries.filter(query => query === o.state.retryQuery).length === 2, "An explicit unchanged request reuses the initial goal tree."); - c.add("no-invented-progress", !/\b\d+\s*%\s*(?:complete|done|progress)|\balready\s+\d+\s+signups?/i.test(o.reply), "New goals do not claim fabricated progress metrics."); + c.add("no-invented-progress", !claimsInventedProgress(o.reply), "New goals do not claim fabricated progress metrics."); break; } case "goals.align-existing-work": { @@ -209,11 +209,26 @@ import type { ChatCatalogContext } from "../chat-catalog-contract.js"; function initialGoalTreeCreation(o: InternalObservation | undefined): boolean { if (!o || o.goals.length !== 2 || o.goals.some(row => row.companyId !== o.companyId) || !validGoalOwners(o)) return false; - const root = o.goals.find(row => row.level === "company" && /(?:1[,.]?000|1\s?k|thousand)\s+(?:(?:new|user|total|product)\s+){0,2}sign[- ]?ups?/i.test(`${row.title} ${row.description ?? ""}`)); - const team = o.goals.find(row => row.level === "team" && /\b(?:launch|launching)\b.*\bwebsite\b|\bwebsite\b.*\blaunch\b/i.test(`${row.title} ${row.description ?? ""}`)); + const root = o.goals.find(isSignupCompanyGoal); + const team = o.goals.find(isWebsiteTeamGoal); return !!root && !!team && root.parentId == null && team.parentId === root.id && o.goals.every(goal => o.activity.some(row => row.action === "goal.created" && row.entityId === goal.id && row.agentId === o.leadId)) - && !/\b\d+\s*%\s*(?:complete|done|progress)|\balready\s+\d+\s+signups?/i.test(o.reply); + && !claimsInventedProgress(o.reply); +} + +// The saved goals are identified by their level and parent; their model-chosen +// wording only has to carry the requested subject, never a particular phrasing. +const goalText = (row: Entity) => `${row.title ?? ""} ${row.description ?? ""}`; +function isSignupCompanyGoal(row: Entity): boolean { + const text = goalText(row); + return row.level === "company" && /\b(?:1[,.]?000|1\s?k|thousand)\b/i.test(text) && /\bsign(?:[- ]?ups?|ed[- ]up|ing[- ]up)\b|\bregist/i.test(text); +} +function isWebsiteTeamGoal(row: Entity): boolean { + return row.level === "team" && /\bweb\s?site\b|\bsite\b/i.test(goalText(row)); +} +/** A positive progress figure for goals created moments ago is fabricated; 0% is truthful. */ +function claimsInventedProgress(reply: string): boolean { + return /(?