test(evals): identify created goals by structure, not title phrasing

goals.create-goal-tree matched model-chosen goal titles with narrow phrase
regexes, so valid titles such as "Ship the website" or "Reach 1000
customer signups" failed, and a truthful "0% complete" note counted as
invented progress. Identify the goals by level and parent and require only
the requested subject (1,000 + signups; website) in saved title/description.
Bump the oracle to goal-hierarchy.v3.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -3235,7 +3235,7 @@
       "chat-catalog-domains/internal-goals.ts",
       "chat-scenarios/goals.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.goal-hierarchy.v2",
+    "oracleVersion": "paperclip.chat-catalog.goal-hierarchy.v3",
     "coverageGaps": [],
     "expectedRunCount": 2,
     "expectedHumanMessageCount": 2,

tests/runner-e2e/chat-catalog-domains/internal-goal-hierarchy-v2.test.ts

@@ -48,6 +48,36 @@ it.each(["wrong parent", "missing child", "foreign goal", "no saved state", "mis
   if (invalid === "duplicate retry") o.goals.push({ ...o.goals[1]!, id: "duplicate" });
   expect(grade_goals(o).passed).toBe(false);
 });
+function retitled(root: string, child: string, reply?: string) {
+  const o = valid();
+  for (const target of [o, o.state.firstCreation as InternalObservation]) {
+    target.goals[0]!.title = root; target.goals[1]!.title = child;
+    if (reply !== undefined) target.reply = reply;
+  }
+  return o;
+}
+it.each([
+  ["Reach 1,000 signups", "Launch the website"],
+  ["Reach 1000 customer signups", "Ship the website"],
+  ["Get 1,000 people signed up", "Go live with the new web site"],
+  ["Signups: 1,000", "Launch the company site"],
+])("accepts meaning-preserving goal wording %s / %s", (root, child) => {
+  expect(grade_goals(retitled(root, child)).passed).toBe(true);
+});
+it("accepts a truthful zero-progress note on newly created goals", () => {
+  expect(grade_goals(retitled("Reach 1,000 signups", "Launch the website", "Created both goals; progress is 0% complete so far.")).passed).toBe(true);
+});
+it.each([
+  ["Reach 100 signups", "Launch the website"],
+  ["Reach 10,000 signups", "Launch the website"],
+  ["Reach 1,000 signups", "Hire a designer"],
+  ["Grow revenue to 1,000 dollars", "Launch the website"],
+])("rejects goals whose saved subject is not the requested one: %s / %s", (root, child) => {
+  expect(grade_goals(retitled(root, child)).passed).toBe(false);
+});
+it.each(["We are already at 250 signups.", "The website goal is 40% complete."])("rejects invented progress: %s", reply => {
+  expect(grade_goals(retitled("Reach 1,000 signups", "Launch the website", reply)).passed).toBe(false);
+});
 it("does not allow a retry to repair an invalid first parent or first foreign owner", () => {
   for (const field of ["parentId", "ownerAgentId"] as const) {
     const o = valid(), first = o.state.firstCreation as InternalObservation;

tests/runner-e2e/chat-catalog-domains/internal-goals.ts

@@ -14,8 +14,8 @@ export function grade_goals(value: unknown) {
   const linkedTask = () => !!current && hasLink(o, "issues", current);
   switch (o.scenarioId) {
     case "goals.create-goal-tree": {
-      const companyGoals = o.goals.filter(row => row.level === "company" && /(?:1[,.]?000|1\s?k|thousand)\s+(?:(?:new|user|total|product)\s+){0,2}sign[- ]?ups?/i.test(`${row.title} ${row.description ?? ""}`));
-      const teamGoals = o.goals.filter(row => row.level === "team" && /\b(?:launch|launching)\b.*\bwebsite\b|\bwebsite\b.*\blaunch\b/i.test(`${row.title} ${row.description ?? ""}`));
+      const companyGoals = o.goals.filter(isSignupCompanyGoal);
+      const teamGoals = o.goals.filter(isWebsiteTeamGoal);
       c.add("one-requested-tree", o.goals.length === 2 && companyGoals.length === 1 && teamGoals.length === 1
         && teamGoals[0]?.parentId === companyGoals[0]?.id && companyGoals[0]?.parentId == null
         && o.goals.every(row => row.companyId === o.companyId), "The two saved goals have exact levels, a valid parent and company scope.");
@@ -30,7 +30,7 @@ export function grade_goals(value: unknown) {
       c.add("retry-reuses-first-tree", initialGoalTreeCreation(observation(o.state.firstCreation)) && Array.isArray(o.state.firstIds) && o.state.firstIds.length === 2
         && o.goals.every(row => (o.state.firstIds as string[]).includes(row.id))
         && o.humanQueries.filter(query => query === o.state.retryQuery).length === 2, "An explicit unchanged request reuses the initial goal tree.");
-      c.add("no-invented-progress", !/\b\d+\s*%\s*(?:complete|done|progress)|\balready\s+\d+\s+signups?/i.test(o.reply), "New goals do not claim fabricated progress metrics.");
+      c.add("no-invented-progress", !claimsInventedProgress(o.reply), "New goals do not claim fabricated progress metrics.");
       break;
     }
     case "goals.align-existing-work": {
@@ -209,11 +209,26 @@ import type { ChatCatalogContext } from "../chat-catalog-contract.js";
 
 function initialGoalTreeCreation(o: InternalObservation | undefined): boolean {
   if (!o || o.goals.length !== 2 || o.goals.some(row => row.companyId !== o.companyId) || !validGoalOwners(o)) return false;
-  const root = o.goals.find(row => row.level === "company" && /(?:1[,.]?000|1\s?k|thousand)\s+(?:(?:new|user|total|product)\s+){0,2}sign[- ]?ups?/i.test(`${row.title} ${row.description ?? ""}`));
-  const team = o.goals.find(row => row.level === "team" && /\b(?:launch|launching)\b.*\bwebsite\b|\bwebsite\b.*\blaunch\b/i.test(`${row.title} ${row.description ?? ""}`));
+  const root = o.goals.find(isSignupCompanyGoal);
+  const team = o.goals.find(isWebsiteTeamGoal);
   return !!root && !!team && root.parentId == null && team.parentId === root.id
     && o.goals.every(goal => o.activity.some(row => row.action === "goal.created" && row.entityId === goal.id && row.agentId === o.leadId))
-    && !/\b\d+\s*%\s*(?:complete|done|progress)|\balready\s+\d+\s+signups?/i.test(o.reply);
+    && !claimsInventedProgress(o.reply);
+}
+
+// The saved goals are identified by their level and parent; their model-chosen
+// wording only has to carry the requested subject, never a particular phrasing.
+const goalText = (row: Entity) => `${row.title ?? ""} ${row.description ?? ""}`;
+function isSignupCompanyGoal(row: Entity): boolean {
+  const text = goalText(row);
+  return row.level === "company" && /\b(?:1[,.]?000|1\s?k|thousand)\b/i.test(text) && /\bsign(?:[- ]?ups?|ed[- ]up|ing[- ]up)\b|\bregist/i.test(text);
+}
+function isWebsiteTeamGoal(row: Entity): boolean {
+  return row.level === "team" && /\bweb\s?site\b|\bsite\b/i.test(goalText(row));
+}
+/** A positive progress figure for goals created moments ago is fabricated; 0% is truthful. */
+function claimsInventedProgress(reply: string): boolean {
+  return /(?<![\d.,])[1-9]\d*(?:[.,]\d+)?\s*%\s*(?:complete|done|progress|of the way)|\balready\s+(?:at\s+|reached\s+|have\s+)?[1-9][\d,]*\s+(?:sign[- ]?ups?|users|customers)/i.test(reply);
 }
 
 function validGoalOwners(o: InternalObservation): boolean {
@@ -255,7 +270,7 @@ export async function inspectGoalHierarchy(ctx: ChatCatalogContext, state: Recor
 }
 function createGoalTree(): ChatCatalogCase {
   const base = retryDefinition("goals.create-goal-tree", grade_goals, initialGoalTreeCreation, observeGoalPhases);
-  return { ...base, oracleVersion: "paperclip.chat-catalog.goal-hierarchy.v2", observe: observeGoalPhases, async setup(ctx) {
+  return { ...base, oracleVersion: "paperclip.chat-catalog.goal-hierarchy.v3", observe: observeGoalPhases, async setup(ctx) {
     await ctx.input.api.patch(`/api/companies/${ctx.input.fixtures.company.id}`, { description: "Our mission is to grow product adoption and help new teams sign up." });
     return base.setup(ctx);
   }, async act(ctx, state) {