test(evals): ground team requirements in actual skill identities

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/external-team-requirements.test.ts

@@ -0,0 +1,47 @@
+import { expect, it } from "vitest";
+import type { CatalogTeam } from "../../../packages/shared/src/types/teams-catalog.js";
+import { describesTeamSkillRequirements, gradeTeams } from "./external-teams.js";
+
+const keys = ["paperclipai/bundled/software-development/github-pr-workflow", "paperclipai/bundled/quality/qa-acceptance",
+  "paperclipai/bundled/paperclip-operations/task-planning", "paperclipai/bundled/docs/doc-maintenance"];
+const team: Pick<CatalogTeam, "requiredSkills"> = { requiredSkills: keys.map(ref => ({ type: "catalog", ref, agentSlugs: [], resolved: true, catalogSkillKey: ref })) };
+const list = "github-pr-workflow, qa-acceptance, task-planning, and doc-maintenance";
+it.each([
+  `It requires ${list}. Accounts still need configuration.`,
+  `Four required skills: ${list}.`,
+  `Required: ${keys.join(", ")}.`,
+  "Requirements: GitHub PR workflow, QA acceptance, task planning and doc maintenance. They do not prove accounts are ready.",
+])("accepts every actual skill identity without requiring an incidental numeral: %s", reply => {
+  expect(describesTeamSkillRequirements(team, reply)).toBe(true);
+});
+it.each([
+  "Required: github-pr-workflow, qa-acceptance and task-planning.",
+  "It needs four skills: foreign/workflows/github-pr-workflow, foreign/quality/qa-acceptance, foreign/planning/task-planning, and foreign/docs/doc-maintenance.",
+  `It requires three skills: ${list}.`,
+  `No required skills. Available examples are ${list}.`,
+  `It does not require ${list}.`,
+  `It requires ${list}. qa-acceptance is optional.`,
+  `It requires ${list}. Never ever include doc-maintenance.`,
+])("rejects missing, foreign, contradictory or falsely counted requirements: %s", reply => {
+  expect(describesTeamSkillRequirements(team, reply)).toBe(false);
+});
+it("does not use an ambiguous short name to stand in for two different real skills", () => {
+  const duplicate = { requiredSkills: [...team.requiredSkills, { ...team.requiredSkills[0]!, ref: "other/catalog/github-pr-workflow", catalogSkillKey: "other/catalog/github-pr-workflow" }] };
+  expect(describesTeamSkillRequirements(duplicate, `Required: ${list}.`)).toBe(false);
+  expect(describesTeamSkillRequirements(duplicate, `Required: ${keys.join(", ")}, other/catalog/github-pr-workflow.`)).toBe(true);
+});
+it("retains truthful empty requirements without accepting an invented count", () => {
+  expect(describesTeamSkillRequirements({ requiredSkills: [] }, "No required skills.")).toBe(true);
+  expect(describesTeamSkillRequirements({ requiredSkills: [] }, "Four required skills.")).toBe(false);
+});
+
+it("uses actual typed skill requirements in the full grader without weakening package or noninstallation checks", () => {
+  const selected = { ...team, envInputs: [], id: "package", name: "Product Engineering", compatibility: "compatible", agentSlugs: ["cto", "senior-coder", "qa"], counts: { projects: 1 } };
+  const o = { scenarioId: "teams.find-team", companyId: "company", reply: `Product Engineering includes a CTO, Senior Coder and QA. Required: ${list}.`,
+    state: { team: selected, installedBefore: [], installedAfter: [], agentIdsBefore: ["ceo"], agentIdsAfter: ["ceo"], projectIdsBefore: [], projectIdsAfter: [] },
+    phaseReplies: [], connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], activity: [], links: [], runs: [], humanQueries: [] };
+  expect(gradeTeams(o).passed).toBe(true);
+  expect(gradeTeams({ ...o, state: { ...o.state, agentIdsAfter: ["ceo", "new-agent"] } }).passed).toBe(false);
+  expect(gradeTeams({ ...o, reply: `Product Engineering includes Senior Coder and QA. Required: ${list}.` }).passed).toBe(false);
+  expect(gradeTeams({ ...o, reply: `Product Engineering includes a CTO, Senior Coder and QA. Required: ${list.replace("qa-acceptance", "foreign/quality/qa-acceptance")}.` }).passed).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/external-teams.ts

@@ -9,7 +9,7 @@ export function gradeTeams(value: unknown) {
     && o.reply.includes(team.name) && team.agentSlugs.every(slug => o.reply.toLowerCase().includes(slug.toLowerCase().replaceAll("-", " "))
       || o.reply.toLowerCase().includes(slug.toLowerCase())), "The response names the real package and every included role.");
   c.add("package-requirements-described", !!team && Array.isArray(team.requiredSkills) && Array.isArray(team.envInputs)
-    && (team.requiredSkills.length === 0 || new RegExp(`\\b${team.requiredSkills.length}\\s+(?:required\\s+|catalog\\s+)?skills?\\b`, "i").test(o.reply))
+    && describesTeamSkillRequirements(team, o.reply)
     && team.envInputs.every(input => o.reply.includes(input.key)), "Visible skill and configuration requirements are included in the explanation.");
   c.add("no-premature-installation", JSON.stringify(o.state.installedBefore) === JSON.stringify(o.state.installedAfter)
     && JSON.stringify(o.state.agentIdsBefore) === JSON.stringify(o.state.agentIdsAfter)
@@ -50,3 +50,25 @@ export const teamExternalCases = ["find-team", "preview-install"].map(id => defi
     return observeExternal(ctx, state);
   },
 }));
+
+export function describesTeamSkillRequirements(team: Pick<CatalogTeam, "requiredSkills">, reply: string): boolean {
+  const plain = reply.replace(/[*`]/g, "");
+  const numbers: Record<string, number> = { no: 0, zero: 0, one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, ten: 10 };
+  for (const match of plain.matchAll(/\b(\d+|no|zero|one|two|three|four|five|six|seven|eight|nine|ten)\s+(?:(?:required|catalog)\s+)?skills?\b/gi)) {
+    if ((numbers[match[1]!.toLowerCase()] ?? Number(match[1])) !== team.requiredSkills.length) return false;
+  }
+  const leaf = (ref: string) => ref.split("/").at(-1)!;
+  const escaped = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const clauses = plain.split(/[.;\n]|\b(?:but|however)\b/i).filter(Boolean);
+  return team.requiredSkills.every(skill => {
+    const refs = [...new Set([skill.ref, skill.catalogSkillKey].filter((ref): ref is string => !!ref))];
+    const short = leaf(skill.catalogSkillKey ?? skill.ref);
+    const unambiguous = team.requiredSkills.filter(other => leaf(other.catalogSkillKey ?? other.ref) === short).length === 1;
+    const aliases = [...refs, ...(unambiguous ? [short, short.replaceAll("-", " ")] : [])];
+    const identity = `(?<![\\w/.-])(?:${aliases.map(escaped).join("|")})(?![\\w/.-])`;
+    const mentions = new RegExp(identity, "i");
+    const negated = new RegExp(`\\b(?:do(?:es)? not|don't|doesn't|never)(?:\\s+(?:ever|actually|really|even))?\\s+(?:require|need|include|use)\\b[^.;]*${identity}|\\b(?:skip|omit|exclude|without)\\s+(?:the\\s+)?${identity}|${identity}\\s+(?:(?:is|are)\\s+)?(?:not required|optional|unnecessary)\\b`, "i");
+    return clauses.some(clause => mentions.test(clause) && !negated.test(clause))
+      && !clauses.some(clause => negated.test(clause));
+  });
+}