diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-requirements.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-requirements.test.ts new file mode 100644 index 00000000000..a3648456aa9 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-team-requirements.test.ts @@ -0,0 +1,47 @@ +import { expect, it } from "vitest"; +import type { CatalogTeam } from "../../../packages/shared/src/types/teams-catalog.js"; +import { describesTeamSkillRequirements, gradeTeams } from "./external-teams.js"; + +const keys = ["paperclipai/bundled/software-development/github-pr-workflow", "paperclipai/bundled/quality/qa-acceptance", + "paperclipai/bundled/paperclip-operations/task-planning", "paperclipai/bundled/docs/doc-maintenance"]; +const team: Pick = { requiredSkills: keys.map(ref => ({ type: "catalog", ref, agentSlugs: [], resolved: true, catalogSkillKey: ref })) }; +const list = "github-pr-workflow, qa-acceptance, task-planning, and doc-maintenance"; +it.each([ + `It requires ${list}. Accounts still need configuration.`, + `Four required skills: ${list}.`, + `Required: ${keys.join(", ")}.`, + "Requirements: GitHub PR workflow, QA acceptance, task planning and doc maintenance. They do not prove accounts are ready.", +])("accepts every actual skill identity without requiring an incidental numeral: %s", reply => { + expect(describesTeamSkillRequirements(team, reply)).toBe(true); +}); +it.each([ + "Required: github-pr-workflow, qa-acceptance and task-planning.", + "It needs four skills: foreign/workflows/github-pr-workflow, foreign/quality/qa-acceptance, foreign/planning/task-planning, and foreign/docs/doc-maintenance.", + `It requires three skills: ${list}.`, + `No required skills. Available examples are ${list}.`, + `It does not require ${list}.`, + `It requires ${list}. qa-acceptance is optional.`, + `It requires ${list}. Never ever include doc-maintenance.`, +])("rejects missing, foreign, contradictory or falsely counted requirements: %s", reply => { + expect(describesTeamSkillRequirements(team, reply)).toBe(false); +}); +it("does not use an ambiguous short name to stand in for two different real skills", () => { + const duplicate = { requiredSkills: [...team.requiredSkills, { ...team.requiredSkills[0]!, ref: "other/catalog/github-pr-workflow", catalogSkillKey: "other/catalog/github-pr-workflow" }] }; + expect(describesTeamSkillRequirements(duplicate, `Required: ${list}.`)).toBe(false); + expect(describesTeamSkillRequirements(duplicate, `Required: ${keys.join(", ")}, other/catalog/github-pr-workflow.`)).toBe(true); +}); +it("retains truthful empty requirements without accepting an invented count", () => { + expect(describesTeamSkillRequirements({ requiredSkills: [] }, "No required skills.")).toBe(true); + expect(describesTeamSkillRequirements({ requiredSkills: [] }, "Four required skills.")).toBe(false); +}); + +it("uses actual typed skill requirements in the full grader without weakening package or noninstallation checks", () => { + const selected = { ...team, envInputs: [], id: "package", name: "Product Engineering", compatibility: "compatible", agentSlugs: ["cto", "senior-coder", "qa"], counts: { projects: 1 } }; + const o = { scenarioId: "teams.find-team", companyId: "company", reply: `Product Engineering includes a CTO, Senior Coder and QA. Required: ${list}.`, + state: { team: selected, installedBefore: [], installedAfter: [], agentIdsBefore: ["ceo"], agentIdsAfter: ["ceo"], projectIdsBefore: [], projectIdsAfter: [] }, + phaseReplies: [], connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], activity: [], links: [], runs: [], humanQueries: [] }; + expect(gradeTeams(o).passed).toBe(true); + expect(gradeTeams({ ...o, state: { ...o.state, agentIdsAfter: ["ceo", "new-agent"] } }).passed).toBe(false); + expect(gradeTeams({ ...o, reply: `Product Engineering includes Senior Coder and QA. Required: ${list}.` }).passed).toBe(false); + expect(gradeTeams({ ...o, reply: `Product Engineering includes a CTO, Senior Coder and QA. Required: ${list.replace("qa-acceptance", "foreign/quality/qa-acceptance")}.` }).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-teams.ts b/tests/runner-e2e/chat-catalog-domains/external-teams.ts index 58cff15458d..0e7371e910d 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-teams.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-teams.ts @@ -9,7 +9,7 @@ export function gradeTeams(value: unknown) { && o.reply.includes(team.name) && team.agentSlugs.every(slug => o.reply.toLowerCase().includes(slug.toLowerCase().replaceAll("-", " ")) || o.reply.toLowerCase().includes(slug.toLowerCase())), "The response names the real package and every included role."); c.add("package-requirements-described", !!team && Array.isArray(team.requiredSkills) && Array.isArray(team.envInputs) - && (team.requiredSkills.length === 0 || new RegExp(`\\b${team.requiredSkills.length}\\s+(?:required\\s+|catalog\\s+)?skills?\\b`, "i").test(o.reply)) + && describesTeamSkillRequirements(team, o.reply) && team.envInputs.every(input => o.reply.includes(input.key)), "Visible skill and configuration requirements are included in the explanation."); c.add("no-premature-installation", JSON.stringify(o.state.installedBefore) === JSON.stringify(o.state.installedAfter) && JSON.stringify(o.state.agentIdsBefore) === JSON.stringify(o.state.agentIdsAfter) @@ -50,3 +50,25 @@ export const teamExternalCases = ["find-team", "preview-install"].map(id => defi return observeExternal(ctx, state); }, })); + +export function describesTeamSkillRequirements(team: Pick, reply: string): boolean { + const plain = reply.replace(/[*`]/g, ""); + const numbers: Record = { no: 0, zero: 0, one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, ten: 10 }; + for (const match of plain.matchAll(/\b(\d+|no|zero|one|two|three|four|five|six|seven|eight|nine|ten)\s+(?:(?:required|catalog)\s+)?skills?\b/gi)) { + if ((numbers[match[1]!.toLowerCase()] ?? Number(match[1])) !== team.requiredSkills.length) return false; + } + const leaf = (ref: string) => ref.split("/").at(-1)!; + const escaped = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const clauses = plain.split(/[.;\n]|\b(?:but|however)\b/i).filter(Boolean); + return team.requiredSkills.every(skill => { + const refs = [...new Set([skill.ref, skill.catalogSkillKey].filter((ref): ref is string => !!ref))]; + const short = leaf(skill.catalogSkillKey ?? skill.ref); + const unambiguous = team.requiredSkills.filter(other => leaf(other.catalogSkillKey ?? other.ref) === short).length === 1; + const aliases = [...refs, ...(unambiguous ? [short, short.replaceAll("-", " ")] : [])]; + const identity = `(? mentions.test(clause) && !negated.test(clause)) + && !clauses.some(clause => negated.test(clause)); + }); +}