test(evals): verify real team installation and member configuration

tests/runner-e2e/chat-catalog-domains/external-definitions.ts

@@ -16,3 +16,5 @@ export { mcpFailureExternalCases } from "./external-mcp-failure.js";
 export { assistantConsentExternalCases } from "./external-assistant-consent.js";
 export { notionReadExternalCases } from "./external-notion-read.js";
 export { secureKeyExternalCases } from "./external-secure-key.js";
+export { teamInstallExternalCases } from "./external-team-install.js";
+export { teamConfigureExternalCases } from "./external-team-configure.js";

tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, it } from "vitest";
+import { teamConfigureExternalCases } from "./external-team-configure.js";
+function valid() {
+  return { scenarioId: "teams.configure-team", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], links: [],
+    reply: "QA has been configured and each team member is ready.", phaseReplies: [{ runId: "initial", body: "QA needs the eligible account; open QA configuration." }, { runId: "verified", body: "QA and the senior coder are now configured and ready." }], runs: [{ id: "initial", status: "succeeded" }, { id: "verified", status: "succeeded" }],
+    activity: [{ id: "repair", action: "agent.updated", actorType: "user", entityId: "qa", agentId: null as string | null }], state: {
+      installedFromCatalog: true, missingId: "qa", workingId: "coder", expectedMembers: 2, memberIdsBefore: ["coder", "qa"], workingRevisionIdsBefore: ["revision"], workingRevisionIdsAfter: ["revision"],
+      initialProbes: [{ agentId: "qa", status: "fail", testedAt: "2026-10-26T10:00:00Z", checks: [{ code: "codex_auth", level: "error" }] }, { agentId: "coder", status: "pass", testedAt: "2026-10-26T10:00:00Z", checks: [] }],
+      afterProbes: [{ agentId: "qa", status: "pass", testedAt: "2026-10-26T10:05:01Z", checks: [] }, { agentId: "coder", status: "pass", testedAt: "2026-10-26T10:05:02Z", checks: [] }],
+      repairVerifiedAfter: Date.parse("2026-10-26T10:05:00Z"), initialMemberNamed: true, initialHandoffSupported: true, repairConfirmed: true, repairChannel: "human-board-configuration", workingHashBefore: "same-config", workingHashAfter: "same-config", memberIdsPreserved: true, initialRunId: "initial", noPrematureWholeTeamClaim: true,
+    } };
+}
+describe("installed team per-member readiness oracle", () => {
+  const definition = teamConfigureExternalCases[0]!;
+  it("accepts real missing-account diagnosis, human repair and fresh separate readiness checks", () => { expect(definition.grade(valid()).passed).toBe(true); });
+  it("also accepts an actually attributed eligible agent configuration repair", () => { const o = valid(); o.state.repairChannel = "agent-configuration"; o.activity[0]!.actorType = "agent"; o.activity[0]!.agentId = "ceo"; expect(definition.grade(o).passed).toBe(true); });
+  it("rejects stale/member-substituted readiness, unrelated config edits and premature success", () => {
+    for (const change of [
+      (o: ReturnType<typeof valid>) => { o.state.afterProbes[0]!.testedAt = "2026-10-26T09:00:00Z"; },
+      (o: ReturnType<typeof valid>) => { o.state.afterProbes[0]!.status = "fail"; },
+      (o: ReturnType<typeof valid>) => { o.state.afterProbes[0]!.agentId = "coder"; },
+      (o: ReturnType<typeof valid>) => { o.state.workingHashAfter = "changed-working-account"; },
+      (o: ReturnType<typeof valid>) => { o.state.workingRevisionIdsAfter = ["masked-env-change"]; },
+      (o: ReturnType<typeof valid>) => { o.state.noPrematureWholeTeamClaim = false; },
+      (o: ReturnType<typeof valid>) => { o.phaseReplies[1]!.runId = "initial"; },
+    ]) { const o = valid(); change(o); expect(definition.grade(o).passed).toBe(false); }
+  });
+  it("missing observations cannot pass", () => { expect(definition.grade(undefined).passed).toBe(false); expect(definition.grade({ reply: "Everyone is ready." }).passed).toBe(false); });
+});

tests/runner-e2e/chat-catalog-domains/external-team-configure.ts

@@ -0,0 +1,107 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { expect } from "@playwright/test";
+import type { Agent, AdapterEnvironmentTestResult } from "../../../packages/shared/src/types/agent.js";
+import type { CompanyPortabilityAdapterOverride } from "../../../packages/shared/src/types/company-portability.js";
+import type { CatalogTeam, CatalogTeamInstallResult } from "../../../packages/shared/src/types/teams-catalog.js";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import { sendChatMessage } from "../chat-flow.js";
+import { codexSubscriptionAgentConfig } from "../codex-subscription-fixture.js";
+import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js";
+import { catalogTeamMembers, observeTeamResources } from "./external-team-install.js";
+import { waitForReply } from "./identity-support.js";
+interface PrivateTeam { emptyHome: string; override: CompanyPortabilityAdapterOverride }
+const fixtures = new WeakMap<ChatCatalogContext, PrivateTeam>();
+function stable(value: unknown): unknown { return Array.isArray(value) ? value.map(stable) : value && typeof value === "object" ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => [key, stable(item)])) : value; }
+const configHash = (agent: Agent) => createHash("sha256").update(JSON.stringify(stable({ adapterType: agent.adapterType, adapterConfig: agent.adapterConfig, runtimeConfig: agent.runtimeConfig, defaultEnvironmentId: agent.defaultEnvironmentId }))).digest("hex");
+interface Probe { agentId: string; status: string; testedAt: string; checks: Array<{code: string; level: string}> }
+async function probe(ctx: ChatCatalogContext, agent: Agent): Promise<Probe> {
+  const result = await ctx.input.api.postSensitive<AdapterEnvironmentTestResult>(`${root(ctx)}/adapters/${agent.adapterType}/test-environment`, { agentId: agent.id, environmentId: agent.defaultEnvironmentId, adapterConfig: agent.adapterConfig });
+  return { agentId: agent.id, status: result.status, testedAt: result.testedAt, checks: result.checks.map(check => ({ code: /^[a-zA-Z0-9_.-]{1,100}$/.test(check.code) ? check.code : "unrecognized", level: check.level })) };
+}
+export function gradeTeamConfigure(value: unknown) {
+  const c = checks(), o = externalObservation(value); c.add("complete-member-configuration-evidence", !!o, "Real installed members, missing versus ready account probes, scoped repair and fresh per-member tests are required."); if (!o) return c.result();
+  const initial = o.state.initialProbes as Probe[] | undefined, after = o.state.afterProbes as Probe[] | undefined;
+  c.add("actual-missing-and-working-member-prerequisite", o.state.installedFromCatalog === true && typeof o.state.missingId === "string" && typeof o.state.workingId === "string" && o.state.missingId !== o.state.workingId
+    && Array.isArray(initial) && initial.some(row => row.agentId === o.state.missingId && row.status === "fail" && row.checks.some(check => check.level === "error" && /auth|credential|login/i.test(check.code)))
+    && initial.some(row => row.agentId === o.state.workingId && row.status === "pass"),
+    "The supported catalog installer creates the team; actual runtime tests prove one exact missing account and a distinct already working member before chat.");
+  c.add("specific-member-diagnosis-and-supported-repair", o.state.initialMemberNamed === true && o.state.initialHandoffSupported === true && o.state.repairConfirmed === true
+    && ((o.state.repairChannel === "agent-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "agent" && row.agentId === o.leadId))
+      || (o.state.repairChannel === "human-board-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "user"))),
+    "Chat identifies the incomplete member and the supported step; repair changes that member through actual agent or human configuration authority.");
+  c.add("already-working-member-preserved", typeof o.state.workingHashBefore === "string" && o.state.workingHashBefore === o.state.workingHashAfter && o.state.memberIdsPreserved === true,
+    "The already working member's semantic configuration and the installed member identities remain unchanged.");
+  c.add("working-configuration-revisions-preserved", Array.isArray(o.state.workingRevisionIdsBefore) && o.state.workingRevisionIdsBefore.length > 0
+    && JSON.stringify(o.state.workingRevisionIdsBefore) === JSON.stringify(o.state.workingRevisionIdsAfter),
+    "Public immutable configuration revision identities also remain unchanged, so redacted environment placeholders cannot hide an account change.");
+  c.add("fresh-readiness-for-every-member", Array.isArray(after) && after.length === o.state.expectedMembers
+    && JSON.stringify(after.map(row => row.agentId).sort()) === JSON.stringify(o.state.memberIdsBefore)
+    && new Set(after.map(row => row.agentId)).size === after.length && after.every(row => row.status === "pass" && row.checks.every(check => check.level !== "error")
+    && Number.isFinite(Date.parse(row.testedAt)) && Date.parse(row.testedAt) >= Number(o.state.repairVerifiedAfter)),
+    "After the real repair every member passes its own fresh saved-runtime test; one healthy member cannot stand for the whole team.");
+  const continuation = o.phaseReplies.filter(row => row.runId !== o.state.initialRunId);
+  c.add("phase-bound-truthful-team-result", continuation.some(row => o.runs.some(run => run.id === row.runId && run.status === "succeeded"))
+    && /ready|configured|verified|account/i.test(continuation.map(row => row.body).join("\n")) && o.state.noPrematureWholeTeamClaim === true,
+    "The final claim belongs to the actual verification followup, and the initial reply did not call an unconfigured team ready.");
+  return c.result();
+}
+export const teamConfigureExternalCases = [definition("teams.configure-team", async ctx => {
+  if (ctx.input.execution.profile.credential !== "CODEX_AUTH_JSON") throw new Error("Setup blocker: this disposable team fixture requires the selected Codex subscription account, not an inferred personal login");
+  const team = (await ctx.input.api.get<CatalogTeam[]>("/api/teams/catalog")).find(row => row.slug === "product-engineering" && row.compatibility === "compatible");
+  if (!team) throw new Error("Setup blocker: compatible real Product Engineering package is unavailable");
+  const built = ctx.input.execution.profile.buildAgent({ environmentId: ctx.input.fixtures.environment.id, environmentFixtureId: ctx.input.execution.environment.id, workspacePath: ctx.input.workspacePath, secretRefs: ctx.input.fixtures.secretRefs, executionId: ctx.input.nonce });
+  if (typeof built.adapterType !== "string") throw new Error("Setup blocker: selected profile does not declare a supported runtime adapter");
+  const configured = codexSubscriptionAgentConfig(built, process.env, ctx.input.fixtures.company.id);
+  const override: CompanyPortabilityAdapterOverride = { adapterType: built.adapterType, adapterConfig: configured.adapterConfig };
+  const emptyHome = await mkdtemp(path.join(tmpdir(), "paperclip-missing-team-login-")); fixtures.set(ctx, { emptyHome, override });
+  try {
+    const install = await ctx.input.api.postSensitive<CatalogTeamInstallResult>(`${root(ctx)}/teams/catalog/${encodeURIComponent(team.id)}/install`, { targetManagerAgentId: ctx.input.fixtures.agent.id, adapterOverrides: Object.fromEntries(team.agentSlugs.map(slug => [slug, override])) });
+    const missingId = install.portabilityImport.agents.find(row => row.slug === "qa")?.id, workingId = install.portabilityImport.agents.find(row => row.slug === "senior-coder")?.id;
+    if (!missingId || !workingId) throw new Error("Setup blocker: installer did not produce the exact QA and senior-coder members");
+    await ctx.input.api.patchSensitive(`/api/agents/${missingId}`, { adapterConfig: { ...override.adapterConfig, env: { ...(override.adapterConfig?.env as Record<string, unknown> ?? {}), CODEX_HOME: emptyHome, OPENAI_API_KEY: "", CODEX_API_KEY: "" } } });
+    const members = catalogTeamMembers(await observeTeamResources(ctx), team.id);
+    const initialProbes = await Promise.all(members.map(member => probe(ctx, member)));
+    if (!initialProbes.some(row => row.agentId === missingId && row.status === "fail" && row.checks.some(check => check.level === "error" && /auth|credential|login/i.test(check.code))) || !initialProbes.some(row => row.agentId === workingId && row.status === "pass")) throw new Error("Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split");
+    const working = await ctx.input.api.get<Agent>(`/api/agents/${workingId}`);
+    const workingRevisionIdsBefore = (await ctx.input.api.get<Array<{ id: string }>>(`/api/agents/${workingId}/config-revisions`)).map(row => row.id).sort();
+    await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue();
+    await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The website team is the installed ${team.name} package. Its QA member (${missingId}) has not selected the eligible company Codex subscription account. The senior coder (${workingId}) is already configured. The existing company subscription is authorized for the QA member too; its value must not enter chat.` });
+    return { teamId: team.id, installedFromCatalog: true, memberIdsBefore: members.map(row => row.id).sort(), expectedMembers: members.length, missingId, missingName: members.find(row => row.id === missingId)!.name,
+      workingId, workingHashBefore: configHash(working), workingRevisionIdsBefore, initialProbes };
+  } catch (error) { fixtures.delete(ctx); await rm(emptyHome, { recursive: true, force: true }); throw error; }
+}, gradeTeamConfigure, {
+  authentication: "human-session", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 600_000,
+  setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["missing-account-fixture", "working-account-fixture"] },
+  async verifySetup(_ctx, state) { const c = checks(), rows = state.initialProbes as Probe[]; c.add("missing-account-fixture", rows.some(row => row.agentId === state.missingId && row.status === "fail"), "Actual saved-runtime probe reports the required missing member account."); c.add("working-account-fixture", rows.some(row => row.agentId === state.workingId && row.status === "pass"), "A distinct installed member already passes its runtime probe."); return c.result(); },
+  sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"],
+  async act(ctx, state) {
+    await waitForReply(ctx); const initial = await observeExternal(ctx, state); state.initialRunId = initial.runs[0]?.id;
+    state.initialMemberNamed = initial.reply.toLowerCase().includes(String(state.missingName).toLowerCase());
+    const members = catalogTeamMembers(await observeTeamResources(ctx), String(state.teamId)), missing = members.find(row => row.id === state.missingId)!;
+    const current = await probe(ctx, missing); const alreadyRepaired = current.status === "pass";
+    const handoff = initial.links.find(href => { try { const url = new URL(href, initial.origin); return url.origin === initial.origin && url.pathname.split("/").some((part, i, parts) => part === "agents" && [missing.id, missing.urlKey].includes(parts[i + 1]!)); } catch { return false; } });
+    state.initialHandoffSupported = alreadyRepaired || !!handoff;
+    state.noPrematureWholeTeamClaim = alreadyRepaired || !/\b(?:team|product engineering)\s+(?:is|are|looks)\s+(?:now\s+)?(?:fully\s+)?ready/i.test(initial.reply);
+    if (!alreadyRepaired) {
+      if (!handoff) throw new Error("The model did not provide a supported configuration handoff for the incomplete member");
+      const source = ctx.input.page.url(); await ctx.input.page.goto(handoff, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("main")).toContainText(missing.name);
+      const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Eligible team account reference missing");
+      await ctx.input.api.patchSensitive(`/api/agents/${missing.id}`, { adapterType: fixture.override.adapterType, adapterConfig: fixture.override.adapterConfig }); state.repairChannel = "human-board-configuration";
+      await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+    } else state.repairChannel = "agent-configuration";
+    state.repairVerifiedAfter = Date.now(); state.repairConfirmed = (await probe(ctx, await ctx.input.api.get<Agent>(`/api/agents/${missing.id}`))).status === "pass";
+    await sendChatMessage(ctx.input.page, "Please verify which installed team members are ready now, including QA, and preserve the already working member's configuration.");
+  },
+  async observe(ctx, state) {
+    const members = catalogTeamMembers(await observeTeamResources(ctx), String(state.teamId)); state.afterProbes = await Promise.all(members.map(member => probe(ctx, member)));
+    state.memberIdsPreserved = JSON.stringify(members.map(row => row.id).sort()) === JSON.stringify(state.memberIdsBefore);
+    state.workingHashAfter = members.some(row => row.id === state.workingId) ? configHash(members.find(row => row.id === state.workingId)!) : null;
+    state.workingRevisionIdsAfter = (await ctx.input.api.get<Array<{ id: string }>>(`/api/agents/${state.workingId}/config-revisions`)).map(row => row.id).sort();
+    return observeExternal(ctx, state);
+  },
+  async verifyResult(ctx, state, value) { await verifyRenderedReply(ctx, value, String(state.initialRunId)); },
+  async cleanup(ctx) { const fixture = fixtures.get(ctx); fixtures.delete(ctx); if (fixture) await rm(fixture.emptyHome, { recursive: true, force: true }); },
+})];

tests/runner-e2e/chat-catalog-domains/external-team-install.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, it } from "vitest";
+import { teamInstallExternalCases } from "./external-team-install.js";
+function valid() {
+  const team = { id: "team", contentHash: "hash", agentSlugs: ["cto", "coder"], counts: { agents: 2, projects: 1, routines: 1 } };
+  const agent = (id: string) => ({ id, companyId: "company", urlKey: id, metadata: { paperclip: { catalogTeam: { catalogId: "team", originHash: "hash" } } } });
+  const before = { agents: [], projects: [], routines: [], installed: [] };
+  const after = { agents: [agent("cto"), agent("coder")], projects: [{ id: "project", companyId: "company", urlKey: "website" }], routines: [{ id: "routine", companyId: "company" }], installed: [{ catalogId: "team", agentCount: 2, outOfDate: false }] };
+  return { scenarioId: "teams.install-team", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", phaseReplies: [{ runId: "initial", body: "Installed the team." }, { runId: "repeat", body: "The existing team is already installed; I kept its same resources." }],
+    connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: after.projects, goals: [], comments: [], documents: [], revisions: [], humanQueries: [], reply: "Installed the team; it already exists.",
+    links: ["/ACORN/agents/cto", "/ACORN/agents/coder", "/ACORN/projects/website"], runs: [{ id: "initial", status: "succeeded" }, { id: "repeat", status: "succeeded" }],
+    activity: [{ id: "install", action: "company.team_catalog_installed", actorType: "agent", agentId: "ceo" }],
+    state: { team, before, first: structuredClone(after), after, companyPrefix: "ACORN", previewAccepted: true, previewErrorCount: 0, actorCanCreateAgents: true, existingAccountReady: true, originalHumanQuery: "Install that website team in this company.", repeatHumanQuery: "Install that website team in this company.", initialRunId: "initial" } };
+}
+describe("real catalog installer oracle", () => {
+  const definition = teamInstallExternalCases[0]!;
+  it("accepts attributed installation and actual repeat reconciliation", () => { expect(definition.grade(valid()).passed).toBe(true); });
+  it("rejects plausible duplicate installations, wrong provenance, human prepopulation and substituted repeat transcript", () => {
+    for (const change of [
+      (o: ReturnType<typeof valid>) => { o.state.after.agents.push({ ...o.state.after.agents[0]!, id: "copy" }); },
+      (o: ReturnType<typeof valid>) => { o.state.after.agents[0]!.metadata.paperclip.catalogTeam.originHash = "other-package"; },
+      (o: ReturnType<typeof valid>) => { o.activity[0]!.actorType = "user"; },
+      (o: ReturnType<typeof valid>) => { o.phaseReplies[1]!.runId = "initial"; },
+      (o: ReturnType<typeof valid>) => { o.links[2] = "https://other.invalid/ACORN/projects/project"; },
+      (o: ReturnType<typeof valid>) => { o.links[2] = "/OTHER/projects/project"; },
+      (o: ReturnType<typeof valid>) => { o.links[2] = "/ACORN/projects"; },
+    ]) { const o = valid(); change(o); expect(definition.grade(o).passed).toBe(false); }
+  });
+  it("missing evidence or a successful-sounding reply alone cannot pass", () => { expect(definition.grade(undefined).passed).toBe(false); expect(definition.grade({ reply: "Installed everything." }).passed).toBe(false); });
+});

tests/runner-e2e/chat-catalog-domains/external-team-install.ts

@@ -0,0 +1,84 @@
+import { expect } from "@playwright/test";
+import type { Agent } from "../../../packages/shared/src/types/agent.js";
+import type { CatalogTeam, CatalogTeamImportPreviewResult, InstalledCatalogTeam } from "../../../packages/shared/src/types/teams-catalog.js";
+import { sendChatMessage } from "../chat-flow.js";
+import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js";
+import { waitForReply } from "./identity-support.js";
+interface Resource { id: string; companyId: string; name?: string; status?: string; urlKey?: string }
+export interface TeamResourceSnapshot { agents: Agent[]; projects: Resource[]; routines: Resource[]; installed: InstalledCatalogTeam[] }
+export function catalogTeamProvenance(agent: Agent) { return (agent.metadata as { paperclip?: { catalogTeam?: { catalogId?: string; originHash?: string; agentSlug?: string } } } | null)?.paperclip?.catalogTeam; }
+export function catalogTeamMembers(snapshot: TeamResourceSnapshot, teamId: string) { return snapshot.agents.filter(agent => catalogTeamProvenance(agent)?.catalogId === teamId); }
+const ids = (rows: Array<{id:string}>) => rows.map(row => row.id).sort();
+function hasResourceLink(links: string[], origin: string, prefix: unknown, kind: "agents" | "projects", resource: { id: string; urlKey?: string }) {
+  return links.some(href => { try { const url = new URL(href, origin), parts = url.pathname.split("/"); const index = parts.indexOf(kind), ref = parts[index + 1]; return url.origin === origin && index === 2
+    && decodeURIComponent(parts[1]!) === prefix && !!ref && [resource.id, resource.urlKey].includes(ref); } catch { return false; } });
+}
+export async function observeTeamResources(ctx: import("../chat-catalog-contract.js").ChatCatalogContext): Promise<TeamResourceSnapshot> {
+  const [agents, projects, routines, installed] = await Promise.all([ctx.input.api.get<Agent[]>(`${root(ctx)}/agents`), ctx.input.api.get<Resource[]>(`${root(ctx)}/projects`),
+    ctx.input.api.get<Resource[]>(`${root(ctx)}/routines`), ctx.input.api.get<InstalledCatalogTeam[]>(`${root(ctx)}/teams/catalog/installed`)]);
+  return { agents, projects, routines, installed };
+}
+export function gradeTeamInstall(value: unknown) {
+  const c = checks(), o = externalObservation(value); c.add("complete-team-install-evidence", !!o, "Accepted immutable preview, effective creator authority, attributed public installation and repeat reconciliation are required."); if (!o) return c.result();
+  const team = o.state.team as CatalogTeam | undefined, first = o.state.first as TeamResourceSnapshot | undefined, after = o.state.after as TeamResourceSnapshot | undefined, before = o.state.before as TeamResourceSnapshot | undefined;
+  c.add("accepted-supported-preview-and-authority", !!team && o.state.previewAccepted === true && o.state.previewErrorCount === 0 && o.state.actorCanCreateAgents === true && o.state.existingAccountReady === true,
+    "The exact real catalog preview is accepted and the acting CEO has actual installation/resource authority with a ready existing runtime account.");
+  const members = team && after ? catalogTeamMembers(after, team.id) : [];
+  c.add("intended-agent-resources-once", !!team && !!after && !!before && members.length === team.agentSlugs.length && members.every(agent => agent.companyId === o.companyId
+    && catalogTeamProvenance(agent)?.originHash === team.contentHash) && new Set(members.map(agent => agent.urlKey)).size === members.length
+    && after.agents.length - before.agents.length === team.counts.agents,
+    "Exactly the package's agents exist in this company with immutable catalog provenance; no unrelated or duplicate hires satisfy the count.");
+  c.add("intended-project-and-routine-resources", !!team && !!after && !!before && after.projects.length - before.projects.length === team.counts.projects
+    && after.routines.length - before.routines.length === team.counts.routines && after.projects.every(project => project.companyId === o.companyId)
+    && after.installed.some(row => row.catalogId === team.id && row.agentCount === team.counts.agents && !row.outOfDate),
+    "Actual project/routine deltas and the server-computed installed-team state agree with the preview in the active company.");
+  c.add("actual-agent-installation-attribution", !!team && o.activity.some(row => row.action === "company.team_catalog_installed" && row.actorType === "agent" && row.agentId === o.leadId),
+    "The requested installer operation is independently attributed to the actual model actor, rather than performed as an oracle setup side effect.");
+  c.add("uncertain-repeat-reconciles-identities", !!team && !!first && !!after && first.installed.some(row => row.catalogId === team.id)
+    && JSON.stringify(ids(first.agents)) === JSON.stringify(ids(after.agents)) && JSON.stringify(ids(first.projects)) === JSON.stringify(ids(after.projects))
+    && JSON.stringify(ids(first.routines)) === JSON.stringify(ids(after.routines)) && o.state.repeatHumanQuery === o.state.originalHumanQuery,
+    "The actual repeated canonical request preserves existing agent/project/routine identities rather than blindly installing renamed copies.");
+  const second = o.phaseReplies.filter(reply => reply.runId !== o.state.initialRunId);
+  c.add("phase-bound-reconciliation-reply", second.some(reply => o.runs.some(run => run.id === reply.runId && run.status === "succeeded"))
+    && /already|existing|installed|same|reconcil/i.test(second.map(reply => reply.body).join("\n")), "The repeat outcome belongs to its own real persisted model continuation.");
+  c.add("actual-installed-resource-links", members.length > 0 && members.every(agent => hasResourceLink(o.links, o.origin, o.state.companyPrefix, "agents", agent))
+    && !!after && !!before && after.projects.filter(project => !before.projects.some(old => old.id === project.id)).every(project => hasResourceLink(o.links, o.origin, o.state.companyPrefix, "projects", project)),
+    "The response links the actual installed members and project, not just the catalog or an intended future installation.");
+  return c.result();
+}
+export const teamInstallExternalCases = [definition("teams.install-team", async ctx => {
+  const teams = await ctx.input.api.get<CatalogTeam[]>("/api/teams/catalog"); const team = teams.find(row => row.slug === "product-engineering" && row.compatibility === "compatible");
+  if (!team) throw new Error("Setup blocker: real compatible Product Engineering team catalog package is unavailable");
+  await ctx.input.api.patch(`/api/agents/${ctx.input.fixtures.agent.id}/permissions`, { canCreateAgents: true });
+  await ctx.input.api.patch(root(ctx), { requireBoardApprovalForNewAgents: false });
+  const actor = await ctx.input.api.get<Agent>(`/api/agents/${ctx.input.fixtures.agent.id}`);
+  if (actor.lifecycleState !== "ready" || actor.permissions?.canCreateAgents !== true) throw new Error("Setup blocker: actual selected installer actor lacks a ready account or creation authority");
+  const preview = await ctx.input.api.post<CatalogTeamImportPreviewResult>(`${root(ctx)}/teams/catalog/${encodeURIComponent(team.id)}/preview`, { targetManagerAgentId: actor.id });
+  if (preview.errors.length || preview.portabilityPreview.errors.length) throw new Error("Setup blocker: accepted team fixture preview has real preparation conflicts");
+  const before = await observeTeamResources(ctx); if (catalogTeamMembers(before, team.id).length) throw new Error("Setup blocker: the selected team is already installed in this disposable company");
+  await ctx.input.api.post(`${root(ctx)}/chats/${actor.id}`); await ctx.refreshIssue();
+  await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The accepted website team preview is ${team.name}, catalog key ${team.key}, content hash ${team.contentHash}. Install the complete selected package under the current CEO using this company's existing eligible runtime account. No team resource is installed yet. The selected runtime uses ${actor.adapterType} in environment ${ctx.input.fixtures.environment.id}.` });
+  return { team, before, companyPrefix: ctx.input.fixtures.company.issuePrefix, previewAccepted: true, previewErrorCount: preview.errors.length + preview.portabilityPreview.errors.length,
+    actorCanCreateAgents: actor.permissions?.canCreateAgents === true, existingAccountReady: actor.lifecycleState === "ready", originalHumanQuery: ctx.scenario.query };
+}, gradeTeamInstall, {
+  expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 480_000,
+  coverageGaps: ["installer-conflict-and-partial-failure-branches"],
+  sourceFiles: ["chat-catalog-domains/external-team-install.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"],
+  async act(ctx, state) {
+    await waitForReply(ctx); state.first = await observeTeamResources(ctx); state.initialRunId = (await ctx.allRuns())[0]?.id;
+    state.repeatHumanQuery = ctx.scenario.query; await sendChatMessage(ctx.input.page, ctx.scenario.query);
+  },
+  async observe(ctx, state) { state.after = await observeTeamResources(ctx); return observeExternal(ctx, state); },
+  async verifyResult(ctx, state, value) {
+    await verifyRenderedReply(ctx, value, String(state.initialRunId)); const team = state.team as CatalogTeam, after = state.after as TeamResourceSnapshot, o = externalObservation(value)!;
+    const source = ctx.input.page.url();
+    for (const member of catalogTeamMembers(after, team.id)) { const href = o.links.find(link => hasResourceLink([link], o.origin, state.companyPrefix, "agents", member)); if (!href) throw new Error("Actual installed member link missing"); await ctx.input.page.goto(new URL(href, o.origin).href, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("main")).toContainText(member.name); }
+    for (const project of after.projects.filter(row => !(state.before as TeamResourceSnapshot).projects.some(old => old.id === row.id))) { const href = o.links.find(link => hasResourceLink([link], o.origin, state.companyPrefix, "projects", project)); if (!href) throw new Error("Actual installed project link missing"); await ctx.input.page.goto(new URL(href, o.origin).href, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("main")).toContainText(project.name ?? ""); }
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+  },
+  async cleanup(ctx, state) {
+    const before = state.before as TeamResourceSnapshot | undefined; if (!before) return;
+    for (const routine of await ctx.input.api.get<Resource[]>(`${root(ctx)}/routines`)) if (!before.routines.some(row => row.id === routine.id)) await ctx.input.api.patch(`/api/routines/${routine.id}`, { status: "paused" });
+    // Complete disposable-company teardown removes imported graph/provenance.
+  },
+})];

tests/runner-e2e/chat-catalog-domains/external.test.ts

@@ -196,10 +196,10 @@ describe("private disposable provider prerequisite contract", () => {
     expect(() => selectedExternalCredential(ctx, binding, "PAPERCLIP_CHAT_EVAL_OTHER_AUTH")).toThrow(/not declared/);
     expect(() => selectedExternalBinding({ ...ctx, input: { ...ctx.input, privateInputs: undefined } })).toThrow(/private inputs/);
   });
-  it("enumerates the 16 named-provider journeys and two authorized repository journeys", () => {
-    expect(Object.keys(externalRequirements)).toHaveLength(18);
+  it("enumerates the 16 named-provider journeys and four authorized repository journeys", () => {
+    expect(Object.keys(externalRequirements)).toHaveLength(20);
     expect(Object.entries(externalRequirements).filter(([, requirement]) => requirement.provider === "github").map(([id]) => id))
-      .toEqual(["projects.add-repository", "projects.replace-repository"]);
+      .toEqual(["projects.add-repository", "workspaces.prepare-repository", "workspaces.private-repository-no-access", "projects.replace-repository"]);
   });
   it("missing input produces field-only receipts, never a configured pass", () => {
     for (const id of Object.keys(externalRequirements)) { const receipt = externalPrerequisiteReceipt(id, undefined, {}); expect(receipt.configured).toBe(false); expect(receipt.missing.length).toBeGreaterThan(1); }