diff --git a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts index e932f0434bf..3cb17b627c8 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts @@ -16,3 +16,5 @@ export { mcpFailureExternalCases } from "./external-mcp-failure.js"; export { assistantConsentExternalCases } from "./external-assistant-consent.js"; export { notionReadExternalCases } from "./external-notion-read.js"; export { secureKeyExternalCases } from "./external-secure-key.js"; +export { teamInstallExternalCases } from "./external-team-install.js"; +export { teamConfigureExternalCases } from "./external-team-configure.js"; diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts new file mode 100644 index 00000000000..fa459b9f061 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { teamConfigureExternalCases } from "./external-team-configure.js"; +function valid() { + return { scenarioId: "teams.configure-team", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], links: [], + reply: "QA has been configured and each team member is ready.", phaseReplies: [{ runId: "initial", body: "QA needs the eligible account; open QA configuration." }, { runId: "verified", body: "QA and the senior coder are now configured and ready." }], runs: [{ id: "initial", status: "succeeded" }, { id: "verified", status: "succeeded" }], + activity: [{ id: "repair", action: "agent.updated", actorType: "user", entityId: "qa", agentId: null as string | null }], state: { + installedFromCatalog: true, missingId: "qa", workingId: "coder", expectedMembers: 2, memberIdsBefore: ["coder", "qa"], workingRevisionIdsBefore: ["revision"], workingRevisionIdsAfter: ["revision"], + initialProbes: [{ agentId: "qa", status: "fail", testedAt: "2026-10-26T10:00:00Z", checks: [{ code: "codex_auth", level: "error" }] }, { agentId: "coder", status: "pass", testedAt: "2026-10-26T10:00:00Z", checks: [] }], + afterProbes: [{ agentId: "qa", status: "pass", testedAt: "2026-10-26T10:05:01Z", checks: [] }, { agentId: "coder", status: "pass", testedAt: "2026-10-26T10:05:02Z", checks: [] }], + repairVerifiedAfter: Date.parse("2026-10-26T10:05:00Z"), initialMemberNamed: true, initialHandoffSupported: true, repairConfirmed: true, repairChannel: "human-board-configuration", workingHashBefore: "same-config", workingHashAfter: "same-config", memberIdsPreserved: true, initialRunId: "initial", noPrematureWholeTeamClaim: true, + } }; +} +describe("installed team per-member readiness oracle", () => { + const definition = teamConfigureExternalCases[0]!; + it("accepts real missing-account diagnosis, human repair and fresh separate readiness checks", () => { expect(definition.grade(valid()).passed).toBe(true); }); + it("also accepts an actually attributed eligible agent configuration repair", () => { const o = valid(); o.state.repairChannel = "agent-configuration"; o.activity[0]!.actorType = "agent"; o.activity[0]!.agentId = "ceo"; expect(definition.grade(o).passed).toBe(true); }); + it("rejects stale/member-substituted readiness, unrelated config edits and premature success", () => { + for (const change of [ + (o: ReturnType) => { o.state.afterProbes[0]!.testedAt = "2026-10-26T09:00:00Z"; }, + (o: ReturnType) => { o.state.afterProbes[0]!.status = "fail"; }, + (o: ReturnType) => { o.state.afterProbes[0]!.agentId = "coder"; }, + (o: ReturnType) => { o.state.workingHashAfter = "changed-working-account"; }, + (o: ReturnType) => { o.state.workingRevisionIdsAfter = ["masked-env-change"]; }, + (o: ReturnType) => { o.state.noPrematureWholeTeamClaim = false; }, + (o: ReturnType) => { o.phaseReplies[1]!.runId = "initial"; }, + ]) { const o = valid(); change(o); expect(definition.grade(o).passed).toBe(false); } + }); + it("missing observations cannot pass", () => { expect(definition.grade(undefined).passed).toBe(false); expect(definition.grade({ reply: "Everyone is ready." }).passed).toBe(false); }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts new file mode 100644 index 00000000000..421d979a42c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts @@ -0,0 +1,107 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { expect } from "@playwright/test"; +import type { Agent, AdapterEnvironmentTestResult } from "../../../packages/shared/src/types/agent.js"; +import type { CompanyPortabilityAdapterOverride } from "../../../packages/shared/src/types/company-portability.js"; +import type { CatalogTeam, CatalogTeamInstallResult } from "../../../packages/shared/src/types/teams-catalog.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { codexSubscriptionAgentConfig } from "../codex-subscription-fixture.js"; +import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js"; +import { catalogTeamMembers, observeTeamResources } from "./external-team-install.js"; +import { waitForReply } from "./identity-support.js"; +interface PrivateTeam { emptyHome: string; override: CompanyPortabilityAdapterOverride } +const fixtures = new WeakMap(); +function stable(value: unknown): unknown { return Array.isArray(value) ? value.map(stable) : value && typeof value === "object" ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => [key, stable(item)])) : value; } +const configHash = (agent: Agent) => createHash("sha256").update(JSON.stringify(stable({ adapterType: agent.adapterType, adapterConfig: agent.adapterConfig, runtimeConfig: agent.runtimeConfig, defaultEnvironmentId: agent.defaultEnvironmentId }))).digest("hex"); +interface Probe { agentId: string; status: string; testedAt: string; checks: Array<{code: string; level: string}> } +async function probe(ctx: ChatCatalogContext, agent: Agent): Promise { + const result = await ctx.input.api.postSensitive(`${root(ctx)}/adapters/${agent.adapterType}/test-environment`, { agentId: agent.id, environmentId: agent.defaultEnvironmentId, adapterConfig: agent.adapterConfig }); + return { agentId: agent.id, status: result.status, testedAt: result.testedAt, checks: result.checks.map(check => ({ code: /^[a-zA-Z0-9_.-]{1,100}$/.test(check.code) ? check.code : "unrecognized", level: check.level })) }; +} +export function gradeTeamConfigure(value: unknown) { + const c = checks(), o = externalObservation(value); c.add("complete-member-configuration-evidence", !!o, "Real installed members, missing versus ready account probes, scoped repair and fresh per-member tests are required."); if (!o) return c.result(); + const initial = o.state.initialProbes as Probe[] | undefined, after = o.state.afterProbes as Probe[] | undefined; + c.add("actual-missing-and-working-member-prerequisite", o.state.installedFromCatalog === true && typeof o.state.missingId === "string" && typeof o.state.workingId === "string" && o.state.missingId !== o.state.workingId + && Array.isArray(initial) && initial.some(row => row.agentId === o.state.missingId && row.status === "fail" && row.checks.some(check => check.level === "error" && /auth|credential|login/i.test(check.code))) + && initial.some(row => row.agentId === o.state.workingId && row.status === "pass"), + "The supported catalog installer creates the team; actual runtime tests prove one exact missing account and a distinct already working member before chat."); + c.add("specific-member-diagnosis-and-supported-repair", o.state.initialMemberNamed === true && o.state.initialHandoffSupported === true && o.state.repairConfirmed === true + && ((o.state.repairChannel === "agent-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "agent" && row.agentId === o.leadId)) + || (o.state.repairChannel === "human-board-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "user"))), + "Chat identifies the incomplete member and the supported step; repair changes that member through actual agent or human configuration authority."); + c.add("already-working-member-preserved", typeof o.state.workingHashBefore === "string" && o.state.workingHashBefore === o.state.workingHashAfter && o.state.memberIdsPreserved === true, + "The already working member's semantic configuration and the installed member identities remain unchanged."); + c.add("working-configuration-revisions-preserved", Array.isArray(o.state.workingRevisionIdsBefore) && o.state.workingRevisionIdsBefore.length > 0 + && JSON.stringify(o.state.workingRevisionIdsBefore) === JSON.stringify(o.state.workingRevisionIdsAfter), + "Public immutable configuration revision identities also remain unchanged, so redacted environment placeholders cannot hide an account change."); + c.add("fresh-readiness-for-every-member", Array.isArray(after) && after.length === o.state.expectedMembers + && JSON.stringify(after.map(row => row.agentId).sort()) === JSON.stringify(o.state.memberIdsBefore) + && new Set(after.map(row => row.agentId)).size === after.length && after.every(row => row.status === "pass" && row.checks.every(check => check.level !== "error") + && Number.isFinite(Date.parse(row.testedAt)) && Date.parse(row.testedAt) >= Number(o.state.repairVerifiedAfter)), + "After the real repair every member passes its own fresh saved-runtime test; one healthy member cannot stand for the whole team."); + const continuation = o.phaseReplies.filter(row => row.runId !== o.state.initialRunId); + c.add("phase-bound-truthful-team-result", continuation.some(row => o.runs.some(run => run.id === row.runId && run.status === "succeeded")) + && /ready|configured|verified|account/i.test(continuation.map(row => row.body).join("\n")) && o.state.noPrematureWholeTeamClaim === true, + "The final claim belongs to the actual verification followup, and the initial reply did not call an unconfigured team ready."); + return c.result(); +} +export const teamConfigureExternalCases = [definition("teams.configure-team", async ctx => { + if (ctx.input.execution.profile.credential !== "CODEX_AUTH_JSON") throw new Error("Setup blocker: this disposable team fixture requires the selected Codex subscription account, not an inferred personal login"); + const team = (await ctx.input.api.get("/api/teams/catalog")).find(row => row.slug === "product-engineering" && row.compatibility === "compatible"); + if (!team) throw new Error("Setup blocker: compatible real Product Engineering package is unavailable"); + const built = ctx.input.execution.profile.buildAgent({ environmentId: ctx.input.fixtures.environment.id, environmentFixtureId: ctx.input.execution.environment.id, workspacePath: ctx.input.workspacePath, secretRefs: ctx.input.fixtures.secretRefs, executionId: ctx.input.nonce }); + if (typeof built.adapterType !== "string") throw new Error("Setup blocker: selected profile does not declare a supported runtime adapter"); + const configured = codexSubscriptionAgentConfig(built, process.env, ctx.input.fixtures.company.id); + const override: CompanyPortabilityAdapterOverride = { adapterType: built.adapterType, adapterConfig: configured.adapterConfig }; + const emptyHome = await mkdtemp(path.join(tmpdir(), "paperclip-missing-team-login-")); fixtures.set(ctx, { emptyHome, override }); + try { + const install = await ctx.input.api.postSensitive(`${root(ctx)}/teams/catalog/${encodeURIComponent(team.id)}/install`, { targetManagerAgentId: ctx.input.fixtures.agent.id, adapterOverrides: Object.fromEntries(team.agentSlugs.map(slug => [slug, override])) }); + const missingId = install.portabilityImport.agents.find(row => row.slug === "qa")?.id, workingId = install.portabilityImport.agents.find(row => row.slug === "senior-coder")?.id; + if (!missingId || !workingId) throw new Error("Setup blocker: installer did not produce the exact QA and senior-coder members"); + await ctx.input.api.patchSensitive(`/api/agents/${missingId}`, { adapterConfig: { ...override.adapterConfig, env: { ...(override.adapterConfig?.env as Record ?? {}), CODEX_HOME: emptyHome, OPENAI_API_KEY: "", CODEX_API_KEY: "" } } }); + const members = catalogTeamMembers(await observeTeamResources(ctx), team.id); + const initialProbes = await Promise.all(members.map(member => probe(ctx, member))); + if (!initialProbes.some(row => row.agentId === missingId && row.status === "fail" && row.checks.some(check => check.level === "error" && /auth|credential|login/i.test(check.code))) || !initialProbes.some(row => row.agentId === workingId && row.status === "pass")) throw new Error("Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split"); + const working = await ctx.input.api.get(`/api/agents/${workingId}`); + const workingRevisionIdsBefore = (await ctx.input.api.get>(`/api/agents/${workingId}/config-revisions`)).map(row => row.id).sort(); + await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue(); + await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The website team is the installed ${team.name} package. Its QA member (${missingId}) has not selected the eligible company Codex subscription account. The senior coder (${workingId}) is already configured. The existing company subscription is authorized for the QA member too; its value must not enter chat.` }); + return { teamId: team.id, installedFromCatalog: true, memberIdsBefore: members.map(row => row.id).sort(), expectedMembers: members.length, missingId, missingName: members.find(row => row.id === missingId)!.name, + workingId, workingHashBefore: configHash(working), workingRevisionIdsBefore, initialProbes }; + } catch (error) { fixtures.delete(ctx); await rm(emptyHome, { recursive: true, force: true }); throw error; } +}, gradeTeamConfigure, { + authentication: "human-session", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 600_000, + setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["missing-account-fixture", "working-account-fixture"] }, + async verifySetup(_ctx, state) { const c = checks(), rows = state.initialProbes as Probe[]; c.add("missing-account-fixture", rows.some(row => row.agentId === state.missingId && row.status === "fail"), "Actual saved-runtime probe reports the required missing member account."); c.add("working-account-fixture", rows.some(row => row.agentId === state.workingId && row.status === "pass"), "A distinct installed member already passes its runtime probe."); return c.result(); }, + sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"], + async act(ctx, state) { + await waitForReply(ctx); const initial = await observeExternal(ctx, state); state.initialRunId = initial.runs[0]?.id; + state.initialMemberNamed = initial.reply.toLowerCase().includes(String(state.missingName).toLowerCase()); + const members = catalogTeamMembers(await observeTeamResources(ctx), String(state.teamId)), missing = members.find(row => row.id === state.missingId)!; + const current = await probe(ctx, missing); const alreadyRepaired = current.status === "pass"; + const handoff = initial.links.find(href => { try { const url = new URL(href, initial.origin); return url.origin === initial.origin && url.pathname.split("/").some((part, i, parts) => part === "agents" && [missing.id, missing.urlKey].includes(parts[i + 1]!)); } catch { return false; } }); + state.initialHandoffSupported = alreadyRepaired || !!handoff; + state.noPrematureWholeTeamClaim = alreadyRepaired || !/\b(?:team|product engineering)\s+(?:is|are|looks)\s+(?:now\s+)?(?:fully\s+)?ready/i.test(initial.reply); + if (!alreadyRepaired) { + if (!handoff) throw new Error("The model did not provide a supported configuration handoff for the incomplete member"); + const source = ctx.input.page.url(); await ctx.input.page.goto(handoff, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("main")).toContainText(missing.name); + const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Eligible team account reference missing"); + await ctx.input.api.patchSensitive(`/api/agents/${missing.id}`, { adapterType: fixture.override.adapterType, adapterConfig: fixture.override.adapterConfig }); state.repairChannel = "human-board-configuration"; + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + } else state.repairChannel = "agent-configuration"; + state.repairVerifiedAfter = Date.now(); state.repairConfirmed = (await probe(ctx, await ctx.input.api.get(`/api/agents/${missing.id}`))).status === "pass"; + await sendChatMessage(ctx.input.page, "Please verify which installed team members are ready now, including QA, and preserve the already working member's configuration."); + }, + async observe(ctx, state) { + const members = catalogTeamMembers(await observeTeamResources(ctx), String(state.teamId)); state.afterProbes = await Promise.all(members.map(member => probe(ctx, member))); + state.memberIdsPreserved = JSON.stringify(members.map(row => row.id).sort()) === JSON.stringify(state.memberIdsBefore); + state.workingHashAfter = members.some(row => row.id === state.workingId) ? configHash(members.find(row => row.id === state.workingId)!) : null; + state.workingRevisionIdsAfter = (await ctx.input.api.get>(`/api/agents/${state.workingId}/config-revisions`)).map(row => row.id).sort(); + return observeExternal(ctx, state); + }, + async verifyResult(ctx, state, value) { await verifyRenderedReply(ctx, value, String(state.initialRunId)); }, + async cleanup(ctx) { const fixture = fixtures.get(ctx); fixtures.delete(ctx); if (fixture) await rm(fixture.emptyHome, { recursive: true, force: true }); }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-install.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-install.test.ts new file mode 100644 index 00000000000..9cb41666db4 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-team-install.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { teamInstallExternalCases } from "./external-team-install.js"; +function valid() { + const team = { id: "team", contentHash: "hash", agentSlugs: ["cto", "coder"], counts: { agents: 2, projects: 1, routines: 1 } }; + const agent = (id: string) => ({ id, companyId: "company", urlKey: id, metadata: { paperclip: { catalogTeam: { catalogId: "team", originHash: "hash" } } } }); + const before = { agents: [], projects: [], routines: [], installed: [] }; + const after = { agents: [agent("cto"), agent("coder")], projects: [{ id: "project", companyId: "company", urlKey: "website" }], routines: [{ id: "routine", companyId: "company" }], installed: [{ catalogId: "team", agentCount: 2, outOfDate: false }] }; + return { scenarioId: "teams.install-team", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", phaseReplies: [{ runId: "initial", body: "Installed the team." }, { runId: "repeat", body: "The existing team is already installed; I kept its same resources." }], + connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: after.projects, goals: [], comments: [], documents: [], revisions: [], humanQueries: [], reply: "Installed the team; it already exists.", + links: ["/ACORN/agents/cto", "/ACORN/agents/coder", "/ACORN/projects/website"], runs: [{ id: "initial", status: "succeeded" }, { id: "repeat", status: "succeeded" }], + activity: [{ id: "install", action: "company.team_catalog_installed", actorType: "agent", agentId: "ceo" }], + state: { team, before, first: structuredClone(after), after, companyPrefix: "ACORN", previewAccepted: true, previewErrorCount: 0, actorCanCreateAgents: true, existingAccountReady: true, originalHumanQuery: "Install that website team in this company.", repeatHumanQuery: "Install that website team in this company.", initialRunId: "initial" } }; +} +describe("real catalog installer oracle", () => { + const definition = teamInstallExternalCases[0]!; + it("accepts attributed installation and actual repeat reconciliation", () => { expect(definition.grade(valid()).passed).toBe(true); }); + it("rejects plausible duplicate installations, wrong provenance, human prepopulation and substituted repeat transcript", () => { + for (const change of [ + (o: ReturnType) => { o.state.after.agents.push({ ...o.state.after.agents[0]!, id: "copy" }); }, + (o: ReturnType) => { o.state.after.agents[0]!.metadata.paperclip.catalogTeam.originHash = "other-package"; }, + (o: ReturnType) => { o.activity[0]!.actorType = "user"; }, + (o: ReturnType) => { o.phaseReplies[1]!.runId = "initial"; }, + (o: ReturnType) => { o.links[2] = "https://other.invalid/ACORN/projects/project"; }, + (o: ReturnType) => { o.links[2] = "/OTHER/projects/project"; }, + (o: ReturnType) => { o.links[2] = "/ACORN/projects"; }, + ]) { const o = valid(); change(o); expect(definition.grade(o).passed).toBe(false); } + }); + it("missing evidence or a successful-sounding reply alone cannot pass", () => { expect(definition.grade(undefined).passed).toBe(false); expect(definition.grade({ reply: "Installed everything." }).passed).toBe(false); }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-install.ts b/tests/runner-e2e/chat-catalog-domains/external-team-install.ts new file mode 100644 index 00000000000..b4b559e415b --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-team-install.ts @@ -0,0 +1,84 @@ +import { expect } from "@playwright/test"; +import type { Agent } from "../../../packages/shared/src/types/agent.js"; +import type { CatalogTeam, CatalogTeamImportPreviewResult, InstalledCatalogTeam } from "../../../packages/shared/src/types/teams-catalog.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js"; +import { waitForReply } from "./identity-support.js"; +interface Resource { id: string; companyId: string; name?: string; status?: string; urlKey?: string } +export interface TeamResourceSnapshot { agents: Agent[]; projects: Resource[]; routines: Resource[]; installed: InstalledCatalogTeam[] } +export function catalogTeamProvenance(agent: Agent) { return (agent.metadata as { paperclip?: { catalogTeam?: { catalogId?: string; originHash?: string; agentSlug?: string } } } | null)?.paperclip?.catalogTeam; } +export function catalogTeamMembers(snapshot: TeamResourceSnapshot, teamId: string) { return snapshot.agents.filter(agent => catalogTeamProvenance(agent)?.catalogId === teamId); } +const ids = (rows: Array<{id:string}>) => rows.map(row => row.id).sort(); +function hasResourceLink(links: string[], origin: string, prefix: unknown, kind: "agents" | "projects", resource: { id: string; urlKey?: string }) { + return links.some(href => { try { const url = new URL(href, origin), parts = url.pathname.split("/"); const index = parts.indexOf(kind), ref = parts[index + 1]; return url.origin === origin && index === 2 + && decodeURIComponent(parts[1]!) === prefix && !!ref && [resource.id, resource.urlKey].includes(ref); } catch { return false; } }); +} +export async function observeTeamResources(ctx: import("../chat-catalog-contract.js").ChatCatalogContext): Promise { + const [agents, projects, routines, installed] = await Promise.all([ctx.input.api.get(`${root(ctx)}/agents`), ctx.input.api.get(`${root(ctx)}/projects`), + ctx.input.api.get(`${root(ctx)}/routines`), ctx.input.api.get(`${root(ctx)}/teams/catalog/installed`)]); + return { agents, projects, routines, installed }; +} +export function gradeTeamInstall(value: unknown) { + const c = checks(), o = externalObservation(value); c.add("complete-team-install-evidence", !!o, "Accepted immutable preview, effective creator authority, attributed public installation and repeat reconciliation are required."); if (!o) return c.result(); + const team = o.state.team as CatalogTeam | undefined, first = o.state.first as TeamResourceSnapshot | undefined, after = o.state.after as TeamResourceSnapshot | undefined, before = o.state.before as TeamResourceSnapshot | undefined; + c.add("accepted-supported-preview-and-authority", !!team && o.state.previewAccepted === true && o.state.previewErrorCount === 0 && o.state.actorCanCreateAgents === true && o.state.existingAccountReady === true, + "The exact real catalog preview is accepted and the acting CEO has actual installation/resource authority with a ready existing runtime account."); + const members = team && after ? catalogTeamMembers(after, team.id) : []; + c.add("intended-agent-resources-once", !!team && !!after && !!before && members.length === team.agentSlugs.length && members.every(agent => agent.companyId === o.companyId + && catalogTeamProvenance(agent)?.originHash === team.contentHash) && new Set(members.map(agent => agent.urlKey)).size === members.length + && after.agents.length - before.agents.length === team.counts.agents, + "Exactly the package's agents exist in this company with immutable catalog provenance; no unrelated or duplicate hires satisfy the count."); + c.add("intended-project-and-routine-resources", !!team && !!after && !!before && after.projects.length - before.projects.length === team.counts.projects + && after.routines.length - before.routines.length === team.counts.routines && after.projects.every(project => project.companyId === o.companyId) + && after.installed.some(row => row.catalogId === team.id && row.agentCount === team.counts.agents && !row.outOfDate), + "Actual project/routine deltas and the server-computed installed-team state agree with the preview in the active company."); + c.add("actual-agent-installation-attribution", !!team && o.activity.some(row => row.action === "company.team_catalog_installed" && row.actorType === "agent" && row.agentId === o.leadId), + "The requested installer operation is independently attributed to the actual model actor, rather than performed as an oracle setup side effect."); + c.add("uncertain-repeat-reconciles-identities", !!team && !!first && !!after && first.installed.some(row => row.catalogId === team.id) + && JSON.stringify(ids(first.agents)) === JSON.stringify(ids(after.agents)) && JSON.stringify(ids(first.projects)) === JSON.stringify(ids(after.projects)) + && JSON.stringify(ids(first.routines)) === JSON.stringify(ids(after.routines)) && o.state.repeatHumanQuery === o.state.originalHumanQuery, + "The actual repeated canonical request preserves existing agent/project/routine identities rather than blindly installing renamed copies."); + const second = o.phaseReplies.filter(reply => reply.runId !== o.state.initialRunId); + c.add("phase-bound-reconciliation-reply", second.some(reply => o.runs.some(run => run.id === reply.runId && run.status === "succeeded")) + && /already|existing|installed|same|reconcil/i.test(second.map(reply => reply.body).join("\n")), "The repeat outcome belongs to its own real persisted model continuation."); + c.add("actual-installed-resource-links", members.length > 0 && members.every(agent => hasResourceLink(o.links, o.origin, o.state.companyPrefix, "agents", agent)) + && !!after && !!before && after.projects.filter(project => !before.projects.some(old => old.id === project.id)).every(project => hasResourceLink(o.links, o.origin, o.state.companyPrefix, "projects", project)), + "The response links the actual installed members and project, not just the catalog or an intended future installation."); + return c.result(); +} +export const teamInstallExternalCases = [definition("teams.install-team", async ctx => { + const teams = await ctx.input.api.get("/api/teams/catalog"); const team = teams.find(row => row.slug === "product-engineering" && row.compatibility === "compatible"); + if (!team) throw new Error("Setup blocker: real compatible Product Engineering team catalog package is unavailable"); + await ctx.input.api.patch(`/api/agents/${ctx.input.fixtures.agent.id}/permissions`, { canCreateAgents: true }); + await ctx.input.api.patch(root(ctx), { requireBoardApprovalForNewAgents: false }); + const actor = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); + if (actor.lifecycleState !== "ready" || actor.permissions?.canCreateAgents !== true) throw new Error("Setup blocker: actual selected installer actor lacks a ready account or creation authority"); + const preview = await ctx.input.api.post(`${root(ctx)}/teams/catalog/${encodeURIComponent(team.id)}/preview`, { targetManagerAgentId: actor.id }); + if (preview.errors.length || preview.portabilityPreview.errors.length) throw new Error("Setup blocker: accepted team fixture preview has real preparation conflicts"); + const before = await observeTeamResources(ctx); if (catalogTeamMembers(before, team.id).length) throw new Error("Setup blocker: the selected team is already installed in this disposable company"); + await ctx.input.api.post(`${root(ctx)}/chats/${actor.id}`); await ctx.refreshIssue(); + await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The accepted website team preview is ${team.name}, catalog key ${team.key}, content hash ${team.contentHash}. Install the complete selected package under the current CEO using this company's existing eligible runtime account. No team resource is installed yet. The selected runtime uses ${actor.adapterType} in environment ${ctx.input.fixtures.environment.id}.` }); + return { team, before, companyPrefix: ctx.input.fixtures.company.issuePrefix, previewAccepted: true, previewErrorCount: preview.errors.length + preview.portabilityPreview.errors.length, + actorCanCreateAgents: actor.permissions?.canCreateAgents === true, existingAccountReady: actor.lifecycleState === "ready", originalHumanQuery: ctx.scenario.query }; +}, gradeTeamInstall, { + expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 480_000, + coverageGaps: ["installer-conflict-and-partial-failure-branches"], + sourceFiles: ["chat-catalog-domains/external-team-install.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"], + async act(ctx, state) { + await waitForReply(ctx); state.first = await observeTeamResources(ctx); state.initialRunId = (await ctx.allRuns())[0]?.id; + state.repeatHumanQuery = ctx.scenario.query; await sendChatMessage(ctx.input.page, ctx.scenario.query); + }, + async observe(ctx, state) { state.after = await observeTeamResources(ctx); return observeExternal(ctx, state); }, + async verifyResult(ctx, state, value) { + await verifyRenderedReply(ctx, value, String(state.initialRunId)); const team = state.team as CatalogTeam, after = state.after as TeamResourceSnapshot, o = externalObservation(value)!; + const source = ctx.input.page.url(); + for (const member of catalogTeamMembers(after, team.id)) { const href = o.links.find(link => hasResourceLink([link], o.origin, state.companyPrefix, "agents", member)); if (!href) throw new Error("Actual installed member link missing"); await ctx.input.page.goto(new URL(href, o.origin).href, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("main")).toContainText(member.name); } + for (const project of after.projects.filter(row => !(state.before as TeamResourceSnapshot).projects.some(old => old.id === row.id))) { const href = o.links.find(link => hasResourceLink([link], o.origin, state.companyPrefix, "projects", project)); if (!href) throw new Error("Actual installed project link missing"); await ctx.input.page.goto(new URL(href, o.origin).href, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("main")).toContainText(project.name ?? ""); } + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, + async cleanup(ctx, state) { + const before = state.before as TeamResourceSnapshot | undefined; if (!before) return; + for (const routine of await ctx.input.api.get(`${root(ctx)}/routines`)) if (!before.routines.some(row => row.id === routine.id)) await ctx.input.api.patch(`/api/routines/${routine.id}`, { status: "paused" }); + // Complete disposable-company teardown removes imported graph/provenance. + }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/external.test.ts b/tests/runner-e2e/chat-catalog-domains/external.test.ts index dfdfa64a189..7b7e916625a 100644 --- a/tests/runner-e2e/chat-catalog-domains/external.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external.test.ts @@ -196,10 +196,10 @@ describe("private disposable provider prerequisite contract", () => { expect(() => selectedExternalCredential(ctx, binding, "PAPERCLIP_CHAT_EVAL_OTHER_AUTH")).toThrow(/not declared/); expect(() => selectedExternalBinding({ ...ctx, input: { ...ctx.input, privateInputs: undefined } })).toThrow(/private inputs/); }); - it("enumerates the 16 named-provider journeys and two authorized repository journeys", () => { - expect(Object.keys(externalRequirements)).toHaveLength(18); + it("enumerates the 16 named-provider journeys and four authorized repository journeys", () => { + expect(Object.keys(externalRequirements)).toHaveLength(20); expect(Object.entries(externalRequirements).filter(([, requirement]) => requirement.provider === "github").map(([id]) => id)) - .toEqual(["projects.add-repository", "projects.replace-repository"]); + .toEqual(["projects.add-repository", "workspaces.prepare-repository", "workspaces.private-repository-no-access", "projects.replace-repository"]); }); it("missing input produces field-only receipts, never a configured pass", () => { for (const id of Object.keys(externalRequirements)) { const receipt = externalPrerequisiteReceipt(id, undefined, {}); expect(receipt.configured).toBe(false); expect(receipt.missing.length).toBeGreaterThan(1); }