diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 210eb94f3be..83e5e41deae 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -6053,6 +6053,7 @@ "chat-catalog-domains/external-team-install.ts", "../../server/src/adapters/registry.ts", "../../packages/adapters/opencode-local/src/server/test.ts", + "../../packages/adapters/opencode-local/src/server/parse.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", @@ -6060,11 +6061,12 @@ "chat-flow.ts", "chat-scenarios/teams.yaml" ], - "oracleVersion": "paperclip.chat-catalog.team-configure.v2", - "expectedRunCount": 2, + "oracleVersion": "paperclip.chat-catalog.team-configure.v6", + "expectedRunCount": 3, + "minimumRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, - "attemptTimeoutMs": 600000, + "attemptTimeoutMs": 720000, "authentication": "human-session", "setupPolicy": { "readiness": "preserve-scenario", diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts index 4ddf64446ab..273ef7fa0ae 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts @@ -14,6 +14,7 @@ function setupFixture(missing: Probe, working: Probe = { status: "pass", testedA if (path.endsWith("/agents")) return members; if (path.endsWith("/config-revisions")) return [{ id: "working-revision" }]; if (path === "/api/agents/coder") return members[1]; + if (path === "/api/auth/get-session") return { user: { id: "person" } }; if (["/projects", "/routines", "/teams/catalog/installed"].some(suffix => path.endsWith(suffix))) return []; throw new Error("Unexpected setup read"); }, @@ -34,6 +35,7 @@ it("the actual setup admits the adapter's exact missing-auth warning and preserv state = await definition.setup(f.ctx); expect(state.initialProbes).toEqual([{ agentId: "qa", ...authWarning }, { agentId: "coder", status: "pass", testedAt: authWarning.testedAt, checks: [] }]); expect((await definition.verifySetup!(f.ctx, state)).passed).toBe(true); + expect(state.currentUserId).toBe("person"); expect(f.probes).toHaveBeenCalledTimes(2); expect(f.evidence).toHaveBeenCalledWith("chat-catalog-team-initial-probes.json", { initialProbes: state.initialProbes }); expect(f.api.patchSensitive).toHaveBeenCalledTimes(1); @@ -51,6 +53,11 @@ it("retains no provider messages or session fields in the precondition witness", expect(Object.keys((f.evidence.mock.calls[0]![1] as { initialProbes: object[] }).initialProbes[0]!).sort()).toEqual(["agentId", "checks", "status", "testedAt"]); } finally { await definition.cleanup!(f.ctx, state ?? {}); } }); +it("names each member's observed readiness when the split does not form", async () => { + const f = setupFixture({ status: "fail", testedAt: authWarning.testedAt, checks: [{ code: "opencode_cwd_valid", level: "info" }, { code: "opencode_hello_probe_failed", level: "error" }] }); + await expect(teamConfigureExternalCases[0]!.setup(f.ctx)).rejects.toThrow( + "Observed: qa (expected missing account): fail [opencode_hello_probe_failed:error]; coder (expected ready): pass"); +}); it.each(["generic warning", "timeout", "model", "wrong level", "other error", "working warning", "working error"])("actual setup rejects %s and retains only closed probe fields before refusing", async fault => { const missing = structuredClone(authWarning), working: Probe = { status: "pass", testedAt: authWarning.testedAt, checks: [] }; if (fault === "generic warning") missing.checks[0]!.code = "opencode_hello_probe_unexpected_output"; diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts index 669bbf669f2..66cd8801854 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; -import { teamConfigureExternalCases } from "./external-team-configure.js"; +import { createHash } from "node:crypto"; +import { teamBindingCandidate, teamConfigureExternalCases, type TeamBindingApproval } from "./external-team-configure.js"; function valid() { return { scenarioId: "teams.configure-team", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], links: [], reply: "QA has been configured and each team member is ready.", phaseReplies: [{ runId: "initial", body: "QA needs the eligible account; open QA configuration." }, { runId: "verified", body: "QA and the senior coder are now configured and ready." }], runs: [{ id: "initial", status: "succeeded" }, { id: "verified", status: "succeeded" }], @@ -7,7 +8,7 @@ function valid() { installedFromCatalog: true, missingId: "qa", workingId: "coder", expectedMembers: 2, memberIdsBefore: ["coder", "qa"], workingRevisionIdsBefore: ["revision"], workingRevisionIdsAfter: ["revision"], initialProbes: [{ agentId: "qa", status: "fail", testedAt: "2026-10-26T10:00:00Z", checks: [{ code: "codex_auth", level: "error" }] }, { agentId: "coder", status: "pass", testedAt: "2026-10-26T10:00:00Z", checks: [] }], afterProbes: [{ agentId: "qa", status: "pass", testedAt: "2026-10-26T10:05:01Z", checks: [] }, { agentId: "coder", status: "pass", testedAt: "2026-10-26T10:05:02Z", checks: [] }], - repairVerifiedAfter: Date.parse("2026-10-26T10:05:00Z"), initialMemberNamed: true, initialHandoffSupported: true, repairConfirmed: true, repairChannel: "human-board-configuration", workingHashBefore: "same-config", workingHashAfter: "same-config", memberIdsPreserved: true, initialRunId: "initial", noPrematureWholeTeamClaim: true, + repairVerifiedAfter: Date.parse("2026-10-26T10:05:00Z"), initialHandoffSupported: true, repairConfirmed: true, repairChannel: "human-board-configuration", workingHashBefore: "same-config", workingHashAfter: "same-config", memberIdsPreserved: true, initialRunId: "initial", preVerificationRunIds: ["initial"], } }; } describe("installed team per-member readiness oracle", () => { @@ -28,9 +29,82 @@ describe("installed team per-member readiness oracle", () => { (o: ReturnType) => { o.state.initialProbes[0]!.agentId = "other-member"; }, (o: ReturnType) => { o.state.workingHashAfter = "changed-working-account"; }, (o: ReturnType) => { o.state.workingRevisionIdsAfter = ["masked-env-change"]; }, - (o: ReturnType) => { o.state.noPrematureWholeTeamClaim = false; }, + (o: ReturnType) => { o.state.initialHandoffSupported = false; }, + (o: ReturnType) => { o.phaseReplies[1]!.body = " "; }, + (o: ReturnType) => { o.runs[1]!.status = "failed"; }, (o: ReturnType) => { o.phaseReplies[1]!.runId = "initial"; }, ]) { const o = valid(); change(o); expect(definition.grade(o).passed).toBe(false); } }); + it("grades the followup structurally, not by its wording", () => { + for (const body of ["Both QA and the senior coder pass their own checks now.", "Fertig: alle Mitglieder bestehen ihre Prüfung."]) { + const o = valid(); o.phaseReplies[1]!.body = body; o.reply = body; expect(definition.grade(o).passed).toBe(true); + } + }); it("missing observations cannot pass", () => { expect(definition.grade(undefined).passed).toBe(false); expect(definition.grade({ reply: "Everyone is ready." }).passed).toBe(false); }); }); + +const cardPayload = { version: 1, prompt: "Bind?", secretProposal: { version: 1, proposalId: "proposal", sourceSecretLabel: "OpenRouter", configPath: "env.OPENROUTER_API_KEY", targetAgentId: "qa", targetAgentName: "QA", justification: "x", expiresAt: "2026-12-01T00:00:00Z" } }; +function card(overrides: Record = {}) { + return { id: "card", companyId: "company", issueId: "source", kind: "request_confirmation", status: "pending", sourceRunId: "initial", createdByAgentId: "ceo", createdByUserId: null, addresseeAgentId: null, + resolvedByUserId: null as string | null, resolvedByAgentId: null as string | null, requestedResolverPolicy: "human_only", effectiveResolverPolicy: "human_only", idempotencyKey: "secret-proposal:proposal", payload: structuredClone(cardPayload), ...overrides }; +} +function proposal(overrides: Record = {}) { + return { id: "proposal", companyId: "company", kind: "binding", status: "pending", secretId: "openrouter-secret", secretProposalId: null, targetType: "agent", target: { id: "qa" }, + configPath: "env.OPENROUTER_API_KEY", originRunId: "initial", proposedBy: { id: "ceo" }, ...overrides }; +} +function candidate(cards: unknown[], proposals: unknown[]) { + return teamBindingCandidate({ interactions: cards as never, proposals: proposals as never, companyId: "company", issueId: "source", leadId: "ceo", initialRunId: "initial", missingId: "qa", + eligible: [{ configPath: "env.OPENROUTER_API_KEY", secretId: "openrouter-secret" }] }); +} +const hash = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex"); +function bindingValid() { + const o = valid(), marker: TeamBindingApproval = { interactionId: "card", proposalId: "proposal", configPath: "env.OPENROUTER_API_KEY", secretId: "openrouter-secret", targetAgentId: "qa", payloadHash: hash(cardPayload) }; + return { ...o, activity: [] as typeof o.activity, interactions: [card({ status: "accepted", resolvedByUserId: "person" })], + runs: [...o.runs, { id: "wake", status: "succeeded" }], phaseReplies: [o.phaseReplies[0]!, { runId: "wake", body: "QA is now bound to the company account." }, o.phaseReplies[1]!], + state: { ...o.state, preVerificationRunIds: ["initial", "wake"], repairChannel: "human-approved-binding", bindingApproval: marker, currentUserId: "person", + bindingsBefore: ["agent:coder:env.OPENROUTER_API_KEY", "agent:ceo:env.OPENROUTER_API_KEY"], bindingsAfter: ["agent:ceo:env.OPENROUTER_API_KEY", "agent:coder:env.OPENROUTER_API_KEY", "agent:qa:env.OPENROUTER_API_KEY"], + missingBindingRef: { type: "secret_ref", secretId: "openrouter-secret" } } }; +} +describe("governed binding approval path", () => { + const definition = teamConfigureExternalCases[0]!; + it("selects only the lead's pending human-only card binding the eligible account to the missing member", () => { + expect(candidate([card()], [proposal()])).toMatchObject({ interactionId: "card", configPath: "env.OPENROUTER_API_KEY", secretId: "openrouter-secret", targetAgentId: "qa" }); + for (const [cards, proposals] of [ + [[card({ payload: { ...cardPayload, secretProposal: { ...cardPayload.secretProposal, targetAgentId: "coder" } } })], [proposal({ target: { id: "coder" } })]], + [[card()], [proposal({ secretId: "other-secret" })]], + [[card()], [proposal({ configPath: "env.OTHER_KEY" })]], + [[card({ resolvedByAgentId: "ceo", status: "accepted" })], [proposal()]], + [[card({ requestedResolverPolicy: "agent_or_human" })], [proposal()]], + [[card({ sourceRunId: "later" })], [proposal()]], + [[], []], + ] as const) expect(candidate([...cards], [...proposals])).toBeUndefined(); + }); + it("accepts the human-approved binding that repairs only the missing member", () => { expect(definition.grade(bindingValid()).passed).toBe(true); }); + it("rejects the wrong member, a different secret, agent self-approval and a missing card", () => { + for (const change of [ + (o: ReturnType) => { o.interactions[0]!.payload.secretProposal.targetAgentId = "coder"; }, + (o: ReturnType) => { o.state.missingBindingRef = { type: "secret_ref", secretId: "other-secret" }; }, + (o: ReturnType) => { o.interactions[0]!.resolvedByAgentId = "ceo"; }, + (o: ReturnType) => { o.interactions[0]!.resolvedByUserId = "someone-else"; }, + (o: ReturnType) => { o.interactions = []; }, + (o: ReturnType) => { o.state.bindingApproval = undefined as never; o.state.initialHandoffSupported = false; }, + (o: ReturnType) => { o.state.bindingsAfter = [...o.state.bindingsAfter, "agent:coder:env.EXTRA"]; }, + (o: ReturnType) => { o.state.workingHashAfter = "changed-working-account"; }, + (o: ReturnType) => { o.phaseReplies = o.phaseReplies.slice(0, 2); }, + (o: ReturnType) => { o.state.preVerificationRunIds = ["initial", "wake", "verified"]; }, + (o: ReturnType) => { o.state.preVerificationRunIds = undefined as never; }, + ]) { const o = bindingValid(); change(o); expect(definition.grade(o).passed).toBe(false); } + }); +}); +it("derives eligible bindings only from the profile's saved secret references", async () => { + const { eligibleTeamBindings } = await import("./external-team-configure.js"); + expect(eligibleTeamBindings({ OPENROUTER_API_KEY: { type: "secret_ref", secretId: "s1", version: "latest" }, OPENCODE_CONFIG_CONTENT: "{}", EMPTY: "" })) + .toEqual([{ configPath: "env.OPENROUTER_API_KEY", secretId: "s1" }]); + expect(eligibleTeamBindings(undefined)).toEqual([]); +}); + +// Paid run 43086418488: a member installed from the catalog has no configuration revisions until it is edited. +it("accepts a working member that still has no configuration revisions after the repair", () => { + const o = valid(); o.state.workingRevisionIdsBefore = []; o.state.workingRevisionIdsAfter = []; expect(teamConfigureExternalCases[0]!.grade(o).passed).toBe(true); + o.state.workingRevisionIdsAfter = ["new-revision"]; expect(teamConfigureExternalCases[0]!.grade(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts index 04c7956e278..ad175bec45d 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts @@ -8,9 +8,12 @@ import type { Agent, AdapterEnvironmentTestResult } from "../../../packages/shar import type { CompanyPortabilityAdapterOverride } from "../../../packages/shared/src/types/company-portability.js"; import type { CatalogTeam, CatalogTeamInstallResult } from "../../../packages/shared/src/types/teams-catalog.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { CompanySecretUsageBinding, SecretProposalView } from "../../../packages/shared/src/types/secrets.js"; +import type { RequestConfirmationInteraction } from "../../../packages/shared/src/types/issue.js"; +import { pollUntil } from "../api.js"; import { sendChatMessage } from "../chat-flow.js"; import { codexSubscriptionAgentConfig } from "../codex-subscription-fixture.js"; -import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js"; +import { definition, externalObservation, observeExternal, root, checks } from "./external-support.js"; import { catalogTeamMembers, observeTeamResources } from "./external-team-install.js"; import { waitForReply } from "./identity-support.js"; interface PrivateTeam { emptyHome: string; override: CompanyPortabilityAdapterOverride } @@ -26,10 +29,55 @@ function missingAccountProbe(row: Probe, agentId: string) { function readyProbe(row: Probe, agentId: string) { return row.agentId === agentId && row.status === "pass" && row.checks.every(check => check.level === "info"); } +export function memberReadinessSummary(rows: readonly Probe[], members: ReadonlyArray>, missingId: string, workingId: string) { + return rows.map(row => { + const role = row.agentId === missingId ? "expected missing account" : row.agentId === workingId ? "expected ready" : "other member"; + const name = members.find(member => member.id === row.agentId)?.name ?? "unknown member"; + const problems = row.checks.filter(check => check.level !== "info").map(check => `${check.code}:${check.level}`); + return `${name} (${role}): ${row.status}${problems.length ? ` [${problems.join(", ")}]` : ""}`; + }).join("; "); +} async function probe(ctx: ChatCatalogContext, agent: Agent): Promise { const result = await ctx.input.api.postSensitive(`${root(ctx)}/adapters/${agent.adapterType}/test-environment`, { agentId: agent.id, environmentId: agent.defaultEnvironmentId, adapterConfig: agent.adapterConfig }); return { agentId: agent.id, status: result.status, testedAt: result.testedAt, checks: result.checks.map(check => ({ code: /^[a-zA-Z0-9_.-]{1,100}$/.test(check.code) ? check.code : "unrecognized", level: check.level })) }; } +export interface EligibleTeamBinding { configPath: string; secretId: string } +export interface TeamBindingApproval { interactionId: string; proposalId: string; configPath: string; secretId: string; targetAgentId: string; payloadHash: string } +const payloadHash = (payload: unknown) => createHash("sha256").update(JSON.stringify(payload)).digest("hex"); +export function eligibleTeamBindings(env: unknown): EligibleTeamBinding[] { + return Object.entries(env && typeof env === "object" ? env as Record : {}).flatMap(([key, value]) => { + const ref = value as { type?: unknown; secretId?: unknown } | null; + return ref && ref.type === "secret_ref" && typeof ref.secretId === "string" ? [{ configPath: `env.${key}`, secretId: ref.secretId }] : []; + }); +} +export function teamBindingCandidate(input: { interactions: readonly RequestConfirmationInteraction[]; proposals: readonly SecretProposalView[]; companyId: string; issueId: string; + leadId: string; initialRunId: string; missingId: string; eligible: readonly EligibleTeamBinding[] }): TeamBindingApproval | undefined { + for (const card of input.interactions) { + const p = card.payload?.secretProposal, proposal = p && input.proposals.find(row => row.id === p.proposalId); + if (!p || !proposal || card.kind !== "request_confirmation" || card.status !== "pending" || card.companyId !== input.companyId || card.issueId !== input.issueId + || card.sourceRunId !== input.initialRunId || card.createdByAgentId !== input.leadId || card.createdByUserId || card.addresseeAgentId || card.resolvedByUserId || card.resolvedByAgentId + || card.requestedResolverPolicy !== "human_only" || card.effectiveResolverPolicy !== "human_only" || card.idempotencyKey !== `secret-proposal:${proposal.id}` + || p.targetAgentId !== input.missingId || p.configPath !== proposal.configPath || proposal.companyId !== input.companyId || proposal.kind !== "binding" || proposal.status !== "pending" + || proposal.targetType !== "agent" || proposal.target?.id !== input.missingId || proposal.secretProposalId || proposal.originRunId !== input.initialRunId || proposal.proposedBy?.id !== input.leadId + || !input.eligible.some(row => row.configPath === proposal.configPath && row.secretId === proposal.secretId)) continue; + return { interactionId: card.id, proposalId: proposal.id, configPath: proposal.configPath!, secretId: proposal.secretId!, targetAgentId: input.missingId, payloadHash: payloadHash(card.payload) }; + } +} +const bindingKey = (row: Pick) => `${row.targetType}:${row.targetId}:${row.configPath}`; +function humanApprovedBinding(o: NonNullable>) { + const marker = o.state.bindingApproval as TeamBindingApproval | undefined, card = marker && o.interactions.find(row => row.id === marker.interactionId) as unknown as RequestConfirmationInteraction | undefined; + const before = o.state.bindingsBefore as string[] | undefined, after = o.state.bindingsAfter as string[] | undefined, ref = o.state.missingBindingRef as { type?: string; secretId?: string } | undefined; + return !!marker && !!card && marker.targetAgentId === o.state.missingId && card.status === "accepted" && card.createdByAgentId === o.leadId && card.sourceRunId === o.state.initialRunId + && typeof o.state.currentUserId === "string" && card.resolvedByUserId === o.state.currentUserId && !card.resolvedByAgentId && payloadHash(card.payload) === marker.payloadHash + && card.payload?.secretProposal?.targetAgentId === o.state.missingId && card.payload.secretProposal.configPath === marker.configPath + && Array.isArray(before) && Array.isArray(after) && !before.includes(`agent:${o.state.missingId}:${marker.configPath}`) + && JSON.stringify([...before, `agent:${o.state.missingId}:${marker.configPath}`].sort()) === JSON.stringify([...after].sort()) + && ref?.type === "secret_ref" && ref.secretId === marker.secretId; +} +function verificationReplies(o: NonNullable>) { + const before = o.state.preVerificationRunIds; + return Array.isArray(before) && before.includes(o.state.initialRunId) ? o.phaseReplies.filter(row => !before.includes(row.runId)) : []; +} export function gradeTeamConfigure(value: unknown) { const c = checks(), o = externalObservation(value); c.add("complete-member-configuration-evidence", !!o, "Real installed members, missing versus ready account probes, scoped repair and fresh per-member tests are required."); if (!o) return c.result(); const initial = o.state.initialProbes as Probe[] | undefined, after = o.state.afterProbes as Probe[] | undefined; @@ -37,13 +85,14 @@ export function gradeTeamConfigure(value: unknown) { && Array.isArray(initial) && initial.some(row => missingAccountProbe(row, String(o.state.missingId))) && initial.some(row => readyProbe(row, String(o.state.workingId))), "The supported catalog installer creates the team; actual runtime tests prove one exact missing account and a distinct already working member before chat."); - c.add("specific-member-diagnosis-and-supported-repair", o.state.initialMemberNamed === true && o.state.initialHandoffSupported === true && o.state.repairConfirmed === true + c.add("specific-member-diagnosis-and-supported-repair", o.state.initialHandoffSupported === true && o.state.repairConfirmed === true && ((o.state.repairChannel === "agent-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "agent" && row.agentId === o.leadId)) - || (o.state.repairChannel === "human-board-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "user"))), - "Chat identifies the incomplete member and the supported step; repair changes that member through actual agent or human configuration authority."); + || (o.state.repairChannel === "human-board-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "user")) + || (o.state.repairChannel === "human-approved-binding" && humanApprovedBinding(o))), + "Chat identifies the incomplete member by linking its own configuration page, proposing a governed binding of the eligible company account to that member, or repairing it itself; the repair changes only that member through actual agent or human authority."); c.add("already-working-member-preserved", typeof o.state.workingHashBefore === "string" && o.state.workingHashBefore === o.state.workingHashAfter && o.state.memberIdsPreserved === true, "The already working member's semantic configuration and the installed member identities remain unchanged."); - c.add("working-configuration-revisions-preserved", Array.isArray(o.state.workingRevisionIdsBefore) && o.state.workingRevisionIdsBefore.length > 0 + c.add("working-configuration-revisions-preserved", Array.isArray(o.state.workingRevisionIdsBefore) && Array.isArray(o.state.workingRevisionIdsAfter) && JSON.stringify(o.state.workingRevisionIdsBefore) === JSON.stringify(o.state.workingRevisionIdsAfter), "Public immutable configuration revision identities also remain unchanged, so redacted environment placeholders cannot hide an account change."); c.add("fresh-readiness-for-every-member", Array.isArray(after) && after.length === o.state.expectedMembers @@ -51,10 +100,9 @@ export function gradeTeamConfigure(value: unknown) { && new Set(after.map(row => row.agentId)).size === after.length && after.every(row => row.status === "pass" && row.checks.every(check => check.level !== "error") && Number.isFinite(Date.parse(row.testedAt)) && Date.parse(row.testedAt) >= Number(o.state.repairVerifiedAfter)), "After the real repair every member passes its own fresh saved-runtime test; one healthy member cannot stand for the whole team."); - const continuation = o.phaseReplies.filter(row => row.runId !== o.state.initialRunId); - c.add("phase-bound-truthful-team-result", continuation.some(row => o.runs.some(run => run.id === row.runId && run.status === "succeeded")) - && /ready|configured|verified|account/i.test(continuation.map(row => row.body).join("\n")) && o.state.noPrematureWholeTeamClaim === true, - "The final claim belongs to the actual verification followup, and the initial reply did not call an unconfigured team ready."); + const continuation = verificationReplies(o); + c.add("phase-bound-team-result", continuation.some(row => row.body.trim().length > 0 && o.runs.some(run => run.id === row.runId && run.status === "succeeded")), + "A run started by the verification message posts its own reply; earlier initial or approval-continuation replies cannot stand in. Its wording is left to human reading against the fresh per-member probes."); return c.result(); } export const teamConfigureExternalCases = [definition("teams.configure-team", async ctx => { @@ -75,35 +123,45 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as const members = catalogTeamMembers(await observeTeamResources(ctx), team.id); const initialProbes = await Promise.all(members.map(member => probe(ctx, member))); await ctx.input.evidence("chat-catalog-team-initial-probes.json", { initialProbes }); - if (!initialProbes.some(row => missingAccountProbe(row, missingId)) || !initialProbes.some(row => readyProbe(row, workingId))) throw new Error("Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split"); + if (!initialProbes.some(row => missingAccountProbe(row, missingId)) || !initialProbes.some(row => readyProbe(row, workingId))) throw new Error(`Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split. Observed: ${memberReadinessSummary(initialProbes, members, missingId, workingId)}`); + const session = await ctx.input.api.get<{ user?: { id?: string } }>("/api/auth/get-session"); const working = await ctx.input.api.get(`/api/agents/${workingId}`); const workingRevisionIdsBefore = (await ctx.input.api.get>(`/api/agents/${workingId}/config-revisions`)).map(row => row.id).sort(); await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue(); await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The website team is the installed ${team.name} package. Its QA member (${missingId}) has not selected the eligible company ${subscription ? "Codex subscription" : "OpenRouter"} account. The senior coder (${workingId}) is already configured. The existing company account is authorized for the QA member too; its value must not enter chat.` }); return { teamId: team.id, installedFromCatalog: true, memberIdsBefore: members.map(row => row.id).sort(), expectedMembers: members.length, missingId, missingName: members.find(row => row.id === missingId)!.name, - workingId, workingHashBefore: configHash(working), workingRevisionIdsBefore, initialProbes }; + workingId, workingHashBefore: configHash(working), workingRevisionIdsBefore, initialProbes, + eligibleBindings: eligibleTeamBindings((override.adapterConfig as { env?: unknown } | undefined)?.env), currentUserId: session.user?.id ?? null }; } catch (error) { fixtures.delete(ctx); await rm(emptyHome, { recursive: true, force: true }); throw error; } }, gradeTeamConfigure, { - authentication: "human-session", oracleVersion: "paperclip.chat-catalog.team-configure.v2", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 600_000, + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.team-configure.v6", expectedRunCount: 3, minimumRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000, setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["missing-account-fixture", "working-account-fixture"] }, async verifySetup(_ctx, state) { const c = checks(), rows = state.initialProbes as Probe[]; c.add("missing-account-fixture", rows.some(row => missingAccountProbe(row, String(state.missingId))), "Actual saved-runtime probe reports the required missing member account."); c.add("working-account-fixture", rows.some(row => readyProbe(row, String(state.workingId))), "A distinct installed member already passes its runtime probe."); return c.result(); }, - sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "../../server/src/adapters/registry.ts", "../../packages/adapters/opencode-local/src/server/test.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"], + sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "../../server/src/adapters/registry.ts", "../../packages/adapters/opencode-local/src/server/test.ts", "../../packages/adapters/opencode-local/src/server/parse.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"], async act(ctx, state) { await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const initial = await observeExternal(ctx, state); state.initialRunId = initial.runs[0]?.id; - state.initialMemberNamed = initial.reply.toLowerCase().includes(String(state.missingName).toLowerCase()); const members = catalogTeamMembers(await observeTeamResources(ctx), String(state.teamId)), missing = members.find(row => row.id === state.missingId)!; const current = await probe(ctx, missing); const alreadyRepaired = current.status === "pass"; const handoff = initial.links.find(href => { try { const url = new URL(href, initial.origin); return url.origin === initial.origin && url.pathname.split("/").some((part, i, parts) => part === "agents" && [missing.id, missing.urlKey].includes(parts[i + 1]!)); } catch { return false; } }); - state.initialHandoffSupported = alreadyRepaired || !!handoff; - state.noPrematureWholeTeamClaim = alreadyRepaired || !/\b(?:team|product engineering)\s+(?:is|are|looks)\s+(?:now\s+)?(?:fully\s+)?ready/i.test(initial.reply); - if (!alreadyRepaired) { - if (!handoff) throw new Error("The model did not provide a supported configuration handoff for the incomplete member"); + const issueId = ctx.issue()!.id, eligible = state.eligibleBindings as EligibleTeamBinding[]; + const binding = alreadyRepaired ? undefined : teamBindingCandidate({ interactions: await ctx.input.api.get(`/api/issues/${issueId}/interactions`), + proposals: await ctx.input.api.get(`${root(ctx)}/secret-proposals`), companyId: ctx.input.fixtures.company.id, issueId, leadId: ctx.input.fixtures.agent.id, + initialRunId: String(state.initialRunId), missingId: missing.id, eligible }); + state.initialHandoffSupported = alreadyRepaired || !!binding || !!handoff; + if (binding) { + state.bindingsBefore = await usageKeys(ctx, binding.secretId); + await acceptTeamBinding(ctx, binding, missing.name, String(state.currentUserId)); + state.bindingApproval = binding; state.repairChannel = "human-approved-binding"; + await waitForBindingContinuation(ctx); + } else if (!alreadyRepaired) { + if (!handoff) throw new Error("The model provided neither a configuration link nor a governed binding proposal for the incomplete member"); const source = ctx.input.page.url(); await ctx.input.page.goto(handoff, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("main")).toContainText(missing.name); const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Eligible team account reference missing"); await ctx.input.api.patchSensitive(`/api/agents/${missing.id}`, { adapterType: fixture.override.adapterType, adapterConfig: fixture.override.adapterConfig }); state.repairChannel = "human-board-configuration"; await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); } else state.repairChannel = "agent-configuration"; state.repairVerifiedAfter = Date.now(); state.repairConfirmed = (await probe(ctx, await ctx.input.api.get(`/api/agents/${missing.id}`))).status === "pass"; + state.preVerificationRunIds = (await ctx.allRuns()).map(run => run.id); await sendChatMessage(ctx.input.page, "Please verify which installed team members are ready now, including QA, and preserve the already working member's configuration.", chatCatalogNativePhaseRemainingBudget(ctx)); }, async observe(ctx, state) { @@ -111,12 +169,45 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as state.memberIdsPreserved = JSON.stringify(members.map(row => row.id).sort()) === JSON.stringify(state.memberIdsBefore); state.workingHashAfter = members.some(row => row.id === state.workingId) ? configHash(members.find(row => row.id === state.workingId)!) : null; state.workingRevisionIdsAfter = (await ctx.input.api.get>(`/api/agents/${state.workingId}/config-revisions`)).map(row => row.id).sort(); + const marker = state.bindingApproval as TeamBindingApproval | undefined; + if (marker) { + state.bindingsAfter = await usageKeys(ctx, marker.secretId); + const qa = await ctx.input.api.get(`/api/agents/${String(state.missingId)}`), ref = (qa.adapterConfig as { env?: Record }).env?.[marker.configPath.slice(4)] as { type?: string; secretId?: string } | undefined; + state.missingBindingRef = ref ? { type: ref.type, secretId: ref.secretId } : null; + } return observeExternal(ctx, state); }, - async verifyResult(ctx, state, value) { await verifyRenderedReply(ctx, value, String(state.initialRunId)); }, + async verifyResult(ctx, _state, value) { + const o = externalObservation(value), reply = o && verificationReplies(o).at(-1); + if (!o || !reply?.commentId || !reply.body.trim()) throw new Error("No persisted reply from the verification run"); + await expect(ctx.input.page.locator(`[id=${JSON.stringify(`comment-${reply.commentId}`)}]`)).toBeVisible(); + await ctx.input.evidence("chat-catalog-external-visible-reply.json", { scenarioId: o.scenarioId, commentId: reply.commentId, runId: reply.runId }); + }, async cleanup(ctx) { const fixture = fixtures.get(ctx); fixtures.delete(ctx); if (fixture) await rm(fixture.emptyHome, { recursive: true, force: true }); }, })]; +async function usageKeys(ctx: ChatCatalogContext, secretId: string) { + return (await ctx.input.api.get<{ bindings: CompanySecretUsageBinding[] }>(`/api/secrets/${secretId}/usage`)).bindings.map(bindingKey).sort(); +} +async function acceptTeamBinding(ctx: ChatCatalogContext, marker: TeamBindingApproval, memberName: string, userId: string) { + const page = ctx.input.page, budget = chatCatalogNativePhaseRemainingBudget(ctx), remaining = () => Math.max(1_000, budget ? budget() : 120_000), issueId = ctx.issue()!.id; + const card = page.locator(`[data-testid="task-chat-interaction"][id=${JSON.stringify(`interaction-${marker.interactionId}`)}]`); + await expect(card).toBeVisible({ timeout: remaining() }); + for (const text of [memberName, marker.configPath]) await expect(card).toContainText(text, { timeout: remaining() }); + const response = page.waitForResponse(res => new URL(res.url()).origin === new URL(page.url()).origin && new URL(res.url()).pathname === `/api/issues/${issueId}/interactions/${marker.interactionId}/accept` && res.request().method() === "POST", { timeout: remaining() }); + await card.getByRole("button", { name: "Create binding", exact: true }).click({ timeout: remaining() }); + const submitted = await response; if (!submitted.ok()) throw new Error("The binding approval did not persist"); + const receipt = await submitted.json() as RequestConfirmationInteraction; + if (receipt.id !== marker.interactionId || receipt.status !== "accepted" || receipt.resolvedByUserId !== userId || receipt.resolvedByAgentId || receipt.issueId !== issueId || payloadHash(receipt.payload) !== marker.payloadHash) + throw new Error("The binding approval receipt differs from the displayed proposal accepted by the current person"); +} +async function waitForBindingContinuation(ctx: ChatCatalogContext) { + const startedAt = Date.now(), budget = chatCatalogNativePhaseRemainingBudget(ctx), deadlineAt = startedAt + Math.min(budget ? budget() : 300_000, 300_000); + await pollUntil({ label: "binding approval continuation", deadlineAt, intervalMs: 1_000, + load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; }, + accept: ({ issue, runs }) => runs.every(run => run.status === "succeeded") && issue?.conversationState === "waiting" && (runs.length >= 2 || Date.now() - startedAt >= 15_000), + reject: ({ runs }) => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "The binding approval continuation failed" : undefined }); +} export function missingTeamAccountEnvironment(subscription: boolean, emptyHome: string) { return subscription ? { CODEX_HOME: emptyHome, OPENAI_API_KEY: "", CODEX_API_KEY: "" } : { OPENROUTER_API_KEY: "" }; }