From ba905c7254556c5bec2887b48e0cfa49711d1705 Mon Sep 17 00:00:00 2001
From: Dotta <34892728+cryppadotta@users.noreply.github.com>
Date: Wed, 11 Nov 2026 17:26:01 -0600
Subject: [PATCH] fix(evals): name each team member's readiness and grade
configure-team structurally (#20879)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Thinking Path
> - Paperclip is the open source app people use to manage AI agents for
work
> - The chat catalog eval `teams.configure-team` checks that chat gets
an installed team ready member by member
> - Its setup installs Product Engineering, removes the QA member's
account, and requires a real probe split: QA missing an account, the
senior coder ready
> - The QA probe returned a generic `opencode_hello_probe_failed` error,
so the setup stopped with "probes do not establish the required
missing-versus-ready account split"
> - The cause is a product classification gap: OpenRouter answers an
empty key with HTTP 401 "Missing Authentication header", which the
OpenCode probe did not treat as authentication (fixed in #20878,
cherry-picked here)
> - This pull request makes the setup blocker name each member's
observed readiness, and replaces reply keyword checks with structural
checks
> - The benefit is that the eval can reach the chat journey and grades
it without matching words
## Linked Issues or Issue Description
Refs #20878
**What happened**
`teams.configure-team` failed in setup. The QA member's probe was `fail`
with `opencode_hello_probe_failed`. The senior coder passed. The setup
error did not say which member was in which state.
**Expected behavior**
The QA member's probe reports the missing-authentication check, so the
split forms and the chat journey runs. When the split does not form, the
setup error names each member's observed status and checks.
**Steps to reproduce**
1. Run `chat-features.runner-opencode.local.teams.configure-team` on
master.
2. Read `snapshots/chat-catalog-team-initial-probes.json`.
3. QA shows `fail` with `opencode_hello_probe_failed:error`; the setup
stops.
## What Changed
- Cherry-pick of #20878: the OpenCode probe classifies a provider 401 as
`opencode_hello_probe_auth_required`.
- The setup blocker now lists each member as `name (expected missing
account | expected ready | other member): status [non-info checks]`.
- Grader: remove the member-name substring check and the "team is ready"
regex. Identification is the rendered link to the missing member's own
page, which the case already required. The followup is graded by a
non-empty reply from its own succeeded run, not by the words
"ready/configured/verified/account". Fresh per-member probes, preserved
working configuration and revisions, and the attributed repair stay
unchanged.
- Calibration: reworded and non-English valid followups pass; missing
handoff, empty followup, and failed followup run fail. A setup test with
the recorded failure shape checks the new error text.
- `oracleVersion` bumped to `paperclip.chat-catalog.team-configure.v3`;
`parse.ts` added to `sourceFiles`.
## Verification
- `pnpm exec vitest run --config tests/runner-e2e/vitest.config.ts
tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts
tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts
tests/runner-e2e/chat-runtime-profile.test.ts
tests/runner-e2e/chat-catalog-contract.test.ts
tests/runner-e2e/chat-catalog-consistency.test.ts` passes.
- `node tests/runner-e2e/source-sdk-public-inputs.mjs
--check`
reports no change.
- `node --test
tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs` passes.
- The paid run
`chat-features.runner-opencode.local.teams.configure-team` is still to
run.
## Risks
- The removed "team is ready" regex failed open. A false whole-team
claim in the reply now needs human reading of the transcript. The
structural checks still reject a missing handoff, a missing repair, and
any member that fails its fresh probe.
## Model Used
- Anthropic Claude Opus 5.5 (`claude-opus-5-5`), tool use and code
execution in Claude Code.
## Checklist
- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either (a) linked existing issues with `Fixes: #` / `Closes
#` / `Refs #` OR (b) described the issue in-PR following the relevant
issue template
- [x] I have not referenced internal/instance-local Paperclip issues or
links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip`
URLs)
- [x] My branch name describes the change (e.g. `docs/...`, `fix/...`)
and contains no internal Paperclip ticket id or instance-derived details
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [ ] All Paperclip CI gates are green
- [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups
- [x] I will address all Greptile and reviewer comments before
requesting merge
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---------
Co-authored-by: Claude Opus 5.5
---
.../runner-e2e/chat-catalog-descriptors.json | 8 +-
...ternal-team-configure-prerequisite.test.ts | 7 +
.../external-team-configure.test.ts | 80 ++++++++++-
.../external-team-configure.ts | 129 +++++++++++++++---
4 files changed, 199 insertions(+), 25 deletions(-)
diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json
index 210eb94f3be..83e5e41deae 100644
--- a/tests/runner-e2e/chat-catalog-descriptors.json
+++ b/tests/runner-e2e/chat-catalog-descriptors.json
@@ -6053,6 +6053,7 @@
"chat-catalog-domains/external-team-install.ts",
"../../server/src/adapters/registry.ts",
"../../packages/adapters/opencode-local/src/server/test.ts",
+ "../../packages/adapters/opencode-local/src/server/parse.ts",
"codex-subscription-fixture.ts",
"chat-catalog-domains/external-support.ts",
"chat-catalog-domains/internal-support.ts",
@@ -6060,11 +6061,12 @@
"chat-flow.ts",
"chat-scenarios/teams.yaml"
],
- "oracleVersion": "paperclip.chat-catalog.team-configure.v2",
- "expectedRunCount": 2,
+ "oracleVersion": "paperclip.chat-catalog.team-configure.v6",
+ "expectedRunCount": 3,
+ "minimumRunCount": 2,
"expectedHumanMessageCount": 2,
"turnTimeoutMs": 120000,
- "attemptTimeoutMs": 600000,
+ "attemptTimeoutMs": 720000,
"authentication": "human-session",
"setupPolicy": {
"readiness": "preserve-scenario",
diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts
index 4ddf64446ab..273ef7fa0ae 100644
--- a/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts
@@ -14,6 +14,7 @@ function setupFixture(missing: Probe, working: Probe = { status: "pass", testedA
if (path.endsWith("/agents")) return members;
if (path.endsWith("/config-revisions")) return [{ id: "working-revision" }];
if (path === "/api/agents/coder") return members[1];
+ if (path === "/api/auth/get-session") return { user: { id: "person" } };
if (["/projects", "/routines", "/teams/catalog/installed"].some(suffix => path.endsWith(suffix))) return [];
throw new Error("Unexpected setup read");
},
@@ -34,6 +35,7 @@ it("the actual setup admits the adapter's exact missing-auth warning and preserv
state = await definition.setup(f.ctx);
expect(state.initialProbes).toEqual([{ agentId: "qa", ...authWarning }, { agentId: "coder", status: "pass", testedAt: authWarning.testedAt, checks: [] }]);
expect((await definition.verifySetup!(f.ctx, state)).passed).toBe(true);
+ expect(state.currentUserId).toBe("person");
expect(f.probes).toHaveBeenCalledTimes(2);
expect(f.evidence).toHaveBeenCalledWith("chat-catalog-team-initial-probes.json", { initialProbes: state.initialProbes });
expect(f.api.patchSensitive).toHaveBeenCalledTimes(1);
@@ -51,6 +53,11 @@ it("retains no provider messages or session fields in the precondition witness",
expect(Object.keys((f.evidence.mock.calls[0]![1] as { initialProbes: object[] }).initialProbes[0]!).sort()).toEqual(["agentId", "checks", "status", "testedAt"]);
} finally { await definition.cleanup!(f.ctx, state ?? {}); }
});
+it("names each member's observed readiness when the split does not form", async () => {
+ const f = setupFixture({ status: "fail", testedAt: authWarning.testedAt, checks: [{ code: "opencode_cwd_valid", level: "info" }, { code: "opencode_hello_probe_failed", level: "error" }] });
+ await expect(teamConfigureExternalCases[0]!.setup(f.ctx)).rejects.toThrow(
+ "Observed: qa (expected missing account): fail [opencode_hello_probe_failed:error]; coder (expected ready): pass");
+});
it.each(["generic warning", "timeout", "model", "wrong level", "other error", "working warning", "working error"])("actual setup rejects %s and retains only closed probe fields before refusing", async fault => {
const missing = structuredClone(authWarning), working: Probe = { status: "pass", testedAt: authWarning.testedAt, checks: [] };
if (fault === "generic warning") missing.checks[0]!.code = "opencode_hello_probe_unexpected_output";
diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts
index 669bbf669f2..66cd8801854 100644
--- a/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts
@@ -1,5 +1,6 @@
import { describe, expect, it } from "vitest";
-import { teamConfigureExternalCases } from "./external-team-configure.js";
+import { createHash } from "node:crypto";
+import { teamBindingCandidate, teamConfigureExternalCases, type TeamBindingApproval } from "./external-team-configure.js";
function valid() {
return { scenarioId: "teams.configure-team", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], links: [],
reply: "QA has been configured and each team member is ready.", phaseReplies: [{ runId: "initial", body: "QA needs the eligible account; open QA configuration." }, { runId: "verified", body: "QA and the senior coder are now configured and ready." }], runs: [{ id: "initial", status: "succeeded" }, { id: "verified", status: "succeeded" }],
@@ -7,7 +8,7 @@ function valid() {
installedFromCatalog: true, missingId: "qa", workingId: "coder", expectedMembers: 2, memberIdsBefore: ["coder", "qa"], workingRevisionIdsBefore: ["revision"], workingRevisionIdsAfter: ["revision"],
initialProbes: [{ agentId: "qa", status: "fail", testedAt: "2026-10-26T10:00:00Z", checks: [{ code: "codex_auth", level: "error" }] }, { agentId: "coder", status: "pass", testedAt: "2026-10-26T10:00:00Z", checks: [] }],
afterProbes: [{ agentId: "qa", status: "pass", testedAt: "2026-10-26T10:05:01Z", checks: [] }, { agentId: "coder", status: "pass", testedAt: "2026-10-26T10:05:02Z", checks: [] }],
- repairVerifiedAfter: Date.parse("2026-10-26T10:05:00Z"), initialMemberNamed: true, initialHandoffSupported: true, repairConfirmed: true, repairChannel: "human-board-configuration", workingHashBefore: "same-config", workingHashAfter: "same-config", memberIdsPreserved: true, initialRunId: "initial", noPrematureWholeTeamClaim: true,
+ repairVerifiedAfter: Date.parse("2026-10-26T10:05:00Z"), initialHandoffSupported: true, repairConfirmed: true, repairChannel: "human-board-configuration", workingHashBefore: "same-config", workingHashAfter: "same-config", memberIdsPreserved: true, initialRunId: "initial", preVerificationRunIds: ["initial"],
} };
}
describe("installed team per-member readiness oracle", () => {
@@ -28,9 +29,82 @@ describe("installed team per-member readiness oracle", () => {
(o: ReturnType) => { o.state.initialProbes[0]!.agentId = "other-member"; },
(o: ReturnType) => { o.state.workingHashAfter = "changed-working-account"; },
(o: ReturnType) => { o.state.workingRevisionIdsAfter = ["masked-env-change"]; },
- (o: ReturnType) => { o.state.noPrematureWholeTeamClaim = false; },
+ (o: ReturnType) => { o.state.initialHandoffSupported = false; },
+ (o: ReturnType) => { o.phaseReplies[1]!.body = " "; },
+ (o: ReturnType) => { o.runs[1]!.status = "failed"; },
(o: ReturnType) => { o.phaseReplies[1]!.runId = "initial"; },
]) { const o = valid(); change(o); expect(definition.grade(o).passed).toBe(false); }
});
+ it("grades the followup structurally, not by its wording", () => {
+ for (const body of ["Both QA and the senior coder pass their own checks now.", "Fertig: alle Mitglieder bestehen ihre Prüfung."]) {
+ const o = valid(); o.phaseReplies[1]!.body = body; o.reply = body; expect(definition.grade(o).passed).toBe(true);
+ }
+ });
it("missing observations cannot pass", () => { expect(definition.grade(undefined).passed).toBe(false); expect(definition.grade({ reply: "Everyone is ready." }).passed).toBe(false); });
});
+
+const cardPayload = { version: 1, prompt: "Bind?", secretProposal: { version: 1, proposalId: "proposal", sourceSecretLabel: "OpenRouter", configPath: "env.OPENROUTER_API_KEY", targetAgentId: "qa", targetAgentName: "QA", justification: "x", expiresAt: "2026-12-01T00:00:00Z" } };
+function card(overrides: Record = {}) {
+ return { id: "card", companyId: "company", issueId: "source", kind: "request_confirmation", status: "pending", sourceRunId: "initial", createdByAgentId: "ceo", createdByUserId: null, addresseeAgentId: null,
+ resolvedByUserId: null as string | null, resolvedByAgentId: null as string | null, requestedResolverPolicy: "human_only", effectiveResolverPolicy: "human_only", idempotencyKey: "secret-proposal:proposal", payload: structuredClone(cardPayload), ...overrides };
+}
+function proposal(overrides: Record = {}) {
+ return { id: "proposal", companyId: "company", kind: "binding", status: "pending", secretId: "openrouter-secret", secretProposalId: null, targetType: "agent", target: { id: "qa" },
+ configPath: "env.OPENROUTER_API_KEY", originRunId: "initial", proposedBy: { id: "ceo" }, ...overrides };
+}
+function candidate(cards: unknown[], proposals: unknown[]) {
+ return teamBindingCandidate({ interactions: cards as never, proposals: proposals as never, companyId: "company", issueId: "source", leadId: "ceo", initialRunId: "initial", missingId: "qa",
+ eligible: [{ configPath: "env.OPENROUTER_API_KEY", secretId: "openrouter-secret" }] });
+}
+const hash = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
+function bindingValid() {
+ const o = valid(), marker: TeamBindingApproval = { interactionId: "card", proposalId: "proposal", configPath: "env.OPENROUTER_API_KEY", secretId: "openrouter-secret", targetAgentId: "qa", payloadHash: hash(cardPayload) };
+ return { ...o, activity: [] as typeof o.activity, interactions: [card({ status: "accepted", resolvedByUserId: "person" })],
+ runs: [...o.runs, { id: "wake", status: "succeeded" }], phaseReplies: [o.phaseReplies[0]!, { runId: "wake", body: "QA is now bound to the company account." }, o.phaseReplies[1]!],
+ state: { ...o.state, preVerificationRunIds: ["initial", "wake"], repairChannel: "human-approved-binding", bindingApproval: marker, currentUserId: "person",
+ bindingsBefore: ["agent:coder:env.OPENROUTER_API_KEY", "agent:ceo:env.OPENROUTER_API_KEY"], bindingsAfter: ["agent:ceo:env.OPENROUTER_API_KEY", "agent:coder:env.OPENROUTER_API_KEY", "agent:qa:env.OPENROUTER_API_KEY"],
+ missingBindingRef: { type: "secret_ref", secretId: "openrouter-secret" } } };
+}
+describe("governed binding approval path", () => {
+ const definition = teamConfigureExternalCases[0]!;
+ it("selects only the lead's pending human-only card binding the eligible account to the missing member", () => {
+ expect(candidate([card()], [proposal()])).toMatchObject({ interactionId: "card", configPath: "env.OPENROUTER_API_KEY", secretId: "openrouter-secret", targetAgentId: "qa" });
+ for (const [cards, proposals] of [
+ [[card({ payload: { ...cardPayload, secretProposal: { ...cardPayload.secretProposal, targetAgentId: "coder" } } })], [proposal({ target: { id: "coder" } })]],
+ [[card()], [proposal({ secretId: "other-secret" })]],
+ [[card()], [proposal({ configPath: "env.OTHER_KEY" })]],
+ [[card({ resolvedByAgentId: "ceo", status: "accepted" })], [proposal()]],
+ [[card({ requestedResolverPolicy: "agent_or_human" })], [proposal()]],
+ [[card({ sourceRunId: "later" })], [proposal()]],
+ [[], []],
+ ] as const) expect(candidate([...cards], [...proposals])).toBeUndefined();
+ });
+ it("accepts the human-approved binding that repairs only the missing member", () => { expect(definition.grade(bindingValid()).passed).toBe(true); });
+ it("rejects the wrong member, a different secret, agent self-approval and a missing card", () => {
+ for (const change of [
+ (o: ReturnType) => { o.interactions[0]!.payload.secretProposal.targetAgentId = "coder"; },
+ (o: ReturnType) => { o.state.missingBindingRef = { type: "secret_ref", secretId: "other-secret" }; },
+ (o: ReturnType) => { o.interactions[0]!.resolvedByAgentId = "ceo"; },
+ (o: ReturnType) => { o.interactions[0]!.resolvedByUserId = "someone-else"; },
+ (o: ReturnType) => { o.interactions = []; },
+ (o: ReturnType) => { o.state.bindingApproval = undefined as never; o.state.initialHandoffSupported = false; },
+ (o: ReturnType) => { o.state.bindingsAfter = [...o.state.bindingsAfter, "agent:coder:env.EXTRA"]; },
+ (o: ReturnType) => { o.state.workingHashAfter = "changed-working-account"; },
+ (o: ReturnType) => { o.phaseReplies = o.phaseReplies.slice(0, 2); },
+ (o: ReturnType) => { o.state.preVerificationRunIds = ["initial", "wake", "verified"]; },
+ (o: ReturnType) => { o.state.preVerificationRunIds = undefined as never; },
+ ]) { const o = bindingValid(); change(o); expect(definition.grade(o).passed).toBe(false); }
+ });
+});
+it("derives eligible bindings only from the profile's saved secret references", async () => {
+ const { eligibleTeamBindings } = await import("./external-team-configure.js");
+ expect(eligibleTeamBindings({ OPENROUTER_API_KEY: { type: "secret_ref", secretId: "s1", version: "latest" }, OPENCODE_CONFIG_CONTENT: "{}", EMPTY: "" }))
+ .toEqual([{ configPath: "env.OPENROUTER_API_KEY", secretId: "s1" }]);
+ expect(eligibleTeamBindings(undefined)).toEqual([]);
+});
+
+// Paid run 43086418488: a member installed from the catalog has no configuration revisions until it is edited.
+it("accepts a working member that still has no configuration revisions after the repair", () => {
+ const o = valid(); o.state.workingRevisionIdsBefore = []; o.state.workingRevisionIdsAfter = []; expect(teamConfigureExternalCases[0]!.grade(o).passed).toBe(true);
+ o.state.workingRevisionIdsAfter = ["new-revision"]; expect(teamConfigureExternalCases[0]!.grade(o).passed).toBe(false);
+});
diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts
index 04c7956e278..ad175bec45d 100644
--- a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts
+++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts
@@ -8,9 +8,12 @@ import type { Agent, AdapterEnvironmentTestResult } from "../../../packages/shar
import type { CompanyPortabilityAdapterOverride } from "../../../packages/shared/src/types/company-portability.js";
import type { CatalogTeam, CatalogTeamInstallResult } from "../../../packages/shared/src/types/teams-catalog.js";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { CompanySecretUsageBinding, SecretProposalView } from "../../../packages/shared/src/types/secrets.js";
+import type { RequestConfirmationInteraction } from "../../../packages/shared/src/types/issue.js";
+import { pollUntil } from "../api.js";
import { sendChatMessage } from "../chat-flow.js";
import { codexSubscriptionAgentConfig } from "../codex-subscription-fixture.js";
-import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js";
+import { definition, externalObservation, observeExternal, root, checks } from "./external-support.js";
import { catalogTeamMembers, observeTeamResources } from "./external-team-install.js";
import { waitForReply } from "./identity-support.js";
interface PrivateTeam { emptyHome: string; override: CompanyPortabilityAdapterOverride }
@@ -26,10 +29,55 @@ function missingAccountProbe(row: Probe, agentId: string) {
function readyProbe(row: Probe, agentId: string) {
return row.agentId === agentId && row.status === "pass" && row.checks.every(check => check.level === "info");
}
+export function memberReadinessSummary(rows: readonly Probe[], members: ReadonlyArray>, missingId: string, workingId: string) {
+ return rows.map(row => {
+ const role = row.agentId === missingId ? "expected missing account" : row.agentId === workingId ? "expected ready" : "other member";
+ const name = members.find(member => member.id === row.agentId)?.name ?? "unknown member";
+ const problems = row.checks.filter(check => check.level !== "info").map(check => `${check.code}:${check.level}`);
+ return `${name} (${role}): ${row.status}${problems.length ? ` [${problems.join(", ")}]` : ""}`;
+ }).join("; ");
+}
async function probe(ctx: ChatCatalogContext, agent: Agent): Promise {
const result = await ctx.input.api.postSensitive(`${root(ctx)}/adapters/${agent.adapterType}/test-environment`, { agentId: agent.id, environmentId: agent.defaultEnvironmentId, adapterConfig: agent.adapterConfig });
return { agentId: agent.id, status: result.status, testedAt: result.testedAt, checks: result.checks.map(check => ({ code: /^[a-zA-Z0-9_.-]{1,100}$/.test(check.code) ? check.code : "unrecognized", level: check.level })) };
}
+export interface EligibleTeamBinding { configPath: string; secretId: string }
+export interface TeamBindingApproval { interactionId: string; proposalId: string; configPath: string; secretId: string; targetAgentId: string; payloadHash: string }
+const payloadHash = (payload: unknown) => createHash("sha256").update(JSON.stringify(payload)).digest("hex");
+export function eligibleTeamBindings(env: unknown): EligibleTeamBinding[] {
+ return Object.entries(env && typeof env === "object" ? env as Record : {}).flatMap(([key, value]) => {
+ const ref = value as { type?: unknown; secretId?: unknown } | null;
+ return ref && ref.type === "secret_ref" && typeof ref.secretId === "string" ? [{ configPath: `env.${key}`, secretId: ref.secretId }] : [];
+ });
+}
+export function teamBindingCandidate(input: { interactions: readonly RequestConfirmationInteraction[]; proposals: readonly SecretProposalView[]; companyId: string; issueId: string;
+ leadId: string; initialRunId: string; missingId: string; eligible: readonly EligibleTeamBinding[] }): TeamBindingApproval | undefined {
+ for (const card of input.interactions) {
+ const p = card.payload?.secretProposal, proposal = p && input.proposals.find(row => row.id === p.proposalId);
+ if (!p || !proposal || card.kind !== "request_confirmation" || card.status !== "pending" || card.companyId !== input.companyId || card.issueId !== input.issueId
+ || card.sourceRunId !== input.initialRunId || card.createdByAgentId !== input.leadId || card.createdByUserId || card.addresseeAgentId || card.resolvedByUserId || card.resolvedByAgentId
+ || card.requestedResolverPolicy !== "human_only" || card.effectiveResolverPolicy !== "human_only" || card.idempotencyKey !== `secret-proposal:${proposal.id}`
+ || p.targetAgentId !== input.missingId || p.configPath !== proposal.configPath || proposal.companyId !== input.companyId || proposal.kind !== "binding" || proposal.status !== "pending"
+ || proposal.targetType !== "agent" || proposal.target?.id !== input.missingId || proposal.secretProposalId || proposal.originRunId !== input.initialRunId || proposal.proposedBy?.id !== input.leadId
+ || !input.eligible.some(row => row.configPath === proposal.configPath && row.secretId === proposal.secretId)) continue;
+ return { interactionId: card.id, proposalId: proposal.id, configPath: proposal.configPath!, secretId: proposal.secretId!, targetAgentId: input.missingId, payloadHash: payloadHash(card.payload) };
+ }
+}
+const bindingKey = (row: Pick) => `${row.targetType}:${row.targetId}:${row.configPath}`;
+function humanApprovedBinding(o: NonNullable>) {
+ const marker = o.state.bindingApproval as TeamBindingApproval | undefined, card = marker && o.interactions.find(row => row.id === marker.interactionId) as unknown as RequestConfirmationInteraction | undefined;
+ const before = o.state.bindingsBefore as string[] | undefined, after = o.state.bindingsAfter as string[] | undefined, ref = o.state.missingBindingRef as { type?: string; secretId?: string } | undefined;
+ return !!marker && !!card && marker.targetAgentId === o.state.missingId && card.status === "accepted" && card.createdByAgentId === o.leadId && card.sourceRunId === o.state.initialRunId
+ && typeof o.state.currentUserId === "string" && card.resolvedByUserId === o.state.currentUserId && !card.resolvedByAgentId && payloadHash(card.payload) === marker.payloadHash
+ && card.payload?.secretProposal?.targetAgentId === o.state.missingId && card.payload.secretProposal.configPath === marker.configPath
+ && Array.isArray(before) && Array.isArray(after) && !before.includes(`agent:${o.state.missingId}:${marker.configPath}`)
+ && JSON.stringify([...before, `agent:${o.state.missingId}:${marker.configPath}`].sort()) === JSON.stringify([...after].sort())
+ && ref?.type === "secret_ref" && ref.secretId === marker.secretId;
+}
+function verificationReplies(o: NonNullable>) {
+ const before = o.state.preVerificationRunIds;
+ return Array.isArray(before) && before.includes(o.state.initialRunId) ? o.phaseReplies.filter(row => !before.includes(row.runId)) : [];
+}
export function gradeTeamConfigure(value: unknown) {
const c = checks(), o = externalObservation(value); c.add("complete-member-configuration-evidence", !!o, "Real installed members, missing versus ready account probes, scoped repair and fresh per-member tests are required."); if (!o) return c.result();
const initial = o.state.initialProbes as Probe[] | undefined, after = o.state.afterProbes as Probe[] | undefined;
@@ -37,13 +85,14 @@ export function gradeTeamConfigure(value: unknown) {
&& Array.isArray(initial) && initial.some(row => missingAccountProbe(row, String(o.state.missingId)))
&& initial.some(row => readyProbe(row, String(o.state.workingId))),
"The supported catalog installer creates the team; actual runtime tests prove one exact missing account and a distinct already working member before chat.");
- c.add("specific-member-diagnosis-and-supported-repair", o.state.initialMemberNamed === true && o.state.initialHandoffSupported === true && o.state.repairConfirmed === true
+ c.add("specific-member-diagnosis-and-supported-repair", o.state.initialHandoffSupported === true && o.state.repairConfirmed === true
&& ((o.state.repairChannel === "agent-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "agent" && row.agentId === o.leadId))
- || (o.state.repairChannel === "human-board-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "user"))),
- "Chat identifies the incomplete member and the supported step; repair changes that member through actual agent or human configuration authority.");
+ || (o.state.repairChannel === "human-board-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "user"))
+ || (o.state.repairChannel === "human-approved-binding" && humanApprovedBinding(o))),
+ "Chat identifies the incomplete member by linking its own configuration page, proposing a governed binding of the eligible company account to that member, or repairing it itself; the repair changes only that member through actual agent or human authority.");
c.add("already-working-member-preserved", typeof o.state.workingHashBefore === "string" && o.state.workingHashBefore === o.state.workingHashAfter && o.state.memberIdsPreserved === true,
"The already working member's semantic configuration and the installed member identities remain unchanged.");
- c.add("working-configuration-revisions-preserved", Array.isArray(o.state.workingRevisionIdsBefore) && o.state.workingRevisionIdsBefore.length > 0
+ c.add("working-configuration-revisions-preserved", Array.isArray(o.state.workingRevisionIdsBefore) && Array.isArray(o.state.workingRevisionIdsAfter)
&& JSON.stringify(o.state.workingRevisionIdsBefore) === JSON.stringify(o.state.workingRevisionIdsAfter),
"Public immutable configuration revision identities also remain unchanged, so redacted environment placeholders cannot hide an account change.");
c.add("fresh-readiness-for-every-member", Array.isArray(after) && after.length === o.state.expectedMembers
@@ -51,10 +100,9 @@ export function gradeTeamConfigure(value: unknown) {
&& new Set(after.map(row => row.agentId)).size === after.length && after.every(row => row.status === "pass" && row.checks.every(check => check.level !== "error")
&& Number.isFinite(Date.parse(row.testedAt)) && Date.parse(row.testedAt) >= Number(o.state.repairVerifiedAfter)),
"After the real repair every member passes its own fresh saved-runtime test; one healthy member cannot stand for the whole team.");
- const continuation = o.phaseReplies.filter(row => row.runId !== o.state.initialRunId);
- c.add("phase-bound-truthful-team-result", continuation.some(row => o.runs.some(run => run.id === row.runId && run.status === "succeeded"))
- && /ready|configured|verified|account/i.test(continuation.map(row => row.body).join("\n")) && o.state.noPrematureWholeTeamClaim === true,
- "The final claim belongs to the actual verification followup, and the initial reply did not call an unconfigured team ready.");
+ const continuation = verificationReplies(o);
+ c.add("phase-bound-team-result", continuation.some(row => row.body.trim().length > 0 && o.runs.some(run => run.id === row.runId && run.status === "succeeded")),
+ "A run started by the verification message posts its own reply; earlier initial or approval-continuation replies cannot stand in. Its wording is left to human reading against the fresh per-member probes.");
return c.result();
}
export const teamConfigureExternalCases = [definition("teams.configure-team", async ctx => {
@@ -75,35 +123,45 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as
const members = catalogTeamMembers(await observeTeamResources(ctx), team.id);
const initialProbes = await Promise.all(members.map(member => probe(ctx, member)));
await ctx.input.evidence("chat-catalog-team-initial-probes.json", { initialProbes });
- if (!initialProbes.some(row => missingAccountProbe(row, missingId)) || !initialProbes.some(row => readyProbe(row, workingId))) throw new Error("Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split");
+ if (!initialProbes.some(row => missingAccountProbe(row, missingId)) || !initialProbes.some(row => readyProbe(row, workingId))) throw new Error(`Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split. Observed: ${memberReadinessSummary(initialProbes, members, missingId, workingId)}`);
+ const session = await ctx.input.api.get<{ user?: { id?: string } }>("/api/auth/get-session");
const working = await ctx.input.api.get(`/api/agents/${workingId}`);
const workingRevisionIdsBefore = (await ctx.input.api.get>(`/api/agents/${workingId}/config-revisions`)).map(row => row.id).sort();
await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue();
await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The website team is the installed ${team.name} package. Its QA member (${missingId}) has not selected the eligible company ${subscription ? "Codex subscription" : "OpenRouter"} account. The senior coder (${workingId}) is already configured. The existing company account is authorized for the QA member too; its value must not enter chat.` });
return { teamId: team.id, installedFromCatalog: true, memberIdsBefore: members.map(row => row.id).sort(), expectedMembers: members.length, missingId, missingName: members.find(row => row.id === missingId)!.name,
- workingId, workingHashBefore: configHash(working), workingRevisionIdsBefore, initialProbes };
+ workingId, workingHashBefore: configHash(working), workingRevisionIdsBefore, initialProbes,
+ eligibleBindings: eligibleTeamBindings((override.adapterConfig as { env?: unknown } | undefined)?.env), currentUserId: session.user?.id ?? null };
} catch (error) { fixtures.delete(ctx); await rm(emptyHome, { recursive: true, force: true }); throw error; }
}, gradeTeamConfigure, {
- authentication: "human-session", oracleVersion: "paperclip.chat-catalog.team-configure.v2", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 600_000,
+ authentication: "human-session", oracleVersion: "paperclip.chat-catalog.team-configure.v6", expectedRunCount: 3, minimumRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000,
setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["missing-account-fixture", "working-account-fixture"] },
async verifySetup(_ctx, state) { const c = checks(), rows = state.initialProbes as Probe[]; c.add("missing-account-fixture", rows.some(row => missingAccountProbe(row, String(state.missingId))), "Actual saved-runtime probe reports the required missing member account."); c.add("working-account-fixture", rows.some(row => readyProbe(row, String(state.workingId))), "A distinct installed member already passes its runtime probe."); return c.result(); },
- sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "../../server/src/adapters/registry.ts", "../../packages/adapters/opencode-local/src/server/test.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"],
+ sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "../../server/src/adapters/registry.ts", "../../packages/adapters/opencode-local/src/server/test.ts", "../../packages/adapters/opencode-local/src/server/parse.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"],
async act(ctx, state) {
await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const initial = await observeExternal(ctx, state); state.initialRunId = initial.runs[0]?.id;
- state.initialMemberNamed = initial.reply.toLowerCase().includes(String(state.missingName).toLowerCase());
const members = catalogTeamMembers(await observeTeamResources(ctx), String(state.teamId)), missing = members.find(row => row.id === state.missingId)!;
const current = await probe(ctx, missing); const alreadyRepaired = current.status === "pass";
const handoff = initial.links.find(href => { try { const url = new URL(href, initial.origin); return url.origin === initial.origin && url.pathname.split("/").some((part, i, parts) => part === "agents" && [missing.id, missing.urlKey].includes(parts[i + 1]!)); } catch { return false; } });
- state.initialHandoffSupported = alreadyRepaired || !!handoff;
- state.noPrematureWholeTeamClaim = alreadyRepaired || !/\b(?:team|product engineering)\s+(?:is|are|looks)\s+(?:now\s+)?(?:fully\s+)?ready/i.test(initial.reply);
- if (!alreadyRepaired) {
- if (!handoff) throw new Error("The model did not provide a supported configuration handoff for the incomplete member");
+ const issueId = ctx.issue()!.id, eligible = state.eligibleBindings as EligibleTeamBinding[];
+ const binding = alreadyRepaired ? undefined : teamBindingCandidate({ interactions: await ctx.input.api.get(`/api/issues/${issueId}/interactions`),
+ proposals: await ctx.input.api.get(`${root(ctx)}/secret-proposals`), companyId: ctx.input.fixtures.company.id, issueId, leadId: ctx.input.fixtures.agent.id,
+ initialRunId: String(state.initialRunId), missingId: missing.id, eligible });
+ state.initialHandoffSupported = alreadyRepaired || !!binding || !!handoff;
+ if (binding) {
+ state.bindingsBefore = await usageKeys(ctx, binding.secretId);
+ await acceptTeamBinding(ctx, binding, missing.name, String(state.currentUserId));
+ state.bindingApproval = binding; state.repairChannel = "human-approved-binding";
+ await waitForBindingContinuation(ctx);
+ } else if (!alreadyRepaired) {
+ if (!handoff) throw new Error("The model provided neither a configuration link nor a governed binding proposal for the incomplete member");
const source = ctx.input.page.url(); await ctx.input.page.goto(handoff, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("main")).toContainText(missing.name);
const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Eligible team account reference missing");
await ctx.input.api.patchSensitive(`/api/agents/${missing.id}`, { adapterType: fixture.override.adapterType, adapterConfig: fixture.override.adapterConfig }); state.repairChannel = "human-board-configuration";
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
} else state.repairChannel = "agent-configuration";
state.repairVerifiedAfter = Date.now(); state.repairConfirmed = (await probe(ctx, await ctx.input.api.get(`/api/agents/${missing.id}`))).status === "pass";
+ state.preVerificationRunIds = (await ctx.allRuns()).map(run => run.id);
await sendChatMessage(ctx.input.page, "Please verify which installed team members are ready now, including QA, and preserve the already working member's configuration.", chatCatalogNativePhaseRemainingBudget(ctx));
},
async observe(ctx, state) {
@@ -111,12 +169,45 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as
state.memberIdsPreserved = JSON.stringify(members.map(row => row.id).sort()) === JSON.stringify(state.memberIdsBefore);
state.workingHashAfter = members.some(row => row.id === state.workingId) ? configHash(members.find(row => row.id === state.workingId)!) : null;
state.workingRevisionIdsAfter = (await ctx.input.api.get>(`/api/agents/${state.workingId}/config-revisions`)).map(row => row.id).sort();
+ const marker = state.bindingApproval as TeamBindingApproval | undefined;
+ if (marker) {
+ state.bindingsAfter = await usageKeys(ctx, marker.secretId);
+ const qa = await ctx.input.api.get(`/api/agents/${String(state.missingId)}`), ref = (qa.adapterConfig as { env?: Record }).env?.[marker.configPath.slice(4)] as { type?: string; secretId?: string } | undefined;
+ state.missingBindingRef = ref ? { type: ref.type, secretId: ref.secretId } : null;
+ }
return observeExternal(ctx, state);
},
- async verifyResult(ctx, state, value) { await verifyRenderedReply(ctx, value, String(state.initialRunId)); },
+ async verifyResult(ctx, _state, value) {
+ const o = externalObservation(value), reply = o && verificationReplies(o).at(-1);
+ if (!o || !reply?.commentId || !reply.body.trim()) throw new Error("No persisted reply from the verification run");
+ await expect(ctx.input.page.locator(`[id=${JSON.stringify(`comment-${reply.commentId}`)}]`)).toBeVisible();
+ await ctx.input.evidence("chat-catalog-external-visible-reply.json", { scenarioId: o.scenarioId, commentId: reply.commentId, runId: reply.runId });
+ },
async cleanup(ctx) { const fixture = fixtures.get(ctx); fixtures.delete(ctx); if (fixture) await rm(fixture.emptyHome, { recursive: true, force: true }); },
})];
+async function usageKeys(ctx: ChatCatalogContext, secretId: string) {
+ return (await ctx.input.api.get<{ bindings: CompanySecretUsageBinding[] }>(`/api/secrets/${secretId}/usage`)).bindings.map(bindingKey).sort();
+}
+async function acceptTeamBinding(ctx: ChatCatalogContext, marker: TeamBindingApproval, memberName: string, userId: string) {
+ const page = ctx.input.page, budget = chatCatalogNativePhaseRemainingBudget(ctx), remaining = () => Math.max(1_000, budget ? budget() : 120_000), issueId = ctx.issue()!.id;
+ const card = page.locator(`[data-testid="task-chat-interaction"][id=${JSON.stringify(`interaction-${marker.interactionId}`)}]`);
+ await expect(card).toBeVisible({ timeout: remaining() });
+ for (const text of [memberName, marker.configPath]) await expect(card).toContainText(text, { timeout: remaining() });
+ const response = page.waitForResponse(res => new URL(res.url()).origin === new URL(page.url()).origin && new URL(res.url()).pathname === `/api/issues/${issueId}/interactions/${marker.interactionId}/accept` && res.request().method() === "POST", { timeout: remaining() });
+ await card.getByRole("button", { name: "Create binding", exact: true }).click({ timeout: remaining() });
+ const submitted = await response; if (!submitted.ok()) throw new Error("The binding approval did not persist");
+ const receipt = await submitted.json() as RequestConfirmationInteraction;
+ if (receipt.id !== marker.interactionId || receipt.status !== "accepted" || receipt.resolvedByUserId !== userId || receipt.resolvedByAgentId || receipt.issueId !== issueId || payloadHash(receipt.payload) !== marker.payloadHash)
+ throw new Error("The binding approval receipt differs from the displayed proposal accepted by the current person");
+}
+async function waitForBindingContinuation(ctx: ChatCatalogContext) {
+ const startedAt = Date.now(), budget = chatCatalogNativePhaseRemainingBudget(ctx), deadlineAt = startedAt + Math.min(budget ? budget() : 300_000, 300_000);
+ await pollUntil({ label: "binding approval continuation", deadlineAt, intervalMs: 1_000,
+ load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; },
+ accept: ({ issue, runs }) => runs.every(run => run.status === "succeeded") && issue?.conversationState === "waiting" && (runs.length >= 2 || Date.now() - startedAt >= 15_000),
+ reject: ({ runs }) => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "The binding approval continuation failed" : undefined });
+}
export function missingTeamAccountEnvironment(subscription: boolean, emptyHome: string) {
return subscription ? { CODEX_HOME: emptyHome, OPENAI_API_KEY: "", CODEX_API_KEY: "" } : { OPENROUTER_API_KEY: "" };
}