fix(evals): name each team member's observed readiness and grade configure-team structurally

teams.configure-team failed setup because the QA member's empty OpenRouter key
surfaced as a generic hello-probe error (OpenRouter 401 "Missing Authentication
header") instead of the missing-authentication check, so the missing-versus-ready
split never formed. The product fix (cherry-picked) classifies a provider 401 as
opencode_hello_probe_auth_required.

- The setup blocker now names every member's observed probe status and non-info checks.
- Replace reply keyword grading: drop the member-name substring and the
  premature-ready regex (identification is the rendered link to the missing
  member's own page, already required), and grade the followup by a non-empty
  reply from its own succeeded run instead of matching ready/configured words.
- Calibrate both ways; bump oracleVersion to v3; add parse.ts to sourceFiles.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -6045,6 +6045,7 @@
       "chat-catalog-domains/external-team-install.ts",
       "../../server/src/adapters/registry.ts",
       "../../packages/adapters/opencode-local/src/server/test.ts",
+      "../../packages/adapters/opencode-local/src/server/parse.ts",
       "codex-subscription-fixture.ts",
       "chat-catalog-domains/external-support.ts",
       "chat-catalog-domains/internal-support.ts",
@@ -6052,7 +6053,7 @@
       "chat-flow.ts",
       "chat-scenarios/teams.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.team-configure.v2",
+    "oracleVersion": "paperclip.chat-catalog.team-configure.v3",
     "expectedRunCount": 2,
     "expectedHumanMessageCount": 2,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts

@@ -51,6 +51,12 @@ it("retains no provider messages or session fields in the precondition witness",
     expect(Object.keys((f.evidence.mock.calls[0]![1] as { initialProbes: object[] }).initialProbes[0]!).sort()).toEqual(["agentId", "checks", "status", "testedAt"]);
   } finally { await definition.cleanup!(f.ctx, state ?? {}); }
 });
+it("names each member's observed readiness when the split does not form", async () => {
+  // The recorded failure: the provider 401 surfaced as a generic probe error.
+  const f = setupFixture({ status: "fail", testedAt: authWarning.testedAt, checks: [{ code: "opencode_cwd_valid", level: "info" }, { code: "opencode_hello_probe_failed", level: "error" }] });
+  await expect(teamConfigureExternalCases[0]!.setup(f.ctx)).rejects.toThrow(
+    "Observed: qa (expected missing account): fail [opencode_hello_probe_failed:error]; coder (expected ready): pass");
+});
 it.each(["generic warning", "timeout", "model", "wrong level", "other error", "working warning", "working error"])("actual setup rejects %s and retains only closed probe fields before refusing", async fault => {
   const missing = structuredClone(authWarning), working: Probe = { status: "pass", testedAt: authWarning.testedAt, checks: [] };
   if (fault === "generic warning") missing.checks[0]!.code = "opencode_hello_probe_unexpected_output";

tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts

@@ -7,7 +7,7 @@ function valid() {
       installedFromCatalog: true, missingId: "qa", workingId: "coder", expectedMembers: 2, memberIdsBefore: ["coder", "qa"], workingRevisionIdsBefore: ["revision"], workingRevisionIdsAfter: ["revision"],
       initialProbes: [{ agentId: "qa", status: "fail", testedAt: "2026-10-26T10:00:00Z", checks: [{ code: "codex_auth", level: "error" }] }, { agentId: "coder", status: "pass", testedAt: "2026-10-26T10:00:00Z", checks: [] }],
       afterProbes: [{ agentId: "qa", status: "pass", testedAt: "2026-10-26T10:05:01Z", checks: [] }, { agentId: "coder", status: "pass", testedAt: "2026-10-26T10:05:02Z", checks: [] }],
-      repairVerifiedAfter: Date.parse("2026-10-26T10:05:00Z"), initialMemberNamed: true, initialHandoffSupported: true, repairConfirmed: true, repairChannel: "human-board-configuration", workingHashBefore: "same-config", workingHashAfter: "same-config", memberIdsPreserved: true, initialRunId: "initial", noPrematureWholeTeamClaim: true,
+      repairVerifiedAfter: Date.parse("2026-10-26T10:05:00Z"), initialHandoffSupported: true, repairConfirmed: true, repairChannel: "human-board-configuration", workingHashBefore: "same-config", workingHashAfter: "same-config", memberIdsPreserved: true, initialRunId: "initial",
     } };
 }
 describe("installed team per-member readiness oracle", () => {
@@ -28,9 +28,16 @@ describe("installed team per-member readiness oracle", () => {
       (o: ReturnType<typeof valid>) => { o.state.initialProbes[0]!.agentId = "other-member"; },
       (o: ReturnType<typeof valid>) => { o.state.workingHashAfter = "changed-working-account"; },
       (o: ReturnType<typeof valid>) => { o.state.workingRevisionIdsAfter = ["masked-env-change"]; },
-      (o: ReturnType<typeof valid>) => { o.state.noPrematureWholeTeamClaim = false; },
+      (o: ReturnType<typeof valid>) => { o.state.initialHandoffSupported = false; },
+      (o: ReturnType<typeof valid>) => { o.phaseReplies[1]!.body = "  "; },
+      (o: ReturnType<typeof valid>) => { o.runs[1]!.status = "failed"; },
       (o: ReturnType<typeof valid>) => { o.phaseReplies[1]!.runId = "initial"; },
     ]) { const o = valid(); change(o); expect(definition.grade(o).passed).toBe(false); }
   });
+  it("grades the followup structurally, not by its wording", () => {
+    for (const body of ["Both QA and the senior coder pass their own checks now.", "Fertig: alle Mitglieder bestehen ihre Prüfung."]) {
+      const o = valid(); o.phaseReplies[1]!.body = body; o.reply = body; expect(definition.grade(o).passed).toBe(true);
+    }
+  });
   it("missing observations cannot pass", () => { expect(definition.grade(undefined).passed).toBe(false); expect(definition.grade({ reply: "Everyone is ready." }).passed).toBe(false); });
 });

tests/runner-e2e/chat-catalog-domains/external-team-configure.ts

@@ -26,6 +26,15 @@ function missingAccountProbe(row: Probe, agentId: string) {
 function readyProbe(row: Probe, agentId: string) {
   return row.agentId === agentId && row.status === "pass" && row.checks.every(check => check.level === "info");
 }
+/** Each member's observed readiness, for a setup blocker that says which side of the split did not form. */
+export function memberReadinessSummary(rows: readonly Probe[], members: ReadonlyArray<Pick<Agent, "id" | "name">>, missingId: string, workingId: string) {
+  return rows.map(row => {
+    const role = row.agentId === missingId ? "expected missing account" : row.agentId === workingId ? "expected ready" : "other member";
+    const name = members.find(member => member.id === row.agentId)?.name ?? "unknown member";
+    const problems = row.checks.filter(check => check.level !== "info").map(check => `${check.code}:${check.level}`);
+    return `${name} (${role}): ${row.status}${problems.length ? ` [${problems.join(", ")}]` : ""}`;
+  }).join("; ");
+}
 async function probe(ctx: ChatCatalogContext, agent: Agent): Promise<Probe> {
   const result = await ctx.input.api.postSensitive<AdapterEnvironmentTestResult>(`${root(ctx)}/adapters/${agent.adapterType}/test-environment`, { agentId: agent.id, environmentId: agent.defaultEnvironmentId, adapterConfig: agent.adapterConfig });
   return { agentId: agent.id, status: result.status, testedAt: result.testedAt, checks: result.checks.map(check => ({ code: /^[a-zA-Z0-9_.-]{1,100}$/.test(check.code) ? check.code : "unrecognized", level: check.level })) };
@@ -37,10 +46,10 @@ export function gradeTeamConfigure(value: unknown) {
     && Array.isArray(initial) && initial.some(row => missingAccountProbe(row, String(o.state.missingId)))
     && initial.some(row => readyProbe(row, String(o.state.workingId))),
     "The supported catalog installer creates the team; actual runtime tests prove one exact missing account and a distinct already working member before chat.");
-  c.add("specific-member-diagnosis-and-supported-repair", o.state.initialMemberNamed === true && o.state.initialHandoffSupported === true && o.state.repairConfirmed === true
+  c.add("specific-member-diagnosis-and-supported-repair", o.state.initialHandoffSupported === true && o.state.repairConfirmed === true
     && ((o.state.repairChannel === "agent-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "agent" && row.agentId === o.leadId))
       || (o.state.repairChannel === "human-board-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "user"))),
-    "Chat identifies the incomplete member and the supported step; repair changes that member through actual agent or human configuration authority.");
+    "Chat identifies the incomplete member by linking its own configuration page (or repairs it itself); repair changes that member through actual agent or human configuration authority.");
   c.add("already-working-member-preserved", typeof o.state.workingHashBefore === "string" && o.state.workingHashBefore === o.state.workingHashAfter && o.state.memberIdsPreserved === true,
     "The already working member's semantic configuration and the installed member identities remain unchanged.");
   c.add("working-configuration-revisions-preserved", Array.isArray(o.state.workingRevisionIdsBefore) && o.state.workingRevisionIdsBefore.length > 0
@@ -52,9 +61,8 @@ export function gradeTeamConfigure(value: unknown) {
     && Number.isFinite(Date.parse(row.testedAt)) && Date.parse(row.testedAt) >= Number(o.state.repairVerifiedAfter)),
     "After the real repair every member passes its own fresh saved-runtime test; one healthy member cannot stand for the whole team.");
   const continuation = o.phaseReplies.filter(row => row.runId !== o.state.initialRunId);
-  c.add("phase-bound-truthful-team-result", continuation.some(row => o.runs.some(run => run.id === row.runId && run.status === "succeeded"))
-    && /ready|configured|verified|account/i.test(continuation.map(row => row.body).join("\n")) && o.state.noPrematureWholeTeamClaim === true,
-    "The final claim belongs to the actual verification followup, and the initial reply did not call an unconfigured team ready.");
+  c.add("phase-bound-team-result", continuation.some(row => row.body.trim().length > 0 && o.runs.some(run => run.id === row.runId && run.status === "succeeded")),
+    "The verification followup posts its own reply from a succeeded run; its wording is left to human reading against the fresh per-member probes.");
   return c.result();
 }
 export const teamConfigureExternalCases = [definition("teams.configure-team", async ctx => {
@@ -75,7 +83,7 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as
     const members = catalogTeamMembers(await observeTeamResources(ctx), team.id);
     const initialProbes = await Promise.all(members.map(member => probe(ctx, member)));
     await ctx.input.evidence("chat-catalog-team-initial-probes.json", { initialProbes });
-    if (!initialProbes.some(row => missingAccountProbe(row, missingId)) || !initialProbes.some(row => readyProbe(row, workingId))) throw new Error("Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split");
+    if (!initialProbes.some(row => missingAccountProbe(row, missingId)) || !initialProbes.some(row => readyProbe(row, workingId))) throw new Error(`Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split. Observed: ${memberReadinessSummary(initialProbes, members, missingId, workingId)}`);
     const working = await ctx.input.api.get<Agent>(`/api/agents/${workingId}`);
     const workingRevisionIdsBefore = (await ctx.input.api.get<Array<{ id: string }>>(`/api/agents/${workingId}/config-revisions`)).map(row => row.id).sort();
     await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue();
@@ -84,18 +92,16 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as
       workingId, workingHashBefore: configHash(working), workingRevisionIdsBefore, initialProbes };
   } catch (error) { fixtures.delete(ctx); await rm(emptyHome, { recursive: true, force: true }); throw error; }
 }, gradeTeamConfigure, {
-  authentication: "human-session", oracleVersion: "paperclip.chat-catalog.team-configure.v2", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 600_000,
+  authentication: "human-session", oracleVersion: "paperclip.chat-catalog.team-configure.v3", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 600_000,
   setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["missing-account-fixture", "working-account-fixture"] },
   async verifySetup(_ctx, state) { const c = checks(), rows = state.initialProbes as Probe[]; c.add("missing-account-fixture", rows.some(row => missingAccountProbe(row, String(state.missingId))), "Actual saved-runtime probe reports the required missing member account."); c.add("working-account-fixture", rows.some(row => readyProbe(row, String(state.workingId))), "A distinct installed member already passes its runtime probe."); return c.result(); },
-  sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "../../server/src/adapters/registry.ts", "../../packages/adapters/opencode-local/src/server/test.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"],
+  sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "../../server/src/adapters/registry.ts", "../../packages/adapters/opencode-local/src/server/test.ts", "../../packages/adapters/opencode-local/src/server/parse.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"],
   async act(ctx, state) {
     await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const initial = await observeExternal(ctx, state); state.initialRunId = initial.runs[0]?.id;
-    state.initialMemberNamed = initial.reply.toLowerCase().includes(String(state.missingName).toLowerCase());
     const members = catalogTeamMembers(await observeTeamResources(ctx), String(state.teamId)), missing = members.find(row => row.id === state.missingId)!;
     const current = await probe(ctx, missing); const alreadyRepaired = current.status === "pass";
     const handoff = initial.links.find(href => { try { const url = new URL(href, initial.origin); return url.origin === initial.origin && url.pathname.split("/").some((part, i, parts) => part === "agents" && [missing.id, missing.urlKey].includes(parts[i + 1]!)); } catch { return false; } });
     state.initialHandoffSupported = alreadyRepaired || !!handoff;
-    state.noPrematureWholeTeamClaim = alreadyRepaired || !/\b(?:team|product engineering)\s+(?:is|are|looks)\s+(?:now\s+)?(?:fully\s+)?ready/i.test(initial.reply);
     if (!alreadyRepaired) {
       if (!handoff) throw new Error("The model did not provide a supported configuration handoff for the incomplete member");
       const source = ctx.input.page.url(); await ctx.input.page.goto(handoff, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("main")).toContainText(missing.name);