test(evals): recognize exact team account probe refusal

Accept the documented OpenCode auth-required warning as a missing-account prerequisite while keeping a distinct member strictly ready. Retain only the existing closed probe projection before setup refusal; no new probes, model calls or clock changes.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts

@@ -0,0 +1,69 @@
+import { expect, it, vi } from "vitest";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import { teamConfigureExternalCases } from "./external-team-configure.js";
+
+type Probe = { status: string; testedAt: string; checks: Array<{ code: string; level: string }> };
+function setupFixture(missing: Probe, working: Probe = { status: "pass", testedAt: "2026-11-03T00:00:00Z", checks: [] }) {
+  const member = (id: string) => ({ id, companyId: "company", name: id, adapterType: "paperclip_runner", adapterConfig: {}, metadata: { paperclip: { catalogTeam: { catalogId: "team" } } } });
+  const members = [member("qa"), member("coder")];
+  const probes = vi.fn(async (_path: string, body: { agentId?: string }) => body.agentId === "qa" ? missing : working);
+  const evidence = vi.fn(async (_name: string, _value: unknown) => {});
+  const api = {
+    get: async (path: string) => {
+      if (path === "/api/teams/catalog") return [{ id: "team", slug: "product-engineering", compatibility: "compatible", agentSlugs: ["qa", "senior-coder"] }];
+      if (path.endsWith("/agents")) return members;
+      if (path.endsWith("/config-revisions")) return [{ id: "working-revision" }];
+      if (path === "/api/agents/coder") return members[1];
+      if (["/projects", "/routines", "/teams/catalog/installed"].some(suffix => path.endsWith(suffix))) return [];
+      throw new Error("Unexpected setup read");
+    },
+    postSensitive: async (path: string, body: { agentId?: string }) => path.endsWith("/install")
+      ? { portabilityImport: { agents: [{ id: "qa", slug: "qa" }, { id: "coder", slug: "senior-coder" }] } }
+      : probes(path, body),
+    patchSensitive: vi.fn(async (_path: string, _body: unknown) => {}),
+    post: vi.fn(async (_path: string) => {}), patch: vi.fn(async (_path: string, _body: unknown) => {}),
+  };
+  const ctx = { input: { api, evidence, nonce: "synthetic-team-prerequisite", workspacePath: "/synthetic/workspace", fixtures: { company: { id: "company" }, agent: { id: "ceo" }, environment: { id: "environment" }, secretRefs: {} }, execution: { profile: { credential: "OPENROUTER_API_KEY", buildAgent: () => ({ adapterType: "paperclip_runner", adapterConfig: { provider: "opencode" } }) }, environment: { id: "local" } } }, refreshIssue: async () => {}, issue: () => ({ id: "source" }) } as unknown as ChatCatalogContext;
+  return { ctx, evidence, probes, api };
+}
+const authWarning: Probe = { status: "warn", testedAt: "2026-11-03T00:00:00Z", checks: [{ code: "opencode_hello_probe_auth_required", level: "warn" }] };
+it("the actual setup admits the adapter's exact missing-auth warning and preserves the independently passing member", async () => {
+  const f = setupFixture(authWarning), definition = teamConfigureExternalCases[0]!;
+  let state: Record<string, unknown> | undefined;
+  try {
+    state = await definition.setup(f.ctx);
+    expect(state.initialProbes).toEqual([{ agentId: "qa", ...authWarning }, { agentId: "coder", status: "pass", testedAt: authWarning.testedAt, checks: [] }]);
+    expect((await definition.verifySetup!(f.ctx, state)).passed).toBe(true);
+    expect(f.probes).toHaveBeenCalledTimes(2);
+    expect(f.evidence).toHaveBeenCalledWith("chat-catalog-team-initial-probes.json", { initialProbes: state.initialProbes });
+    expect(f.api.patchSensitive).toHaveBeenCalledTimes(1);
+    expect(f.api.patch).toHaveBeenCalledTimes(1);
+  } finally { await definition.cleanup!(f.ctx, state ?? {}); }
+});
+it("retains no provider messages or session fields in the precondition witness", async () => {
+  const canary = "synthetic-private-probe-content";
+  const raw = { ...authWarning, providerSession: canary, checks: authWarning.checks.map(check => ({ ...check, message: canary, detail: canary })) };
+  const f = setupFixture(raw), definition = teamConfigureExternalCases[0]!;
+  let state: Record<string, unknown> | undefined;
+  try {
+    state = await definition.setup(f.ctx);
+    expect(JSON.stringify(f.evidence.mock.calls)).not.toContain(canary);
+    expect(Object.keys((f.evidence.mock.calls[0]![1] as { initialProbes: object[] }).initialProbes[0]!).sort()).toEqual(["agentId", "checks", "status", "testedAt"]);
+  } finally { await definition.cleanup!(f.ctx, state ?? {}); }
+});
+it.each(["generic warning", "timeout", "model", "wrong level", "other error", "working warning", "working error"])("actual setup rejects %s and retains only closed probe fields before refusing", async fault => {
+  const missing = structuredClone(authWarning), working: Probe = { status: "pass", testedAt: authWarning.testedAt, checks: [] };
+  if (fault === "generic warning") missing.checks[0]!.code = "opencode_hello_probe_unexpected_output";
+  if (fault === "timeout") missing.checks[0]!.code = "opencode_hello_probe_timed_out";
+  if (fault === "model") missing.checks[0]!.code = "opencode_hello_probe_model_unavailable";
+  if (fault === "wrong level") missing.checks[0]!.level = "info";
+  if (fault === "other error") missing.checks.push({ code: "opencode_hello_probe_failed", level: "error" });
+  if (fault === "working warning") working.status = "warn";
+  if (fault === "working error") working.checks.push({ code: "opencode_hello_probe_failed", level: "error" });
+  const f = setupFixture(missing, working);
+  await expect(teamConfigureExternalCases[0]!.setup(f.ctx)).rejects.toThrow("actual per-member runtime probes");
+  expect(f.probes).toHaveBeenCalledTimes(2);
+  expect(f.evidence).toHaveBeenCalledTimes(1);
+  expect(f.evidence.mock.calls[0]![1]).toEqual({ initialProbes: [{ agentId: "qa", ...missing }, { agentId: "coder", ...working }] });
+  expect(f.api.post).not.toHaveBeenCalled();
+});

tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts

@@ -13,12 +13,19 @@ function valid() {
 describe("installed team per-member readiness oracle", () => {
   const definition = teamConfigureExternalCases[0]!;
   it("accepts real missing-account diagnosis, human repair and fresh separate readiness checks", () => { expect(definition.grade(valid()).passed).toBe(true); });
+  it("accepts the exact OpenCode missing-auth warning without treating the member as ready", () => {
+    const o = valid(); o.state.initialProbes[0]!.status = "warn"; o.state.initialProbes[0]!.checks = [{ code: "opencode_hello_probe_auth_required", level: "warn" }];
+    expect(definition.grade(o).passed).toBe(true);
+    o.state.afterProbes[0]!.status = "warn";
+    expect(definition.grade(o).passed).toBe(false);
+  });
   it("also accepts an actually attributed eligible agent configuration repair", () => { const o = valid(); o.state.repairChannel = "agent-configuration"; o.activity[0]!.actorType = "agent"; o.activity[0]!.agentId = "ceo"; expect(definition.grade(o).passed).toBe(true); });
   it("rejects stale/member-substituted readiness, unrelated config edits and premature success", () => {
     for (const change of [
       (o: ReturnType<typeof valid>) => { o.state.afterProbes[0]!.testedAt = "2026-10-26T09:00:00Z"; },
       (o: ReturnType<typeof valid>) => { o.state.afterProbes[0]!.status = "fail"; },
       (o: ReturnType<typeof valid>) => { o.state.afterProbes[0]!.agentId = "coder"; },
+      (o: ReturnType<typeof valid>) => { o.state.initialProbes[0]!.agentId = "other-member"; },
       (o: ReturnType<typeof valid>) => { o.state.workingHashAfter = "changed-working-account"; },
       (o: ReturnType<typeof valid>) => { o.state.workingRevisionIdsAfter = ["masked-env-change"]; },
       (o: ReturnType<typeof valid>) => { o.state.noPrematureWholeTeamClaim = false; },

tests/runner-e2e/chat-catalog-domains/external-team-configure.ts

@@ -18,6 +18,14 @@ const fixtures = new WeakMap<ChatCatalogContext, PrivateTeam>();
 function stable(value: unknown): unknown { return Array.isArray(value) ? value.map(stable) : value && typeof value === "object" ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => [key, stable(item)])) : value; }
 const configHash = (agent: Agent) => createHash("sha256").update(JSON.stringify(stable({ adapterType: agent.adapterType, adapterConfig: agent.adapterConfig, runtimeConfig: agent.runtimeConfig, defaultEnvironmentId: agent.defaultEnvironmentId }))).digest("hex");
 interface Probe { agentId: string; status: string; testedAt: string; checks: Array<{code: string; level: string}> }
+function missingAccountProbe(row: Probe, agentId: string) {
+  return row.agentId === agentId && (row.status === "fail" && row.checks.some(check => check.level === "error" && /auth|credential|login/i.test(check.code))
+    || row.status === "warn" && row.checks.some(check => check.code === "opencode_hello_probe_auth_required" && check.level === "warn")
+      && row.checks.every(check => check.level === "info" || check.code === "opencode_hello_probe_auth_required" && check.level === "warn"));
+}
+function readyProbe(row: Probe, agentId: string) {
+  return row.agentId === agentId && row.status === "pass" && row.checks.every(check => check.level === "info");
+}
 async function probe(ctx: ChatCatalogContext, agent: Agent): Promise<Probe> {
   const result = await ctx.input.api.postSensitive<AdapterEnvironmentTestResult>(`${root(ctx)}/adapters/${agent.adapterType}/test-environment`, { agentId: agent.id, environmentId: agent.defaultEnvironmentId, adapterConfig: agent.adapterConfig });
   return { agentId: agent.id, status: result.status, testedAt: result.testedAt, checks: result.checks.map(check => ({ code: /^[a-zA-Z0-9_.-]{1,100}$/.test(check.code) ? check.code : "unrecognized", level: check.level })) };
@@ -26,8 +34,8 @@ export function gradeTeamConfigure(value: unknown) {
   const c = checks(), o = externalObservation(value); c.add("complete-member-configuration-evidence", !!o, "Real installed members, missing versus ready account probes, scoped repair and fresh per-member tests are required."); if (!o) return c.result();
   const initial = o.state.initialProbes as Probe[] | undefined, after = o.state.afterProbes as Probe[] | undefined;
   c.add("actual-missing-and-working-member-prerequisite", o.state.installedFromCatalog === true && typeof o.state.missingId === "string" && typeof o.state.workingId === "string" && o.state.missingId !== o.state.workingId
-    && Array.isArray(initial) && initial.some(row => row.agentId === o.state.missingId && row.status === "fail" && row.checks.some(check => check.level === "error" && /auth|credential|login/i.test(check.code)))
-    && initial.some(row => row.agentId === o.state.workingId && row.status === "pass"),
+    && Array.isArray(initial) && initial.some(row => missingAccountProbe(row, String(o.state.missingId)))
+    && initial.some(row => readyProbe(row, String(o.state.workingId))),
     "The supported catalog installer creates the team; actual runtime tests prove one exact missing account and a distinct already working member before chat.");
   c.add("specific-member-diagnosis-and-supported-repair", o.state.initialMemberNamed === true && o.state.initialHandoffSupported === true && o.state.repairConfirmed === true
     && ((o.state.repairChannel === "agent-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "agent" && row.agentId === o.leadId))
@@ -66,7 +74,8 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as
     await ctx.input.api.patchSensitive(`/api/agents/${missingId}`, { adapterConfig: { ...override.adapterConfig, env: { ...(override.adapterConfig?.env as Record<string, unknown> ?? {}), ...missingTeamAccountEnvironment(subscription, emptyHome) } } });
     const members = catalogTeamMembers(await observeTeamResources(ctx), team.id);
     const initialProbes = await Promise.all(members.map(member => probe(ctx, member)));
-    if (!initialProbes.some(row => row.agentId === missingId && row.status === "fail" && row.checks.some(check => check.level === "error" && /auth|credential|login/i.test(check.code))) || !initialProbes.some(row => row.agentId === workingId && row.status === "pass")) throw new Error("Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split");
+    await ctx.input.evidence("chat-catalog-team-initial-probes.json", { initialProbes });
+    if (!initialProbes.some(row => missingAccountProbe(row, missingId)) || !initialProbes.some(row => readyProbe(row, workingId))) throw new Error("Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split");
     const working = await ctx.input.api.get<Agent>(`/api/agents/${workingId}`);
     const workingRevisionIdsBefore = (await ctx.input.api.get<Array<{ id: string }>>(`/api/agents/${workingId}/config-revisions`)).map(row => row.id).sort();
     await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue();
@@ -75,10 +84,10 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as
       workingId, workingHashBefore: configHash(working), workingRevisionIdsBefore, initialProbes };
   } catch (error) { fixtures.delete(ctx); await rm(emptyHome, { recursive: true, force: true }); throw error; }
 }, gradeTeamConfigure, {
-  authentication: "human-session", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 600_000,
+  authentication: "human-session", oracleVersion: "paperclip.chat-catalog.team-configure.v2", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 600_000,
   setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["missing-account-fixture", "working-account-fixture"] },
-  async verifySetup(_ctx, state) { const c = checks(), rows = state.initialProbes as Probe[]; c.add("missing-account-fixture", rows.some(row => row.agentId === state.missingId && row.status === "fail"), "Actual saved-runtime probe reports the required missing member account."); c.add("working-account-fixture", rows.some(row => row.agentId === state.workingId && row.status === "pass"), "A distinct installed member already passes its runtime probe."); return c.result(); },
-  sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"],
+  async verifySetup(_ctx, state) { const c = checks(), rows = state.initialProbes as Probe[]; c.add("missing-account-fixture", rows.some(row => missingAccountProbe(row, String(state.missingId))), "Actual saved-runtime probe reports the required missing member account."); c.add("working-account-fixture", rows.some(row => readyProbe(row, String(state.workingId))), "A distinct installed member already passes its runtime probe."); return c.result(); },
+  sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "../../server/src/adapters/registry.ts", "../../packages/adapters/opencode-local/src/server/test.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"],
   async act(ctx, state) {
     await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const initial = await observeExternal(ctx, state); state.initialRunId = initial.runs[0]?.id;
     state.initialMemberNamed = initial.reply.toLowerCase().includes(String(state.missingName).toLowerCase());