From d12be2755bb178bbcb6ebc7b52215bf1ceddad61 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 21:31:19 -0600 Subject: [PATCH] test(evals): recognize exact team account probe refusal Accept the documented OpenCode auth-required warning as a missing-account prerequisite while keeping a distinct member strictly ready. Retain only the existing closed probe projection before setup refusal; no new probes, model calls or clock changes. Co-Authored-By: Paperclip --- ...ternal-team-configure-prerequisite.test.ts | 69 +++++++++++++++++++ .../external-team-configure.test.ts | 7 ++ .../external-team-configure.ts | 21 ++++-- 3 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts new file mode 100644 index 00000000000..4ddf64446ab --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts @@ -0,0 +1,69 @@ +import { expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { teamConfigureExternalCases } from "./external-team-configure.js"; + +type Probe = { status: string; testedAt: string; checks: Array<{ code: string; level: string }> }; +function setupFixture(missing: Probe, working: Probe = { status: "pass", testedAt: "2026-11-03T00:00:00Z", checks: [] }) { + const member = (id: string) => ({ id, companyId: "company", name: id, adapterType: "paperclip_runner", adapterConfig: {}, metadata: { paperclip: { catalogTeam: { catalogId: "team" } } } }); + const members = [member("qa"), member("coder")]; + const probes = vi.fn(async (_path: string, body: { agentId?: string }) => body.agentId === "qa" ? missing : working); + const evidence = vi.fn(async (_name: string, _value: unknown) => {}); + const api = { + get: async (path: string) => { + if (path === "/api/teams/catalog") return [{ id: "team", slug: "product-engineering", compatibility: "compatible", agentSlugs: ["qa", "senior-coder"] }]; + if (path.endsWith("/agents")) return members; + if (path.endsWith("/config-revisions")) return [{ id: "working-revision" }]; + if (path === "/api/agents/coder") return members[1]; + if (["/projects", "/routines", "/teams/catalog/installed"].some(suffix => path.endsWith(suffix))) return []; + throw new Error("Unexpected setup read"); + }, + postSensitive: async (path: string, body: { agentId?: string }) => path.endsWith("/install") + ? { portabilityImport: { agents: [{ id: "qa", slug: "qa" }, { id: "coder", slug: "senior-coder" }] } } + : probes(path, body), + patchSensitive: vi.fn(async (_path: string, _body: unknown) => {}), + post: vi.fn(async (_path: string) => {}), patch: vi.fn(async (_path: string, _body: unknown) => {}), + }; + const ctx = { input: { api, evidence, nonce: "synthetic-team-prerequisite", workspacePath: "/synthetic/workspace", fixtures: { company: { id: "company" }, agent: { id: "ceo" }, environment: { id: "environment" }, secretRefs: {} }, execution: { profile: { credential: "OPENROUTER_API_KEY", buildAgent: () => ({ adapterType: "paperclip_runner", adapterConfig: { provider: "opencode" } }) }, environment: { id: "local" } } }, refreshIssue: async () => {}, issue: () => ({ id: "source" }) } as unknown as ChatCatalogContext; + return { ctx, evidence, probes, api }; +} +const authWarning: Probe = { status: "warn", testedAt: "2026-11-03T00:00:00Z", checks: [{ code: "opencode_hello_probe_auth_required", level: "warn" }] }; +it("the actual setup admits the adapter's exact missing-auth warning and preserves the independently passing member", async () => { + const f = setupFixture(authWarning), definition = teamConfigureExternalCases[0]!; + let state: Record | undefined; + try { + state = await definition.setup(f.ctx); + expect(state.initialProbes).toEqual([{ agentId: "qa", ...authWarning }, { agentId: "coder", status: "pass", testedAt: authWarning.testedAt, checks: [] }]); + expect((await definition.verifySetup!(f.ctx, state)).passed).toBe(true); + expect(f.probes).toHaveBeenCalledTimes(2); + expect(f.evidence).toHaveBeenCalledWith("chat-catalog-team-initial-probes.json", { initialProbes: state.initialProbes }); + expect(f.api.patchSensitive).toHaveBeenCalledTimes(1); + expect(f.api.patch).toHaveBeenCalledTimes(1); + } finally { await definition.cleanup!(f.ctx, state ?? {}); } +}); +it("retains no provider messages or session fields in the precondition witness", async () => { + const canary = "synthetic-private-probe-content"; + const raw = { ...authWarning, providerSession: canary, checks: authWarning.checks.map(check => ({ ...check, message: canary, detail: canary })) }; + const f = setupFixture(raw), definition = teamConfigureExternalCases[0]!; + let state: Record | undefined; + try { + state = await definition.setup(f.ctx); + expect(JSON.stringify(f.evidence.mock.calls)).not.toContain(canary); + expect(Object.keys((f.evidence.mock.calls[0]![1] as { initialProbes: object[] }).initialProbes[0]!).sort()).toEqual(["agentId", "checks", "status", "testedAt"]); + } finally { await definition.cleanup!(f.ctx, state ?? {}); } +}); +it.each(["generic warning", "timeout", "model", "wrong level", "other error", "working warning", "working error"])("actual setup rejects %s and retains only closed probe fields before refusing", async fault => { + const missing = structuredClone(authWarning), working: Probe = { status: "pass", testedAt: authWarning.testedAt, checks: [] }; + if (fault === "generic warning") missing.checks[0]!.code = "opencode_hello_probe_unexpected_output"; + if (fault === "timeout") missing.checks[0]!.code = "opencode_hello_probe_timed_out"; + if (fault === "model") missing.checks[0]!.code = "opencode_hello_probe_model_unavailable"; + if (fault === "wrong level") missing.checks[0]!.level = "info"; + if (fault === "other error") missing.checks.push({ code: "opencode_hello_probe_failed", level: "error" }); + if (fault === "working warning") working.status = "warn"; + if (fault === "working error") working.checks.push({ code: "opencode_hello_probe_failed", level: "error" }); + const f = setupFixture(missing, working); + await expect(teamConfigureExternalCases[0]!.setup(f.ctx)).rejects.toThrow("actual per-member runtime probes"); + expect(f.probes).toHaveBeenCalledTimes(2); + expect(f.evidence).toHaveBeenCalledTimes(1); + expect(f.evidence.mock.calls[0]![1]).toEqual({ initialProbes: [{ agentId: "qa", ...missing }, { agentId: "coder", ...working }] }); + expect(f.api.post).not.toHaveBeenCalled(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts index fa459b9f061..669bbf669f2 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts @@ -13,12 +13,19 @@ function valid() { describe("installed team per-member readiness oracle", () => { const definition = teamConfigureExternalCases[0]!; it("accepts real missing-account diagnosis, human repair and fresh separate readiness checks", () => { expect(definition.grade(valid()).passed).toBe(true); }); + it("accepts the exact OpenCode missing-auth warning without treating the member as ready", () => { + const o = valid(); o.state.initialProbes[0]!.status = "warn"; o.state.initialProbes[0]!.checks = [{ code: "opencode_hello_probe_auth_required", level: "warn" }]; + expect(definition.grade(o).passed).toBe(true); + o.state.afterProbes[0]!.status = "warn"; + expect(definition.grade(o).passed).toBe(false); + }); it("also accepts an actually attributed eligible agent configuration repair", () => { const o = valid(); o.state.repairChannel = "agent-configuration"; o.activity[0]!.actorType = "agent"; o.activity[0]!.agentId = "ceo"; expect(definition.grade(o).passed).toBe(true); }); it("rejects stale/member-substituted readiness, unrelated config edits and premature success", () => { for (const change of [ (o: ReturnType) => { o.state.afterProbes[0]!.testedAt = "2026-10-26T09:00:00Z"; }, (o: ReturnType) => { o.state.afterProbes[0]!.status = "fail"; }, (o: ReturnType) => { o.state.afterProbes[0]!.agentId = "coder"; }, + (o: ReturnType) => { o.state.initialProbes[0]!.agentId = "other-member"; }, (o: ReturnType) => { o.state.workingHashAfter = "changed-working-account"; }, (o: ReturnType) => { o.state.workingRevisionIdsAfter = ["masked-env-change"]; }, (o: ReturnType) => { o.state.noPrematureWholeTeamClaim = false; }, diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts index b5e8f4ba98a..04c7956e278 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts @@ -18,6 +18,14 @@ const fixtures = new WeakMap(); function stable(value: unknown): unknown { return Array.isArray(value) ? value.map(stable) : value && typeof value === "object" ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => [key, stable(item)])) : value; } const configHash = (agent: Agent) => createHash("sha256").update(JSON.stringify(stable({ adapterType: agent.adapterType, adapterConfig: agent.adapterConfig, runtimeConfig: agent.runtimeConfig, defaultEnvironmentId: agent.defaultEnvironmentId }))).digest("hex"); interface Probe { agentId: string; status: string; testedAt: string; checks: Array<{code: string; level: string}> } +function missingAccountProbe(row: Probe, agentId: string) { + return row.agentId === agentId && (row.status === "fail" && row.checks.some(check => check.level === "error" && /auth|credential|login/i.test(check.code)) + || row.status === "warn" && row.checks.some(check => check.code === "opencode_hello_probe_auth_required" && check.level === "warn") + && row.checks.every(check => check.level === "info" || check.code === "opencode_hello_probe_auth_required" && check.level === "warn")); +} +function readyProbe(row: Probe, agentId: string) { + return row.agentId === agentId && row.status === "pass" && row.checks.every(check => check.level === "info"); +} async function probe(ctx: ChatCatalogContext, agent: Agent): Promise { const result = await ctx.input.api.postSensitive(`${root(ctx)}/adapters/${agent.adapterType}/test-environment`, { agentId: agent.id, environmentId: agent.defaultEnvironmentId, adapterConfig: agent.adapterConfig }); return { agentId: agent.id, status: result.status, testedAt: result.testedAt, checks: result.checks.map(check => ({ code: /^[a-zA-Z0-9_.-]{1,100}$/.test(check.code) ? check.code : "unrecognized", level: check.level })) }; @@ -26,8 +34,8 @@ export function gradeTeamConfigure(value: unknown) { const c = checks(), o = externalObservation(value); c.add("complete-member-configuration-evidence", !!o, "Real installed members, missing versus ready account probes, scoped repair and fresh per-member tests are required."); if (!o) return c.result(); const initial = o.state.initialProbes as Probe[] | undefined, after = o.state.afterProbes as Probe[] | undefined; c.add("actual-missing-and-working-member-prerequisite", o.state.installedFromCatalog === true && typeof o.state.missingId === "string" && typeof o.state.workingId === "string" && o.state.missingId !== o.state.workingId - && Array.isArray(initial) && initial.some(row => row.agentId === o.state.missingId && row.status === "fail" && row.checks.some(check => check.level === "error" && /auth|credential|login/i.test(check.code))) - && initial.some(row => row.agentId === o.state.workingId && row.status === "pass"), + && Array.isArray(initial) && initial.some(row => missingAccountProbe(row, String(o.state.missingId))) + && initial.some(row => readyProbe(row, String(o.state.workingId))), "The supported catalog installer creates the team; actual runtime tests prove one exact missing account and a distinct already working member before chat."); c.add("specific-member-diagnosis-and-supported-repair", o.state.initialMemberNamed === true && o.state.initialHandoffSupported === true && o.state.repairConfirmed === true && ((o.state.repairChannel === "agent-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "agent" && row.agentId === o.leadId)) @@ -66,7 +74,8 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as await ctx.input.api.patchSensitive(`/api/agents/${missingId}`, { adapterConfig: { ...override.adapterConfig, env: { ...(override.adapterConfig?.env as Record ?? {}), ...missingTeamAccountEnvironment(subscription, emptyHome) } } }); const members = catalogTeamMembers(await observeTeamResources(ctx), team.id); const initialProbes = await Promise.all(members.map(member => probe(ctx, member))); - if (!initialProbes.some(row => row.agentId === missingId && row.status === "fail" && row.checks.some(check => check.level === "error" && /auth|credential|login/i.test(check.code))) || !initialProbes.some(row => row.agentId === workingId && row.status === "pass")) throw new Error("Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split"); + await ctx.input.evidence("chat-catalog-team-initial-probes.json", { initialProbes }); + if (!initialProbes.some(row => missingAccountProbe(row, missingId)) || !initialProbes.some(row => readyProbe(row, workingId))) throw new Error("Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split"); const working = await ctx.input.api.get(`/api/agents/${workingId}`); const workingRevisionIdsBefore = (await ctx.input.api.get>(`/api/agents/${workingId}/config-revisions`)).map(row => row.id).sort(); await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue(); @@ -75,10 +84,10 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as workingId, workingHashBefore: configHash(working), workingRevisionIdsBefore, initialProbes }; } catch (error) { fixtures.delete(ctx); await rm(emptyHome, { recursive: true, force: true }); throw error; } }, gradeTeamConfigure, { - authentication: "human-session", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 600_000, + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.team-configure.v2", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 600_000, setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["missing-account-fixture", "working-account-fixture"] }, - async verifySetup(_ctx, state) { const c = checks(), rows = state.initialProbes as Probe[]; c.add("missing-account-fixture", rows.some(row => row.agentId === state.missingId && row.status === "fail"), "Actual saved-runtime probe reports the required missing member account."); c.add("working-account-fixture", rows.some(row => row.agentId === state.workingId && row.status === "pass"), "A distinct installed member already passes its runtime probe."); return c.result(); }, - sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"], + async verifySetup(_ctx, state) { const c = checks(), rows = state.initialProbes as Probe[]; c.add("missing-account-fixture", rows.some(row => missingAccountProbe(row, String(state.missingId))), "Actual saved-runtime probe reports the required missing member account."); c.add("working-account-fixture", rows.some(row => readyProbe(row, String(state.workingId))), "A distinct installed member already passes its runtime probe."); return c.result(); }, + sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "../../server/src/adapters/registry.ts", "../../packages/adapters/opencode-local/src/server/test.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"], async act(ctx, state) { await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const initial = await observeExternal(ctx, state); state.initialRunId = initial.runs[0]?.id; state.initialMemberNamed = initial.reply.toLowerCase().includes(String(state.missingName).toLowerCase());