From 2b194d77e2f8384d6fc789b6a46f3b66754652a0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 11 Nov 2026 11:50:49 -0600 Subject: [PATCH] fix(evals): name each team member's observed readiness and grade configure-team structurally teams.configure-team failed setup because the QA member's empty OpenRouter key surfaced as a generic hello-probe error (OpenRouter 401 "Missing Authentication header") instead of the missing-authentication check, so the missing-versus-ready split never formed. The product fix (cherry-picked) classifies a provider 401 as opencode_hello_probe_auth_required. - The setup blocker now names every member's observed probe status and non-info checks. - Replace reply keyword grading: drop the member-name substring and the premature-ready regex (identification is the rendered link to the missing member's own page, already required), and grade the followup by a non-empty reply from its own succeeded run instead of matching ready/configured words. - Calibrate both ways; bump oracleVersion to v3; add parse.ts to sourceFiles. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 3 ++- ...ternal-team-configure-prerequisite.test.ts | 6 +++++ .../external-team-configure.test.ts | 11 ++++++-- .../external-team-configure.ts | 26 ++++++++++++------- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 98e7b3817a4..e90d93bbba6 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -6045,6 +6045,7 @@ "chat-catalog-domains/external-team-install.ts", "../../server/src/adapters/registry.ts", "../../packages/adapters/opencode-local/src/server/test.ts", + "../../packages/adapters/opencode-local/src/server/parse.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", @@ -6052,7 +6053,7 @@ "chat-flow.ts", "chat-scenarios/teams.yaml" ], - "oracleVersion": "paperclip.chat-catalog.team-configure.v2", + "oracleVersion": "paperclip.chat-catalog.team-configure.v3", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts index 4ddf64446ab..21070d0f359 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure-prerequisite.test.ts @@ -51,6 +51,12 @@ it("retains no provider messages or session fields in the precondition witness", expect(Object.keys((f.evidence.mock.calls[0]![1] as { initialProbes: object[] }).initialProbes[0]!).sort()).toEqual(["agentId", "checks", "status", "testedAt"]); } finally { await definition.cleanup!(f.ctx, state ?? {}); } }); +it("names each member's observed readiness when the split does not form", async () => { + // The recorded failure: the provider 401 surfaced as a generic probe error. + const f = setupFixture({ status: "fail", testedAt: authWarning.testedAt, checks: [{ code: "opencode_cwd_valid", level: "info" }, { code: "opencode_hello_probe_failed", level: "error" }] }); + await expect(teamConfigureExternalCases[0]!.setup(f.ctx)).rejects.toThrow( + "Observed: qa (expected missing account): fail [opencode_hello_probe_failed:error]; coder (expected ready): pass"); +}); it.each(["generic warning", "timeout", "model", "wrong level", "other error", "working warning", "working error"])("actual setup rejects %s and retains only closed probe fields before refusing", async fault => { const missing = structuredClone(authWarning), working: Probe = { status: "pass", testedAt: authWarning.testedAt, checks: [] }; if (fault === "generic warning") missing.checks[0]!.code = "opencode_hello_probe_unexpected_output"; diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts index 669bbf669f2..5bc0f8fa563 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure.test.ts @@ -7,7 +7,7 @@ function valid() { installedFromCatalog: true, missingId: "qa", workingId: "coder", expectedMembers: 2, memberIdsBefore: ["coder", "qa"], workingRevisionIdsBefore: ["revision"], workingRevisionIdsAfter: ["revision"], initialProbes: [{ agentId: "qa", status: "fail", testedAt: "2026-10-26T10:00:00Z", checks: [{ code: "codex_auth", level: "error" }] }, { agentId: "coder", status: "pass", testedAt: "2026-10-26T10:00:00Z", checks: [] }], afterProbes: [{ agentId: "qa", status: "pass", testedAt: "2026-10-26T10:05:01Z", checks: [] }, { agentId: "coder", status: "pass", testedAt: "2026-10-26T10:05:02Z", checks: [] }], - repairVerifiedAfter: Date.parse("2026-10-26T10:05:00Z"), initialMemberNamed: true, initialHandoffSupported: true, repairConfirmed: true, repairChannel: "human-board-configuration", workingHashBefore: "same-config", workingHashAfter: "same-config", memberIdsPreserved: true, initialRunId: "initial", noPrematureWholeTeamClaim: true, + repairVerifiedAfter: Date.parse("2026-10-26T10:05:00Z"), initialHandoffSupported: true, repairConfirmed: true, repairChannel: "human-board-configuration", workingHashBefore: "same-config", workingHashAfter: "same-config", memberIdsPreserved: true, initialRunId: "initial", } }; } describe("installed team per-member readiness oracle", () => { @@ -28,9 +28,16 @@ describe("installed team per-member readiness oracle", () => { (o: ReturnType) => { o.state.initialProbes[0]!.agentId = "other-member"; }, (o: ReturnType) => { o.state.workingHashAfter = "changed-working-account"; }, (o: ReturnType) => { o.state.workingRevisionIdsAfter = ["masked-env-change"]; }, - (o: ReturnType) => { o.state.noPrematureWholeTeamClaim = false; }, + (o: ReturnType) => { o.state.initialHandoffSupported = false; }, + (o: ReturnType) => { o.phaseReplies[1]!.body = " "; }, + (o: ReturnType) => { o.runs[1]!.status = "failed"; }, (o: ReturnType) => { o.phaseReplies[1]!.runId = "initial"; }, ]) { const o = valid(); change(o); expect(definition.grade(o).passed).toBe(false); } }); + it("grades the followup structurally, not by its wording", () => { + for (const body of ["Both QA and the senior coder pass their own checks now.", "Fertig: alle Mitglieder bestehen ihre Prüfung."]) { + const o = valid(); o.phaseReplies[1]!.body = body; o.reply = body; expect(definition.grade(o).passed).toBe(true); + } + }); it("missing observations cannot pass", () => { expect(definition.grade(undefined).passed).toBe(false); expect(definition.grade({ reply: "Everyone is ready." }).passed).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts index 04c7956e278..52f0c431ee4 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts @@ -26,6 +26,15 @@ function missingAccountProbe(row: Probe, agentId: string) { function readyProbe(row: Probe, agentId: string) { return row.agentId === agentId && row.status === "pass" && row.checks.every(check => check.level === "info"); } +/** Each member's observed readiness, for a setup blocker that says which side of the split did not form. */ +export function memberReadinessSummary(rows: readonly Probe[], members: ReadonlyArray>, missingId: string, workingId: string) { + return rows.map(row => { + const role = row.agentId === missingId ? "expected missing account" : row.agentId === workingId ? "expected ready" : "other member"; + const name = members.find(member => member.id === row.agentId)?.name ?? "unknown member"; + const problems = row.checks.filter(check => check.level !== "info").map(check => `${check.code}:${check.level}`); + return `${name} (${role}): ${row.status}${problems.length ? ` [${problems.join(", ")}]` : ""}`; + }).join("; "); +} async function probe(ctx: ChatCatalogContext, agent: Agent): Promise { const result = await ctx.input.api.postSensitive(`${root(ctx)}/adapters/${agent.adapterType}/test-environment`, { agentId: agent.id, environmentId: agent.defaultEnvironmentId, adapterConfig: agent.adapterConfig }); return { agentId: agent.id, status: result.status, testedAt: result.testedAt, checks: result.checks.map(check => ({ code: /^[a-zA-Z0-9_.-]{1,100}$/.test(check.code) ? check.code : "unrecognized", level: check.level })) }; @@ -37,10 +46,10 @@ export function gradeTeamConfigure(value: unknown) { && Array.isArray(initial) && initial.some(row => missingAccountProbe(row, String(o.state.missingId))) && initial.some(row => readyProbe(row, String(o.state.workingId))), "The supported catalog installer creates the team; actual runtime tests prove one exact missing account and a distinct already working member before chat."); - c.add("specific-member-diagnosis-and-supported-repair", o.state.initialMemberNamed === true && o.state.initialHandoffSupported === true && o.state.repairConfirmed === true + c.add("specific-member-diagnosis-and-supported-repair", o.state.initialHandoffSupported === true && o.state.repairConfirmed === true && ((o.state.repairChannel === "agent-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "agent" && row.agentId === o.leadId)) || (o.state.repairChannel === "human-board-configuration" && o.activity.some(row => row.entityId === o.state.missingId && row.action === "agent.updated" && row.actorType === "user"))), - "Chat identifies the incomplete member and the supported step; repair changes that member through actual agent or human configuration authority."); + "Chat identifies the incomplete member by linking its own configuration page (or repairs it itself); repair changes that member through actual agent or human configuration authority."); c.add("already-working-member-preserved", typeof o.state.workingHashBefore === "string" && o.state.workingHashBefore === o.state.workingHashAfter && o.state.memberIdsPreserved === true, "The already working member's semantic configuration and the installed member identities remain unchanged."); c.add("working-configuration-revisions-preserved", Array.isArray(o.state.workingRevisionIdsBefore) && o.state.workingRevisionIdsBefore.length > 0 @@ -52,9 +61,8 @@ export function gradeTeamConfigure(value: unknown) { && Number.isFinite(Date.parse(row.testedAt)) && Date.parse(row.testedAt) >= Number(o.state.repairVerifiedAfter)), "After the real repair every member passes its own fresh saved-runtime test; one healthy member cannot stand for the whole team."); const continuation = o.phaseReplies.filter(row => row.runId !== o.state.initialRunId); - c.add("phase-bound-truthful-team-result", continuation.some(row => o.runs.some(run => run.id === row.runId && run.status === "succeeded")) - && /ready|configured|verified|account/i.test(continuation.map(row => row.body).join("\n")) && o.state.noPrematureWholeTeamClaim === true, - "The final claim belongs to the actual verification followup, and the initial reply did not call an unconfigured team ready."); + c.add("phase-bound-team-result", continuation.some(row => row.body.trim().length > 0 && o.runs.some(run => run.id === row.runId && run.status === "succeeded")), + "The verification followup posts its own reply from a succeeded run; its wording is left to human reading against the fresh per-member probes."); return c.result(); } export const teamConfigureExternalCases = [definition("teams.configure-team", async ctx => { @@ -75,7 +83,7 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as const members = catalogTeamMembers(await observeTeamResources(ctx), team.id); const initialProbes = await Promise.all(members.map(member => probe(ctx, member))); await ctx.input.evidence("chat-catalog-team-initial-probes.json", { initialProbes }); - if (!initialProbes.some(row => missingAccountProbe(row, missingId)) || !initialProbes.some(row => readyProbe(row, workingId))) throw new Error("Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split"); + if (!initialProbes.some(row => missingAccountProbe(row, missingId)) || !initialProbes.some(row => readyProbe(row, workingId))) throw new Error(`Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split. Observed: ${memberReadinessSummary(initialProbes, members, missingId, workingId)}`); const working = await ctx.input.api.get(`/api/agents/${workingId}`); const workingRevisionIdsBefore = (await ctx.input.api.get>(`/api/agents/${workingId}/config-revisions`)).map(row => row.id).sort(); await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue(); @@ -84,18 +92,16 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as workingId, workingHashBefore: configHash(working), workingRevisionIdsBefore, initialProbes }; } catch (error) { fixtures.delete(ctx); await rm(emptyHome, { recursive: true, force: true }); throw error; } }, gradeTeamConfigure, { - authentication: "human-session", oracleVersion: "paperclip.chat-catalog.team-configure.v2", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 600_000, + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.team-configure.v3", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 600_000, setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["missing-account-fixture", "working-account-fixture"] }, async verifySetup(_ctx, state) { const c = checks(), rows = state.initialProbes as Probe[]; c.add("missing-account-fixture", rows.some(row => missingAccountProbe(row, String(state.missingId))), "Actual saved-runtime probe reports the required missing member account."); c.add("working-account-fixture", rows.some(row => readyProbe(row, String(state.workingId))), "A distinct installed member already passes its runtime probe."); return c.result(); }, - sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "../../server/src/adapters/registry.ts", "../../packages/adapters/opencode-local/src/server/test.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"], + sourceFiles: ["chat-catalog-domains/external-team-configure.ts", "chat-catalog-domains/external-team-install.ts", "../../server/src/adapters/registry.ts", "../../packages/adapters/opencode-local/src/server/test.ts", "../../packages/adapters/opencode-local/src/server/parse.ts", "codex-subscription-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", "chat-scenarios/teams.yaml"], async act(ctx, state) { await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const initial = await observeExternal(ctx, state); state.initialRunId = initial.runs[0]?.id; - state.initialMemberNamed = initial.reply.toLowerCase().includes(String(state.missingName).toLowerCase()); const members = catalogTeamMembers(await observeTeamResources(ctx), String(state.teamId)), missing = members.find(row => row.id === state.missingId)!; const current = await probe(ctx, missing); const alreadyRepaired = current.status === "pass"; const handoff = initial.links.find(href => { try { const url = new URL(href, initial.origin); return url.origin === initial.origin && url.pathname.split("/").some((part, i, parts) => part === "agents" && [missing.id, missing.urlKey].includes(parts[i + 1]!)); } catch { return false; } }); state.initialHandoffSupported = alreadyRepaired || !!handoff; - state.noPrematureWholeTeamClaim = alreadyRepaired || !/\b(?:team|product engineering)\s+(?:is|are|looks)\s+(?:now\s+)?(?:fully\s+)?ready/i.test(initial.reply); if (!alreadyRepaired) { if (!handoff) throw new Error("The model did not provide a supported configuration handoff for the incomplete member"); const source = ctx.input.page.url(); await ctx.input.page.goto(handoff, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("main")).toContainText(missing.name);