From 8e57349a43de7a6cdf2c907d35273e5d9a72cfa1 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 20:38:47 -0500 Subject: [PATCH] test(evals): bind Morgan handoff chat to a real engineer principal --- tests/runner-e2e/chat-catalog-contract.ts | 1 + tests/runner-e2e/chat-catalog-registry.ts | 5 ++- .../chat-catalog-respondent.test.ts | 29 +++++++++++++++ tests/runner-e2e/chat-catalog-respondent.ts | 35 +++++++++++++++++++ tests/runner-e2e/chat-flow.ts | 2 ++ 5 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 tests/runner-e2e/chat-catalog-respondent.test.ts create mode 100644 tests/runner-e2e/chat-catalog-respondent.ts diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 2f96a60a9f1..35a66a2bd14 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -36,6 +36,7 @@ export interface ChatCatalogCase { scenarioId: string; /** Real browser locale context for the canonical person's schedule. */ browserTimezone?: "America/Chicago"; + respondingAgent?: { kind: "case-primary-owner"; name: "Morgan"; role: "engineer"; canAssignTasks: true }; requiredCredentials?: readonly CredentialName[]; externalInput?: ChatExternalInputRequirement; installedSource?: "published-global-npm"; diff --git a/tests/runner-e2e/chat-catalog-registry.ts b/tests/runner-e2e/chat-catalog-registry.ts index 77b2d6a455a..108df6709ce 100644 --- a/tests/runner-e2e/chat-catalog-registry.ts +++ b/tests/runner-e2e/chat-catalog-registry.ts @@ -24,6 +24,9 @@ export function findChatCatalogCase(scenarioId: string): ChatCatalogDescriptor | if (!descriptor) return; if (descriptor.browserTimezone && (scenarioId !== "routines.create-daily-brief" || descriptor.browserTimezone !== "America/Chicago")) throw new Error("Browser timezone is declared only for the canonical Chicago weekday routine"); + if (descriptor.respondingAgent && (scenarioId !== "cases.handoff-same-task" || Object.keys(descriptor.respondingAgent).length !== 4 + || descriptor.respondingAgent.kind !== "case-primary-owner" || descriptor.respondingAgent.name !== "Morgan" || descriptor.respondingAgent.role !== "engineer" || descriptor.respondingAgent.canAssignTasks !== true)) + throw new Error("A non-CEO responding fixture is declared only for the canonical Morgan primary-owner handoff"); if (descriptor.installedSource && (descriptor.installedSource !== "published-global-npm" || !(installedChatScenarioIds as readonly string[]).includes(scenarioId))) throw new Error("Installed ordinary Chat is allowlisted only for the two published-source operator scenarios"); if (!known.has(scenarioId) || !descriptor.oracleVersion || !descriptor.sourceFiles.length @@ -50,7 +53,7 @@ export async function loadChatCatalogImplementation( const selected = implementations.filter((row: ChatCatalogCase) => row.scenarioId === descriptor.scenarioId); if (selected.length !== 1) throw new Error(`Missing or duplicate runtime case: ${descriptor.scenarioId}`); const implementation = selected[0] as ChatCatalogCase; - for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments", "requiredCredentials", "externalInput", "installedSource", "browserTimezone"] as const) { + for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments", "requiredCredentials", "externalInput", "installedSource", "browserTimezone", "respondingAgent"] as const) { if (JSON.stringify(implementation[key]) !== JSON.stringify(descriptor[key])) throw new Error(`Runtime descriptor drift: ${descriptor.scenarioId}.${key}`); } if ((descriptor.setupPolicy?.funding === "preserve-scenario" || descriptor.setupPolicy?.readiness === "preserve-scenario") diff --git a/tests/runner-e2e/chat-catalog-respondent.test.ts b/tests/runner-e2e/chat-catalog-respondent.test.ts new file mode 100644 index 00000000000..dd6cfdae1bd --- /dev/null +++ b/tests/runner-e2e/chat-catalog-respondent.test.ts @@ -0,0 +1,29 @@ +import { expect, it, vi } from "vitest"; +import { prepareChatCatalogRespondent } from "./chat-catalog-respondent.js"; +import type { ChatFlowInput } from "./chat-flow.js"; +const descriptor = { scenarioId: "cases.handoff-same-task", respondingAgent: { kind: "case-primary-owner", name: "Morgan", role: "engineer", canAssignTasks: true } } as const; +function fixture() { + const persisted = { id: "morgan", companyId: "company", name: "Morgan", role: "engineer", lifecycleState: "ready", status: "idle", permissions: { canCreateAgents: false }, access: { grants: [{ permissionKey: "tasks:assign" }] } }; + const api = { post: vi.fn(async (_path: string, _payload: unknown) => persisted), patch: vi.fn(async (_path: string, _payload: unknown) => persisted), get: vi.fn(async () => persisted), delete: vi.fn(async (_path: string, _options: unknown) => {}) }; + const input = { api, nonce: "fixture", workspacePath: "/owned/synthetic", evidence: vi.fn(async () => {}), fixtures: { company: { id: "company" }, agent: { id: "ceo", name: "CEO", companyId: "company" }, environment: { id: "environment" }, secretRefs: {} }, + execution: { environment: { id: "local" }, profile: { credential: "OPENAI_API_KEY", buildAgent: () => ({ name: "CEO", role: "ceo", permissions: { canCreateAgents: true } }) } } } as unknown as ChatFlowInput; + return { input, api, persisted }; +} +it("uses a separate actual engineer principal and only the declared assignment grant", async () => { + const { input, api } = fixture(); const originalFixtures = input.fixtures; + await prepareChatCatalogRespondent(input, descriptor); + expect(input.fixtures).toBe(originalFixtures); expect(input.fixtures.agent).toEqual({ id: "morgan", name: "Morgan", companyId: "company" }); + expect(api.post.mock.calls[0]?.[1]).toMatchObject({ name: "Morgan", role: "engineer", reportsTo: "ceo", permissions: { canCreateAgents: false } }); + expect(api.patch.mock.calls[0]?.[1]).toEqual({ canCreateAgents: false, canAssignTasks: true }); +}); +it("preserves default CEO fixtures and refuses another scenario before any mutation", async () => { + const { input, api } = fixture(); await prepareChatCatalogRespondent(input); + expect(input.fixtures.agent.id).toBe("ceo"); expect(api.post).not.toHaveBeenCalled(); + await expect(prepareChatCatalogRespondent(input, { ...descriptor, scenarioId: "cases.create-case" })).rejects.toThrow("Unsupported responding-agent"); + expect(api.post).not.toHaveBeenCalled(); +}); +it("retires a partially prepared respondent and preserves CEO identity on grant failure", async () => { + const { input, api } = fixture(); api.patch.mockRejectedValueOnce(new Error("public grant failed")); + await expect(prepareChatCatalogRespondent(input, descriptor)).rejects.toThrow("public grant failed"); + expect(input.fixtures.agent.id).toBe("ceo"); expect(api.delete).toHaveBeenCalledWith("/api/agents/morgan", { allowNotFound: true }); +}); diff --git a/tests/runner-e2e/chat-catalog-respondent.ts b/tests/runner-e2e/chat-catalog-respondent.ts new file mode 100644 index 00000000000..46360107148 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-respondent.ts @@ -0,0 +1,35 @@ +import { pollUntil } from "./api.js"; +import { codexSubscriptionAgentConfig } from "./codex-subscription-fixture.js"; +import type { ChatFlowInput } from "./chat-flow.js"; +import type { ChatCatalogCase } from "./chat-catalog-contract.js"; +type Respondent = ChatFlowInput["fixtures"]["agent"] & { + role: string; lifecycleState: string; status: string; permissions?: { canCreateAgents?: boolean }; + access?: { grants?: Array<{ permissionKey: string }> }; +}; +/** The original CEO remains a company fixture. This narrowly selected case gets + * a separate real engineer principal; no visible-name or authority substitution. */ +export async function prepareChatCatalogRespondent(input: ChatFlowInput, descriptor?: Pick) { + if (!descriptor?.respondingAgent) return; + const policy = descriptor.respondingAgent; + if (descriptor.scenarioId !== "cases.handoff-same-task" || Object.keys(policy).length !== 4 || policy.kind !== "case-primary-owner" || policy.name !== "Morgan" || policy.role !== "engineer" || policy.canAssignTasks !== true) + throw new Error("Unsupported responding-agent fixture policy"); + const original = input.fixtures.agent; + const base = input.execution.profile.buildAgent({ environmentId: input.fixtures.environment.id, environmentFixtureId: input.execution.environment.id, + workspacePath: input.workspacePath, secretRefs: input.fixtures.secretRefs, executionId: `${input.nonce}-case-owner` }); + const payload = { ...base, name: policy.name, role: policy.role, title: "Case investigator", reportsTo: original.id, + capabilities: "Investigates customer cases and preserves findings when handing the same task to a reviewer.", permissions: { canCreateAgents: false }, + runtimeConfig: { ...(base.runtimeConfig as Record), heartbeat: { enabled: false, maxConcurrentRuns: 1, maxDailyRuns: 3, maxDailyCostCents: 500 } } }; + const created = await input.api.post(`/api/companies/${input.fixtures.company.id}/agents`, input.execution.profile.credential === "CODEX_AUTH_JSON" + ? codexSubscriptionAgentConfig(payload, process.env, input.fixtures.company.id) : payload); + try { + await input.api.patch(`/api/agents/${created.id}/permissions`, { canCreateAgents: false, canAssignTasks: true }); + const actual = await pollUntil({ label: "actual Morgan case-owner respondent", deadlineAt: Date.now() + 45_000, + load: () => input.api.get(`/api/agents/${created.id}`), + accept: row => row.lifecycleState === "ready" && ["idle", "active"].includes(row.status) + && row.name === "Morgan" && row.role === "engineer" && row.companyId === input.fixtures.company.id + && row.permissions?.canCreateAgents === false && row.access?.grants?.some(grant => grant.permissionKey === "tasks:assign") === true }); + input.fixtures.agent = { id: actual.id, name: actual.name, companyId: actual.companyId }; + await input.evidence("chat-catalog-responding-agent.json", { originalCompanyCEOId: original.id, actualRespondingAgentId: actual.id, + actualName: actual.name, actualRole: actual.role, canCreateAgents: actual.permissions?.canCreateAgents, requiredGrant: "tasks:assign", fixturePolicy: policy }); + } catch (error) { input.fixtures.agent = original; await input.api.delete(`/api/agents/${created.id}`, { allowNotFound: true }); throw error; } +} diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.ts index 9c50be4e6dd..8d02ed073a5 100644 --- a/tests/runner-e2e/chat-flow.ts +++ b/tests/runner-e2e/chat-flow.ts @@ -1,3 +1,4 @@ +import { prepareChatCatalogRespondent } from "./chat-catalog-respondent.js"; import type { ChatPrivateInputs } from "./chat-private-inputs.js"; import { chatCatalogAllowedRunStatuses } from "./chat-catalog-run-policy.js"; import { runHiringTemplateFlow } from "./hiring-template-flow.js"; @@ -374,6 +375,7 @@ export interface ChatFlowInput { check?: (id: string, passed: boolean, detail: string) => void; } export async function runChatFlow(input: ChatFlowInput) { + if (input.execution.suite.id === CHAT_FEATURE_SUITE) await prepareChatCatalogRespondent(input, findChatCatalogCase(input.execution.task.id)); const { page, api, fixtures: f, execution, nonce } = input; const chatPath = `/api/companies/${f.company.id}/chats/${f.agent.id}`; const route = `/${f.company.issuePrefix}/chats/${f.agent.id}`;