diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts index c9c05ddb683..f03defd6da6 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts @@ -1,13 +1,17 @@ -import { describe, expect, it } from "vitest"; -import { companyModelIdentityCases } from "./identity-company-models.js"; +import { describe, expect, it, vi } from "vitest"; +import { companyModelIdentityCases, requestedAuxiliaryServiceModel } from "./identity-company-models.js"; import type { IdentityObservation } from "./identity-support.js"; +vi.mock("./identity-support.js", async importOriginal => ({ + ...await importOriginal(), probeAgentPermission: async () => 403, +})); function valid(scenarioId: string): IdentityObservation { + const model = scenarioId === "companies.configure-decision-model" ? "typesafe/jev-1.13" : requestedAuxiliaryServiceModel; const usage = { inputTokens: 17, outputTokens: 8, costStatus: "reported", costCents: "0.02" }; - const history = { id: "test", connectionId: "shared", model: "typesafe/jev-1.13", actorType: "user", responsibleUserId: "alex", feature: "settings.test", finishedAt: "2026-10-26T12:00:00Z", status: "succeeded", ...usage }; + const history = { id: "test", connectionId: "shared", model, actorType: "user", responsibleUserId: "alex", feature: "settings.test", finishedAt: "2026-10-26T12:00:00Z", status: "succeeded", ...usage }; const o: IdentityObservation = { scenarioId, origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], interactions: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], activity: [], links: [], runs: [], humanQueries: [], reply: "Open Connections in Settings to configure and test the optional fast acknowledgement or classifier model. An acknowledgement is only a receipt, not task work.", - state: { connection: { id: "shared", grantId: "grant", companyId: "acorn", ownership: "shared", provider: "openrouter", method: "api_key", status: "connected" }, humanId: "alex", - saved: { companyId: "acorn", enabled: true, connectionId: "shared", grantId: "grant", model: "typesafe/jev-1.13" }, beforeEnabled: false, afterReplyEnabled: false, agentWriteStatus: 403, agentTestStatus: 403, + state: { connection: { id: "shared", name: "Shared QA", grantId: "grant", companyId: "acorn", ownership: "shared", provider: "openrouter", method: "api_key", status: "connected" }, humanId: "alex", + saved: { companyId: "acorn", enabled: true, connectionId: "shared", grantId: "grant", model }, modelBeforeTest: model, beforeEnabled: false, afterReplyEnabled: false, agentWriteStatus: 403, agentTestStatus: 403, persistedAfterReload: true, testHttpStatus: 200, visibleTestReceipt: true, history: [history], testCostEvents: 2 } }; if (scenarioId.endsWith("configure-decision-model")) o.state.testResult = { status: "succeeded", invocationId: "test", answers: {}, usage }; else if (scenarioId.endsWith("fast-responses")) o.state.testResult = { status: "succeeded", invocationId: "test", text: "I will check that for you.", durationMs: 140, usage }; @@ -23,6 +27,32 @@ describe("company model service configuration and actual tests", () => { it(`${d.scenarioId}: wrong shared account or authority`, () => { const o = valid(d.scenarioId); (o.state.saved as { grantId: string }).grantId = "other-grant"; expect(d.grade(o).passed).toBe(false); const wrong = valid(d.scenarioId); wrong.state.agentWriteStatus = 200; expect(d.grade(wrong).passed).toBe(false); }); it(`${d.scenarioId}: configuration without an actual result is insufficient`, () => { const o = valid(d.scenarioId); delete o.state.testResult; expect(d.grade(o).passed).toBe(false); }); } + it.each(["companies.prompt-injection-guard", "companies.fast-responses"])("%s rejects saved/default/changed/missing model evidence", id => { + const definition = companyModelIdentityCases.find(row => row.scenarioId === id)!; + for (const model of ["openai/gpt-oss-safeguard-20b", "openai/gpt-oss-120b", "openrouter/deepseek/deepseek-v4.1-flash-1017", "deepseek/another-model"]) { + const wrong = valid(id); (wrong.state.saved as { model: string }).model = model; expect(definition.grade(wrong).passed).toBe(false); + const earlier = valid(id); earlier.state.modelBeforeTest = model; expect(definition.grade(earlier).passed).toBe(false); + } + const absent = valid(id); delete absent.state.modelBeforeTest; expect(definition.grade(absent).passed).toBe(false); + }); + it("rejects fast-service history from a different actual model despite requested saved config", () => { + const definition = companyModelIdentityCases.find(row => row.scenarioId === "companies.fast-responses")!; + const wrong = valid(definition.scenarioId); (wrong.state.history as Array<{ model: string }>)[0]!.model = "openai/gpt-oss-120b"; + expect(definition.grade(wrong).passed).toBe(false); + }); + it.each(["companies.prompt-injection-guard", "companies.fast-responses"])("%s actual human act stops before test if saved model differs", async id => { + const definition = companyModelIdentityCases.find(row => row.scenarioId === id)!; + const fill = vi.fn(async () => {}), save = vi.fn(async () => {}), test = vi.fn(async () => {}); + const section = { getByLabel: () => ({ fill }), getByRole: (_role: string, options: { name: string }) => ({ + click: options.name.startsWith("Test") ? test : save, setChecked: async () => {}, + }) }; + const get = vi.fn(async () => ({ settings: { enabled: false, model: "openai/wrong-default" } })); + const ctx = { refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }), allRuns: async () => [{ id: "source", status: "succeeded" }], + input: { fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get }, page: { url: () => "https://fixture.test", goto: async () => {}, getByRole: () => section } } }; + await expect(definition.act!(ctx as never, valid(id).state)).rejects.toThrow(/Requested auxiliary model saved before any service test/); + expect(fill).toHaveBeenCalledExactlyOnceWith(requestedAuxiliaryServiceModel); expect(save).toHaveBeenCalledTimes(2); + expect(get).toHaveBeenCalledTimes(2); expect(test).not.toHaveBeenCalled(); + }); it("rejects an ordinary agent run standing in for a decision invocation", () => { const d = companyModelIdentityCases[0]!; const o = valid(d.scenarioId); o.runs = [{ id: "test", companyId: "acorn", agentId: "ceo", status: "succeeded" }]; o.state.history = []; expect(d.grade(o).passed).toBe(false); }); it("rejects builtin fallback being reported as model classification", () => { const d = companyModelIdentityCases[1]!; const o = valid(d.scenarioId); (o.state.testResult as { samples: Array<{ invocationId?: string }> }).samples[0]!.invocationId = undefined; expect(d.grade(o).passed).toBe(false); }); it("rejects classifier model samples without their independent cost events", () => { const d = companyModelIdentityCases[1]!; const o = valid(d.scenarioId); o.state.testCostEvents = 0; expect(d.grade(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts index 2ff9952e1bc..bf6d6e047b0 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts @@ -8,6 +8,8 @@ import type { CostSummary } from "../../../packages/shared/src/types/cost.js"; import { checks } from "./internal-support.js"; import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission, waitForReply } from "./identity-support.js"; +export const requestedAuxiliaryServiceModel = "deepseek/deepseek-v4.1-flash-1017"; + type Settings = DecisionModelSettings | PromptInjectionPreventionSettings | FastResponseSettings; const fastExperiments = new WeakMap(); const specs = [ @@ -25,6 +27,10 @@ export function gradeCompanyModel(value: unknown) { && o.state.testHttpStatus === 200 && o.state.visibleTestReceipt === true, "The canonical chat leaves the disabled board-managed service unchanged; the actual human settings controls save the intended grant and the result survives reload."); c.add("honest-human-settings-guidance", /settings|connections/i.test(o.reply) && !/\bI(?:'ve| have)?\s+(?:enabled|configured|turned on|tested)\b/i.test(o.reply), "The chat names the human settings surface and does not claim a board-only change or test it could not perform."); + if (["companies.prompt-injection-guard", "companies.fast-responses"].includes(o.scenarioId)) { + c.add("requested-service-model-persisted-before-test", o.state.modelBeforeTest === requestedAuxiliaryServiceModel && saved?.model === requestedAuxiliaryServiceModel, + "The actual public human configuration uses the requested OpenRouter API model before service testing and after readback; default auxiliary models are not substituted."); + } if (o.scenarioId === "companies.configure-decision-model") { const result = o.state.testResult as DecisionResult | undefined; const history = o.state.history as DecisionHistoryEntry[] | undefined; c.add("actual-decision-test-and-ledger", (saved as DecisionModelSettings | undefined)?.model === DECISION_MODELS.openrouter.id @@ -43,7 +49,7 @@ export function gradeCompanyModel(value: unknown) { } else if (o.scenarioId === "companies.fast-responses") { const result = o.state.testResult as FastResponseTestResult | undefined; const history = o.state.history as Array | undefined; c.add("actual-fast-response-receipt", !!result && result.status === "succeeded" && !!result.text.trim() && result.durationMs >= 0 && !!history?.some(row => row.id === result.invocationId - && row.connectionId === connection?.id && row.responsibleUserId === o.state.humanId && row.feature === "settings.test" && row.finishedAt != null + && row.connectionId === connection?.id && row.model === requestedAuxiliaryServiceModel && row.responsibleUserId === o.state.humanId && row.feature === "settings.test" && row.finishedAt != null && row.inputTokens === result.usage.inputTokens && row.outputTokens === result.usage.outputTokens && row.costStatus === result.usage.costStatus), "The real settings test generates an actual model acknowledgement and matching durable usage/failure history; elapsed worker time alone cannot qualify."); c.add("acknowledgement-is-not-work", /acknowledg|receipt|quick|fast/i.test(o.reply) && !/\b(?:work|task)\s+(?:is|has been)\s+(?:finished|completed|started)/i.test(o.reply), "The response does not present a fast acknowledgement as task completion or proof of worker execution."); @@ -74,13 +80,19 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s await waitForReply(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url(); state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled; state.agentWriteStatus = await probeAgentPermission(ctx, `${root}/${spec.path}`, "PUT", { enabled: true, connectionId: (state.connection as AiManagedConnectionSummary).id, grantId: (state.connection as AiManagedConnectionSummary).grantId, - ...(spec.path === "decision-model" ? { allowBackground: false } : spec.path === "fast-response" ? { model: "openai/gpt-oss-120b", allowSponsored: false } : { model: "openai/gpt-oss-safeguard-20b" }) }); + ...(spec.path === "decision-model" ? { allowBackground: false } : spec.path === "fast-response" ? { model: requestedAuxiliaryServiceModel, allowSponsored: false } : { model: requestedAuxiliaryServiceModel }) }); state.agentTestStatus = await probeAgentPermission(ctx, `${root}/${spec.path}/test`, "POST", {}); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/connections`, { waitUntil: "domcontentloaded" }); const section = spec.path === "decision-model" ? ctx.input.page.locator('section[aria-labelledby="decision-model-heading"]') : ctx.input.page.getByRole("region", { name: spec.section, exact: true }); await section.getByRole("button", { name: (state.connection as AiManagedConnectionSummary).name, exact: true }).click(); + if (spec.path !== "decision-model") await section.getByLabel("Model", { exact: true }).fill(requestedAuxiliaryServiceModel); await section.getByRole("switch", { name: spec.toggle, exact: true }).setChecked(true); await section.getByRole("button", { name: spec.save, exact: true }).click(); + if (spec.path !== "decision-model") { + const persisted = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings; + expect(persisted.model, "Requested auxiliary model saved before any service test").toBe(requestedAuxiliaryServiceModel); + state.modelBeforeTest = persisted.model; + } await expect(section.getByRole("button", { name: spec.test, exact: true })).toBeEnabled(); const before = await ctx.input.api.get(`${root}/costs/summary`); const response = ctx.input.page.waitForResponse(row => row.request().method() === "POST" && new URL(row.url()).pathname === `${root}/${spec.path}/test`); @@ -89,7 +101,9 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s await expect(section.locator('[role="status"],[role="alert"]').last()).toBeVisible(); state.visibleTestReceipt = true; state.testCostEvents = (await ctx.input.api.get(`${root}/costs/summary`)).eventCount - before.eventCount; state.saved = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings; - await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(section.getByRole("switch", { name: spec.toggle, exact: true })).toBeChecked(); state.persistedAfterReload = true; + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(section.getByRole("switch", { name: spec.toggle, exact: true })).toBeChecked(); + if (spec.path !== "decision-model") await expect(section.getByLabel("Model", { exact: true })).toHaveValue(requestedAuxiliaryServiceModel); + state.persistedAfterReload = true; if (spec.path !== "prompt-injection-prevention") state.history = await ctx.input.api.get(`${root}/${spec.path}/history?limit=100`); await ctx.input.evidence("identity-company-model-test.json", { service: spec.path, connectionId: (state.connection as AiManagedConnectionSummary).id, saved: state.saved, result: state.testResult, history: state.history, independentlyAddedCostEvents: state.testCostEvents, nativeChatRunUsedAsServiceProof: false });