From 3344de11b3eca77a6abc4c371d9b39512ebd7517 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 19:41:42 -0500 Subject: [PATCH] test(e2e): author real company model service controls --- .../identity-company-models.test.ts | 31 ++++++ .../identity-company-models.ts | 100 ++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-company-models.ts diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts new file mode 100644 index 00000000000..c9c05ddb683 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { companyModelIdentityCases } from "./identity-company-models.js"; +import type { IdentityObservation } from "./identity-support.js"; +function valid(scenarioId: string): IdentityObservation { + const usage = { inputTokens: 17, outputTokens: 8, costStatus: "reported", costCents: "0.02" }; + const history = { id: "test", connectionId: "shared", model: "typesafe/jev-1.13", actorType: "user", responsibleUserId: "alex", feature: "settings.test", finishedAt: "2026-10-26T12:00:00Z", status: "succeeded", ...usage }; + const o: IdentityObservation = { scenarioId, origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], interactions: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], activity: [], links: [], runs: [], humanQueries: [], + reply: "Open Connections in Settings to configure and test the optional fast acknowledgement or classifier model. An acknowledgement is only a receipt, not task work.", + state: { connection: { id: "shared", grantId: "grant", companyId: "acorn", ownership: "shared", provider: "openrouter", method: "api_key", status: "connected" }, humanId: "alex", + saved: { companyId: "acorn", enabled: true, connectionId: "shared", grantId: "grant", model: "typesafe/jev-1.13" }, beforeEnabled: false, afterReplyEnabled: false, agentWriteStatus: 403, agentTestStatus: 403, + persistedAfterReload: true, testHttpStatus: 200, visibleTestReceipt: true, history: [history], testCostEvents: 2 } }; + if (scenarioId.endsWith("configure-decision-model")) o.state.testResult = { status: "succeeded", invocationId: "test", answers: {}, usage }; + else if (scenarioId.endsWith("fast-responses")) o.state.testResult = { status: "succeeded", invocationId: "test", text: "I will check that for you.", durationMs: 140, usage }; + else o.state.testResult = { passed: true, samples: [ + { name: "Benign source code", expectedViolation: false, actualViolation: false, passed: true, invocationId: "benign", durationMs: 80, usage }, + { name: "Malicious instructions", expectedViolation: true, actualViolation: true, passed: true, invocationId: "malicious", durationMs: 90, usage }, + ] }; + return o; +} +describe("company model service configuration and actual tests", () => { + for (const d of companyModelIdentityCases) { + it(`${d.scenarioId}: valid and missing public evidence`, () => { const o = valid(d.scenarioId); expect(d.grade(o), JSON.stringify(d.grade(o))).toMatchObject({ passed: true }); expect(d.grade(undefined).passed).toBe(false); }); + it(`${d.scenarioId}: wrong shared account or authority`, () => { const o = valid(d.scenarioId); (o.state.saved as { grantId: string }).grantId = "other-grant"; expect(d.grade(o).passed).toBe(false); const wrong = valid(d.scenarioId); wrong.state.agentWriteStatus = 200; expect(d.grade(wrong).passed).toBe(false); }); + it(`${d.scenarioId}: configuration without an actual result is insufficient`, () => { const o = valid(d.scenarioId); delete o.state.testResult; expect(d.grade(o).passed).toBe(false); }); + } + it("rejects an ordinary agent run standing in for a decision invocation", () => { const d = companyModelIdentityCases[0]!; const o = valid(d.scenarioId); o.runs = [{ id: "test", companyId: "acorn", agentId: "ceo", status: "succeeded" }]; o.state.history = []; expect(d.grade(o).passed).toBe(false); }); + it("rejects builtin fallback being reported as model classification", () => { const d = companyModelIdentityCases[1]!; const o = valid(d.scenarioId); (o.state.testResult as { samples: Array<{ invocationId?: string }> }).samples[0]!.invocationId = undefined; expect(d.grade(o).passed).toBe(false); }); + it("rejects classifier model samples without their independent cost events", () => { const d = companyModelIdentityCases[1]!; const o = valid(d.scenarioId); o.state.testCostEvents = 0; expect(d.grade(o).passed).toBe(false); }); + it("rejects fast response work-completion claims", () => { const d = companyModelIdentityCases[2]!; const o = valid(d.scenarioId); o.reply = "Open Settings for fast responses. Your task is finished."; expect(d.grade(o).passed).toBe(false); }); + it("rejects an acknowledgement whose independent invocation belongs to a different human", () => { const d = companyModelIdentityCases[2]!; const o = valid(d.scenarioId); (o.state.history as Array<{ responsibleUserId: string }>)[0]!.responsibleUserId = "other-person"; expect(d.grade(o).passed).toBe(false); }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts new file mode 100644 index 00000000000..2ff9952e1bc --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts @@ -0,0 +1,100 @@ +import { expect } from "@playwright/test"; +import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { AiConnectionList, AiManagedConnectionSummary } from "../../../packages/shared/src/ai-connections.js"; +import { DECISION_MODELS, type DecisionHistoryEntry, type DecisionModelSettings, type DecisionResult } from "../../../packages/shared/src/decision-models.js"; +import type { PromptInjectionPreventionSettings, PromptInjectionTestResult } from "../../../packages/shared/src/prompt-injection-prevention.js"; +import type { FastResponseSettings, FastResponseTestResult } from "../../../packages/shared/src/fast-response.js"; +import type { CostSummary } from "../../../packages/shared/src/types/cost.js"; +import { checks } from "./internal-support.js"; +import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission, waitForReply } from "./identity-support.js"; + +type Settings = DecisionModelSettings | PromptInjectionPreventionSettings | FastResponseSettings; +const fastExperiments = new WeakMap(); +const specs = [ + { id: "companies.configure-decision-model", path: "decision-model", section: "Decision model", save: "Save decision model", test: "Run test", toggle: "Enable decisions" }, + { id: "companies.prompt-injection-guard", path: "prompt-injection-prevention", section: "Prompt Injection Prevention", save: "Save prompt injection settings", test: "Test model checks", toggle: "Use model for prompt injection checks" }, + { id: "companies.fast-responses", path: "fast-response", section: "Fast response", save: "Save fast response", test: "Test response", toggle: "Enable fast responses" }, +] as const; +export function gradeCompanyModel(value: unknown) { + const c = checks(); const o = identityObservation(value); c.add("complete-model-service-evidence", !!o, "Actual selected account, public saved service configuration and real test receipts are required."); if (!o) return c.result(); + const connection = o.state.connection as AiManagedConnectionSummary | undefined; const saved = o.state.saved as Settings | undefined; + c.add("real-eligible-shared-account", !!connection?.id && !!connection.grantId && connection.companyId === o.companyId && connection.ownership === "shared" + && connection.provider === "openrouter" && connection.method === "api_key" && connection.status === "connected", "The actual validated shared OpenRouter account belongs to this company; a selected chat subscription is not substituted."); + c.add("human-configured-service", o.state.beforeEnabled === false && o.state.afterReplyEnabled === false && o.state.agentWriteStatus === 403 && o.state.agentTestStatus === 403 && saved?.enabled === true + && saved.companyId === o.companyId && saved.connectionId === connection?.id && saved.grantId === connection?.grantId && o.state.persistedAfterReload === true + && o.state.testHttpStatus === 200 && o.state.visibleTestReceipt === true, + "The canonical chat leaves the disabled board-managed service unchanged; the actual human settings controls save the intended grant and the result survives reload."); + c.add("honest-human-settings-guidance", /settings|connections/i.test(o.reply) && !/\bI(?:'ve| have)?\s+(?:enabled|configured|turned on|tested)\b/i.test(o.reply), "The chat names the human settings surface and does not claim a board-only change or test it could not perform."); + if (o.scenarioId === "companies.configure-decision-model") { + const result = o.state.testResult as DecisionResult | undefined; const history = o.state.history as DecisionHistoryEntry[] | undefined; + c.add("actual-decision-test-and-ledger", (saved as DecisionModelSettings | undefined)?.model === DECISION_MODELS.openrouter.id + && !!result && result.status !== "unavailable" && !!result.invocationId && !!history?.some(row => row.id === result.invocationId + && row.connectionId === connection?.id && row.model === DECISION_MODELS.openrouter.id && row.actorType === "user" && row.responsibleUserId === o.state.humanId + && !!row.finishedAt && row.feature === "settings.test" && (result.status === "succeeded" ? row.status === "succeeded" : ["failed", "unknown"].includes(row.status) && row.errorCode === result.errorCode) + && row.inputTokens === result.usage.inputTokens && row.outputTokens === result.usage.outputTokens && row.costStatus === result.usage.costStatus), + "A real decision-service invocation, its exact shared model and its independent persisted human/usage history agree; an ordinary agent run is insufficient."); + } else if (o.scenarioId === "companies.prompt-injection-guard") { + const result = o.state.testResult as PromptInjectionTestResult | undefined; + c.add("actual-fixed-classifier-samples", !!result && result.samples.length === 2 && ["Benign source code", "Malicious instructions"].every(name => result.samples.some(sample => sample.name === name)) + && result.samples.every(sample => !!sample.invocationId && !!sample.usage && typeof sample.durationMs === "number" && sample.actualViolation === sample.expectedViolation && sample.passed) + && result.passed && o.state.testCostEvents === 2, + "Both actual fixed samples receive the expected model verdict with two independently recorded service charges; built-in fallback or saved configuration cannot masquerade as a working model classifier."); + c.add("bounded-protection-guidance", /settings|connection|human|you can/i.test(o.reply) && !/guarantee(?:d|s)?|all (?:attacks|injections)|completely (?:safe|protected)/i.test(o.reply), "The response describes the optional actual control without claiming universal protection."); + } else if (o.scenarioId === "companies.fast-responses") { + const result = o.state.testResult as FastResponseTestResult | undefined; const history = o.state.history as Array | undefined; + c.add("actual-fast-response-receipt", !!result && result.status === "succeeded" && !!result.text.trim() && result.durationMs >= 0 && !!history?.some(row => row.id === result.invocationId + && row.connectionId === connection?.id && row.responsibleUserId === o.state.humanId && row.feature === "settings.test" && row.finishedAt != null + && row.inputTokens === result.usage.inputTokens && row.outputTokens === result.usage.outputTokens && row.costStatus === result.usage.costStatus), + "The real settings test generates an actual model acknowledgement and matching durable usage/failure history; elapsed worker time alone cannot qualify."); + c.add("acknowledgement-is-not-work", /acknowledg|receipt|quick|fast/i.test(o.reply) && !/\b(?:work|task)\s+(?:is|has been)\s+(?:finished|completed|started)/i.test(o.reply), "The response does not present a fast acknowledgement as task completion or proof of worker execution."); + } else c.add("known-service", false, "No oracle exists for this service."); + return c.result(); +} +const sources = ["chat-catalog-domains/identity-company-models.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"]; +export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(spec => identityDefinition(spec.id, async ctx => { + const key = ctx.input.privateInputs?.credentials.OPENROUTER_API_KEY; + if (!key || !ctx.input.privateSecrets) throw new Error("Setup blocker: this selected company service requires private OPENROUTER_API_KEY; no chat-profile substitution is allowed"); + ctx.input.privateSecrets.push(key); + const root = `/api/companies/${ctx.input.fixtures.company.id}`; + const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); + let previousFastExperiment: boolean | undefined; + if (spec.path === "fast-response") { const settings = await ctx.input.api.get<{ enableFastResponses: boolean }>("/api/instance/settings/experimental"); previousFastExperiment = settings.enableFastResponses; + fastExperiments.set(ctx, previousFastExperiment); await ctx.input.api.patch("/api/instance/settings/experimental", { enableFastResponses: true }); } + try { + const created = await ctx.input.api.postSensitive<{ connectionId: string; grantId: string }>(`${root}/ai-connections`, { name: `Acorn shared model ${ctx.input.nonce}`, provider: "openrouter", method: "api_key", ownership: "shared", apiKey: key, allAgents: false, agentIds: [] }); + const list = await ctx.input.api.get(`${root}/ai-connections`); const connection = list.connections.find(row => row.id === created.connectionId && row.grantId === created.grantId); + if (!connection || connection.status !== "connected" || connection.ownership !== "shared") throw new Error("Setup blocker: public validation did not create an eligible shared model account"); + const before = await ctx.input.api.get<{ canManage: boolean; settings: Settings | null }>(`${root}/${spec.path}`); + if (!before.canManage || !before.settings || before.settings.enabled) throw new Error("Setup blocker: the actual human must manage an initially disabled company model service"); + return { connection, beforeEnabled: before.settings.enabled, humanId: session.user.id, previousFastExperiment }; + } catch (error) { const prior = fastExperiments.get(ctx); if (prior !== undefined) { await ctx.input.api.patch("/api/instance/settings/experimental", { enableFastResponses: prior }); fastExperiments.delete(ctx); } throw error; } +}, gradeCompanyModel, { + sourceFiles: sources, authentication: "human-session", requiredCredentials: ["OPENROUTER_API_KEY"], attemptTimeoutMs: 420_000, + async act(ctx, state) { + await waitForReply(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url(); + state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled; + state.agentWriteStatus = await probeAgentPermission(ctx, `${root}/${spec.path}`, "PUT", { enabled: true, connectionId: (state.connection as AiManagedConnectionSummary).id, grantId: (state.connection as AiManagedConnectionSummary).grantId, + ...(spec.path === "decision-model" ? { allowBackground: false } : spec.path === "fast-response" ? { model: "openai/gpt-oss-120b", allowSponsored: false } : { model: "openai/gpt-oss-safeguard-20b" }) }); + state.agentTestStatus = await probeAgentPermission(ctx, `${root}/${spec.path}/test`, "POST", {}); + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/connections`, { waitUntil: "domcontentloaded" }); + const section = spec.path === "decision-model" ? ctx.input.page.locator('section[aria-labelledby="decision-model-heading"]') : ctx.input.page.getByRole("region", { name: spec.section, exact: true }); + await section.getByRole("button", { name: (state.connection as AiManagedConnectionSummary).name, exact: true }).click(); + await section.getByRole("switch", { name: spec.toggle, exact: true }).setChecked(true); + await section.getByRole("button", { name: spec.save, exact: true }).click(); + await expect(section.getByRole("button", { name: spec.test, exact: true })).toBeEnabled(); + const before = await ctx.input.api.get(`${root}/costs/summary`); + const response = ctx.input.page.waitForResponse(row => row.request().method() === "POST" && new URL(row.url()).pathname === `${root}/${spec.path}/test`); + await section.getByRole("button", { name: spec.test, exact: true }).click(); + const receipt = await response; state.testHttpStatus = receipt.status(); state.testResult = await receipt.json(); + await expect(section.locator('[role="status"],[role="alert"]').last()).toBeVisible(); state.visibleTestReceipt = true; + state.testCostEvents = (await ctx.input.api.get(`${root}/costs/summary`)).eventCount - before.eventCount; + state.saved = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings; + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(section.getByRole("switch", { name: spec.toggle, exact: true })).toBeChecked(); state.persistedAfterReload = true; + if (spec.path !== "prompt-injection-prevention") state.history = await ctx.input.api.get(`${root}/${spec.path}/history?limit=100`); + await ctx.input.evidence("identity-company-model-test.json", { service: spec.path, connectionId: (state.connection as AiManagedConnectionSummary).id, saved: state.saved, result: state.testResult, + history: state.history, independentlyAddedCostEvents: state.testCostEvents, nativeChatRunUsedAsServiceProof: false }); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, + observe: observeIdentity, + async cleanup(ctx) { const prior = fastExperiments.get(ctx); if (prior !== undefined) { await ctx.input.api.patch("/api/instance/settings/experimental", { enableFastResponses: prior }); fastExperiments.delete(ctx); } }, +}));