test(e2e): author real company model service controls

tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts

@@ -0,0 +1,31 @@
+import { describe, expect, it } from "vitest";
+import { companyModelIdentityCases } from "./identity-company-models.js";
+import type { IdentityObservation } from "./identity-support.js";
+function valid(scenarioId: string): IdentityObservation {
+  const usage = { inputTokens: 17, outputTokens: 8, costStatus: "reported", costCents: "0.02" };
+  const history = { id: "test", connectionId: "shared", model: "typesafe/jev-1.13", actorType: "user", responsibleUserId: "alex", feature: "settings.test", finishedAt: "2026-10-26T12:00:00Z", status: "succeeded", ...usage };
+  const o: IdentityObservation = { scenarioId, origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], interactions: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], activity: [], links: [], runs: [], humanQueries: [],
+    reply: "Open Connections in Settings to configure and test the optional fast acknowledgement or classifier model. An acknowledgement is only a receipt, not task work.",
+    state: { connection: { id: "shared", grantId: "grant", companyId: "acorn", ownership: "shared", provider: "openrouter", method: "api_key", status: "connected" }, humanId: "alex",
+      saved: { companyId: "acorn", enabled: true, connectionId: "shared", grantId: "grant", model: "typesafe/jev-1.13" }, beforeEnabled: false, afterReplyEnabled: false, agentWriteStatus: 403, agentTestStatus: 403,
+      persistedAfterReload: true, testHttpStatus: 200, visibleTestReceipt: true, history: [history], testCostEvents: 2 } };
+  if (scenarioId.endsWith("configure-decision-model")) o.state.testResult = { status: "succeeded", invocationId: "test", answers: {}, usage };
+  else if (scenarioId.endsWith("fast-responses")) o.state.testResult = { status: "succeeded", invocationId: "test", text: "I will check that for you.", durationMs: 140, usage };
+  else o.state.testResult = { passed: true, samples: [
+    { name: "Benign source code", expectedViolation: false, actualViolation: false, passed: true, invocationId: "benign", durationMs: 80, usage },
+    { name: "Malicious instructions", expectedViolation: true, actualViolation: true, passed: true, invocationId: "malicious", durationMs: 90, usage },
+  ] };
+  return o;
+}
+describe("company model service configuration and actual tests", () => {
+  for (const d of companyModelIdentityCases) {
+    it(`${d.scenarioId}: valid and missing public evidence`, () => { const o = valid(d.scenarioId); expect(d.grade(o), JSON.stringify(d.grade(o))).toMatchObject({ passed: true }); expect(d.grade(undefined).passed).toBe(false); });
+    it(`${d.scenarioId}: wrong shared account or authority`, () => { const o = valid(d.scenarioId); (o.state.saved as { grantId: string }).grantId = "other-grant"; expect(d.grade(o).passed).toBe(false); const wrong = valid(d.scenarioId); wrong.state.agentWriteStatus = 200; expect(d.grade(wrong).passed).toBe(false); });
+    it(`${d.scenarioId}: configuration without an actual result is insufficient`, () => { const o = valid(d.scenarioId); delete o.state.testResult; expect(d.grade(o).passed).toBe(false); });
+  }
+  it("rejects an ordinary agent run standing in for a decision invocation", () => { const d = companyModelIdentityCases[0]!; const o = valid(d.scenarioId); o.runs = [{ id: "test", companyId: "acorn", agentId: "ceo", status: "succeeded" }]; o.state.history = []; expect(d.grade(o).passed).toBe(false); });
+  it("rejects builtin fallback being reported as model classification", () => { const d = companyModelIdentityCases[1]!; const o = valid(d.scenarioId); (o.state.testResult as { samples: Array<{ invocationId?: string }> }).samples[0]!.invocationId = undefined; expect(d.grade(o).passed).toBe(false); });
+  it("rejects classifier model samples without their independent cost events", () => { const d = companyModelIdentityCases[1]!; const o = valid(d.scenarioId); o.state.testCostEvents = 0; expect(d.grade(o).passed).toBe(false); });
+  it("rejects fast response work-completion claims", () => { const d = companyModelIdentityCases[2]!; const o = valid(d.scenarioId); o.reply = "Open Settings for fast responses. Your task is finished."; expect(d.grade(o).passed).toBe(false); });
+  it("rejects an acknowledgement whose independent invocation belongs to a different human", () => { const d = companyModelIdentityCases[2]!; const o = valid(d.scenarioId); (o.state.history as Array<{ responsibleUserId: string }>)[0]!.responsibleUserId = "other-person"; expect(d.grade(o).passed).toBe(false); });
+});

tests/runner-e2e/chat-catalog-domains/identity-company-models.ts

@@ -0,0 +1,100 @@
+import { expect } from "@playwright/test";
+import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { AiConnectionList, AiManagedConnectionSummary } from "../../../packages/shared/src/ai-connections.js";
+import { DECISION_MODELS, type DecisionHistoryEntry, type DecisionModelSettings, type DecisionResult } from "../../../packages/shared/src/decision-models.js";
+import type { PromptInjectionPreventionSettings, PromptInjectionTestResult } from "../../../packages/shared/src/prompt-injection-prevention.js";
+import type { FastResponseSettings, FastResponseTestResult } from "../../../packages/shared/src/fast-response.js";
+import type { CostSummary } from "../../../packages/shared/src/types/cost.js";
+import { checks } from "./internal-support.js";
+import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission, waitForReply } from "./identity-support.js";
+
+type Settings = DecisionModelSettings | PromptInjectionPreventionSettings | FastResponseSettings;
+const fastExperiments = new WeakMap<ChatCatalogContext, boolean>();
+const specs = [
+  { id: "companies.configure-decision-model", path: "decision-model", section: "Decision model", save: "Save decision model", test: "Run test", toggle: "Enable decisions" },
+  { id: "companies.prompt-injection-guard", path: "prompt-injection-prevention", section: "Prompt Injection Prevention", save: "Save prompt injection settings", test: "Test model checks", toggle: "Use model for prompt injection checks" },
+  { id: "companies.fast-responses", path: "fast-response", section: "Fast response", save: "Save fast response", test: "Test response", toggle: "Enable fast responses" },
+] as const;
+export function gradeCompanyModel(value: unknown) {
+  const c = checks(); const o = identityObservation(value); c.add("complete-model-service-evidence", !!o, "Actual selected account, public saved service configuration and real test receipts are required."); if (!o) return c.result();
+  const connection = o.state.connection as AiManagedConnectionSummary | undefined; const saved = o.state.saved as Settings | undefined;
+  c.add("real-eligible-shared-account", !!connection?.id && !!connection.grantId && connection.companyId === o.companyId && connection.ownership === "shared"
+    && connection.provider === "openrouter" && connection.method === "api_key" && connection.status === "connected", "The actual validated shared OpenRouter account belongs to this company; a selected chat subscription is not substituted.");
+  c.add("human-configured-service", o.state.beforeEnabled === false && o.state.afterReplyEnabled === false && o.state.agentWriteStatus === 403 && o.state.agentTestStatus === 403 && saved?.enabled === true
+    && saved.companyId === o.companyId && saved.connectionId === connection?.id && saved.grantId === connection?.grantId && o.state.persistedAfterReload === true
+    && o.state.testHttpStatus === 200 && o.state.visibleTestReceipt === true,
+    "The canonical chat leaves the disabled board-managed service unchanged; the actual human settings controls save the intended grant and the result survives reload.");
+  c.add("honest-human-settings-guidance", /settings|connections/i.test(o.reply) && !/\bI(?:'ve| have)?\s+(?:enabled|configured|turned on|tested)\b/i.test(o.reply), "The chat names the human settings surface and does not claim a board-only change or test it could not perform.");
+  if (o.scenarioId === "companies.configure-decision-model") {
+    const result = o.state.testResult as DecisionResult | undefined; const history = o.state.history as DecisionHistoryEntry[] | undefined;
+    c.add("actual-decision-test-and-ledger", (saved as DecisionModelSettings | undefined)?.model === DECISION_MODELS.openrouter.id
+      && !!result && result.status !== "unavailable" && !!result.invocationId && !!history?.some(row => row.id === result.invocationId
+      && row.connectionId === connection?.id && row.model === DECISION_MODELS.openrouter.id && row.actorType === "user" && row.responsibleUserId === o.state.humanId
+      && !!row.finishedAt && row.feature === "settings.test" && (result.status === "succeeded" ? row.status === "succeeded" : ["failed", "unknown"].includes(row.status) && row.errorCode === result.errorCode)
+      && row.inputTokens === result.usage.inputTokens && row.outputTokens === result.usage.outputTokens && row.costStatus === result.usage.costStatus),
+      "A real decision-service invocation, its exact shared model and its independent persisted human/usage history agree; an ordinary agent run is insufficient.");
+  } else if (o.scenarioId === "companies.prompt-injection-guard") {
+    const result = o.state.testResult as PromptInjectionTestResult | undefined;
+    c.add("actual-fixed-classifier-samples", !!result && result.samples.length === 2 && ["Benign source code", "Malicious instructions"].every(name => result.samples.some(sample => sample.name === name))
+      && result.samples.every(sample => !!sample.invocationId && !!sample.usage && typeof sample.durationMs === "number" && sample.actualViolation === sample.expectedViolation && sample.passed)
+      && result.passed && o.state.testCostEvents === 2,
+      "Both actual fixed samples receive the expected model verdict with two independently recorded service charges; built-in fallback or saved configuration cannot masquerade as a working model classifier.");
+    c.add("bounded-protection-guidance", /settings|connection|human|you can/i.test(o.reply) && !/guarantee(?:d|s)?|all (?:attacks|injections)|completely (?:safe|protected)/i.test(o.reply), "The response describes the optional actual control without claiming universal protection.");
+  } else if (o.scenarioId === "companies.fast-responses") {
+    const result = o.state.testResult as FastResponseTestResult | undefined; const history = o.state.history as Array<DecisionHistoryEntry & { publicationStatus: string }> | undefined;
+    c.add("actual-fast-response-receipt", !!result && result.status === "succeeded" && !!result.text.trim() && result.durationMs >= 0 && !!history?.some(row => row.id === result.invocationId
+      && row.connectionId === connection?.id && row.responsibleUserId === o.state.humanId && row.feature === "settings.test" && row.finishedAt != null
+      && row.inputTokens === result.usage.inputTokens && row.outputTokens === result.usage.outputTokens && row.costStatus === result.usage.costStatus),
+      "The real settings test generates an actual model acknowledgement and matching durable usage/failure history; elapsed worker time alone cannot qualify.");
+    c.add("acknowledgement-is-not-work", /acknowledg|receipt|quick|fast/i.test(o.reply) && !/\b(?:work|task)\s+(?:is|has been)\s+(?:finished|completed|started)/i.test(o.reply), "The response does not present a fast acknowledgement as task completion or proof of worker execution.");
+  } else c.add("known-service", false, "No oracle exists for this service.");
+  return c.result();
+}
+const sources = ["chat-catalog-domains/identity-company-models.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"];
+export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(spec => identityDefinition(spec.id, async ctx => {
+  const key = ctx.input.privateInputs?.credentials.OPENROUTER_API_KEY;
+  if (!key || !ctx.input.privateSecrets) throw new Error("Setup blocker: this selected company service requires private OPENROUTER_API_KEY; no chat-profile substitution is allowed");
+  ctx.input.privateSecrets.push(key);
+  const root = `/api/companies/${ctx.input.fixtures.company.id}`;
+  const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session");
+  let previousFastExperiment: boolean | undefined;
+  if (spec.path === "fast-response") { const settings = await ctx.input.api.get<{ enableFastResponses: boolean }>("/api/instance/settings/experimental"); previousFastExperiment = settings.enableFastResponses;
+    fastExperiments.set(ctx, previousFastExperiment); await ctx.input.api.patch("/api/instance/settings/experimental", { enableFastResponses: true }); }
+  try {
+  const created = await ctx.input.api.postSensitive<{ connectionId: string; grantId: string }>(`${root}/ai-connections`, { name: `Acorn shared model ${ctx.input.nonce}`, provider: "openrouter", method: "api_key", ownership: "shared", apiKey: key, allAgents: false, agentIds: [] });
+  const list = await ctx.input.api.get<AiConnectionList>(`${root}/ai-connections`); const connection = list.connections.find(row => row.id === created.connectionId && row.grantId === created.grantId);
+  if (!connection || connection.status !== "connected" || connection.ownership !== "shared") throw new Error("Setup blocker: public validation did not create an eligible shared model account");
+  const before = await ctx.input.api.get<{ canManage: boolean; settings: Settings | null }>(`${root}/${spec.path}`);
+  if (!before.canManage || !before.settings || before.settings.enabled) throw new Error("Setup blocker: the actual human must manage an initially disabled company model service");
+  return { connection, beforeEnabled: before.settings.enabled, humanId: session.user.id, previousFastExperiment };
+  } catch (error) { const prior = fastExperiments.get(ctx); if (prior !== undefined) { await ctx.input.api.patch("/api/instance/settings/experimental", { enableFastResponses: prior }); fastExperiments.delete(ctx); } throw error; }
+}, gradeCompanyModel, {
+  sourceFiles: sources, authentication: "human-session", requiredCredentials: ["OPENROUTER_API_KEY"], attemptTimeoutMs: 420_000,
+  async act(ctx, state) {
+    await waitForReply(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
+    state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled;
+    state.agentWriteStatus = await probeAgentPermission(ctx, `${root}/${spec.path}`, "PUT", { enabled: true, connectionId: (state.connection as AiManagedConnectionSummary).id, grantId: (state.connection as AiManagedConnectionSummary).grantId,
+      ...(spec.path === "decision-model" ? { allowBackground: false } : spec.path === "fast-response" ? { model: "openai/gpt-oss-120b", allowSponsored: false } : { model: "openai/gpt-oss-safeguard-20b" }) });
+    state.agentTestStatus = await probeAgentPermission(ctx, `${root}/${spec.path}/test`, "POST", {});
+    await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/connections`, { waitUntil: "domcontentloaded" });
+    const section = spec.path === "decision-model" ? ctx.input.page.locator('section[aria-labelledby="decision-model-heading"]') : ctx.input.page.getByRole("region", { name: spec.section, exact: true });
+    await section.getByRole("button", { name: (state.connection as AiManagedConnectionSummary).name, exact: true }).click();
+    await section.getByRole("switch", { name: spec.toggle, exact: true }).setChecked(true);
+    await section.getByRole("button", { name: spec.save, exact: true }).click();
+    await expect(section.getByRole("button", { name: spec.test, exact: true })).toBeEnabled();
+    const before = await ctx.input.api.get<CostSummary>(`${root}/costs/summary`);
+    const response = ctx.input.page.waitForResponse(row => row.request().method() === "POST" && new URL(row.url()).pathname === `${root}/${spec.path}/test`);
+    await section.getByRole("button", { name: spec.test, exact: true }).click();
+    const receipt = await response; state.testHttpStatus = receipt.status(); state.testResult = await receipt.json();
+    await expect(section.locator('[role="status"],[role="alert"]').last()).toBeVisible(); state.visibleTestReceipt = true;
+    state.testCostEvents = (await ctx.input.api.get<CostSummary>(`${root}/costs/summary`)).eventCount - before.eventCount;
+    state.saved = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings;
+    await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(section.getByRole("switch", { name: spec.toggle, exact: true })).toBeChecked(); state.persistedAfterReload = true;
+    if (spec.path !== "prompt-injection-prevention") state.history = await ctx.input.api.get<DecisionHistoryEntry[]>(`${root}/${spec.path}/history?limit=100`);
+    await ctx.input.evidence("identity-company-model-test.json", { service: spec.path, connectionId: (state.connection as AiManagedConnectionSummary).id, saved: state.saved, result: state.testResult,
+      history: state.history, independentlyAddedCostEvents: state.testCostEvents, nativeChatRunUsedAsServiceProof: false });
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+  },
+  observe: observeIdentity,
+  async cleanup(ctx) { const prior = fastExperiments.get(ctx); if (prior !== undefined) { await ctx.input.api.patch("/api/instance/settings/experimental", { enableFastResponses: prior }); fastExperiments.delete(ctx); } },
+}));