diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts index 9c4b298a80b..37f81cc26d8 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts @@ -22,6 +22,25 @@ function valid(scenarioId: string): IdentityObservation { return o; } describe("company model service configuration and actual tests", () => { + it.each(["companies.configure-decision-model", "companies.prompt-injection-guard"])("actual %s initial native observation uses the remaining original case clock", async scenario => { + vi.useFakeTimers(); vi.setSystemTime(50_000); + try { + const get = vi.fn(async () => { throw new Error("Reached actual human settings phase"); }); + const ctx = { observationDeadlineAt: 440_000, refreshIssue: async () => {}, + issue: () => ({ conversationState: Date.now() >= 210_000 ? "waiting" : "running" }), + allRuns: async () => [{ status: Date.now() >= 210_000 ? "succeeded" : "running" }], + input: { execution: { id: `chat-features.runner-opencode.local.${scenario}` }, + fixtures: { company: { id: "acorn" } }, page: { url: () => "https://fixture.test" }, api: { get } }, + }; + const d = companyModelIdentityCases.find(row => row.scenarioId === scenario)!; + const pending = Promise.resolve(d.act!(ctx as never, {})).then(() => null, error => error as Error); + await vi.dynamicImportSettled(); + await vi.advanceTimersByTimeAsync(160_000); + expect((await pending)?.message).toBe("Reached actual human settings phase"); + expect(get).toHaveBeenCalledOnce(); expect(ctx.observationDeadlineAt).toBe(440_000); + } finally { vi.useRealTimers(); } + }); + for (const d of companyModelIdentityCases) { it(`${d.scenarioId}: valid and missing public evidence`, () => { const o = valid(d.scenarioId); expect(d.grade(o), JSON.stringify(d.grade(o))).toMatchObject({ passed: true }); expect(d.grade(undefined).passed).toBe(false); }); it(`${d.scenarioId}: wrong shared account or authority`, () => { const o = valid(d.scenarioId); (o.state.saved as { grantId: string }).grantId = "other-grant"; expect(d.grade(o).passed).toBe(false); const wrong = valid(d.scenarioId); wrong.state.agentWriteStatus = 200; expect(d.grade(wrong).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts index 82d6a131ce6..3232dccd611 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts @@ -1,4 +1,4 @@ -import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; +import { waitForCompanySettingsTurn } from "./identity-company-observer.js"; import { expect } from "@playwright/test"; import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; import type { AiConnectionList, AiManagedConnectionSummary } from "../../../packages/shared/src/ai-connections.js"; @@ -7,7 +7,7 @@ import type { PromptInjectionPreventionSettings, PromptInjectionTestResult } fro import type { FastResponseSettings, FastResponseTestResult } from "../../../packages/shared/src/fast-response.js"; import type { CostSummary } from "../../../packages/shared/src/types/cost.js"; import { checks } from "./internal-support.js"; -import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission, waitForReply } from "./identity-support.js"; +import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission } from "./identity-support.js"; export const requestedAuxiliaryServiceModel = "deepseek/deepseek-v4.1-flash-1017"; @@ -84,7 +84,7 @@ export function gradeCompanyModel(value: unknown) { } else c.add("known-service", false, "No oracle exists for this service."); return c.result(); } -const sources = ["chat-catalog-domains/identity-company-models.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"]; +const sources = ["chat-catalog-domains/identity-company-models.ts", "chat-catalog-domains/identity-company-observer.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"]; export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(spec => identityDefinition(spec.id, async ctx => { const key = ctx.input.privateInputs?.credentials.OPENROUTER_API_KEY; if (!key || !ctx.input.privateSecrets) throw new Error("Setup blocker: this selected company service requires private OPENROUTER_API_KEY; no chat-profile substitution is allowed"); @@ -105,7 +105,7 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s }, gradeCompanyModel, { sourceFiles: sources, authentication: "human-session", requiredCredentials: ["OPENROUTER_API_KEY"], attemptTimeoutMs: 420_000, async act(ctx, state) { - await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url(); + await waitForCompanySettingsTurn(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url(); state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled; state.agentWriteStatus = await probeAgentPermission(ctx, `${root}/${spec.path}`, "PUT", { enabled: true, connectionId: (state.connection as AiManagedConnectionSummary).id, grantId: (state.connection as AiManagedConnectionSummary).grantId, ...(spec.path === "decision-model" ? { allowBackground: false } : spec.path === "fast-response" ? { model: requestedAuxiliaryServiceModel, allowSponsored: false } : { model: requestedAuxiliaryServiceModel }) }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-observer.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-observer.test.ts new file mode 100644 index 00000000000..d90273f4a3d --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-observer.test.ts @@ -0,0 +1,57 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { waitForCompanySettingsTurn } from "./identity-company-observer.js"; + +const start = 50_000; +afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); }); +function fixture(count = 1, allowance = 390_000) { + vi.useFakeTimers(); vi.setSystemTime(start); + let completesAt = start + 160_000, status = "succeeded", actualCount = count; + const refreshIssue = vi.fn(async () => {}); + const ctx = { observationDeadlineAt: start + allowance, refreshIssue, + issue: () => ({ conversationState: Date.now() >= completesAt ? "waiting" : "running" }), + allRuns: async () => Array.from({ length: actualCount }, () => ({ status: Date.now() >= completesAt ? status : "running" })), + } as unknown as ChatCatalogContext; + return { ctx, refreshIssue, completesAt: (value: number) => { completesAt = value; }, + status: (value: string) => { status = value; }, count: (value: number) => { actualCount = value; } }; +} +it.each([1, 2, 3])("observes valid declared turn %s after the old150s cap within the unchanged original clock", async count => { + const f = fixture(count, count === 1 ? 390_000 : 570_000); + const pending = waitForCompanySettingsTurn(f.ctx, count); + const result = expect(pending).resolves.toBeUndefined(); + await vi.advanceTimersByTimeAsync(160_000); await result; + expect(f.ctx.observationDeadlineAt).toBe(start + (count === 1 ? 390_000 : 570_000)); +}); +it.each([1, 2, 3])("does not reset the original deadline for late turn %s", async count => { + const f = fixture(count, 10_000); f.completesAt(start + 11_000); + const result = expect(waitForCompanySettingsTurn(f.ctx, count)).rejects.toThrow(/Timed out waiting/); + await vi.advanceTimersByTimeAsync(12_000); await result; + expect(f.ctx.observationDeadlineAt).toBe(start + 10_000); +}); +it.each([undefined, NaN, Infinity, start, start - 1])("rejects missing, invalid or exhausted original deadline %s before observing", async deadline => { + const f = fixture(); f.ctx.observationDeadlineAt = deadline; + await expect(waitForCompanySettingsTurn(f.ctx)).rejects.toThrow(/Original identity observation budget/); + expect(f.refreshIssue).not.toHaveBeenCalled(); +}); +it.each(["failed", "cancelled", "timed_out"])("rejects actual stopped/failed native work %s", async status => { + for (const count of [1, 2, 3]) { + const f = fixture(count); f.completesAt(start); f.status(status); + await expect(waitForCompanySettingsTurn(f.ctx, count)).rejects.toThrow(/Initial response failed|Declared company settings turn failed/); + } +}); +it("rejects a late asynchronous observation instead of accepting after the original deadline", async () => { + const f = fixture(2, 1000); f.completesAt(start); + f.refreshIssue.mockImplementation(async () => { vi.setSystemTime(start + 1001); }); + const result = expect(waitForCompanySettingsTurn(f.ctx, 2)).rejects.toThrow(/Timed out waiting/); + await vi.advanceTimersByTimeAsync(2000); await result; +}); +it("rejects extra native work instead of fitting a completed successor to the bound", async () => { + const f = fixture(2); f.completesAt(start); f.count(3); + await expect(waitForCompanySettingsTurn(f.ctx, 2)).rejects.toThrow(/declared native run bound/); +}); + +it.each([1, 2, 3])("does not accept a waiting chat with no native work for declared turn %s", async count => { + const f = fixture(count, 1000); f.completesAt(start); f.count(0); + const result = expect(waitForCompanySettingsTurn(f.ctx, count)).rejects.toThrow(/Timed out waiting/); + await vi.advanceTimersByTimeAsync(2000); await result; +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-observer.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-observer.ts new file mode 100644 index 00000000000..4439f6d3376 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-observer.ts @@ -0,0 +1,19 @@ +import { pollUntil } from "../api.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { remainingIdentityObservationBudget, waitForReply } from "./identity-support.js"; + +/** Company settings setup and all declared native turns share the original case clock. */ +export async function waitForCompanySettingsTurn(ctx: ChatCatalogContext, count = 1): Promise { + remainingIdentityObservationBudget(ctx); + const deadlineAt = ctx.observationDeadlineAt!; + if (count === 1) await waitForReply(ctx, deadlineAt); + else await pollUntil({ + label: `company settings native turn ${count}`, deadlineAt, + load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, + accept: value => Date.now() < deadlineAt && value.source?.conversationState === "waiting" + && value.runs.length === count && value.runs.every(run => run.status === "succeeded"), + reject: value => value.runs.length > count ? "Company settings exceeded its declared native run bound" + : value.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Declared company settings turn failed" : undefined, + }); + remainingIdentityObservationBudget(ctx); +} diff --git a/tests/runner-e2e/chat-catalog-domains/identity-observability.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-observability.test.ts index 12041e2ee37..da96ad99c74 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-observability.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-observability.test.ts @@ -1,5 +1,6 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { gradeObservability } from "./identity-observability.js"; +vi.mock("../chat-flow.js", () => ({ sendChatMessage: vi.fn() })); import type { IdentityObservation } from "./identity-support.js"; function valid(): IdentityObservation { return { scenarioId: "companies.agent-observability", origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], interactions: [], documents: [], revisions: [], activity: [], links: [], humanQueries: [], reply: "Configure optional trace export in company settings. First-party Telemetry and the local run log are separate data paths.", diff --git a/tests/runner-e2e/chat-catalog-domains/identity-observability.ts b/tests/runner-e2e/chat-catalog-domains/identity-observability.ts index 4b499f94e7c..01260e2975b 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-observability.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-observability.ts @@ -1,3 +1,4 @@ +import { waitForCompanySettingsTurn } from "./identity-company-observer.js"; import { privateIdentityBrowser } from "./identity-private-browser.js"; import { createServer, type Server } from "node:http"; import { randomBytes, createHash } from "node:crypto"; @@ -8,7 +9,7 @@ import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contra import { pollUntil } from "../api.js"; import { sendChatMessage } from "../chat-flow.js"; import { checks } from "./internal-support.js"; -import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission, waitForReply } from "./identity-support.js"; +import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission } from "./identity-support.js"; interface Received { path: string; authenticated: boolean; body: Buffer; sha256: string } interface Collector { server: Server; url: string; key: string; received: Received[] } @@ -57,9 +58,9 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD return { beforeSettings: before.settings, collectorUrl: target.url, syntheticCollector: true }; }, gradeObservability, { authentication: "human-session", expectedRunCount: 3, expectedHumanMessageCount: 3, attemptTimeoutMs: 600_000, - sourceFiles: ["chat-catalog-domains/identity-observability.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"], + sourceFiles: ["chat-catalog-domains/identity-observability.ts", "chat-catalog-domains/identity-company-observer.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"], async act(ctx, state) { - await waitForReply(ctx); const target = collectors.get(ctx); if (!target) throw new Error("Owned synthetic collector is missing"); + await waitForCompanySettingsTurn(ctx); const target = collectors.get(ctx); if (!target) throw new Error("Owned synthetic collector is missing"); const root = `/api/companies/${ctx.input.fixtures.company.id}/observability`; const source = ctx.input.page.url(); state.afterReplySettings = (await ctx.input.api.get(root)).settings; state.beforeConfiguredCalls = target.received.length; state.agentWriteStatus = await probeAgentPermission(ctx, root, "PUT", { expectedRevision: null, enabled: true, destination: { provider: "otlp", endpointBaseUrl: target.url }, credentials: { mode: "replace", headers: [] }, capture: { userPrompts: false, assistantResponses: false, toolDetails: false, toolContent: false } }); @@ -75,7 +76,7 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD state.collectorMatchedTestSignals = ["/v1/logs", "/v1/traces"].filter(signal => target.received.some(row => row.path === signal && row.authenticated && row.body.includes(receipt.marker) && row.body.includes(ctx.input.fixtures.company.id) && row.body.includes(receipt.revision))).length; await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await sendChatMessage(ctx.input.page, "Please acknowledge that our test collector configuration is saved. Do not change its settings."); - await settled(ctx, 2); state.enabledNativeRunCount = (await ctx.allRuns()).length; + await waitForCompanySettingsTurn(ctx, 2); state.enabledNativeRunCount = (await ctx.allRuns()).length; state.nativeExportMatched = await pollUntil({ label: "actual native harness trace export", deadlineAt: Date.now() + 30_000, load: async () => target.received.some(row => row.authenticated && row.body.includes(ctx.input.fixtures.company.id) && row.body.includes(ctx.input.fixtures.agent.id) && row.body.includes(receipt.revision)), accept: Boolean }); @@ -83,7 +84,7 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD await section.getByRole("button", { name: "Disable export", exact: true }).click(); await expect.poll(async () => (await ctx.input.api.get(root)).settings?.enabled).toBe(false); state.disabledSaved = true; state.callsAtDisable = target.received.length; await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await sendChatMessage(ctx.input.page, "Please acknowledge that export is now disabled. Do not change it or start any additional work."); - await settled(ctx, 3); state.disabledNativeRunCount = (await ctx.allRuns()).length; state.callsAfterDisabledRun = target.received.length; state.disabledCallsDidNotIncrease = state.callsAfterDisabledRun === state.callsAtDisable; + await waitForCompanySettingsTurn(ctx, 3); state.disabledNativeRunCount = (await ctx.allRuns()).length; state.callsAfterDisabledRun = target.received.length; state.disabledCallsDidNotIncrease = state.callsAfterDisabledRun === state.callsAtDisable; await ctx.input.evidence("identity-synthetic-otlp-delivery.json", { syntheticCollector: true, savedRevision: receipt.revision, controlPlaneTestMarker: receipt.marker, requests: target.received.map(row => ({ path: row.path, authenticated: row.authenticated, sha256: row.sha256 })), nativeExportMatched: state.nativeExportMatched, enabledNativeRunCount: state.enabledNativeRunCount, disabledNativeRunCount: state.disabledNativeRunCount, callsAtDisable: state.callsAtDisable, callsAfterDisabledRun: state.callsAfterDisabledRun, @@ -92,7 +93,3 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD observe: observeIdentity, async cleanup(ctx) { const target = collectors.get(ctx); if (target) { target.server.closeAllConnections(); await new Promise((resolve, reject) => target.server.close(error => error ? reject(error) : resolve())); target.received.length = 0; } collectors.delete(ctx); }, })]; -async function settled(ctx: ChatCatalogContext, count: number) { - await pollUntil({ label: `native observability turn ${count}`, deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, - accept: value => value.source?.conversationState === "waiting" && value.runs.length === count && value.runs.every(run => run.status === "succeeded") }); -}