test(evals): share the original company settings clock

Observe declared company settings native turns within the existing setup-inclusive case deadline instead of the unrelated 150-second helper cap. Keep the original case limits, model, queries and feature oracles.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts

@@ -22,6 +22,25 @@ function valid(scenarioId: string): IdentityObservation {
   return o;
 }
 describe("company model service configuration and actual tests", () => {
+  it.each(["companies.configure-decision-model", "companies.prompt-injection-guard"])("actual %s initial native observation uses the remaining original case clock", async scenario => {
+    vi.useFakeTimers(); vi.setSystemTime(50_000);
+    try {
+      const get = vi.fn(async () => { throw new Error("Reached actual human settings phase"); });
+      const ctx = { observationDeadlineAt: 440_000, refreshIssue: async () => {},
+        issue: () => ({ conversationState: Date.now() >= 210_000 ? "waiting" : "running" }),
+        allRuns: async () => [{ status: Date.now() >= 210_000 ? "succeeded" : "running" }],
+        input: { execution: { id: `chat-features.runner-opencode.local.${scenario}` },
+          fixtures: { company: { id: "acorn" } }, page: { url: () => "https://fixture.test" }, api: { get } },
+      };
+      const d = companyModelIdentityCases.find(row => row.scenarioId === scenario)!;
+      const pending = Promise.resolve(d.act!(ctx as never, {})).then(() => null, error => error as Error);
+      await vi.dynamicImportSettled();
+      await vi.advanceTimersByTimeAsync(160_000);
+      expect((await pending)?.message).toBe("Reached actual human settings phase");
+      expect(get).toHaveBeenCalledOnce(); expect(ctx.observationDeadlineAt).toBe(440_000);
+    } finally { vi.useRealTimers(); }
+  });
+
   for (const d of companyModelIdentityCases) {
     it(`${d.scenarioId}: valid and missing public evidence`, () => { const o = valid(d.scenarioId); expect(d.grade(o), JSON.stringify(d.grade(o))).toMatchObject({ passed: true }); expect(d.grade(undefined).passed).toBe(false); });
     it(`${d.scenarioId}: wrong shared account or authority`, () => { const o = valid(d.scenarioId); (o.state.saved as { grantId: string }).grantId = "other-grant"; expect(d.grade(o).passed).toBe(false); const wrong = valid(d.scenarioId); wrong.state.agentWriteStatus = 200; expect(d.grade(wrong).passed).toBe(false); });

tests/runner-e2e/chat-catalog-domains/identity-company-models.ts

@@ -1,4 +1,4 @@
-import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
+import { waitForCompanySettingsTurn } from "./identity-company-observer.js";
 import { expect } from "@playwright/test";
 import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
 import type { AiConnectionList, AiManagedConnectionSummary } from "../../../packages/shared/src/ai-connections.js";
@@ -7,7 +7,7 @@ import type { PromptInjectionPreventionSettings, PromptInjectionTestResult } fro
 import type { FastResponseSettings, FastResponseTestResult } from "../../../packages/shared/src/fast-response.js";
 import type { CostSummary } from "../../../packages/shared/src/types/cost.js";
 import { checks } from "./internal-support.js";
-import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission, waitForReply } from "./identity-support.js";
+import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission } from "./identity-support.js";
 
 export const requestedAuxiliaryServiceModel = "deepseek/deepseek-v4.1-flash-1017";
 
@@ -84,7 +84,7 @@ export function gradeCompanyModel(value: unknown) {
   } else c.add("known-service", false, "No oracle exists for this service.");
   return c.result();
 }
-const sources = ["chat-catalog-domains/identity-company-models.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"];
+const sources = ["chat-catalog-domains/identity-company-models.ts", "chat-catalog-domains/identity-company-observer.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"];
 export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(spec => identityDefinition(spec.id, async ctx => {
   const key = ctx.input.privateInputs?.credentials.OPENROUTER_API_KEY;
   if (!key || !ctx.input.privateSecrets) throw new Error("Setup blocker: this selected company service requires private OPENROUTER_API_KEY; no chat-profile substitution is allowed");
@@ -105,7 +105,7 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s
 }, gradeCompanyModel, {
   sourceFiles: sources, authentication: "human-session", requiredCredentials: ["OPENROUTER_API_KEY"], attemptTimeoutMs: 420_000,
   async act(ctx, state) {
-    await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
+    await waitForCompanySettingsTurn(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
     state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled;
     state.agentWriteStatus = await probeAgentPermission(ctx, `${root}/${spec.path}`, "PUT", { enabled: true, connectionId: (state.connection as AiManagedConnectionSummary).id, grantId: (state.connection as AiManagedConnectionSummary).grantId,
       ...(spec.path === "decision-model" ? { allowBackground: false } : spec.path === "fast-response" ? { model: requestedAuxiliaryServiceModel, allowSponsored: false } : { model: requestedAuxiliaryServiceModel }) });

tests/runner-e2e/chat-catalog-domains/identity-company-observer.test.ts

@@ -0,0 +1,57 @@
+import { afterEach, expect, it, vi } from "vitest";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import { waitForCompanySettingsTurn } from "./identity-company-observer.js";
+
+const start = 50_000;
+afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); });
+function fixture(count = 1, allowance = 390_000) {
+  vi.useFakeTimers(); vi.setSystemTime(start);
+  let completesAt = start + 160_000, status = "succeeded", actualCount = count;
+  const refreshIssue = vi.fn(async () => {});
+  const ctx = { observationDeadlineAt: start + allowance, refreshIssue,
+    issue: () => ({ conversationState: Date.now() >= completesAt ? "waiting" : "running" }),
+    allRuns: async () => Array.from({ length: actualCount }, () => ({ status: Date.now() >= completesAt ? status : "running" })),
+  } as unknown as ChatCatalogContext;
+  return { ctx, refreshIssue, completesAt: (value: number) => { completesAt = value; },
+    status: (value: string) => { status = value; }, count: (value: number) => { actualCount = value; } };
+}
+it.each([1, 2, 3])("observes valid declared turn %s after the old150s cap within the unchanged original clock", async count => {
+  const f = fixture(count, count === 1 ? 390_000 : 570_000);
+  const pending = waitForCompanySettingsTurn(f.ctx, count);
+  const result = expect(pending).resolves.toBeUndefined();
+  await vi.advanceTimersByTimeAsync(160_000); await result;
+  expect(f.ctx.observationDeadlineAt).toBe(start + (count === 1 ? 390_000 : 570_000));
+});
+it.each([1, 2, 3])("does not reset the original deadline for late turn %s", async count => {
+  const f = fixture(count, 10_000); f.completesAt(start + 11_000);
+  const result = expect(waitForCompanySettingsTurn(f.ctx, count)).rejects.toThrow(/Timed out waiting/);
+  await vi.advanceTimersByTimeAsync(12_000); await result;
+  expect(f.ctx.observationDeadlineAt).toBe(start + 10_000);
+});
+it.each([undefined, NaN, Infinity, start, start - 1])("rejects missing, invalid or exhausted original deadline %s before observing", async deadline => {
+  const f = fixture(); f.ctx.observationDeadlineAt = deadline;
+  await expect(waitForCompanySettingsTurn(f.ctx)).rejects.toThrow(/Original identity observation budget/);
+  expect(f.refreshIssue).not.toHaveBeenCalled();
+});
+it.each(["failed", "cancelled", "timed_out"])("rejects actual stopped/failed native work %s", async status => {
+  for (const count of [1, 2, 3]) {
+    const f = fixture(count); f.completesAt(start); f.status(status);
+    await expect(waitForCompanySettingsTurn(f.ctx, count)).rejects.toThrow(/Initial response failed|Declared company settings turn failed/);
+  }
+});
+it("rejects a late asynchronous observation instead of accepting after the original deadline", async () => {
+  const f = fixture(2, 1000); f.completesAt(start);
+  f.refreshIssue.mockImplementation(async () => { vi.setSystemTime(start + 1001); });
+  const result = expect(waitForCompanySettingsTurn(f.ctx, 2)).rejects.toThrow(/Timed out waiting/);
+  await vi.advanceTimersByTimeAsync(2000); await result;
+});
+it("rejects extra native work instead of fitting a completed successor to the bound", async () => {
+  const f = fixture(2); f.completesAt(start); f.count(3);
+  await expect(waitForCompanySettingsTurn(f.ctx, 2)).rejects.toThrow(/declared native run bound/);
+});
+
+it.each([1, 2, 3])("does not accept a waiting chat with no native work for declared turn %s", async count => {
+  const f = fixture(count, 1000); f.completesAt(start); f.count(0);
+  const result = expect(waitForCompanySettingsTurn(f.ctx, count)).rejects.toThrow(/Timed out waiting/);
+  await vi.advanceTimersByTimeAsync(2000); await result;
+});

tests/runner-e2e/chat-catalog-domains/identity-company-observer.ts

@@ -0,0 +1,19 @@
+import { pollUntil } from "../api.js";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import { remainingIdentityObservationBudget, waitForReply } from "./identity-support.js";
+
+/** Company settings setup and all declared native turns share the original case clock. */
+export async function waitForCompanySettingsTurn(ctx: ChatCatalogContext, count = 1): Promise<void> {
+  remainingIdentityObservationBudget(ctx);
+  const deadlineAt = ctx.observationDeadlineAt!;
+  if (count === 1) await waitForReply(ctx, deadlineAt);
+  else await pollUntil({
+    label: `company settings native turn ${count}`, deadlineAt,
+    load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
+    accept: value => Date.now() < deadlineAt && value.source?.conversationState === "waiting"
+      && value.runs.length === count && value.runs.every(run => run.status === "succeeded"),
+    reject: value => value.runs.length > count ? "Company settings exceeded its declared native run bound"
+      : value.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Declared company settings turn failed" : undefined,
+  });
+  remainingIdentityObservationBudget(ctx);
+}

tests/runner-e2e/chat-catalog-domains/identity-observability.test.ts

@@ -1,5 +1,6 @@
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 import { gradeObservability } from "./identity-observability.js";
+vi.mock("../chat-flow.js", () => ({ sendChatMessage: vi.fn() }));
 import type { IdentityObservation } from "./identity-support.js";
 function valid(): IdentityObservation { return { scenarioId: "companies.agent-observability", origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], interactions: [], documents: [], revisions: [], activity: [], links: [], humanQueries: [],
   reply: "Configure optional trace export in company settings. First-party Telemetry and the local run log are separate data paths.",

tests/runner-e2e/chat-catalog-domains/identity-observability.ts

@@ -1,3 +1,4 @@
+import { waitForCompanySettingsTurn } from "./identity-company-observer.js";
 import { privateIdentityBrowser } from "./identity-private-browser.js";
 import { createServer, type Server } from "node:http";
 import { randomBytes, createHash } from "node:crypto";
@@ -8,7 +9,7 @@ import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contra
 import { pollUntil } from "../api.js";
 import { sendChatMessage } from "../chat-flow.js";
 import { checks } from "./internal-support.js";
-import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission, waitForReply } from "./identity-support.js";
+import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission } from "./identity-support.js";
 
 interface Received { path: string; authenticated: boolean; body: Buffer; sha256: string }
 interface Collector { server: Server; url: string; key: string; received: Received[] }
@@ -57,9 +58,9 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD
   return { beforeSettings: before.settings, collectorUrl: target.url, syntheticCollector: true };
 }, gradeObservability, {
   authentication: "human-session", expectedRunCount: 3, expectedHumanMessageCount: 3, attemptTimeoutMs: 600_000,
-  sourceFiles: ["chat-catalog-domains/identity-observability.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"],
+  sourceFiles: ["chat-catalog-domains/identity-observability.ts", "chat-catalog-domains/identity-company-observer.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"],
   async act(ctx, state) {
-    await waitForReply(ctx); const target = collectors.get(ctx); if (!target) throw new Error("Owned synthetic collector is missing");
+    await waitForCompanySettingsTurn(ctx); const target = collectors.get(ctx); if (!target) throw new Error("Owned synthetic collector is missing");
     const root = `/api/companies/${ctx.input.fixtures.company.id}/observability`; const source = ctx.input.page.url();
     state.afterReplySettings = (await ctx.input.api.get<AgentObservabilitySettingsResponse>(root)).settings; state.beforeConfiguredCalls = target.received.length;
     state.agentWriteStatus = await probeAgentPermission(ctx, root, "PUT", { expectedRevision: null, enabled: true, destination: { provider: "otlp", endpointBaseUrl: target.url }, credentials: { mode: "replace", headers: [] }, capture: { userPrompts: false, assistantResponses: false, toolDetails: false, toolContent: false } });
@@ -75,7 +76,7 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD
     state.collectorMatchedTestSignals = ["/v1/logs", "/v1/traces"].filter(signal => target.received.some(row => row.path === signal && row.authenticated && row.body.includes(receipt.marker)
       && row.body.includes(ctx.input.fixtures.company.id) && row.body.includes(receipt.revision))).length;
     await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await sendChatMessage(ctx.input.page, "Please acknowledge that our test collector configuration is saved. Do not change its settings.");
-    await settled(ctx, 2); state.enabledNativeRunCount = (await ctx.allRuns()).length;
+    await waitForCompanySettingsTurn(ctx, 2); state.enabledNativeRunCount = (await ctx.allRuns()).length;
     state.nativeExportMatched = await pollUntil({ label: "actual native harness trace export", deadlineAt: Date.now() + 30_000,
       load: async () => target.received.some(row => row.authenticated && row.body.includes(ctx.input.fixtures.company.id)
         && row.body.includes(ctx.input.fixtures.agent.id) && row.body.includes(receipt.revision)), accept: Boolean });
@@ -83,7 +84,7 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD
     await section.getByRole("button", { name: "Disable export", exact: true }).click();
     await expect.poll(async () => (await ctx.input.api.get<AgentObservabilitySettingsResponse>(root)).settings?.enabled).toBe(false); state.disabledSaved = true; state.callsAtDisable = target.received.length;
     await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await sendChatMessage(ctx.input.page, "Please acknowledge that export is now disabled. Do not change it or start any additional work.");
-    await settled(ctx, 3); state.disabledNativeRunCount = (await ctx.allRuns()).length; state.callsAfterDisabledRun = target.received.length; state.disabledCallsDidNotIncrease = state.callsAfterDisabledRun === state.callsAtDisable;
+    await waitForCompanySettingsTurn(ctx, 3); state.disabledNativeRunCount = (await ctx.allRuns()).length; state.callsAfterDisabledRun = target.received.length; state.disabledCallsDidNotIncrease = state.callsAfterDisabledRun === state.callsAtDisable;
     await ctx.input.evidence("identity-synthetic-otlp-delivery.json", { syntheticCollector: true, savedRevision: receipt.revision, controlPlaneTestMarker: receipt.marker,
       requests: target.received.map(row => ({ path: row.path, authenticated: row.authenticated, sha256: row.sha256 })), nativeExportMatched: state.nativeExportMatched,
       enabledNativeRunCount: state.enabledNativeRunCount, disabledNativeRunCount: state.disabledNativeRunCount, callsAtDisable: state.callsAtDisable, callsAfterDisabledRun: state.callsAfterDisabledRun,
@@ -92,7 +93,3 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD
   observe: observeIdentity,
   async cleanup(ctx) { const target = collectors.get(ctx); if (target) { target.server.closeAllConnections(); await new Promise<void>((resolve, reject) => target.server.close(error => error ? reject(error) : resolve())); target.received.length = 0; } collectors.delete(ctx); },
 })];
-async function settled(ctx: ChatCatalogContext, count: number) {
-  await pollUntil({ label: `native observability turn ${count}`, deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
-    accept: value => value.source?.conversationState === "waiting" && value.runs.length === count && value.runs.every(run => run.status === "succeeded") });
-}