fix(evals): await fast response save before model readback

tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it, vi } from "vitest";
-import { companyModelIdentityCases, requestedAuxiliaryServiceModel } from "./identity-company-models.js";
+import { companyModelIdentityCases, requestedAuxiliaryServiceModel, saveFastResponseModelBeforeTest } from "./identity-company-models.js";
 import type { IdentityObservation } from "./identity-support.js";
 vi.mock("./identity-support.js", async importOriginal => ({
   ...await importOriginal<typeof import("./identity-support.js")>(), probeAgentPermission: async () => 403,
@@ -48,10 +48,10 @@ describe("company model service configuration and actual tests", () => {
     }) };
     const get = vi.fn(async () => ({ settings: { enabled: false, model: "openai/wrong-default" } }));
     const ctx = { refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }), allRuns: async () => [{ id: "source", status: "succeeded" }],
-      input: { fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get }, page: { url: () => "https://fixture.test", goto: async () => {}, getByRole: () => section } } };
+      observationDeadlineAt: Date.now() + 390_000, input: { fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get }, page: { waitForResponse: async () => ({ status: () => 200, json: async () => ({ model: "wrong" }) }), url: () => "https://fixture.test", goto: async () => {}, getByRole: () => section } } };
     await expect(definition.act!(ctx as never, valid(id).state)).rejects.toThrow(/Requested auxiliary model saved before any service test/);
     expect(fill).toHaveBeenCalledExactlyOnceWith(requestedAuxiliaryServiceModel); expect(save).toHaveBeenCalledTimes(2);
-    expect(get).toHaveBeenCalledTimes(2); expect(test).not.toHaveBeenCalled();
+    expect(get).toHaveBeenCalledTimes(id.endsWith("fast-responses") ? 1 : 2); expect(test).not.toHaveBeenCalled();
   });
   it("rejects an ordinary agent run standing in for a decision invocation", () => { const d = companyModelIdentityCases[0]!; const o = valid(d.scenarioId); o.runs = [{ id: "test", companyId: "acorn", agentId: "ceo", status: "succeeded" }]; o.state.history = []; expect(d.grade(o).passed).toBe(false); });
   it("rejects builtin fallback being reported as model classification", () => { const d = companyModelIdentityCases[1]!; const o = valid(d.scenarioId); (o.state.testResult as { samples: Array<{ invocationId?: string }> }).samples[0]!.invocationId = undefined; expect(d.grade(o).passed).toBe(false); });
@@ -59,3 +59,54 @@ describe("company model service configuration and actual tests", () => {
   it("rejects fast response work-completion claims", () => { const d = companyModelIdentityCases[2]!; const o = valid(d.scenarioId); o.reply = "Open Settings for fast responses. Your task is finished."; expect(d.grade(o).passed).toBe(false); });
   it("rejects an acknowledgement whose independent invocation belongs to a different human", () => { const d = companyModelIdentityCases[2]!; const o = valid(d.scenarioId); (o.state.history as Array<{ responsibleUserId: string }>)[0]!.responsibleUserId = "other-person"; expect(d.grade(o).passed).toBe(false); });
 });
+
+describe("fast settings save completion before any service invocation", () => {
+  function fixture() {
+    const saved = { companyId: "acorn", enabled: true, connectionId: "shared", grantId: "grant", model: requestedAuxiliaryServiceModel };
+    let resolve!: (value: unknown) => void;
+    const pending = new Promise(resolveValue => { resolve = resolveValue; });
+    const get = vi.fn(async () => ({ settings: saved }));
+    const ctx = { observationDeadlineAt: Date.now() + 390_000, input: { fixtures: { company: { id: "acorn" } }, api: { get }, page: {
+      url: () => "https://fixture.test",
+      waitForResponse: vi.fn((predicate: (row: unknown) => boolean) => {
+        expect(predicate({ request: () => ({ method: () => "PUT" }), url: () => "https://fixture.test/api/companies/acorn/fast-response" })).toBe(true);
+        for (const [method, path] of [["POST", "/api/companies/acorn/fast-response/test"], ["PUT", "/api/companies/foreign/fast-response"]])
+          expect(predicate({ request: () => ({ method: () => method }), url: () => `https://fixture.test${path}` })).toBe(false);
+        expect(predicate({ request: () => ({ method: () => "PUT" }), url: () => "https://foreign.test/api/companies/acorn/fast-response" })).toBe(false);
+        return pending;
+      }) } } };
+    const reply = (status = 200, body = saved) => resolve({ status: () => status, json: async () => body });
+    return { saved, ctx, get, reply, connection: { id: "shared", grantId: "grant" } };
+  }
+  it("waits for the real matching delayed PUT before reading persisted model", async () => {
+    const f = fixture(); const click = vi.fn(async () => {});
+    f.get.mockImplementation(async () => ({ settings: { ...f.saved, model: "stale" } }));
+    const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, click);
+    await Promise.resolve(); expect(f.get).not.toHaveBeenCalled();
+    f.get.mockResolvedValue({ settings: f.saved }); f.reply();
+    await expect(pending).resolves.toEqual(f.saved); expect(click).toHaveBeenCalledOnce(); expect(f.get).toHaveBeenCalledOnce();
+  });
+  it.each([403, 422, 500])("does not read or test after rejected save %s", async status => {
+    const f = fixture(); const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, async () => {}); f.reply(status);
+    await expect(pending).rejects.toThrow(/save completed successfully/); expect(f.get).not.toHaveBeenCalled();
+  });
+  it("rejects wrong PUT model and independently wrong persisted model", async () => {
+    for (const responseWrong of [true, false]) {
+      const f = fixture(); const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, async () => {});
+      if (!responseWrong) f.get.mockResolvedValue({ settings: { ...f.saved, model: "wrong" } });
+      f.reply(200, responseWrong ? { ...f.saved, model: "wrong" } : f.saved);
+      await expect(pending).rejects.toThrow(/Requested auxiliary model saved/);
+    }
+  });
+  it.each([{ companyId: "foreign" }, { connectionId: "foreign" }, { grantId: "foreign" }, { enabled: false }])("rejects wrong saved identity or disabled state %j", async wrong => {
+    const f = fixture(); const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, async () => {});
+    f.reply(200, { ...f.saved, ...wrong }); await expect(pending).rejects.toThrow(/Requested auxiliary model saved/);
+    expect(f.get).not.toHaveBeenCalled();
+  });
+  it("never grants a fresh or unlimited deadline after delayed save", async () => {
+    const f = fixture(); const now = Date.now(); const clock = vi.spyOn(Date, "now").mockReturnValue(now);
+    const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, async () => {});
+    clock.mockReturnValue(now + 400_000); f.reply();
+    await expect(pending).rejects.toThrow(/budget exhausted/); expect(f.get).not.toHaveBeenCalled(); clock.mockRestore();
+  });
+});

tests/runner-e2e/chat-catalog-domains/identity-company-models.ts

@@ -10,6 +10,33 @@ import { identityDefinition, identityObservation, observeIdentity, probeAgentPer
 
 export const requestedAuxiliaryServiceModel = "deepseek/deepseek-v4.1-flash-1017";
 
+function remainingCompanyServiceObservationBudget(ctx: ChatCatalogContext): number {
+  const now = Date.now(), deadline = ctx.observationDeadlineAt;
+  if (!Number.isFinite(deadline) || deadline! <= now) throw new Error("Original company service observation budget exhausted or missing");
+  return deadline! - now;
+}
+
+/** The settings click starts an async PUT; observe its completion before independent readback. */
+export async function saveFastResponseModelBeforeTest(ctx: ChatCatalogContext, connection: AiManagedConnectionSummary,
+  clickSave: (options: { timeout: number }) => Promise<unknown>): Promise<FastResponseSettings> {
+  const remaining = () => remainingCompanyServiceObservationBudget(ctx);
+  const path = `/api/companies/${ctx.input.fixtures.company.id}/fast-response`;
+  const origin = new URL(ctx.input.page.url()).origin;
+  const response = ctx.input.page.waitForResponse(row => row.request().method() === "PUT"
+    && new URL(row.url()).origin === origin && new URL(row.url()).pathname === path, { timeout: remaining() });
+  const [, receipt] = await Promise.all([clickSave({ timeout: remaining() }), response]);
+  remaining();
+  expect(receipt.status(), "Actual fast response save completed successfully before service test").toBe(200);
+  const saved = await receipt.json() as FastResponseSettings;
+  const expected = { companyId: ctx.input.fixtures.company.id, enabled: true, connectionId: connection.id,
+    grantId: connection.grantId, model: requestedAuxiliaryServiceModel };
+  expect(saved, "Requested auxiliary model saved before any service test").toMatchObject(expected);
+  const persisted = (await ctx.input.api.get<{ settings: FastResponseSettings }>(path, { timeout: remaining() })).settings;
+  remaining();
+  expect(persisted, "Requested auxiliary model saved before any service test").toMatchObject(expected);
+  return persisted;
+}
+
 type Settings = DecisionModelSettings | PromptInjectionPreventionSettings | FastResponseSettings;
 const fastExperiments = new WeakMap<ChatCatalogContext, boolean>();
 const specs = [
@@ -87,17 +114,22 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s
     await section.getByRole("button", { name: (state.connection as AiManagedConnectionSummary).name, exact: true }).click();
     if (spec.path !== "decision-model") await section.getByLabel("Model", { exact: true }).fill(requestedAuxiliaryServiceModel);
     await section.getByRole("switch", { name: spec.toggle, exact: true }).setChecked(true);
-    await section.getByRole("button", { name: spec.save, exact: true }).click();
-    if (spec.path !== "decision-model") {
+    if (spec.path === "fast-response") {
+      const persisted = await saveFastResponseModelBeforeTest(ctx, state.connection as AiManagedConnectionSummary,
+        options => section.getByRole("button", { name: spec.save, exact: true }).click(options));
+      state.modelBeforeTest = persisted.model;
+    } else await section.getByRole("button", { name: spec.save, exact: true }).click();
+    if (spec.path === "prompt-injection-prevention") {
       const persisted = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings;
       expect(persisted.model, "Requested auxiliary model saved before any service test").toBe(requestedAuxiliaryServiceModel);
       state.modelBeforeTest = persisted.model;
     }
-    await expect(section.getByRole("button", { name: spec.test, exact: true })).toBeEnabled();
-    const before = await ctx.input.api.get<CostSummary>(`${root}/costs/summary`);
-    const response = ctx.input.page.waitForResponse(row => row.request().method() === "POST" && new URL(row.url()).pathname === `${root}/${spec.path}/test`);
-    await section.getByRole("button", { name: spec.test, exact: true }).click();
-    const receipt = await response; state.testHttpStatus = receipt.status(); state.testResult = await receipt.json();
+    const testOptions = () => spec.path === "fast-response" ? { timeout: remainingCompanyServiceObservationBudget(ctx) } : {};
+    await expect(section.getByRole("button", { name: spec.test, exact: true })).toBeEnabled(testOptions());
+    const before = await ctx.input.api.get<CostSummary>(`${root}/costs/summary`, spec.path === "fast-response" ? { timeout: remainingCompanyServiceObservationBudget(ctx) } : undefined);
+    const response = ctx.input.page.waitForResponse(row => row.request().method() === "POST" && new URL(row.url()).pathname === `${root}/${spec.path}/test`, testOptions());
+    await section.getByRole("button", { name: spec.test, exact: true }).click(testOptions());
+    const receipt = await response; if (spec.path === "fast-response") remainingCompanyServiceObservationBudget(ctx); state.testHttpStatus = receipt.status(); state.testResult = await receipt.json();
     await expect(section.locator('[role="status"],[role="alert"]').last()).toBeVisible(); state.visibleTestReceipt = true;
     state.testCostEvents = (await ctx.input.api.get<CostSummary>(`${root}/costs/summary`)).eventCount - before.eventCount;
     state.saved = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings;