From af01c4f6d3809e1d2be575bf0fad09ca62686ccd Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 28 Oct 2026 02:06:38 -0500 Subject: [PATCH] fix(evals): await fast response save before model readback --- .../identity-company-models.test.ts | 57 ++++++++++++++++++- .../identity-company-models.ts | 46 ++++++++++++--- 2 files changed, 93 insertions(+), 10 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts index f03defd6da6..9c4b298a80b 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { companyModelIdentityCases, requestedAuxiliaryServiceModel } from "./identity-company-models.js"; +import { companyModelIdentityCases, requestedAuxiliaryServiceModel, saveFastResponseModelBeforeTest } from "./identity-company-models.js"; import type { IdentityObservation } from "./identity-support.js"; vi.mock("./identity-support.js", async importOriginal => ({ ...await importOriginal(), probeAgentPermission: async () => 403, @@ -48,10 +48,10 @@ describe("company model service configuration and actual tests", () => { }) }; const get = vi.fn(async () => ({ settings: { enabled: false, model: "openai/wrong-default" } })); const ctx = { refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }), allRuns: async () => [{ id: "source", status: "succeeded" }], - input: { fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get }, page: { url: () => "https://fixture.test", goto: async () => {}, getByRole: () => section } } }; + observationDeadlineAt: Date.now() + 390_000, input: { fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get }, page: { waitForResponse: async () => ({ status: () => 200, json: async () => ({ model: "wrong" }) }), url: () => "https://fixture.test", goto: async () => {}, getByRole: () => section } } }; await expect(definition.act!(ctx as never, valid(id).state)).rejects.toThrow(/Requested auxiliary model saved before any service test/); expect(fill).toHaveBeenCalledExactlyOnceWith(requestedAuxiliaryServiceModel); expect(save).toHaveBeenCalledTimes(2); - expect(get).toHaveBeenCalledTimes(2); expect(test).not.toHaveBeenCalled(); + expect(get).toHaveBeenCalledTimes(id.endsWith("fast-responses") ? 1 : 2); expect(test).not.toHaveBeenCalled(); }); it("rejects an ordinary agent run standing in for a decision invocation", () => { const d = companyModelIdentityCases[0]!; const o = valid(d.scenarioId); o.runs = [{ id: "test", companyId: "acorn", agentId: "ceo", status: "succeeded" }]; o.state.history = []; expect(d.grade(o).passed).toBe(false); }); it("rejects builtin fallback being reported as model classification", () => { const d = companyModelIdentityCases[1]!; const o = valid(d.scenarioId); (o.state.testResult as { samples: Array<{ invocationId?: string }> }).samples[0]!.invocationId = undefined; expect(d.grade(o).passed).toBe(false); }); @@ -59,3 +59,54 @@ describe("company model service configuration and actual tests", () => { it("rejects fast response work-completion claims", () => { const d = companyModelIdentityCases[2]!; const o = valid(d.scenarioId); o.reply = "Open Settings for fast responses. Your task is finished."; expect(d.grade(o).passed).toBe(false); }); it("rejects an acknowledgement whose independent invocation belongs to a different human", () => { const d = companyModelIdentityCases[2]!; const o = valid(d.scenarioId); (o.state.history as Array<{ responsibleUserId: string }>)[0]!.responsibleUserId = "other-person"; expect(d.grade(o).passed).toBe(false); }); }); + +describe("fast settings save completion before any service invocation", () => { + function fixture() { + const saved = { companyId: "acorn", enabled: true, connectionId: "shared", grantId: "grant", model: requestedAuxiliaryServiceModel }; + let resolve!: (value: unknown) => void; + const pending = new Promise(resolveValue => { resolve = resolveValue; }); + const get = vi.fn(async () => ({ settings: saved })); + const ctx = { observationDeadlineAt: Date.now() + 390_000, input: { fixtures: { company: { id: "acorn" } }, api: { get }, page: { + url: () => "https://fixture.test", + waitForResponse: vi.fn((predicate: (row: unknown) => boolean) => { + expect(predicate({ request: () => ({ method: () => "PUT" }), url: () => "https://fixture.test/api/companies/acorn/fast-response" })).toBe(true); + for (const [method, path] of [["POST", "/api/companies/acorn/fast-response/test"], ["PUT", "/api/companies/foreign/fast-response"]]) + expect(predicate({ request: () => ({ method: () => method }), url: () => `https://fixture.test${path}` })).toBe(false); + expect(predicate({ request: () => ({ method: () => "PUT" }), url: () => "https://foreign.test/api/companies/acorn/fast-response" })).toBe(false); + return pending; + }) } } }; + const reply = (status = 200, body = saved) => resolve({ status: () => status, json: async () => body }); + return { saved, ctx, get, reply, connection: { id: "shared", grantId: "grant" } }; + } + it("waits for the real matching delayed PUT before reading persisted model", async () => { + const f = fixture(); const click = vi.fn(async () => {}); + f.get.mockImplementation(async () => ({ settings: { ...f.saved, model: "stale" } })); + const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, click); + await Promise.resolve(); expect(f.get).not.toHaveBeenCalled(); + f.get.mockResolvedValue({ settings: f.saved }); f.reply(); + await expect(pending).resolves.toEqual(f.saved); expect(click).toHaveBeenCalledOnce(); expect(f.get).toHaveBeenCalledOnce(); + }); + it.each([403, 422, 500])("does not read or test after rejected save %s", async status => { + const f = fixture(); const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, async () => {}); f.reply(status); + await expect(pending).rejects.toThrow(/save completed successfully/); expect(f.get).not.toHaveBeenCalled(); + }); + it("rejects wrong PUT model and independently wrong persisted model", async () => { + for (const responseWrong of [true, false]) { + const f = fixture(); const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, async () => {}); + if (!responseWrong) f.get.mockResolvedValue({ settings: { ...f.saved, model: "wrong" } }); + f.reply(200, responseWrong ? { ...f.saved, model: "wrong" } : f.saved); + await expect(pending).rejects.toThrow(/Requested auxiliary model saved/); + } + }); + it.each([{ companyId: "foreign" }, { connectionId: "foreign" }, { grantId: "foreign" }, { enabled: false }])("rejects wrong saved identity or disabled state %j", async wrong => { + const f = fixture(); const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, async () => {}); + f.reply(200, { ...f.saved, ...wrong }); await expect(pending).rejects.toThrow(/Requested auxiliary model saved/); + expect(f.get).not.toHaveBeenCalled(); + }); + it("never grants a fresh or unlimited deadline after delayed save", async () => { + const f = fixture(); const now = Date.now(); const clock = vi.spyOn(Date, "now").mockReturnValue(now); + const pending = saveFastResponseModelBeforeTest(f.ctx as never, f.connection as never, async () => {}); + clock.mockReturnValue(now + 400_000); f.reply(); + await expect(pending).rejects.toThrow(/budget exhausted/); expect(f.get).not.toHaveBeenCalled(); clock.mockRestore(); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts index bf6d6e047b0..390af6b20f3 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts @@ -10,6 +10,33 @@ import { identityDefinition, identityObservation, observeIdentity, probeAgentPer export const requestedAuxiliaryServiceModel = "deepseek/deepseek-v4.1-flash-1017"; +function remainingCompanyServiceObservationBudget(ctx: ChatCatalogContext): number { + const now = Date.now(), deadline = ctx.observationDeadlineAt; + if (!Number.isFinite(deadline) || deadline! <= now) throw new Error("Original company service observation budget exhausted or missing"); + return deadline! - now; +} + +/** The settings click starts an async PUT; observe its completion before independent readback. */ +export async function saveFastResponseModelBeforeTest(ctx: ChatCatalogContext, connection: AiManagedConnectionSummary, + clickSave: (options: { timeout: number }) => Promise): Promise { + const remaining = () => remainingCompanyServiceObservationBudget(ctx); + const path = `/api/companies/${ctx.input.fixtures.company.id}/fast-response`; + const origin = new URL(ctx.input.page.url()).origin; + const response = ctx.input.page.waitForResponse(row => row.request().method() === "PUT" + && new URL(row.url()).origin === origin && new URL(row.url()).pathname === path, { timeout: remaining() }); + const [, receipt] = await Promise.all([clickSave({ timeout: remaining() }), response]); + remaining(); + expect(receipt.status(), "Actual fast response save completed successfully before service test").toBe(200); + const saved = await receipt.json() as FastResponseSettings; + const expected = { companyId: ctx.input.fixtures.company.id, enabled: true, connectionId: connection.id, + grantId: connection.grantId, model: requestedAuxiliaryServiceModel }; + expect(saved, "Requested auxiliary model saved before any service test").toMatchObject(expected); + const persisted = (await ctx.input.api.get<{ settings: FastResponseSettings }>(path, { timeout: remaining() })).settings; + remaining(); + expect(persisted, "Requested auxiliary model saved before any service test").toMatchObject(expected); + return persisted; +} + type Settings = DecisionModelSettings | PromptInjectionPreventionSettings | FastResponseSettings; const fastExperiments = new WeakMap(); const specs = [ @@ -87,17 +114,22 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s await section.getByRole("button", { name: (state.connection as AiManagedConnectionSummary).name, exact: true }).click(); if (spec.path !== "decision-model") await section.getByLabel("Model", { exact: true }).fill(requestedAuxiliaryServiceModel); await section.getByRole("switch", { name: spec.toggle, exact: true }).setChecked(true); - await section.getByRole("button", { name: spec.save, exact: true }).click(); - if (spec.path !== "decision-model") { + if (spec.path === "fast-response") { + const persisted = await saveFastResponseModelBeforeTest(ctx, state.connection as AiManagedConnectionSummary, + options => section.getByRole("button", { name: spec.save, exact: true }).click(options)); + state.modelBeforeTest = persisted.model; + } else await section.getByRole("button", { name: spec.save, exact: true }).click(); + if (spec.path === "prompt-injection-prevention") { const persisted = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings; expect(persisted.model, "Requested auxiliary model saved before any service test").toBe(requestedAuxiliaryServiceModel); state.modelBeforeTest = persisted.model; } - await expect(section.getByRole("button", { name: spec.test, exact: true })).toBeEnabled(); - const before = await ctx.input.api.get(`${root}/costs/summary`); - const response = ctx.input.page.waitForResponse(row => row.request().method() === "POST" && new URL(row.url()).pathname === `${root}/${spec.path}/test`); - await section.getByRole("button", { name: spec.test, exact: true }).click(); - const receipt = await response; state.testHttpStatus = receipt.status(); state.testResult = await receipt.json(); + const testOptions = () => spec.path === "fast-response" ? { timeout: remainingCompanyServiceObservationBudget(ctx) } : {}; + await expect(section.getByRole("button", { name: spec.test, exact: true })).toBeEnabled(testOptions()); + const before = await ctx.input.api.get(`${root}/costs/summary`, spec.path === "fast-response" ? { timeout: remainingCompanyServiceObservationBudget(ctx) } : undefined); + const response = ctx.input.page.waitForResponse(row => row.request().method() === "POST" && new URL(row.url()).pathname === `${root}/${spec.path}/test`, testOptions()); + await section.getByRole("button", { name: spec.test, exact: true }).click(testOptions()); + const receipt = await response; if (spec.path === "fast-response") remainingCompanyServiceObservationBudget(ctx); state.testHttpStatus = receipt.status(); state.testResult = await receipt.json(); await expect(section.locator('[role="status"],[role="alert"]').last()).toBeVisible(); state.visibleTestReceipt = true; state.testCostEvents = (await ctx.input.api.get(`${root}/costs/summary`)).eventCount - before.eventCount; state.saved = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings;