From b70439f3874df9461569ab43c81d51ecb4f57238 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 28 Oct 2026 01:18:57 -0500 Subject: [PATCH] fix(evals): keep deadline boundary timeout strictly positive --- .../chat-catalog-domains/privileged-budget.test.ts | 7 ++++++- .../runner-e2e/chat-catalog-domains/privileged-support.ts | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-budget.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-budget.test.ts index 8d3cdd98d8e..662eefd394c 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-budget.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-budget.test.ts @@ -9,7 +9,7 @@ function context(settlesAt = 180_000, deadline = 390_000): ChatCatalogContext { observed.load.mockImplementation(async () => ({ runs: [{ status: Date.now() >= settlesAt ? "succeeded" : "running" }], reply: Date.now() >= settlesAt ? "Use the operator experimental settings." : "" })); return ctx; } -afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); +afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); vi.restoreAllMocks(); }); describe("instance settings consume the original observation clock", () => { it("waits beyond the incidental150s only for a genuinely settled reply", async () => { vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); let done = false; @@ -65,6 +65,11 @@ describe("instance settings consume the original observation clock", () => { await expect(privilegedConfigurationCases[0]!.act!(ctx, {})).rejects.toThrow(/exhausted/); expect(goto).toHaveBeenCalledOnce(); expect(click).not.toHaveBeenCalled(); }); + it("never returns Playwright's unlimited zero timeout at the deadline boundary", () => { + const now = vi.spyOn(Date, "now").mockReturnValueOnce(389_999).mockReturnValueOnce(390_000); + expect(remainingPrivilegedObservationBudget({ observationDeadlineAt: 390_000 } as ChatCatalogContext)).toBe(1); + expect(now).toHaveBeenCalledOnce(); + }); it("missing, expired and nonfinite original deadlines fail before any UI action", async () => { vi.useFakeTimers(); vi.setSystemTime(0); for (const deadline of [undefined, NaN, Infinity, 0, -1]) { diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-support.ts b/tests/runner-e2e/chat-catalog-domains/privileged-support.ts index d8d1a99b653..356a537c618 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-support.ts @@ -11,8 +11,9 @@ export function privilegedDefinition(scenarioId: string, module: string, extra: } export function remainingPrivilegedObservationBudget(ctx: ChatCatalogContext): number { const deadline = ctx.observationDeadlineAt; - if (!Number.isFinite(deadline) || deadline! <= Date.now()) throw new Error("Original privileged observation budget exhausted or missing"); - return deadline! - Date.now(); + const now = Date.now(); + if (!Number.isFinite(deadline) || deadline! <= now) throw new Error("Original privileged observation budget exhausted or missing"); + return deadline! - now; } export async function settlePrivilegedChat(ctx: ChatCatalogContext, observationDeadlineAt?: number): Promise { return pollUntil({ label: "Canonical privileged guidance settled", deadlineAt: observationDeadlineAt ?? Date.now() + 150_000,