fix(evals): keep deadline boundary timeout strictly positive

This commit is contained in:
Dotta committed 2026-10-28 01:32:20 -05:00
1 parent b0eff403b8
commit b70439f387
2 files changed
+9 -3

No files matched your search

@@ -9,7 +9,7 @@ function context(settlesAt = 180_000, deadline = 390_000): ChatCatalogContext {
observed.load.mockImplementation(async () => ({ runs: [{ status: Date.now() >= settlesAt ? "succeeded" : "running" }], reply: Date.now() >= settlesAt ? "Use the operator experimental settings." : "" }));
return ctx;
}
afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); });
afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); vi.restoreAllMocks(); });
describe("instance settings consume the original observation clock", () => {
it("waits beyond the incidental150s only for a genuinely settled reply", async () => {
vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); let done = false;
@@ -65,6 +65,11 @@ describe("instance settings consume the original observation clock", () => {
await expect(privilegedConfigurationCases[0]!.act!(ctx, {})).rejects.toThrow(/exhausted/);
expect(goto).toHaveBeenCalledOnce(); expect(click).not.toHaveBeenCalled();
});
it("never returns Playwright's unlimited zero timeout at the deadline boundary", () => {
const now = vi.spyOn(Date, "now").mockReturnValueOnce(389_999).mockReturnValueOnce(390_000);
expect(remainingPrivilegedObservationBudget({ observationDeadlineAt: 390_000 } as ChatCatalogContext)).toBe(1);
expect(now).toHaveBeenCalledOnce();
});
it("missing, expired and nonfinite original deadlines fail before any UI action", async () => {
vi.useFakeTimers(); vi.setSystemTime(0);
for (const deadline of [undefined, NaN, Infinity, 0, -1]) {
@@ -11,8 +11,9 @@ export function privilegedDefinition(scenarioId: string, module: string, extra:
}
export function remainingPrivilegedObservationBudget(ctx: ChatCatalogContext): number {
const deadline = ctx.observationDeadlineAt;
if (!Number.isFinite(deadline) || deadline! <= Date.now()) throw new Error("Original privileged observation budget exhausted or missing");
return deadline! - Date.now();
const now = Date.now();
if (!Number.isFinite(deadline) || deadline! <= now) throw new Error("Original privileged observation budget exhausted or missing");
return deadline! - now;
}
export async function settlePrivilegedChat(ctx: ChatCatalogContext, observationDeadlineAt?: number): Promise<InternalObservation> {
return pollUntil({ label: "Canonical privileged guidance settled", deadlineAt: observationDeadlineAt ?? Date.now() + 150_000,