mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-05 21:02:34 +01:00
fix(evals): keep deadline boundary timeout strictly positive
This commit is contained in:
1 parent
b0eff403b8
commit
b70439f387
2 files changed
+9
-3
No files matched your search
@@ -9,7 +9,7 @@ function context(settlesAt = 180_000, deadline = 390_000): ChatCatalogContext {
|
||||
observed.load.mockImplementation(async () => ({ runs: [{ status: Date.now() >= settlesAt ? "succeeded" : "running" }], reply: Date.now() >= settlesAt ? "Use the operator experimental settings." : "" }));
|
||||
return ctx;
|
||||
}
|
||||
afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); });
|
||||
afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); vi.restoreAllMocks(); });
|
||||
describe("instance settings consume the original observation clock", () => {
|
||||
it("waits beyond the incidental150s only for a genuinely settled reply", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); let done = false;
|
||||
@@ -65,6 +65,11 @@ describe("instance settings consume the original observation clock", () => {
|
||||
await expect(privilegedConfigurationCases[0]!.act!(ctx, {})).rejects.toThrow(/exhausted/);
|
||||
expect(goto).toHaveBeenCalledOnce(); expect(click).not.toHaveBeenCalled();
|
||||
});
|
||||
it("never returns Playwright's unlimited zero timeout at the deadline boundary", () => {
|
||||
const now = vi.spyOn(Date, "now").mockReturnValueOnce(389_999).mockReturnValueOnce(390_000);
|
||||
expect(remainingPrivilegedObservationBudget({ observationDeadlineAt: 390_000 } as ChatCatalogContext)).toBe(1);
|
||||
expect(now).toHaveBeenCalledOnce();
|
||||
});
|
||||
it("missing, expired and nonfinite original deadlines fail before any UI action", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(0);
|
||||
for (const deadline of [undefined, NaN, Infinity, 0, -1]) {
|
||||
|
||||
@@ -11,8 +11,9 @@ export function privilegedDefinition(scenarioId: string, module: string, extra:
|
||||
}
|
||||
export function remainingPrivilegedObservationBudget(ctx: ChatCatalogContext): number {
|
||||
const deadline = ctx.observationDeadlineAt;
|
||||
if (!Number.isFinite(deadline) || deadline! <= Date.now()) throw new Error("Original privileged observation budget exhausted or missing");
|
||||
return deadline! - Date.now();
|
||||
const now = Date.now();
|
||||
if (!Number.isFinite(deadline) || deadline! <= now) throw new Error("Original privileged observation budget exhausted or missing");
|
||||
return deadline! - now;
|
||||
}
|
||||
export async function settlePrivilegedChat(ctx: ChatCatalogContext, observationDeadlineAt?: number): Promise<InternalObservation> {
|
||||
return pollUntil({ label: "Canonical privileged guidance settled", deadlineAt: observationDeadlineAt ?? Date.now() + 150_000,
|
||||
|
||||
Reference in new issue
Block a user