From 104a23be5ffa6a1bd43a52fef9a6c20bf3619eca Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 28 Oct 2026 11:21:06 -0500 Subject: [PATCH] fix(evals): retain completion stability window within original handoff deadline --- .../chat-catalog-domains/runtime-handoffs.ts | 4 ++-- .../chat-historical-native-clock.test.ts | 3 +++ tests/runner-e2e/completion-update-flow.ts | 15 ++++++++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-handoffs.ts b/tests/runner-e2e/chat-catalog-domains/runtime-handoffs.ts index b01f291db6e..e0810971a41 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-handoffs.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-handoffs.ts @@ -1,4 +1,4 @@ -import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js"; +import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; import { writeFile } from "node:fs/promises"; import { pollUntil } from "../api.js"; @@ -94,7 +94,7 @@ export const runtimeHandoffCases = [runtimeDefinition("agent-chat.track-delegate state.queryLinks = links; state.workerStatusAtQueryReply = phase.runs.find(run => run.id === state.workerRunId)!.status; await writeFile((state.gate as { gate: string }).gate, `The real website brief: use a blue page with concise copy. Save a short task delivery document that includes exactly ${state.marker}, then complete this existing Website task.`, "utf8"); const delivered = await observeCompletionUpdate({ page: ctx.input.page, api: ctx.input.api, sourceId: String(state.sourceId), workerId: (state.task as Entity).id, - marker: String(state.marker), fixtureRequest: String(state.prompt), allRuns: () => ctx.allRuns(), evidence: ctx.input.evidence, capture: ctx.input.capture }); + marker: String(state.marker), fixtureRequest: String(state.prompt), originalObservationDeadlineAt: chatCatalogOriginalPhaseDeadline(ctx), allRuns: () => ctx.allRuns(), evidence: ctx.input.evidence, capture: ctx.input.capture }); const delivery = completionDelivery(delivered); state.delivery = delivery; state.completionDelivered = !!delivery.response; }, async observe(ctx, state) { diff --git a/tests/runner-e2e/chat-historical-native-clock.test.ts b/tests/runner-e2e/chat-historical-native-clock.test.ts index eff4fc9c99a..4eba384af73 100644 --- a/tests/runner-e2e/chat-historical-native-clock.test.ts +++ b/tests/runner-e2e/chat-historical-native-clock.test.ts @@ -1,6 +1,7 @@ import {afterEach,expect,it,vi} from "vitest"; import type {ChatCatalogContext} from "./chat-catalog-contract.js"; import {chatCatalogNativePhaseDeadline,chatCatalogNativePhaseIsTimely,chatCatalogOriginalPhaseDeadline} from "./chat-catalog-contract.js"; +import {completionObservationBounds} from "./completion-update-flow.js"; import {pollUntil} from "./api.js"; void pollUntil; import {waitForReply} from "./chat-catalog-domains/identity-support.js"; @@ -11,3 +12,5 @@ it.each(ids)("%s consumes the original setup-inclusive clock with no new grace", it("actual selected identity reply wait survives its artificial150s cap but keeps the original deadline",async()=>{vi.useFakeTimers();vi.setSystemTime(0);const c=ctx("budgets-costs.set-budget");const result=waitForReply(c,chatCatalogOriginalPhaseDeadline(c)).then(()=>"accepted",()=>"expired");await vi.advanceTimersByTimeAsync(0);await vi.advanceTimersByTimeAsync(181000);expect(await result).toBe("accepted");expect(c.observationDeadlineAt).toBe(270000);}); it("selected identity helper rejects a success read after the original clock",async()=>{vi.useFakeTimers();vi.setSystemTime(269999);const c=ctx("access.cli-authorization");c.refreshIssue=async()=>{vi.setSystemTime(270001);};c.allRuns=async()=>[{status:"succeeded"}] as never;const result=waitForReply(c,chatCatalogOriginalPhaseDeadline(c)).then(()=>"accepted",()=>"expired");await vi.advanceTimersByTimeAsync(1000);expect(await result).toBe("expired");}); it("keeps exhausted cases, provenance controls and other provider lanes outside this new clock scope",()=>{vi.useFakeTimers();vi.setSystemTime(200000);for(const id of ["chat-channels.provider-disabled","access.sign-in","agents.hire-denied","agent-chat.chat-disabled","tasks.create-backlog","activity.audit-change","cli-operations.choose-cli-context","companies.rename-company"]){const c=ctx(id);expect(chatCatalogOriginalPhaseDeadline(c)).toBeUndefined();expect(chatCatalogNativePhaseDeadline(c,350000)).toBe(350000);}const c=ctx(ids[0]!);Object.assign(c.input.execution,{id:`chat-features.runner-codex.local.${ids[0]}`});expect(chatCatalogOriginalPhaseDeadline(c)).toBeUndefined();}); + +it("completion observation retains its full120s stability minimum without restarting or extending the original clock",()=>{expect(completionObservationBounds(10000)).toEqual({observationWindowEndsAt:130000,deadlineAt:135000});expect(completionObservationBounds(10000,570000)).toEqual({observationWindowEndsAt:130000,deadlineAt:570000});expect(completionObservationBounds(200000,570000)).toEqual({observationWindowEndsAt:320000,deadlineAt:570000});expect(completionObservationBounds(560000,570000)).toEqual({observationWindowEndsAt:680000,deadlineAt:570000});for(const d of [NaN,Infinity,10000,9999])expect(()=>completionObservationBounds(10000,d)).toThrow(/exhausted or invalid/);}); diff --git a/tests/runner-e2e/completion-update-flow.ts b/tests/runner-e2e/completion-update-flow.ts index b3a852f823a..432e91be91c 100644 --- a/tests/runner-e2e/completion-update-flow.ts +++ b/tests/runner-e2e/completion-update-flow.ts @@ -9,23 +9,30 @@ import { prepareChatBrief } from "./chat-stories.js"; import { completionDelivery, completionOutputUsesReleasedBrief, type CompletionObservation } from "./completion-updates.js"; type Row = Record; +/** Keep the full stability window while selected catalog callers retain their original outer clock. */ +export function completionObservationBounds(startedAtMs: number, originalDeadlineAt?: number) { + if (originalDeadlineAt !== undefined && (!Number.isFinite(originalDeadlineAt) || originalDeadlineAt <= startedAtMs)) + throw new Error("Original completion observation budget exhausted or invalid"); + return { observationWindowEndsAt: startedAtMs + 120_000, deadlineAt: originalDeadlineAt ?? startedAtMs + 125_000 }; +} export async function observeCompletionUpdate(input: { page: Page; api: RunnerApi; sourceId: string; workerId: string; marker: string; fixtureRequest: string; + originalObservationDeadlineAt?: number; relatedWorkerIds?: string[]; allRuns(): Promise; evidence(name: string, data: unknown): Promise; capture(id: string, label: string, file: string): Promise; }) { const startedAt = new Date().toISOString(); - const observationWindowEndsAt = Date.now() + 120_000; + const { observationWindowEndsAt, deadlineAt } = completionObservationBounds(Date.now(), input.originalObservationDeadlineAt); let observation: CompletionObservation | undefined; let failure: unknown; try { await pollUntil({ label: "unsolicited source-thread completion reply and result access", // Keep observing even after an early reply, so a later correction is retained. - deadlineAt: observationWindowEndsAt + 5_000, intervalMs: 1000, + deadlineAt, intervalMs: 1000, load: async () => { const worker = await input.api.get(`/api/issues/${input.workerId}`); const documents = await input.api.get(`/api/issues/${input.workerId}/documents`); @@ -49,7 +56,9 @@ export async function observeCompletionUpdate(input: { } return completionDelivery(observation); }, - accept: result => Date.now() >= observationWindowEndsAt && result.checks.every(c => c.passed), + accept: result => Date.now() >= observationWindowEndsAt + && (input.originalObservationDeadlineAt === undefined || Date.now() < input.originalObservationDeadlineAt) + && result.checks.every(c => c.passed), reject: () => observation!.runs.length > 12 ? "completion probe exceeded 12 runs" : undefined, timeoutDetail: result => result?.checks.filter(c => !c.passed).map(c => c.id).join(", "), });