From 8cd9e8bc6b47a297832b2cba27dde3e01cb33dcc Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 28 Oct 2026 07:26:09 -0500 Subject: [PATCH] fix(evals): keep fast response initial observation on original clock --- tests/runner-e2e/chat-catalog-contract.ts | 1 + .../chat-catalog-domains/identity-company-models.ts | 3 ++- tests/runner-e2e/chat-catalog-original-clock.test.ts | 10 +++++----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 52b7b14faa8..66c536ef249 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -48,6 +48,7 @@ const originalClockExecutionIds = new Set([ "chat-features.runner-opencode.local.secrets.configure-company-secret", "chat-features.runner-opencode.local.teams.configure-team", "chat-features.runner-opencode.local.steering.interrupt-with-direction", + "chat-features.runner-opencode.local.companies.fast-responses", ]); function usesOriginalNativePhaseClock(ctx: ChatCatalogContext): boolean { return originalClockExecutionIds.has(ctx.input?.execution?.id); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts index 390af6b20f3..82d6a131ce6 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts @@ -1,3 +1,4 @@ +import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; import type { AiConnectionList, AiManagedConnectionSummary } from "../../../packages/shared/src/ai-connections.js"; @@ -104,7 +105,7 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s }, gradeCompanyModel, { sourceFiles: sources, authentication: "human-session", requiredCredentials: ["OPENROUTER_API_KEY"], attemptTimeoutMs: 420_000, async act(ctx, state) { - await waitForReply(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url(); + await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url(); state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled; state.agentWriteStatus = await probeAgentPermission(ctx, `${root}/${spec.path}`, "PUT", { enabled: true, connectionId: (state.connection as AiManagedConnectionSummary).id, grantId: (state.connection as AiManagedConnectionSummary).grantId, ...(spec.path === "decision-model" ? { allowBackground: false } : spec.path === "fast-response" ? { model: requestedAuxiliaryServiceModel, allowSponsored: false } : { model: requestedAuxiliaryServiceModel }) }); diff --git a/tests/runner-e2e/chat-catalog-original-clock.test.ts b/tests/runner-e2e/chat-catalog-original-clock.test.ts index dcf620f8561..3492ac83a24 100644 --- a/tests/runner-e2e/chat-catalog-original-clock.test.ts +++ b/tests/runner-e2e/chat-catalog-original-clock.test.ts @@ -17,22 +17,22 @@ describe("selected native phases share the once-created original flow clock", () vi.setSystemTime(850_000); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toBe(870_000); expect(ctx.observationDeadlineAt).toBe(870_000); }); - it.each(["instance-operations.backup-and-restore", "instance-operations.restart-service", "routines.webhook-trigger", "status-cards.explain-card", "secrets.configure-company-secret", "teams.configure-team", "steering.interrupt-with-direction"])("future13 %s shares the setup-inclusive original clock", scenario => { + it.each(["instance-operations.backup-and-restore", "instance-operations.restart-service", "routines.webhook-trigger", "status-cards.explain-card", "secrets.configure-company-secret", "teams.configure-team", "steering.interrupt-with-direction", "companies.fast-responses"])("future13 %s shares the setup-inclusive original clock", scenario => { vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000)).toBe(870_000); expect(chatCatalogOriginalPhaseDeadline(ctx)).toBe(870_000); vi.setSystemTime(869_999); expect(chatCatalogNativePhaseRemainingBudget(ctx)!()).toBe(1); vi.setSystemTime(870_000); expect(() => chatCatalogOriginalPhaseDeadline(ctx)).toThrow(/exhausted/); }); - it("actual identity helper settles after its old150s cap using only the original remaining clock", async () => { - vi.useFakeTimers(); vi.setSystemTime(50_000); const ctx = context("secrets.configure-company-secret", 450_000); + it.each(["secrets.configure-company-secret", "companies.fast-responses"])("actual %s identity helper settles after its old150s cap using only the original remaining clock", async scenario => { + vi.useFakeTimers(); vi.setSystemTime(50_000); const originalDeadline = scenario === "companies.fast-responses" ? 390_000 : 450_000; const ctx = context(scenario, originalDeadline); Object.assign(ctx, { refreshIssue: async () => {}, allRuns: async () => [{ status: Date.now() >= 230_000 ? "succeeded" : "running" }] }); let completed = false; const pending = waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)).then(() => { completed = true; }); await vi.advanceTimersByTimeAsync(152_000); expect(completed).toBe(false); await vi.advanceTimersByTimeAsync(28_000); await pending; expect(completed).toBe(true); - expect(ctx.observationDeadlineAt).toBe(450_000); + expect(ctx.observationDeadlineAt).toBe(originalDeadline); }); - it.each(["onboarding.first-work", "routines.plugin-managed-routine", "status-cards.archive-card", "status-cards.other-authors-card"])("future13 %s does not receive new phase or UI behavior", scenario => { + it.each(["onboarding.first-work", "routines.plugin-managed-routine", "status-cards.archive-card", "status-cards.other-authors-card", "companies.prompt-injection-guard", "companies.configure-decision-model"])("future13 %s does not receive new phase or UI behavior", scenario => { vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario, 1); expect(chatCatalogNativePhaseDeadline(ctx, 900_000)).toBe(900_000); expect(chatCatalogOriginalPhaseDeadline(ctx)).toBeUndefined();