fix(evals): keep fast response initial observation on original clock

This commit is contained in:
Dotta committed 2026-10-28 07:26:09 -05:00
1 parent 1c7cbdc25b
commit 8cd9e8bc6b
3 files changed
+8 -6

No files matched your search

@@ -48,6 +48,7 @@ const originalClockExecutionIds = new Set([
"chat-features.runner-opencode.local.secrets.configure-company-secret",
"chat-features.runner-opencode.local.teams.configure-team",
"chat-features.runner-opencode.local.steering.interrupt-with-direction",
"chat-features.runner-opencode.local.companies.fast-responses",
]);
function usesOriginalNativePhaseClock(ctx: ChatCatalogContext): boolean {
return originalClockExecutionIds.has(ctx.input?.execution?.id);
@@ -1,3 +1,4 @@
import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
import type { AiConnectionList, AiManagedConnectionSummary } from "../../../packages/shared/src/ai-connections.js";
@@ -104,7 +105,7 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s
}, gradeCompanyModel, {
sourceFiles: sources, authentication: "human-session", requiredCredentials: ["OPENROUTER_API_KEY"], attemptTimeoutMs: 420_000,
async act(ctx, state) {
await waitForReply(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled;
state.agentWriteStatus = await probeAgentPermission(ctx, `${root}/${spec.path}`, "PUT", { enabled: true, connectionId: (state.connection as AiManagedConnectionSummary).id, grantId: (state.connection as AiManagedConnectionSummary).grantId,
...(spec.path === "decision-model" ? { allowBackground: false } : spec.path === "fast-response" ? { model: requestedAuxiliaryServiceModel, allowSponsored: false } : { model: requestedAuxiliaryServiceModel }) });
@@ -17,22 +17,22 @@ describe("selected native phases share the once-created original flow clock", ()
vi.setSystemTime(850_000); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toBe(870_000);
expect(ctx.observationDeadlineAt).toBe(870_000);
});
it.each(["instance-operations.backup-and-restore", "instance-operations.restart-service", "routines.webhook-trigger", "status-cards.explain-card", "secrets.configure-company-secret", "teams.configure-team", "steering.interrupt-with-direction"])("future13 %s shares the setup-inclusive original clock", scenario => {
it.each(["instance-operations.backup-and-restore", "instance-operations.restart-service", "routines.webhook-trigger", "status-cards.explain-card", "secrets.configure-company-secret", "teams.configure-team", "steering.interrupt-with-direction", "companies.fast-responses"])("future13 %s shares the setup-inclusive original clock", scenario => {
vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario);
expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000)).toBe(870_000);
expect(chatCatalogOriginalPhaseDeadline(ctx)).toBe(870_000);
vi.setSystemTime(869_999); expect(chatCatalogNativePhaseRemainingBudget(ctx)!()).toBe(1);
vi.setSystemTime(870_000); expect(() => chatCatalogOriginalPhaseDeadline(ctx)).toThrow(/exhausted/);
});
it("actual identity helper settles after its old150s cap using only the original remaining clock", async () => {
vi.useFakeTimers(); vi.setSystemTime(50_000); const ctx = context("secrets.configure-company-secret", 450_000);
it.each(["secrets.configure-company-secret", "companies.fast-responses"])("actual %s identity helper settles after its old150s cap using only the original remaining clock", async scenario => {
vi.useFakeTimers(); vi.setSystemTime(50_000); const originalDeadline = scenario === "companies.fast-responses" ? 390_000 : 450_000; const ctx = context(scenario, originalDeadline);
Object.assign(ctx, { refreshIssue: async () => {}, allRuns: async () => [{ status: Date.now() >= 230_000 ? "succeeded" : "running" }] });
let completed = false; const pending = waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)).then(() => { completed = true; });
await vi.advanceTimersByTimeAsync(152_000); expect(completed).toBe(false);
await vi.advanceTimersByTimeAsync(28_000); await pending; expect(completed).toBe(true);
expect(ctx.observationDeadlineAt).toBe(450_000);
expect(ctx.observationDeadlineAt).toBe(originalDeadline);
});
it.each(["onboarding.first-work", "routines.plugin-managed-routine", "status-cards.archive-card", "status-cards.other-authors-card"])("future13 %s does not receive new phase or UI behavior", scenario => {
it.each(["onboarding.first-work", "routines.plugin-managed-routine", "status-cards.archive-card", "status-cards.other-authors-card", "companies.prompt-injection-guard", "companies.configure-decision-model"])("future13 %s does not receive new phase or UI behavior", scenario => {
vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario, 1);
expect(chatCatalogNativePhaseDeadline(ctx, 900_000)).toBe(900_000);
expect(chatCatalogOriginalPhaseDeadline(ctx)).toBeUndefined();