fix(evals): preserve original task and inbox observation budgets

This commit is contained in:
Dotta committed 2026-10-28 08:17:14 -05:00
1 parent efa3382b21
commit a477b2c5d8
4 files changed
+9 -5

No files matched your search

@@ -26,6 +26,8 @@ export interface ChatCatalogContext {
/** These exact selected OpenCode/local journeys use the original flow clock for
* native observation phases. Other cases/lanes keep their authored phase caps. */
const originalClockExecutionIds = new Set([
"chat-features.runner-opencode.local.tasks.website-without-project",
"chat-features.runner-opencode.local.inbox-search.archive-partial-failure",
"chat-features.runner-opencode.local.cli-operations.feedback-bundle",
"chat-features.runner-opencode.local.cli-operations.inspect-run-cli",
"chat-features.runner-opencode.local.cli-operations.owned-dev-instance",
@@ -1,4 +1,5 @@
import { expect } from "@playwright/test";
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
import { pollUntil } from "../api.js";
import { makeDefinition } from "./internal-definition.js";
@@ -106,10 +107,10 @@ function archivePartial(): ChatCatalogCase {
};
return { ...base, coverageGaps: [], authentication: "human-session", expectedRunCount: 2, minimumRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 480000,
sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-archive-partial.ts", "chat-catalog-domains/internal-archive-transport.ts", "../e2e/fetch-allowed-port.ts", "chat-archive-fault-control.ts"],
async act(ctx, state) { const first = await pollUntil({ label: "Real initial partial archive turn settles", deadlineAt: Date.now() + 150000, load: () => observe(ctx, state), accept: o => o.runs.length === 1 && o.runs[0]?.status === "succeeded" && ctx.issue()?.conversationState === "waiting" });
async act(ctx, state) { const first = await pollUntil({ label: "Real initial partial archive turn settles", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150000), load: () => observe(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && o.runs.length === 1 && o.runs[0]?.status === "succeeded" && ctx.issue()?.conversationState === "waiting" });
state.firstPartial = { ...first, state: structuredClone(state) }; await ctx.input.evidence("chat-catalog-before-partial-archive-retry.json", state.firstPartial);
if (!firstIsValid(first)) throw new Error("Initial partial archive evidence failed before declared retry");
await transport(ctx)!.releaseForExplicitRetry(); const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, ctx.scenario.query); },
await transport(ctx)!.releaseForExplicitRetry(); const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, ctx.scenario.query, chatCatalogNativePhaseRemainingBudget(ctx)); },
observe,
async verifyResult(ctx, state) { const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "task").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "task").title!, exact: true }).first()).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } },
// Launcher owns the pre-bootstrap transport lifetime; it closes only after server shutdown.
@@ -1,4 +1,5 @@
import { createHash } from "node:crypto";
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js";
import { verifyDistinctTaskFixtures } from "./internal-distinct-fixtures.js";
import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
import type { ChatRun, ChatIssue } from "../chat-flow.js";
@@ -118,8 +119,8 @@ function creationDefinition(scenarioId: string): ChatCatalogCase {
if (scenarioId !== "tasks.website-without-project") return;
if (!interception) throw new Error("Lost acknowledgement fixture was not armed");
const sent = await interception.committed;
const first = await pollUntil({ label: "Committed project and task before lost-ack replay", deadlineAt: Date.now() + 150_000,
load: () => observeCreation(ctx, state), accept: o => newTasks(o).length === 1 && o.projects.filter(row => row.name === "Launch").length === 1
const first = await pollUntil({ label: "Committed project and task before lost-ack replay", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: () => observeCreation(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && newTasks(o).length === 1 && o.projects.filter(row => row.name === "Launch").length === 1
&& ctx.issue()?.conversationState === "waiting" });
await interception.dispose();
const replay = await ctx.input.api.post<{ id: string }>(sent.path, sent.data);
@@ -11,7 +11,7 @@ function context(scenarioId = "cli-operations.owned-dev-instance", deadline = 87
}
afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); });
describe("selected native phases share the once-created original flow clock", () => {
it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => {
it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => {
vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario);
expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000)).toBe(870_000);
vi.setSystemTime(850_000); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toBe(870_000);