test(e2e): keep long-thread measurements within original clock

This commit is contained in:
Dotta committed 2026-10-28 09:46:16 -05:00
1 parent ed24a4a444
commit 8bbf313552
4 files changed
+36 -4

No files matched your search

@@ -26,6 +26,7 @@ export interface ChatCatalogContext {
/** These exact selected OpenCode/local journeys use the original flow clock for
* native observation phases. Other cases/lanes keep their authored phase caps. */
const originalClockExecutionIds = new Set([
"chat-features.runner-opencode.local.developer-labs.long-thread-performance",
"chat-features.runner-opencode.local.questions-and-approvals.approve-tool-once",
"chat-features.runner-opencode.local.runs-adapters.reset-session",
"chat-features.runner-opencode.local.tasks.website-without-project",
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
@@ -76,10 +77,10 @@ const performance = privilegedDefinition("developer-labs.long-thread-performance
} finally { await page.goto(source, { waitUntil: "domcontentloaded" }); }
// Declared operator companion step: facts only, no desired conclusion or threshold.
const message = `Here is the report from my owned documented long-thread fixture. Please interpret the measured behavior and its limits:\n\n${JSON.stringify(report!)}`;
await sendChatMessage(page, message);
const final = await pollUntil({ label: "Actual native interpretation of the measured report", deadlineAt: Date.now() + 150_000,
await sendChatMessage(page, message, chatCatalogNativePhaseRemainingBudget(ctx));
const final = await pollUntil({ label: "Actual native interpretation of the measured report", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; },
accept: o => o.issue?.conversationState === "waiting" && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded") });
accept: o => chatCatalogNativePhaseIsTimely(ctx) && o.issue?.conversationState === "waiting" && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded") });
const run = final.runs.find(run => run.id !== first.runs[0]!.id)!;
const comments = await ctx.input.api.get<Entity[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`);
const human = comments.filter(comment => !comment.authorAgentId && comment.body === message);
@@ -11,7 +11,7 @@ function context(scenarioId = "cli-operations.owned-dev-instance", deadline = 87
}
afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); });
describe("selected native phases share the once-created original flow clock", () => {
it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure", "questions-and-approvals.approve-tool-once", "runs-adapters.reset-session"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => {
it.each(["developer-labs.long-thread-performance", "cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure", "questions-and-approvals.approve-tool-once", "runs-adapters.reset-session"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => {
vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario);
expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000)).toBe(870_000);
vi.setSystemTime(850_000); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toBe(870_000);
@@ -0,0 +1,30 @@
import { afterEach, expect, it, vi } from "vitest";
import type { ChatCatalogContext } from "./chat-catalog-contract.js";
const { poll, settle, send } = vi.hoisted(() => ({ poll: vi.fn(), settle: vi.fn(), send: vi.fn() }));
vi.mock("./api.js", async original => ({ ...await original<object>(), pollUntil: poll }));
vi.mock("./chat-flow.js", async original => ({ ...await original<object>(), sendChatMessage: send }));
vi.mock("./chat-catalog-domains/privileged-support.js", async original => ({ ...await original<object>(), settlePrivilegedChat: settle }));
vi.mock("@playwright/test", async original => ({ ...await original<object>(), expect: Object.assign(() => ({ toBeVisible: async () => {} }), { poll: () => ({ toBeGreaterThanOrEqual: async () => {} }) }) }));
import { privilegedPerformanceCases } from "./chat-catalog-domains/privileged-performance.js";
afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); });
function context() {
const values: Record<string, number> = { "perf-fixture-row-target": 487, "perf-fixture-markdown-rows": 150, "perf-mount-duration": 12.3, "perf-latest-duration": 8.1, "perf-max-duration": 12.3, "perf-total-duration": 20.4, "perf-commit-count": 2 };
const page = { url: () => "http://owned.invalid/chat", goto: vi.fn(async () => {}), getByTestId: (id: string) => ({ innerText: async () => String(values[id]) }), evaluate: async () => "unit browser", viewportSize: () => ({ width: 1280, height: 720 }) };
return { observationDeadlineAt: 570_000, scenario: { id: "developer-labs.long-thread-performance" }, input: { execution: { id: "chat-features.runner-opencode.local.developer-labs.long-thread-performance" }, page, api: { baseURL: "http://owned.invalid" }, evidence: vi.fn(), capture: vi.fn() }, refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }) } as unknown as ChatCatalogContext;
}
it("actual measured-report followup and interpretation consume the original clock after browser work", async () => {
vi.useFakeTimers(); vi.setSystemTime(200_000); const ctx = context(), state: Record<string, unknown> = {};
settle.mockResolvedValue({ runs: [{ id: "first" }] }); send.mockImplementation(async (_page, _text, remaining) => { expect(remaining()).toBe(370_000); }); poll.mockRejectedValue(new Error("captured interpretation"));
await expect(privilegedPerformanceCases[0]!.act!(ctx, state)).rejects.toThrow("captured interpretation");
expect(send).toHaveBeenCalledExactlyOnceWith(ctx.input.page, expect.stringContaining('"commentCount":469'), expect.any(Function));
const options = poll.mock.calls[0]![0]; expect(options.deadlineAt).toBe(570_000);
const valid = { issue: { conversationState: "waiting" }, runs: [{ status: "succeeded" }, { status: "succeeded" }] };
expect(options.accept(valid)).toBe(true); expect(options.accept({ ...valid, runs: [{ status: "succeeded" }, { status: "running" }] })).toBe(false);
vi.setSystemTime(570_000); expect(options.accept(valid)).toBe(false);
expect(state.performanceReport).toMatchObject({ commentCount: 469, rowCount: 487, markdownCount: 150, mountMs: 12.3 });
});
it("elapsed browser measurement cannot reset the report send budget", async () => {
vi.useFakeTimers(); vi.setSystemTime(570_000); const ctx = context(); settle.mockResolvedValue({ runs: [{ id: "first" }] }); send.mockImplementation(async (_page, _text, remaining) => remaining());
await expect(privilegedPerformanceCases[0]!.act!(ctx, {})).rejects.toThrow(/exhausted/); expect(poll).not.toHaveBeenCalled();
expect(ctx.observationDeadlineAt).toBe(570_000);
});