diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 6cb9e63de0f..2c7de8cb352 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -26,6 +26,7 @@ export interface ChatCatalogContext { /** These exact selected OpenCode/local journeys use the original flow clock for * native observation phases. Other cases/lanes keep their authored phase caps. */ const originalClockExecutionIds = new Set([ + "chat-features.runner-opencode.local.developer-labs.long-thread-performance", "chat-features.runner-opencode.local.questions-and-approvals.approve-tool-once", "chat-features.runner-opencode.local.runs-adapters.reset-session", "chat-features.runner-opencode.local.tasks.website-without-project", diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts b/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts index ac2df752e1e..1b8b122689c 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts @@ -1,3 +1,4 @@ +import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; import { createHash } from "node:crypto"; import { readFile } from "node:fs/promises"; @@ -76,10 +77,10 @@ const performance = privilegedDefinition("developer-labs.long-thread-performance } finally { await page.goto(source, { waitUntil: "domcontentloaded" }); } // Declared operator companion step: facts only, no desired conclusion or threshold. const message = `Here is the report from my owned documented long-thread fixture. Please interpret the measured behavior and its limits:\n\n${JSON.stringify(report!)}`; - await sendChatMessage(page, message); - const final = await pollUntil({ label: "Actual native interpretation of the measured report", deadlineAt: Date.now() + 150_000, + await sendChatMessage(page, message, chatCatalogNativePhaseRemainingBudget(ctx)); + const final = await pollUntil({ label: "Actual native interpretation of the measured report", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; }, - accept: o => o.issue?.conversationState === "waiting" && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded") }); + accept: o => chatCatalogNativePhaseIsTimely(ctx) && o.issue?.conversationState === "waiting" && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded") }); const run = final.runs.find(run => run.id !== first.runs[0]!.id)!; const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); const human = comments.filter(comment => !comment.authorAgentId && comment.body === message); diff --git a/tests/runner-e2e/chat-catalog-original-clock.test.ts b/tests/runner-e2e/chat-catalog-original-clock.test.ts index 0e9da0ccd55..05e7745b9fb 100644 --- a/tests/runner-e2e/chat-catalog-original-clock.test.ts +++ b/tests/runner-e2e/chat-catalog-original-clock.test.ts @@ -11,7 +11,7 @@ function context(scenarioId = "cli-operations.owned-dev-instance", deadline = 87 } afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); }); describe("selected native phases share the once-created original flow clock", () => { - it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure", "questions-and-approvals.approve-tool-once", "runs-adapters.reset-session"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => { + it.each(["developer-labs.long-thread-performance", "cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure", "questions-and-approvals.approve-tool-once", "runs-adapters.reset-session"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => { vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000)).toBe(870_000); vi.setSystemTime(850_000); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toBe(870_000); diff --git a/tests/runner-e2e/chat-long-thread-clock.test.ts b/tests/runner-e2e/chat-long-thread-clock.test.ts new file mode 100644 index 00000000000..46e4328f480 --- /dev/null +++ b/tests/runner-e2e/chat-long-thread-clock.test.ts @@ -0,0 +1,30 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "./chat-catalog-contract.js"; +const { poll, settle, send } = vi.hoisted(() => ({ poll: vi.fn(), settle: vi.fn(), send: vi.fn() })); +vi.mock("./api.js", async original => ({ ...await original(), pollUntil: poll })); +vi.mock("./chat-flow.js", async original => ({ ...await original(), sendChatMessage: send })); +vi.mock("./chat-catalog-domains/privileged-support.js", async original => ({ ...await original(), settlePrivilegedChat: settle })); +vi.mock("@playwright/test", async original => ({ ...await original(), expect: Object.assign(() => ({ toBeVisible: async () => {} }), { poll: () => ({ toBeGreaterThanOrEqual: async () => {} }) }) })); +import { privilegedPerformanceCases } from "./chat-catalog-domains/privileged-performance.js"; +afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); +function context() { + const values: Record = { "perf-fixture-row-target": 487, "perf-fixture-markdown-rows": 150, "perf-mount-duration": 12.3, "perf-latest-duration": 8.1, "perf-max-duration": 12.3, "perf-total-duration": 20.4, "perf-commit-count": 2 }; + const page = { url: () => "http://owned.invalid/chat", goto: vi.fn(async () => {}), getByTestId: (id: string) => ({ innerText: async () => String(values[id]) }), evaluate: async () => "unit browser", viewportSize: () => ({ width: 1280, height: 720 }) }; + return { observationDeadlineAt: 570_000, scenario: { id: "developer-labs.long-thread-performance" }, input: { execution: { id: "chat-features.runner-opencode.local.developer-labs.long-thread-performance" }, page, api: { baseURL: "http://owned.invalid" }, evidence: vi.fn(), capture: vi.fn() }, refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }) } as unknown as ChatCatalogContext; +} +it("actual measured-report followup and interpretation consume the original clock after browser work", async () => { + vi.useFakeTimers(); vi.setSystemTime(200_000); const ctx = context(), state: Record = {}; + settle.mockResolvedValue({ runs: [{ id: "first" }] }); send.mockImplementation(async (_page, _text, remaining) => { expect(remaining()).toBe(370_000); }); poll.mockRejectedValue(new Error("captured interpretation")); + await expect(privilegedPerformanceCases[0]!.act!(ctx, state)).rejects.toThrow("captured interpretation"); + expect(send).toHaveBeenCalledExactlyOnceWith(ctx.input.page, expect.stringContaining('"commentCount":469'), expect.any(Function)); + const options = poll.mock.calls[0]![0]; expect(options.deadlineAt).toBe(570_000); + const valid = { issue: { conversationState: "waiting" }, runs: [{ status: "succeeded" }, { status: "succeeded" }] }; + expect(options.accept(valid)).toBe(true); expect(options.accept({ ...valid, runs: [{ status: "succeeded" }, { status: "running" }] })).toBe(false); + vi.setSystemTime(570_000); expect(options.accept(valid)).toBe(false); + expect(state.performanceReport).toMatchObject({ commentCount: 469, rowCount: 487, markdownCount: 150, mountMs: 12.3 }); +}); +it("elapsed browser measurement cannot reset the report send budget", async () => { + vi.useFakeTimers(); vi.setSystemTime(570_000); const ctx = context(); settle.mockResolvedValue({ runs: [{ id: "first" }] }); send.mockImplementation(async (_page, _text, remaining) => remaining()); + await expect(privilegedPerformanceCases[0]!.act!(ctx, {})).rejects.toThrow(/exhausted/); expect(poll).not.toHaveBeenCalled(); + expect(ctx.observationDeadlineAt).toBe(570_000); +});