From b0eff403b89ce6f1bac37d626cc570a7fa0bb7c6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 28 Oct 2026 01:16:01 -0500 Subject: [PATCH] fix(evals): share original budget for instance and confirmation waits --- .../identity-confirmation-budget.test.ts | 50 ++++++++++++ .../identity-confirmation-workflows.test.ts | 6 ++ .../identity-confirmation-workflows.ts | 26 +++++-- .../chat-catalog-domains/internal-support.ts | 16 ++-- .../privileged-budget.test.ts | 76 +++++++++++++++++++ .../privileged-configuration.ts | 34 +++++---- .../privileged-support.ts | 11 ++- tests/runner-e2e/chat-flow.ts | 12 +-- .../runner-e2e/chat-scoped-ui-budget.test.ts | 32 ++++++++ 9 files changed, 227 insertions(+), 36 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-confirmation-budget.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-budget.test.ts create mode 100644 tests/runner-e2e/chat-scoped-ui-budget.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-budget.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-budget.test.ts new file mode 100644 index 00000000000..2123ce25808 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-budget.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +const repeated = vi.hoisted(() => vi.fn()); +vi.mock("../chat-flow.js", async importOriginal => ({ ...await importOriginal(), sendChatMessage: repeated })); +import { confirmationWorkflowIdentityCases } from "./identity-confirmation-workflows.js"; +const definition = confirmationWorkflowIdentityCases.find(row => row.scenarioId === "questions-and-approvals.accept-one-proposal")!; +function context(settlesAt = 180_000): ChatCatalogContext { + return { observationDeadlineAt: 390_000, scenario: { query: "Yes, go ahead with that." }, refreshIssue: async () => {}, + issue: () => ({ id: "chat", conversationState: Date.now() >= settlesAt ? "waiting" : "active" }), + allRuns: async () => [{ status: Date.now() >= settlesAt ? "succeeded" : "running" }], + } as unknown as ChatCatalogContext; +} +afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); +describe("confirmation first reply and declared repeat share the original420s outer", () => { + it("actual act can reach real product readback after150s without changing its original deadline", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); + const get = vi.fn(async () => { throw new Error("independent saved-task boundary"); }); + ctx.input = { api: { get }, fixtures: { company: { id: "acorn" } } } as never; + const pending = definition.act!(ctx, {}).catch(String); + await vi.advanceTimersByTimeAsync(151_000); expect(get).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(29_000); expect(await pending).toContain("independent saved-task boundary"); + expect(get).toHaveBeenCalledExactlyOnceWith("/api/companies/acorn/issues?includeAgentCreatedTasks=true", { timeout: 210_000 }); + expect(ctx.observationDeadlineAt).toBe(390_000); expect(repeated).not.toHaveBeenCalled(); + }); + it("slow reload cannot start a fresh read or send the declared repeat after expiry", async () => { + vi.useFakeTimers(); vi.setSystemTime(389_000); const ctx = context(0); + const get = vi.fn().mockResolvedValueOnce([{ id: "task", title: "Acorn checklist" }]).mockResolvedValueOnce([{ id: "canonical", authorUserId: "person", body: ctx.scenario.query }]); + const reload = vi.fn(async () => { vi.setSystemTime(391_000); }); + ctx.input = { api: { get }, fixtures: { company: { id: "acorn" } }, page: { reload } } as never; + await expect(definition.act!(ctx, { taskTitle: "Acorn checklist" })).rejects.toThrow(/exhausted/); + expect(get).toHaveBeenCalledTimes(2); expect(reload).toHaveBeenCalledExactlyOnceWith({ waitUntil: "domcontentloaded", timeout: 1000 }); + expect(repeated).not.toHaveBeenCalled(); + }); + it("the existing one-run settlement gate does not waive an extra queued task run", async () => { + vi.useFakeTimers(); vi.setSystemTime(389_000); const ctx = context(0); + ctx.allRuns = async () => [{ status: "succeeded" }, { status: "queued" }] as never; + const get = vi.fn(); ctx.input = { api: { get } } as never; + const pending = definition.act!(ctx, {}).then(() => "accepted", () => "timeout"); + await vi.advanceTimersByTimeAsync(1500); expect(await pending).toBe("timeout"); + expect(get).not.toHaveBeenCalled(); expect(repeated).not.toHaveBeenCalled(); + }); + it("passes the same remaining clock to the one declared repeat", async () => { + vi.useFakeTimers(); vi.setSystemTime(380_000); const ctx = context(0); + const get = vi.fn().mockResolvedValueOnce([{ id: "task", title: "Acorn checklist" }]).mockResolvedValueOnce([{ id: "canonical", authorUserId: "person", body: ctx.scenario.query }]).mockResolvedValueOnce({ id: "task" }); + ctx.input = { api: { get }, fixtures: { company: { id: "acorn" } }, page: { reload: async () => {} }, evidence: async () => {} } as never; + repeated.mockImplementation(async (_page, _query, remaining) => { expect(remaining()).toBe(10_000); vi.setSystemTime(391_000); expect(() => remaining()).toThrow(/exhausted/); }); + await expect(definition.act!(ctx, { taskTitle: "Acorn checklist", original: { id: "proposal" } })).rejects.toThrow(/exhausted/); + expect(repeated).toHaveBeenCalledOnce(); expect(repeated.mock.calls[0]![1]).toBe(ctx.scenario.query); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts index 76af69184e7..db659a43459 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts @@ -34,6 +34,12 @@ describe("conversational proposal and simultaneous human answer oracles", () => else wrong.state.submitStatuses = [200, 200]; expect(definition.grade(wrong).passed).toBe(false); expect(definition.grade(undefined).passed).toBe(false); }); + it("a settled non-backlog execution task is not rescued by longer observation", () => { + for (const status of ["todo", "in_progress", "done"]) { + const o = valid("questions-and-approvals.accept-one-proposal"); o.tasks[0]!.status = status; + expect(gradeConfirmationWorkflow(o).passed).toBe(false); + } + }); it("accepting the proposal twice cannot produce two durable tasks", () => { const o = valid("questions-and-approvals.accept-one-proposal"); o.tasks.push({ ...o.tasks[0]!, id: "duplicate" }); expect(gradeConfirmationWorkflow(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts index bbf133edaec..9ffa7aaffcd 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts @@ -5,7 +5,7 @@ import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contra import { pollUntil } from "../api.js"; import { sendChatMessage } from "../chat-flow.js"; import { checks, entity, hasLink, verifyInternalResult, type Entity } from "./internal-support.js"; -import { identityDefinition, identityObservation, identityWorker, observeIdentity, openStandingChat, waitForReply, type Interaction } from "./identity-support.js"; +import { identityDefinition, identityObservation, identityWorker, observeIdentity, openStandingChat, waitForReply, remainingIdentityObservationBudget, type Interaction } from "./identity-support.js"; interface SourceComment extends Entity { authorType?: string } const tabs = new WeakMap(); @@ -75,16 +75,26 @@ export const confirmationWorkflowIdentityCases: readonly ChatCatalogCase[] = [ authentication: "human-session", sourceFiles: sources, expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000, observe: observeWorkflow, async act(ctx, state) { - await waitForReply(ctx); - const created = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/issues?includeAgentCreatedTasks=true`); + const remaining = () => remainingIdentityObservationBudget(ctx); + remaining(); + await waitForReply(ctx, ctx.observationDeadlineAt); + remaining(); + const created = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/issues?includeAgentCreatedTasks=true`, { timeout: remaining() }); const first = created.filter(row => row.id !== ctx.issue()!.id); if (first.length !== 1 || first[0]?.title !== state.taskTitle) throw new Error("The approved named task was not actually created once"); state.firstTask = first[0]; - const comments = await sourceComments(ctx); state.canonicalCommentId = comments.find(row => row.authorUserId && row.body?.trim() === ctx.scenario.query)?.id; - await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); - state.afterReloadTaskId = (await ctx.input.api.get(`/api/issues/${first[0]!.id}`)).id; - await sendChatMessage(ctx.input.page, ctx.scenario.query); + const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`, { timeout: remaining() }); state.canonicalCommentId = comments.find(row => row.authorUserId && row.body?.trim() === ctx.scenario.query)?.id; + await ctx.input.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() }); + state.afterReloadTaskId = (await ctx.input.api.get(`/api/issues/${first[0]!.id}`, { timeout: remaining() })).id; + await sendChatMessage(ctx.input.page, ctx.scenario.query, remaining); await ctx.input.evidence("identity-repeat-original-confirmation.json", { requestId: entity(state, "original").id, firstTaskId: first[0]!.id, canonicalCommentId: state.canonicalCommentId, repeatedOriginalReply: true }); - }, verifyResult: verifyInternalResult, + remaining(); + }, + async verifyResult(ctx, state, value) { + const remaining = () => remainingIdentityObservationBudget(ctx); + remaining(); + await verifyInternalResult(ctx, state, value, remaining); + remaining(); + }, }), identityDefinition("questions-and-approvals.duplicate-answer", async ctx => { const source = await openStandingChat(ctx); const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-support.ts b/tests/runner-e2e/chat-catalog-domains/internal-support.ts index ba192c38b68..c6b682521a7 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-support.ts @@ -99,7 +99,8 @@ export function sameTask(before: Entity, after: Entity | undefined, except: read } /** Follow the exact rendered chat links, check the real result view, and return to the source. */ -export async function verifyInternalResult(ctx: ChatCatalogContext, _state: Record, value: unknown) { +export async function verifyInternalResult(ctx: ChatCatalogContext, _state: Record, value: unknown, remainingBudget?: () => number) { + const options = () => remainingBudget ? { timeout: remainingBudget() } : {}; const o = observation(value); if (!o) throw new Error("Result navigation requires a complete observation"); const sourceUrl = ctx.input.page.url(); @@ -112,15 +113,16 @@ export async function verifyInternalResult(ctx: ChatCatalogContext, _state: Reco for (const { kind, row } of candidates) { const href = o.links.find(link => hasLink({ ...o, links: [link] }, kind, row))!; const link = ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first(); - await expect(link).toBeVisible(); - await link.click(); + await expect(link).toBeVisible(options()); + await link.click(options()); await ctx.input.page.waitForURL(url => { const parts = url.pathname.split("/").map(decodeURIComponent); const at = parts.indexOf(kind); return at >= 0 && [row.id, row.identifier, row.urlKey].includes(parts[at + 1]); - }); - await expect(ctx.input.page.getByRole("heading", { name: row.title ?? row.name!, exact: true }).first()).toBeVisible(); + }, options()); + await expect(ctx.input.page.getByRole("heading", { name: row.title ?? row.name!, exact: true }).first()).toBeVisible(options()); await ctx.input.evidence(`chat-catalog-opened-${kind}-${row.id}.json`, { id: row.id, kind, url: ctx.input.page.url(), title: row.title ?? row.name }); - await ctx.input.page.goto(sourceUrl, { waitUntil: "domcontentloaded" }); - await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); + await ctx.input.page.goto(sourceUrl, { waitUntil: "domcontentloaded", ...options() }); + await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(options()); + remainingBudget?.(); } } diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-budget.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-budget.test.ts new file mode 100644 index 00000000000..8d3cdd98d8e --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-budget.test.ts @@ -0,0 +1,76 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +const observed = vi.hoisted(() => ({ load: vi.fn() })); +vi.mock("./internal-support.js", async importOriginal => ({ ...await importOriginal(), observeInternal: observed.load })); +import { remainingPrivilegedObservationBudget, settlePrivilegedChat } from "./privileged-support.js"; +import { privilegedConfigurationCases } from "./privileged-configuration.js"; +function context(settlesAt = 180_000, deadline = 390_000): ChatCatalogContext { + const ctx = { observationDeadlineAt: deadline, issue: () => ({ conversationState: Date.now() >= settlesAt ? "waiting" : "active" }) } as unknown as ChatCatalogContext; + observed.load.mockImplementation(async () => ({ runs: [{ status: Date.now() >= settlesAt ? "succeeded" : "running" }], reply: Date.now() >= settlesAt ? "Use the operator experimental settings." : "" })); + return ctx; +} +afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); +describe("instance settings consume the original observation clock", () => { + it("waits beyond the incidental150s only for a genuinely settled reply", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); let done = false; + const pending = settlePrivilegedChat(ctx, ctx.observationDeadlineAt).then(() => { done = true; }); + await vi.advanceTimersByTimeAsync(151_000); expect(done).toBe(false); + await vi.advanceTimersByTimeAsync(29_000); await pending; + expect(done).toBe(true); expect(remainingPrivilegedObservationBudget(ctx)).toBe(210_000); + }); + it("does not change the default150s for other privileged callers", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); + const pending = settlePrivilegedChat(ctx).then(() => "accepted", () => "timeout"); + await vi.advanceTimersByTimeAsync(151_000); expect(await pending).toBe("timeout"); + }); + it("charges elapsed setup against the same390s clock and retains the30s reserve", async () => { + vi.useFakeTimers(); vi.setSystemTime(300_000); const ctx = context(400_000); + expect(remainingPrivilegedObservationBudget(ctx)).toBe(90_000); + const pending = settlePrivilegedChat(ctx, ctx.observationDeadlineAt).then(() => "accepted", () => "timeout"); + await vi.advanceTimersByTimeAsync(91_000); expect(await pending).toBe("timeout"); + expect(() => remainingPrivilegedObservationBudget(ctx)).toThrow(/exhausted/); + }); + it("cannot accept late successful reads or still-running native work", async () => { + vi.useFakeTimers(); vi.setSystemTime(389_000); const ctx = context(0); + observed.load.mockImplementation(async () => { vi.setSystemTime(391_000); return { runs: [{ status: "succeeded" }], reply: "Settings" }; }); + const pending = settlePrivilegedChat(ctx, ctx.observationDeadlineAt).then(() => "accepted", () => "timeout"); + await vi.advanceTimersByTimeAsync(3_000); expect(await pending).toBe("timeout"); + vi.setSystemTime(389_000); const running = context(999_000); + const another = settlePrivilegedChat(running, running.observationDeadlineAt).then(() => "accepted", () => "timeout"); + await vi.advanceTimersByTimeAsync(2_000); expect(await another).toBe("timeout"); + }); + it("keeps failure, queued-extra-run and missing reply gates", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(0); + observed.load.mockResolvedValue({ runs: [{ status: "failed" }], reply: "Settings" }); + await expect(settlePrivilegedChat(ctx, ctx.observationDeadlineAt)).rejects.toThrow(/run failed/); + for (const value of [{ runs: [{ status: "succeeded" }, { status: "queued" }], reply: "Settings" }, { runs: [{ status: "succeeded" }], reply: "" }]) { + observed.load.mockResolvedValue(value); + const pending = settlePrivilegedChat(ctx, Date.now() + 1000).then(() => "accepted", () => "timeout"); + await vi.advanceTimersByTimeAsync(2100); expect(await pending).toBe("timeout"); + } + }); + it("actual settings act opts in and forwards only the remaining original budget", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); + const goto = vi.fn(async () => { throw new Error("actual navigation boundary"); }); + ctx.input = { page: { url: () => "https://fixture.test/RUN/chats/ceo", goto }, fixtures: { company: { issuePrefix: "RUN" } }, api: { baseURL: "https://fixture.test" } } as never; + const pending = privilegedConfigurationCases[0]!.act!(ctx, {}).catch(String); + await vi.advanceTimersByTimeAsync(151_000); expect(goto).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(29_000); expect(await pending).toContain("actual navigation boundary"); + expect(goto).toHaveBeenCalledExactlyOnceWith("https://fixture.test/RUN/company/settings/instance/experimental", { waitUntil: "domcontentloaded", timeout: 210_000 }); + }); + it("slow navigation cannot start the next mutation or reset the original budget", async () => { + vi.useFakeTimers(); vi.setSystemTime(389_000); const ctx = context(0); const click = vi.fn(); + const goto = vi.fn(async () => { vi.setSystemTime(391_000); }); + ctx.input = { page: { url: () => "https://fixture.test/RUN/chats/ceo", goto, getByRole: () => ({ click }) }, fixtures: { company: { issuePrefix: "RUN" } }, api: { baseURL: "https://fixture.test" } } as never; + await expect(privilegedConfigurationCases[0]!.act!(ctx, {})).rejects.toThrow(/exhausted/); + expect(goto).toHaveBeenCalledOnce(); expect(click).not.toHaveBeenCalled(); + }); + it("missing, expired and nonfinite original deadlines fail before any UI action", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); + for (const deadline of [undefined, NaN, Infinity, 0, -1]) { + const ctx = { observationDeadlineAt: deadline } as ChatCatalogContext; + expect(() => remainingPrivilegedObservationBudget(ctx)).toThrow(); + await expect(privilegedConfigurationCases[0]!.act!(ctx, {})).rejects.toThrow(/exhausted or missing/); + } + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts b/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts index c18e01f713a..47697070c1b 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts @@ -1,7 +1,7 @@ import { expect } from "@playwright/test"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { checks, observeInternal } from "./internal-support.js"; -import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat, remainingPrivilegedObservationBudget } from "./privileged-support.js"; const probeKeys = new WeakMap(); const endpoint = "/api/instance/settings/experimental"; async function probe(ctx: ChatCatalogContext, method: "GET" | "PATCH") { @@ -37,27 +37,35 @@ const settings = privilegedDefinition("instance-operations.instance-settings", " return { agentReadStatus, agentPatchStatus, gateUnchangedAfterAgent: before.enableAgentChat === after.enableAgentChat }; }, async act(ctx, state) { - await settlePrivilegedChat(ctx); + const remaining = () => remainingPrivilegedObservationBudget(ctx); + remaining(); + await settlePrivilegedChat(ctx, ctx.observationDeadlineAt); + remaining(); const page = ctx.input.page, source = page.url(); const settingsUrl = new URL(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/instance/experimental`, ctx.input.api.baseURL).href; - await page.goto(settingsUrl, { waitUntil: "domcontentloaded" }); + await page.goto(settingsUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); const toggle = page.getByRole("switch", { name: "Toggle agent chat experimental setting", exact: true }); - await expect(toggle).toHaveAttribute("aria-checked", "true"); - await toggle.click(); - await expect.poll(async () => (await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint)).enableAgentChat).toBe(false); + await expect(toggle).toHaveAttribute("aria-checked", "true", { timeout: remaining() }); + await toggle.click({ timeout: remaining() }); + await expect.poll(async () => (await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint, { timeout: remaining() })).enableAgentChat, { timeout: remaining() }).toBe(false); state.savedDisabled = true; - await page.goto(source, { waitUntil: "domcontentloaded" }); - await expect(page.getByText(/Agent Chat is disabled(?:\.| in Experimental settings)/).first()).toBeVisible(); + await page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(page.getByText(/Agent Chat is disabled(?:\.| in Experimental settings)/).first()).toBeVisible({ timeout: remaining() }); state.disabledRouteClear = true; - await page.goto(settingsUrl, { waitUntil: "domcontentloaded" }); - await expect(toggle).toHaveAttribute("aria-checked", "false"); await toggle.click(); - await expect.poll(async () => (await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint)).enableAgentChat).toBe(true); + await page.goto(settingsUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(toggle).toHaveAttribute("aria-checked", "false", { timeout: remaining() }); await toggle.click({ timeout: remaining() }); + await expect.poll(async () => (await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint, { timeout: remaining() })).enableAgentChat, { timeout: remaining() }).toBe(true); state.savedEnabled = true; - await page.goto(source, { waitUntil: "domcontentloaded" }); await expect(page.getByTestId("task-chat-composer-input")).toBeVisible(); + await page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); await expect(page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: remaining() }); + remaining(); state.enabledRouteVisible = true; await ctx.input.evidence("chat-catalog-instance-gate-ui.json", { savedDisabled: true, disabledRouteClear: true, savedEnabled: true, enabledRouteVisible: true }); + remaining(); }, grade: gradeInstanceSettings, - async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, + async verifyResult(ctx) { + await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: remainingPrivilegedObservationBudget(ctx) }); + remainingPrivilegedObservationBudget(ctx); + }, }); settings.observe = async (ctx, state) => { const o = await observeInternal(ctx, state), key = probeKeys.get(ctx); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-support.ts b/tests/runner-e2e/chat-catalog-domains/privileged-support.ts index 50f4c796063..d8d1a99b653 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-support.ts @@ -9,10 +9,15 @@ export function privilegedDefinition(scenarioId: string, module: string, extra: sourceFiles: ["chat-catalog-domains/privileged-support.ts", `chat-catalog-domains/${module}.ts`, `chat-scenarios/${scenarioId.split(".")[0]}.yaml`], observe: observeInternal, ...extra }; } -export async function settlePrivilegedChat(ctx: ChatCatalogContext): Promise { - return pollUntil({ label: "Canonical privileged guidance settled", deadlineAt: Date.now() + 150_000, +export function remainingPrivilegedObservationBudget(ctx: ChatCatalogContext): number { + const deadline = ctx.observationDeadlineAt; + if (!Number.isFinite(deadline) || deadline! <= Date.now()) throw new Error("Original privileged observation budget exhausted or missing"); + return deadline! - Date.now(); +} +export async function settlePrivilegedChat(ctx: ChatCatalogContext, observationDeadlineAt?: number): Promise { + return pollUntil({ label: "Canonical privileged guidance settled", deadlineAt: observationDeadlineAt ?? Date.now() + 150_000, load: () => observeInternal(ctx, {}), - accept: o => o.runs.length === 1 && o.runs[0]?.status === "succeeded" && !!o.reply && ctx.issue()?.conversationState === "waiting", + accept: o => (observationDeadlineAt === undefined || Date.now() < observationDeadlineAt) && o.runs.length === 1 && o.runs[0]?.status === "succeeded" && !!o.reply && ctx.issue()?.conversationState === "waiting", reject: o => o.runs.some(run => ["failed", "timed_out", "cancelled"].includes(run.status)) ? "Canonical guidance run failed" : undefined }); } export const privilegedObservation = observation; diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.ts index 93ed63ce508..de08868cd55 100644 --- a/tests/runner-e2e/chat-flow.ts +++ b/tests/runner-e2e/chat-flow.ts @@ -329,19 +329,21 @@ export function assertChatHandoff( ); } } -export async function sendChatMessage(page: Page, message: string) { +export async function sendChatMessage(page: Page, message: string, remainingBudget?: () => number) { + const options = () => remainingBudget ? { timeout: remainingBudget() } : {}; const composer = page.getByTestId("task-chat-composer-input").last(); const takeover = page.getByTestId("task-chat-composer-takeover").last(); - await expect(composer.or(takeover).first()).toBeVisible(); + await expect(composer.or(takeover).first()).toBeVisible(options()); if (await takeover.isVisible()) { // Close the card's composer view without answering, skipping, or approving it. - await takeover.getByRole("button", { name: /^Dismiss / }).click(); + await takeover.getByRole("button", { name: /^Dismiss / }).click(options()); } await composer .locator('[contenteditable="true"], textarea') .first() - .fill(message); - await page.getByTestId("task-chat-composer-send").last().click(); + .fill(message, options()); + await page.getByTestId("task-chat-composer-send").last().click(options()); + remainingBudget?.(); } diff --git a/tests/runner-e2e/chat-scoped-ui-budget.test.ts b/tests/runner-e2e/chat-scoped-ui-budget.test.ts new file mode 100644 index 00000000000..1a36b66a08c --- /dev/null +++ b/tests/runner-e2e/chat-scoped-ui-budget.test.ts @@ -0,0 +1,32 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +const visibility = vi.hoisted(() => vi.fn(async () => {})); +vi.mock("@playwright/test", async importOriginal => ({ ...await importOriginal(), expect: () => ({ toBeVisible: visibility }) })); +import { sendChatMessage } from "./chat-flow.js"; +import { verifyInternalResult } from "./chat-catalog-domains/internal-support.js"; +const remaining = () => { const value = 390_000 - Date.now(); if (value <= 0) throw new Error("original budget exhausted"); return value; }; +afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); +describe("optional original-clock callbacks reach the real shared UI helpers", () => { + it("slow composer work cannot send a human message after the original deadline", async () => { + vi.useFakeTimers(); vi.setSystemTime(389_000); + const fill = vi.fn(async () => { vi.setSystemTime(391_000); }); const click = vi.fn(); + const node: any = { last: () => node, first: () => node, or: () => node, locator: () => node, isVisible: async () => false, fill, click }; + await expect(sendChatMessage({ getByTestId: () => node } as never, "Yes, go ahead with that.", remaining)).rejects.toThrow(/exhausted/); + expect(fill).toHaveBeenCalledExactlyOnceWith("Yes, go ahead with that.", { timeout: 1000 }); + expect(click).not.toHaveBeenCalled(); + }); + it("unchanged chat callers keep default UI timeouts", async () => { + const fill = vi.fn(); const click = vi.fn(); + const node: any = { last: () => node, first: () => node, or: () => node, locator: () => node, isVisible: async () => false, fill, click }; + await sendChatMessage({ getByTestId: () => node } as never, "Hello"); + expect(fill).toHaveBeenCalledExactlyOnceWith("Hello", {}); expect(click).toHaveBeenCalledExactlyOnceWith({}); + }); + it("slow result-link click cannot obtain a fresh verification/navigation budget", async () => { + vi.useFakeTimers(); vi.setSystemTime(389_000); + const click = vi.fn(async () => { vi.setSystemTime(391_000); }); const waitForURL = vi.fn(); + const node: any = { first: () => node, click }; + const ctx = { input: { page: { url: () => "https://fixture.test/RUN/chats/ceo", locator: () => node, waitForURL } } }; + const observation = { scenarioId: "questions-and-approvals.accept-one-proposal", origin: "https://fixture.test", companyId: "company", leadId: "lead", state: {}, reply: "Created", links: ["/RUN/issues/task"], tasks: [{ id: "task", title: "Checklist" }], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], runs: [], humanQueries: [] }; + await expect(verifyInternalResult(ctx as never, {}, observation, remaining)).rejects.toThrow(/exhausted/); + expect(click).toHaveBeenCalledExactlyOnceWith({ timeout: 1000 }); expect(waitForURL).not.toHaveBeenCalled(); + }); +});