From 7170ca4b8133f2691e9d920d5b58535fdd64953c Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 28 Oct 2026 09:11:34 -0500 Subject: [PATCH] test(e2e): preserve original approval and session clocks --- .../chat-approval-reset-clock.test.ts | 33 +++++++++++++++++++ tests/runner-e2e/chat-catalog-contract.ts | 2 ++ .../identity-tool-review.ts | 13 +++++--- .../chat-catalog-domains/runtime-session.ts | 12 ++++--- .../chat-catalog-original-clock.test.ts | 2 +- .../runner-e2e/chat-scoped-ui-budget.test.ts | 18 ++++++++++ tests/runner-e2e/user-actions.ts | 19 +++++++---- 7 files changed, 82 insertions(+), 17 deletions(-) create mode 100644 tests/runner-e2e/chat-approval-reset-clock.test.ts diff --git a/tests/runner-e2e/chat-approval-reset-clock.test.ts b/tests/runner-e2e/chat-approval-reset-clock.test.ts new file mode 100644 index 00000000000..b4d4c741f3a --- /dev/null +++ b/tests/runner-e2e/chat-approval-reset-clock.test.ts @@ -0,0 +1,33 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "./chat-catalog-contract.js"; +const poll = vi.hoisted(() => vi.fn()); +vi.mock("./api.js", async original => ({ ...await original(), pollUntil: poll })); +vi.mock("../fixtures/connection-review-provider.js", () => ({ startReviewProvider: async () => ({ url: "http://fixture.invalid", captures: [], close: async () => {} }) })); +import { toolReviewIdentityCases } from "./chat-catalog-domains/identity-tool-review.js"; +import { runtimeSessionCases } from "./chat-catalog-domains/runtime-session.js"; + +afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); +function context(id: string, deadline: number) { + return { observationDeadlineAt: deadline, scenario: { id }, input: { nonce: "unit", privateSecrets: [], execution: { id: `chat-features.runner-opencode.local.${id}` }, fixtures: { agent: { id: "ceo" }, company: { id: "company" } }, api: { + get: async () => ({ runtimeConfig: {} }), patch: async () => ({}), post: async () => ({}), putSensitive: async () => ({}), + postSensitive: async () => ({ connectionId: "connection", actions: { readOnly: [], canMakeChanges: [] } }), + } }, issue: () => ({ id: "chat", conversationState: "waiting" }), refreshIssue: async () => {}, allRuns: async () => [] } as unknown as ChatCatalogContext; +} +it("actual approval request wait consumes elapsed setup and rejects a late pending card without changing review UI", async () => { + vi.useFakeTimers(); vi.setSystemTime(200_000); + const ctx = context("questions-and-approvals.approve-tool-once", 570_000), definition = toolReviewIdentityCases[0]!; + const state = await definition.setup(ctx); poll.mockRejectedValue(new Error("captured phase")); + await expect(definition.act!(ctx, state)).rejects.toThrow("captured phase"); + const options = poll.mock.calls[0]![0]; expect(options.deadlineAt).toBe(570_000); + const pending = [{ status: "pending", payload: { toolAction: { connectionId: "connection" } } }]; + expect(options.accept(pending)).toBe(true); vi.setSystemTime(570_000); expect(options.accept(pending)).toBe(false); +}); +it("actual reset canonical wait uses the same setup-inclusive clock and retains exact successful-run prerequisites", async () => { + vi.useFakeTimers(); vi.setSystemTime(200_000); + const ctx = context("runs-adapters.reset-session", 690_000); poll.mockRejectedValue(new Error("captured phase")); + await expect(runtimeSessionCases[0]!.act!(ctx, {})).rejects.toThrow("captured phase"); + const options = poll.mock.calls[0]![0]; expect(options.deadlineAt).toBe(690_000); + const valid = { source: { conversationState: "waiting" }, runs: [{ status: "succeeded" }, { status: "succeeded" }] }; + expect(options.accept(valid)).toBe(true); expect(options.accept({ ...valid, runs: [{ status: "succeeded" }, { status: "running" }] })).toBe(false); + vi.setSystemTime(690_000); expect(options.accept(valid)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 0d955e5f969..6cb9e63de0f 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -26,6 +26,8 @@ export interface ChatCatalogContext { /** These exact selected OpenCode/local journeys use the original flow clock for * native observation phases. Other cases/lanes keep their authored phase caps. */ const originalClockExecutionIds = new Set([ + "chat-features.runner-opencode.local.questions-and-approvals.approve-tool-once", + "chat-features.runner-opencode.local.runs-adapters.reset-session", "chat-features.runner-opencode.local.tasks.website-without-project", "chat-features.runner-opencode.local.inbox-search.archive-partial-failure", "chat-features.runner-opencode.local.cli-operations.feedback-bundle", diff --git a/tests/runner-e2e/chat-catalog-domains/identity-tool-review.ts b/tests/runner-e2e/chat-catalog-domains/identity-tool-review.ts index bd8c5806942..a94cc8fdd03 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-tool-review.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-tool-review.ts @@ -1,3 +1,4 @@ +import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js"; import { randomBytes } from "node:crypto"; import { expect, type Page } from "@playwright/test"; import { startReviewProvider } from "../../fixtures/connection-review-provider.js"; @@ -105,19 +106,21 @@ export const toolReviewIdentityCases = ["approve-tool-once", "decline-tool"].map state.beforeDecisionCalls = calls(provider).length; const review = await openIdentityToolReview(ctx.input.page, card, String(state.connectionId)); state.reviewVisible = true; state.beforeDecisionCommentIds = (await comments(ctx)).map(row => row.id); + chatCatalogNativePhaseRemainingBudget(ctx)?.(); await review.getByRole("button", { name: id === "decline-tool" ? "Decline" : "Approve & run", exact: true }).click(); if (id === "approve-tool-once") { - await pollUntil({ label: "first approved synthetic read completed", deadlineAt: Date.now() + 150_000, + await pollUntil({ label: "first approved synthetic read completed", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), load: async () => { await ctx.refreshIssue(); const rows = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/interactions`); return { rows, runs: await ctx.allRuns(), source: ctx.issue(), comments: await comments(ctx) }; }, - accept: value => value.rows.some(row => row.id === card.id && row.result?.toolAction?.status === "executed") + accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.rows.some(row => row.id === card.id && row.result?.toolAction?.status === "executed") && value.source?.conversationState === "waiting" && value.runs.every(run => run.status === "succeeded") && value.comments.some(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.body.includes(String(state.marker))) }); state.firstReply = (await comments(ctx)).filter(row => row.authorAgentId === ctx.input.fixtures.agent.id).map(row => row.body).join("\n"); - await sendChatMessage(ctx.input.page, "Read the launch notes again using the app's diagnostic query value fail, so we can check its unavailable-service response."); + await sendChatMessage(ctx.input.page, "Read the launch notes again using the app's diagnostic query value fail, so we can check its unavailable-service response.", chatCatalogNativePhaseRemainingBudget(ctx)); const second = await pendingReview(ctx, state, provider, 1); state.secondDecisionId = second.id; state.beforeSecondDecisionCalls = calls(provider).length; const secondReview = await openIdentityToolReview(ctx.input.page, second, String(state.connectionId)); state.secondReviewVisible = true; state.beforeDecisionCommentIds = (await comments(ctx)).map(row => row.id); + chatCatalogNativePhaseRemainingBudget(ctx)?.(); await secondReview.getByRole("button", { name: "Approve & run", exact: true }).click(); } }, @@ -141,9 +144,9 @@ export const toolReviewIdentityCases = ["approve-tool-once", "decline-tool"].map })); async function comments(ctx: ChatCatalogContext) { return ctx.input.api.get>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); } async function pendingReview(ctx: ChatCatalogContext, state: Record, provider: Provider, previousCalls = 0) { - const rows = await pollUntil({ label: "scoped synthetic service tool review", deadlineAt: Date.now() + 150_000, + const rows = await pollUntil({ label: "scoped synthetic service tool review", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), load: async () => { await ctx.refreshIssue(); return ctx.issue() ? ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/interactions`) : []; }, - accept: rows => rows.some(row => row.status === "pending" && row.payload.toolAction?.connectionId === state.connectionId), + accept: rows => chatCatalogNativePhaseIsTimely(ctx) && rows.some(row => row.status === "pending" && row.payload.toolAction?.connectionId === state.connectionId), reject: () => calls(provider).length > previousCalls ? "Synthetic provider was called before this action's approval" : undefined }); return pendingStoryDecision(rows, { kind: "tool", connectionId: String(state.connectionId) }); } diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-session.ts b/tests/runner-e2e/chat-catalog-domains/runtime-session.ts index 6d3cc63deee..adaf2a55026 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-session.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-session.ts @@ -1,3 +1,4 @@ +import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; import { pollUntil } from "../api.js"; import { submitTaskReply } from "../user-actions.js"; @@ -44,9 +45,9 @@ export const runtimeSessionCases = [runtimeDefinition("runs-adapters.reset-sessi sourceFiles: ["chat-catalog-domains/runtime-session.ts", "chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/runs-adapters.yaml", "../../ui/src/components/AgentActionButtons.tsx", "../../ui/src/pages/AgentDetail.tsx"], expectedRunCount: 3, attemptTimeoutMs: 720_000, async act(ctx, state) { - await pollUntil({ label: "Canonical session guidance settled", deadlineAt: Date.now() + 150_000, + await pollUntil({ label: "Canonical session guidance settled", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, - accept: value => value.source?.conversationState === "waiting" && value.runs.length === 2 && value.runs.every(run => run.status === "succeeded") }); + accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.runs.length === 2 && value.runs.every(run => run.status === "succeeded") }); const worker = state.worker as RuntimeAgent; const task = state.task as Entity; const query = await observeRuntime(ctx, state); state.queryReply = query.reply; state.agentResetStatus = await probeAgentPermission(ctx, `/api/agents/${worker.id}/runtime-state/reset-session`, "POST", {}); @@ -55,6 +56,7 @@ export const runtimeSessionCases = [runtimeDefinition("runs-adapters.reset-sessi const profile = ctx.input.page.locator("#main-content"); await expect(profile.getByRole("heading", { name: worker.name!, exact: true })).toBeVisible(); await profile.getByRole("button", { name: `Open actions for ${worker.name}`, exact: true }).click(); + chatCatalogNativePhaseRemainingBudget(ctx)?.(); await ctx.input.page.getByRole("button", { name: "Reset Sessions", exact: true }).click(); await pollUntil({ label: "Human reset clears the real task session", deadlineAt: Date.now() + 30_000, load: () => ctx.input.api.get(`/api/agents/${worker.id}/task-sessions`), accept: rows => rows.length === 0 }); @@ -62,10 +64,10 @@ export const runtimeSessionCases = [runtimeDefinition("runs-adapters.reset-sessi const beforeRuns = await ctx.allRuns(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); const followup = "Please continue this same task using your fresh runtime session. Read the existing website-fixture.txt without changing its bytes, inspect our saved task history, and add a brief task comment confirming it is still readable. Keep existing documents and attachments unchanged, then finish this task. Do not create another task."; - await submitTaskReply(ctx.input.page, followup); - const future = await pollUntil({ label: "Actual fresh-session continuation", deadlineAt: Date.now() + 180_000, + await submitTaskReply(ctx.input.page, followup, chatCatalogNativePhaseRemainingBudget(ctx)); + const future = await pollUntil({ label: "Actual fresh-session continuation", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000), load: async () => ({ runs: await ctx.allRuns(), task: await ctx.input.api.get(`/api/issues/${task.id}`) }), - accept: value => value.task.status === "done" && value.runs.length === beforeRuns.length + 1 + accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.task.status === "done" && value.runs.length === beforeRuns.length + 1 && value.runs.filter(run => !beforeRuns.some(prior => prior.id === run.id)).every(run => run.status === "succeeded"), reject: value => value.runs.some(run => !beforeRuns.some(prior => prior.id === run.id) && ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Fresh-session continuation failed" : undefined }); const next = future.runs.find(run => !beforeRuns.some(prior => prior.id === run.id))!; diff --git a/tests/runner-e2e/chat-catalog-original-clock.test.ts b/tests/runner-e2e/chat-catalog-original-clock.test.ts index cff9490f41a..0e9da0ccd55 100644 --- a/tests/runner-e2e/chat-catalog-original-clock.test.ts +++ b/tests/runner-e2e/chat-catalog-original-clock.test.ts @@ -11,7 +11,7 @@ function context(scenarioId = "cli-operations.owned-dev-instance", deadline = 87 } afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); }); describe("selected native phases share the once-created original flow clock", () => { - it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => { + it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure", "questions-and-approvals.approve-tool-once", "runs-adapters.reset-session"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => { vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000)).toBe(870_000); vi.setSystemTime(850_000); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toBe(870_000); diff --git a/tests/runner-e2e/chat-scoped-ui-budget.test.ts b/tests/runner-e2e/chat-scoped-ui-budget.test.ts index 1a36b66a08c..d499ad1584f 100644 --- a/tests/runner-e2e/chat-scoped-ui-budget.test.ts +++ b/tests/runner-e2e/chat-scoped-ui-budget.test.ts @@ -2,10 +2,28 @@ import { afterEach, describe, expect, it, vi } from "vitest"; const visibility = vi.hoisted(() => vi.fn(async () => {})); vi.mock("@playwright/test", async importOriginal => ({ ...await importOriginal(), expect: () => ({ toBeVisible: visibility }) })); import { sendChatMessage } from "./chat-flow.js"; +import { submitTaskReply } from "./user-actions.js"; import { verifyInternalResult } from "./chat-catalog-domains/internal-support.js"; const remaining = () => { const value = 390_000 - Date.now(); if (value <= 0) throw new Error("original budget exhausted"); return value; }; afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); describe("optional original-clock callbacks reach the real shared UI helpers", () => { + it("task continuation cannot send after slow composer work exhausts the original clock", async () => { + vi.useFakeTimers(); vi.setSystemTime(389_000); + const fill = vi.fn(async () => { vi.setSystemTime(391_000); }); const click = vi.fn(); + const node: any = { last: () => node, first: () => node, locator: () => node, fill, click }; + await expect(submitTaskReply({ getByTestId: () => node } as never, "Continue the same task.", remaining)).rejects.toThrow(/exhausted/); + expect(fill).toHaveBeenCalledExactlyOnceWith("Continue the same task.", { timeout: 1000 }); expect(click).not.toHaveBeenCalled(); + }); + it.each([0, -1, NaN, Infinity])("task continuation rejects invalid remaining budget %s before UI work", async budget => { + const node: any = { last: () => node }; const prior = visibility.mock.calls.length; + await expect(submitTaskReply({ getByTestId: () => node } as never, "Continue.", () => budget)).rejects.toThrow(/exhausted/); + expect(visibility.mock.calls.length).toBe(prior); + }); + it("ordinary task reply callers retain existing UI defaults", async () => { + const fill = vi.fn(); const click = vi.fn(); const node: any = { last: () => node, first: () => node, locator: () => node, fill, click }; + await submitTaskReply({ getByTestId: () => node } as never, "Continue."); + expect(visibility).toHaveBeenCalledWith({ timeout: 30_000 }); expect(fill).toHaveBeenCalledExactlyOnceWith("Continue."); expect(click).toHaveBeenCalledExactlyOnceWith(); + }); it("slow composer work cannot send a human message after the original deadline", async () => { vi.useFakeTimers(); vi.setSystemTime(389_000); const fill = vi.fn(async () => { vi.setSystemTime(391_000); }); const click = vi.fn(); diff --git a/tests/runner-e2e/user-actions.ts b/tests/runner-e2e/user-actions.ts index 7ba0d18ecff..64e5c68f424 100644 --- a/tests/runner-e2e/user-actions.ts +++ b/tests/runner-e2e/user-actions.ts @@ -130,14 +130,21 @@ export async function createTaskThroughUi(input: { export async function submitTaskReply( page: Page, body: string, + remainingBudget?: () => number, ): Promise { const composer = page.getByTestId("task-chat-composer-input").last(); - await expect(composer).toBeVisible({ timeout: 30_000 }); - await composer - .locator('[contenteditable="true"], textarea') - .first() - .fill(body); + const options = () => { + const remaining = remainingBudget!(); + if (!Number.isFinite(remaining) || remaining <= 0) throw new Error("Task reply original observation budget exhausted"); + return { timeout: Math.min(30_000, remaining) }; + }; + await expect(composer).toBeVisible(remainingBudget ? options() : { timeout: 30_000 }); + const editor = composer.locator('[contenteditable="true"], textarea').first(); + if (remainingBudget) await editor.fill(body, options()); + else await editor.fill(body); const submittedAtMs = Date.now(); - await page.getByTestId("task-chat-composer-send").last().click(); + const send = page.getByTestId("task-chat-composer-send").last(); + if (remainingBudget) await send.click(options()); + else await send.click(); return submittedAtMs; }