test(e2e): preserve original approval and session clocks

This commit is contained in:
Dotta committed 2026-10-28 09:12:28 -05:00
1 parent a477b2c5d8
commit 7170ca4b81
7 files changed
+82 -17

No files matched your search

@@ -0,0 +1,33 @@
import { afterEach, expect, it, vi } from "vitest";
import type { ChatCatalogContext } from "./chat-catalog-contract.js";
const poll = vi.hoisted(() => vi.fn());
vi.mock("./api.js", async original => ({ ...await original<object>(), pollUntil: poll }));
vi.mock("../fixtures/connection-review-provider.js", () => ({ startReviewProvider: async () => ({ url: "http://fixture.invalid", captures: [], close: async () => {} }) }));
import { toolReviewIdentityCases } from "./chat-catalog-domains/identity-tool-review.js";
import { runtimeSessionCases } from "./chat-catalog-domains/runtime-session.js";
afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); });
function context(id: string, deadline: number) {
return { observationDeadlineAt: deadline, scenario: { id }, input: { nonce: "unit", privateSecrets: [], execution: { id: `chat-features.runner-opencode.local.${id}` }, fixtures: { agent: { id: "ceo" }, company: { id: "company" } }, api: {
get: async () => ({ runtimeConfig: {} }), patch: async () => ({}), post: async () => ({}), putSensitive: async () => ({}),
postSensitive: async () => ({ connectionId: "connection", actions: { readOnly: [], canMakeChanges: [] } }),
} }, issue: () => ({ id: "chat", conversationState: "waiting" }), refreshIssue: async () => {}, allRuns: async () => [] } as unknown as ChatCatalogContext;
}
it("actual approval request wait consumes elapsed setup and rejects a late pending card without changing review UI", async () => {
vi.useFakeTimers(); vi.setSystemTime(200_000);
const ctx = context("questions-and-approvals.approve-tool-once", 570_000), definition = toolReviewIdentityCases[0]!;
const state = await definition.setup(ctx); poll.mockRejectedValue(new Error("captured phase"));
await expect(definition.act!(ctx, state)).rejects.toThrow("captured phase");
const options = poll.mock.calls[0]![0]; expect(options.deadlineAt).toBe(570_000);
const pending = [{ status: "pending", payload: { toolAction: { connectionId: "connection" } } }];
expect(options.accept(pending)).toBe(true); vi.setSystemTime(570_000); expect(options.accept(pending)).toBe(false);
});
it("actual reset canonical wait uses the same setup-inclusive clock and retains exact successful-run prerequisites", async () => {
vi.useFakeTimers(); vi.setSystemTime(200_000);
const ctx = context("runs-adapters.reset-session", 690_000); poll.mockRejectedValue(new Error("captured phase"));
await expect(runtimeSessionCases[0]!.act!(ctx, {})).rejects.toThrow("captured phase");
const options = poll.mock.calls[0]![0]; expect(options.deadlineAt).toBe(690_000);
const valid = { source: { conversationState: "waiting" }, runs: [{ status: "succeeded" }, { status: "succeeded" }] };
expect(options.accept(valid)).toBe(true); expect(options.accept({ ...valid, runs: [{ status: "succeeded" }, { status: "running" }] })).toBe(false);
vi.setSystemTime(690_000); expect(options.accept(valid)).toBe(false);
});
@@ -26,6 +26,8 @@ export interface ChatCatalogContext {
/** These exact selected OpenCode/local journeys use the original flow clock for
* native observation phases. Other cases/lanes keep their authored phase caps. */
const originalClockExecutionIds = new Set([
"chat-features.runner-opencode.local.questions-and-approvals.approve-tool-once",
"chat-features.runner-opencode.local.runs-adapters.reset-session",
"chat-features.runner-opencode.local.tasks.website-without-project",
"chat-features.runner-opencode.local.inbox-search.archive-partial-failure",
"chat-features.runner-opencode.local.cli-operations.feedback-bundle",
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import { randomBytes } from "node:crypto";
import { expect, type Page } from "@playwright/test";
import { startReviewProvider } from "../../fixtures/connection-review-provider.js";
@@ -105,19 +106,21 @@ export const toolReviewIdentityCases = ["approve-tool-once", "decline-tool"].map
state.beforeDecisionCalls = calls(provider).length;
const review = await openIdentityToolReview(ctx.input.page, card, String(state.connectionId)); state.reviewVisible = true;
state.beforeDecisionCommentIds = (await comments(ctx)).map(row => row.id);
chatCatalogNativePhaseRemainingBudget(ctx)?.();
await review.getByRole("button", { name: id === "decline-tool" ? "Decline" : "Approve & run", exact: true }).click();
if (id === "approve-tool-once") {
await pollUntil({ label: "first approved synthetic read completed", deadlineAt: Date.now() + 150_000,
await pollUntil({ label: "first approved synthetic read completed", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); const rows = await ctx.input.api.get<Interaction[]>(`/api/issues/${ctx.issue()!.id}/interactions`);
return { rows, runs: await ctx.allRuns(), source: ctx.issue(), comments: await comments(ctx) }; },
accept: value => value.rows.some(row => row.id === card.id && row.result?.toolAction?.status === "executed")
accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.rows.some(row => row.id === card.id && row.result?.toolAction?.status === "executed")
&& value.source?.conversationState === "waiting" && value.runs.every(run => run.status === "succeeded")
&& value.comments.some(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.body.includes(String(state.marker))) });
state.firstReply = (await comments(ctx)).filter(row => row.authorAgentId === ctx.input.fixtures.agent.id).map(row => row.body).join("\n");
await sendChatMessage(ctx.input.page, "Read the launch notes again using the app's diagnostic query value fail, so we can check its unavailable-service response.");
await sendChatMessage(ctx.input.page, "Read the launch notes again using the app's diagnostic query value fail, so we can check its unavailable-service response.", chatCatalogNativePhaseRemainingBudget(ctx));
const second = await pendingReview(ctx, state, provider, 1); state.secondDecisionId = second.id; state.beforeSecondDecisionCalls = calls(provider).length;
const secondReview = await openIdentityToolReview(ctx.input.page, second, String(state.connectionId)); state.secondReviewVisible = true;
state.beforeDecisionCommentIds = (await comments(ctx)).map(row => row.id);
chatCatalogNativePhaseRemainingBudget(ctx)?.();
await secondReview.getByRole("button", { name: "Approve & run", exact: true }).click();
}
},
@@ -141,9 +144,9 @@ export const toolReviewIdentityCases = ["approve-tool-once", "decline-tool"].map
}));
async function comments(ctx: ChatCatalogContext) { return ctx.input.api.get<Array<{ id: string; authorAgentId?: string; body: string }>>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); }
async function pendingReview(ctx: ChatCatalogContext, state: Record<string, unknown>, provider: Provider, previousCalls = 0) {
const rows = await pollUntil({ label: "scoped synthetic service tool review", deadlineAt: Date.now() + 150_000,
const rows = await pollUntil({ label: "scoped synthetic service tool review", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return ctx.issue() ? ctx.input.api.get<Interaction[]>(`/api/issues/${ctx.issue()!.id}/interactions`) : []; },
accept: rows => rows.some(row => row.status === "pending" && row.payload.toolAction?.connectionId === state.connectionId),
accept: rows => chatCatalogNativePhaseIsTimely(ctx) && rows.some(row => row.status === "pending" && row.payload.toolAction?.connectionId === state.connectionId),
reject: () => calls(provider).length > previousCalls ? "Synthetic provider was called before this action's approval" : undefined });
return pendingStoryDecision(rows, { kind: "tool", connectionId: String(state.connectionId) });
}
@@ -1,3 +1,4 @@
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { pollUntil } from "../api.js";
import { submitTaskReply } from "../user-actions.js";
@@ -44,9 +45,9 @@ export const runtimeSessionCases = [runtimeDefinition("runs-adapters.reset-sessi
sourceFiles: ["chat-catalog-domains/runtime-session.ts", "chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/runs-adapters.yaml", "../../ui/src/components/AgentActionButtons.tsx", "../../ui/src/pages/AgentDetail.tsx"],
expectedRunCount: 3, attemptTimeoutMs: 720_000,
async act(ctx, state) {
await pollUntil({ label: "Canonical session guidance settled", deadlineAt: Date.now() + 150_000,
await pollUntil({ label: "Canonical session guidance settled", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
accept: value => value.source?.conversationState === "waiting" && value.runs.length === 2 && value.runs.every(run => run.status === "succeeded") });
accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.runs.length === 2 && value.runs.every(run => run.status === "succeeded") });
const worker = state.worker as RuntimeAgent; const task = state.task as Entity;
const query = await observeRuntime(ctx, state); state.queryReply = query.reply;
state.agentResetStatus = await probeAgentPermission(ctx, `/api/agents/${worker.id}/runtime-state/reset-session`, "POST", {});
@@ -55,6 +56,7 @@ export const runtimeSessionCases = [runtimeDefinition("runs-adapters.reset-sessi
const profile = ctx.input.page.locator("#main-content");
await expect(profile.getByRole("heading", { name: worker.name!, exact: true })).toBeVisible();
await profile.getByRole("button", { name: `Open actions for ${worker.name}`, exact: true }).click();
chatCatalogNativePhaseRemainingBudget(ctx)?.();
await ctx.input.page.getByRole("button", { name: "Reset Sessions", exact: true }).click();
await pollUntil({ label: "Human reset clears the real task session", deadlineAt: Date.now() + 30_000,
load: () => ctx.input.api.get<unknown[]>(`/api/agents/${worker.id}/task-sessions`), accept: rows => rows.length === 0 });
@@ -62,10 +64,10 @@ export const runtimeSessionCases = [runtimeDefinition("runs-adapters.reset-sessi
const beforeRuns = await ctx.allRuns();
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
const followup = "Please continue this same task using your fresh runtime session. Read the existing website-fixture.txt without changing its bytes, inspect our saved task history, and add a brief task comment confirming it is still readable. Keep existing documents and attachments unchanged, then finish this task. Do not create another task.";
await submitTaskReply(ctx.input.page, followup);
const future = await pollUntil({ label: "Actual fresh-session continuation", deadlineAt: Date.now() + 180_000,
await submitTaskReply(ctx.input.page, followup, chatCatalogNativePhaseRemainingBudget(ctx));
const future = await pollUntil({ label: "Actual fresh-session continuation", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000),
load: async () => ({ runs: await ctx.allRuns(), task: await ctx.input.api.get<Entity>(`/api/issues/${task.id}`) }),
accept: value => value.task.status === "done" && value.runs.length === beforeRuns.length + 1
accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.task.status === "done" && value.runs.length === beforeRuns.length + 1
&& value.runs.filter(run => !beforeRuns.some(prior => prior.id === run.id)).every(run => run.status === "succeeded"),
reject: value => value.runs.some(run => !beforeRuns.some(prior => prior.id === run.id) && ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Fresh-session continuation failed" : undefined });
const next = future.runs.find(run => !beforeRuns.some(prior => prior.id === run.id))!;
@@ -11,7 +11,7 @@ function context(scenarioId = "cli-operations.owned-dev-instance", deadline = 87
}
afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); });
describe("selected native phases share the once-created original flow clock", () => {
it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => {
it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure", "questions-and-approvals.approve-tool-once", "runs-adapters.reset-session"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => {
vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario);
expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000)).toBe(870_000);
vi.setSystemTime(850_000); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toBe(870_000);
@@ -2,10 +2,28 @@ import { afterEach, describe, expect, it, vi } from "vitest";
const visibility = vi.hoisted(() => vi.fn(async () => {}));
vi.mock("@playwright/test", async importOriginal => ({ ...await importOriginal<object>(), expect: () => ({ toBeVisible: visibility }) }));
import { sendChatMessage } from "./chat-flow.js";
import { submitTaskReply } from "./user-actions.js";
import { verifyInternalResult } from "./chat-catalog-domains/internal-support.js";
const remaining = () => { const value = 390_000 - Date.now(); if (value <= 0) throw new Error("original budget exhausted"); return value; };
afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); });
describe("optional original-clock callbacks reach the real shared UI helpers", () => {
it("task continuation cannot send after slow composer work exhausts the original clock", async () => {
vi.useFakeTimers(); vi.setSystemTime(389_000);
const fill = vi.fn(async () => { vi.setSystemTime(391_000); }); const click = vi.fn();
const node: any = { last: () => node, first: () => node, locator: () => node, fill, click };
await expect(submitTaskReply({ getByTestId: () => node } as never, "Continue the same task.", remaining)).rejects.toThrow(/exhausted/);
expect(fill).toHaveBeenCalledExactlyOnceWith("Continue the same task.", { timeout: 1000 }); expect(click).not.toHaveBeenCalled();
});
it.each([0, -1, NaN, Infinity])("task continuation rejects invalid remaining budget %s before UI work", async budget => {
const node: any = { last: () => node }; const prior = visibility.mock.calls.length;
await expect(submitTaskReply({ getByTestId: () => node } as never, "Continue.", () => budget)).rejects.toThrow(/exhausted/);
expect(visibility.mock.calls.length).toBe(prior);
});
it("ordinary task reply callers retain existing UI defaults", async () => {
const fill = vi.fn(); const click = vi.fn(); const node: any = { last: () => node, first: () => node, locator: () => node, fill, click };
await submitTaskReply({ getByTestId: () => node } as never, "Continue.");
expect(visibility).toHaveBeenCalledWith({ timeout: 30_000 }); expect(fill).toHaveBeenCalledExactlyOnceWith("Continue."); expect(click).toHaveBeenCalledExactlyOnceWith();
});
it("slow composer work cannot send a human message after the original deadline", async () => {
vi.useFakeTimers(); vi.setSystemTime(389_000);
const fill = vi.fn(async () => { vi.setSystemTime(391_000); }); const click = vi.fn();
+13 -6
View File
@@ -130,14 +130,21 @@ export async function createTaskThroughUi(input: {
export async function submitTaskReply(
page: Page,
body: string,
remainingBudget?: () => number,
): Promise<number> {
const composer = page.getByTestId("task-chat-composer-input").last();
await expect(composer).toBeVisible({ timeout: 30_000 });
await composer
.locator('[contenteditable="true"], textarea')
.first()
.fill(body);
const options = () => {
const remaining = remainingBudget!();
if (!Number.isFinite(remaining) || remaining <= 0) throw new Error("Task reply original observation budget exhausted");
return { timeout: Math.min(30_000, remaining) };
};
await expect(composer).toBeVisible(remainingBudget ? options() : { timeout: 30_000 });
const editor = composer.locator('[contenteditable="true"], textarea').first();
if (remainingBudget) await editor.fill(body, options());
else await editor.fill(body);
const submittedAtMs = Date.now();
await page.getByTestId("task-chat-composer-send").last().click();
const send = page.getByTestId("task-chat-composer-send").last();
if (remainingBudget) await send.click(options());
else await send.click();
return submittedAtMs;
}