From fa7304ae5ef2ad35f62855e5402127a78b560168 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 28 Oct 2026 00:28:54 -0500 Subject: [PATCH] test(e2e): share original budget across company followups --- tests/runner-e2e/chat-catalog-contract.ts | 2 + .../identity-company-budget.test.ts | 70 +++++++++++++++++++ .../identity-company-lifecycle.ts | 52 ++++++++------ .../chat-catalog-domains/identity-support.ts | 13 +++- tests/runner-e2e/chat-catalog-flow.ts | 2 +- 5 files changed, 114 insertions(+), 25 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-company-budget.test.ts diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index f6934b44be1..a02848a9123 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -6,6 +6,8 @@ import type { ChatScenario } from "./chat-feature-cases.js"; export interface ChatCatalogCheck { id: string; passed: boolean; detail: string } export interface ChatCatalogGrade { passed: boolean; checks: ChatCatalogCheck[] } export interface ChatCatalogContext { + /** Original flow observation deadline; optional opt-in for bounded human followup acts. */ + readonly observationDeadlineAt?: number; input: ChatFlowInput; scenario: ChatScenario; issue(): ChatIssue | undefined; diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-budget.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-budget.test.ts new file mode 100644 index 00000000000..63295e290fc --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-budget.test.ts @@ -0,0 +1,70 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { remainingIdentityObservationBudget, waitForReply } from "./identity-support.js"; +import { companyLifecycleIdentityCases } from "./identity-company-lifecycle.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +function context(replyAt = 180_000, deadline = 270_000): ChatCatalogContext { + return { observationDeadlineAt: deadline, refreshIssue: async () => {}, + issue: () => ({ conversationState: Date.now() >= replyAt ? "waiting" : "active" }), + allRuns: async () => [{ id: "source", status: Date.now() >= replyAt ? "succeeded" : "running" }], + } as unknown as ChatCatalogContext; +} +afterEach(() => vi.useRealTimers()); +describe("two company acts share the original observation clock", () => { + it("accepts a genuinely settled reply past150s within the unchanged original deadline", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); let done = false; + const pending = waitForReply(ctx, ctx.observationDeadlineAt).then(() => { done = true; }); + await vi.advanceTimersByTimeAsync(151_000); expect(done).toBe(false); + await vi.advanceTimersByTimeAsync(29_000); await pending; expect(done).toBe(true); + expect(remainingIdentityObservationBudget(ctx)).toBe(90_000); + }); + it("preserves the default150s behavior for unchanged callers", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); + const pending = waitForReply(ctx).then(() => "accepted", () => "timed out"); + await vi.advanceTimersByTimeAsync(150_500); expect(await pending).toBe("timed out"); + }); + it("elapsed setup is consumed, not replaced with a new300s clock", async () => { + vi.useFakeTimers(); vi.setSystemTime(200_000); const ctx = context(280_000); + expect(remainingIdentityObservationBudget(ctx)).toBe(70_000); + const pending = waitForReply(ctx, ctx.observationDeadlineAt).then(() => "accepted", () => "timed out"); + await vi.advanceTimersByTimeAsync(70_500); expect(await pending).toBe("timed out"); + expect(() => remainingIdentityObservationBudget(ctx)).toThrow(/exhausted/); + }); + it("cannot accept a still-running native or a late successful read crossing the deadline", async () => { + vi.useFakeTimers(); vi.setSystemTime(269_000); const ctx = context(0); + ctx.refreshIssue = async () => { vi.setSystemTime(271_000); }; + const pending = waitForReply(ctx, ctx.observationDeadlineAt).then(() => "accepted", () => "timed out"); + await vi.advanceTimersByTimeAsync(500); expect(await pending).toBe("timed out"); + vi.setSystemTime(269_000); const running = context(999_000); + const another = waitForReply(running, running.observationDeadlineAt).then(() => "accepted", () => "timed out"); + await vi.advanceTimersByTimeAsync(1_500); expect(await another).toBe("timed out"); + }); + it("rejects actual terminal failure immediately and missing/expired budgets fail closed", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); + ctx.allRuns = async () => [{ id: "source", status: "failed" }] as never; + await expect(waitForReply(ctx, ctx.observationDeadlineAt)).rejects.toThrow(/Initial response failed/); + for (const deadline of [undefined, NaN, Infinity, 0, -1]) expect(() => remainingIdentityObservationBudget({ observationDeadlineAt: deadline } as never)).toThrow(); + }); + it.each(["companies.switch-company", "companies.archive-company"])("%s followup cannot receive a fresh budget after a slow action", async id => { + vi.useFakeTimers(); vi.setSystemTime(260_000); const ctx = context(0); + const first = vi.fn(async () => { vi.setSystemTime(271_000); return { status: "active" }; }); + const forbidden = vi.fn(() => { throw new Error("must not start another followup"); }); + const control = { click: first, filter: () => ({ click: forbidden }) }; + ctx.input = { page: { url: () => "https://fixture.test/RUN/chats/ceo", getByRole: () => control }, + fixtures: { company: { id: "company" } }, api: { get: first, post: forbidden } } as never; + const act = companyLifecycleIdentityCases.find(row => row.scenarioId === id)!.act!; + await expect(act(ctx, {})).rejects.toThrow(/exhausted/); + expect(first).toHaveBeenCalledOnce(); expect(forbidden).not.toHaveBeenCalled(); + }); + it.each(["companies.switch-company", "companies.archive-company"])("actual %s act opts in and does not start followup after expiry", async id => { + vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); const followup = vi.fn(() => { throw new Error("real followup boundary"); }); + ctx.input = { page: { url: () => "https://fixture.test/RUN/chats/ceo", getByRole: () => ({ click: followup }) }, + fixtures: { company: { id: "company" } }, api: { get: followup } } as never; + const act = companyLifecycleIdentityCases.find(row => row.scenarioId === id)!.act!; + const pending = act(ctx, {}).then(() => "accepted", error => String(error)); + await vi.advanceTimersByTimeAsync(151_000); expect(followup).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(29_000); expect(await pending).toContain("real followup boundary"); + expect(followup).toHaveBeenCalledOnce(); + expect(followup.mock.calls[0]).toEqual(id === "companies.switch-company" ? [{ timeout: 90_000 }] : ["/api/companies/company", { timeout: 90_000 }]); + followup.mockClear(); vi.setSystemTime(270_000); await expect(act(ctx, {})).rejects.toThrow(/exhausted/); expect(followup).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts index bef720c6549..5f734b7cbd4 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts @@ -2,7 +2,7 @@ import { expect } from "@playwright/test"; import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; import { checks, entity, project, task, sameTask, type Entity } from "./internal-support.js"; -import { identityDefinition, identityObservation, observeIdentity, readAgentResource, probeAgentPermission, waitForReply } from "./identity-support.js"; +import { identityDefinition, identityObservation, observeIdentity, readAgentResource, probeAgentPermission, waitForReply, remainingIdentityObservationBudget } from "./identity-support.js"; export function gradeCompanyLifecycle(value: unknown) { const c = checks(); const o = identityObservation(value); @@ -55,19 +55,23 @@ export const companyLifecycleIdentityCases: readonly ChatCatalogCase[] = [ }, gradeCompanyLifecycle, { authentication: "human-session", sourceFiles: sources, async act(ctx, state) { - await waitForReply(ctx); const source = ctx.input.page.url(); - await ctx.input.page.getByRole("button", { name: /^Open .+ (?:organization|workspace|company) switcher$/i }).click(); - await ctx.input.page.getByRole("menuitem").filter({ hasText: /^Birch/ }).click(); + const remaining = () => remainingIdentityObservationBudget(ctx); + remaining(); + await waitForReply(ctx, ctx.observationDeadlineAt); + remaining(); + const source = ctx.input.page.url(); + await ctx.input.page.getByRole("button", { name: /^Open .+ (?:organization|workspace|company) switcher$/i }).click({ timeout: remaining() }); + await ctx.input.page.getByRole("menuitem").filter({ hasText: /^Birch/ }).click({ timeout: remaining() }); const prefix = String(state.foreignPrefix); - await expect(ctx.input.page).toHaveURL(new RegExp(`/${prefix}/`)); + await expect(ctx.input.page).toHaveURL(new RegExp(`/${prefix}/`), { timeout: remaining() }); state.switchedCompanyId = entity(state, "otherCompany").id; - await ctx.input.page.goto(`/${prefix}/projects/${entity(state, "foreignProject").id}`, { waitUntil: "domcontentloaded" }); - await expect(ctx.input.page.getByText(entity(state, "foreignTask").title!, { exact: true }).first()).toBeVisible(); + await ctx.input.page.goto(`/${prefix}/projects/${entity(state, "foreignProject").id}`, { waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(ctx.input.page.getByText(entity(state, "foreignTask").title!, { exact: true }).first()).toBeVisible({ timeout: remaining() }); state.foreignTaskVisible = true; state.browserCompanyPrefix = new URL(ctx.input.page.url()).pathname.split("/")[1]; - await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + await ctx.input.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() }); state.afterReloadVisible = await ctx.input.page.getByText(entity(state, "foreignTask").title!, { exact: true }).first().isVisible(); await ctx.input.evidence("identity-human-company-switch.json", { companyId: state.switchedCompanyId, prefix: state.browserCompanyPrefix, taskId: entity(state, "foreignTask").id, afterReloadVisible: state.afterReloadVisible }); - await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); remaining(); }, async observe(ctx, state) { state.foreignAfter = await ctx.input.api.get(`/api/issues/${entity(state, "foreignTask").id}`); return observeIdentity(ctx, state); }, async cleanup(ctx, state) { await ctx.input.api.delete(`/api/companies/${entity(state, "otherCompany").id}`, { allowNotFound: true }); }, @@ -79,24 +83,30 @@ export const companyLifecycleIdentityCases: readonly ChatCatalogCase[] = [ }, gradeCompanyLifecycle, { authentication: "human-session", sourceFiles: sources, async act(ctx, state) { - await waitForReply(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url(); - state.beforeHumanStatus = (await ctx.input.api.get(root)).status; + const remaining = () => remainingIdentityObservationBudget(ctx); + remaining(); + await waitForReply(ctx, ctx.observationDeadlineAt); + remaining(); + const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url(); + state.beforeHumanStatus = (await ctx.input.api.get(root, { timeout: remaining() })).status; + remaining(); state.agentArchiveStatus = await probeAgentPermission(ctx, `${root}/archive`, "POST", {}); - await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings`, { waitUntil: "domcontentloaded" }); + remaining(); + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings`, { waitUntil: "domcontentloaded", timeout: remaining() }); ctx.input.page.once("dialog", dialog => dialog.accept()); - await ctx.input.page.getByRole("button", { name: "Archive organization", exact: true }).click(); - state.archivedCompany = await pollUntil({ label: "human company archive persisted", deadlineAt: Date.now() + 30_000, load: () => ctx.input.api.get(root), accept: row => row.status === "archived" }); - state.archivedAgents = await pollUntil({ label: "archive agent gates", deadlineAt: Date.now() + 30_000, - load: () => ctx.input.api.get>(`${root}/agents`), accept: rows => rows.length > 0 && rows.every(row => row.status === "paused" && row.pauseReason === "company_archived") }); + await ctx.input.page.getByRole("button", { name: "Archive organization", exact: true }).click({ timeout: remaining() }); + state.archivedCompany = await pollUntil({ label: "human company archive persisted", deadlineAt: Math.min(ctx.observationDeadlineAt!, Date.now() + 30_000), load: () => ctx.input.api.get(root, { timeout: remaining() }), accept: row => row.status === "archived" }); + state.archivedAgents = await pollUntil({ label: "archive agent gates", deadlineAt: Math.min(ctx.observationDeadlineAt!, Date.now() + 30_000), + load: () => ctx.input.api.get>(`${root}/agents`, { timeout: remaining() }), accept: rows => rows.length > 0 && rows.every(row => row.status === "paused" && row.pauseReason === "company_archived") }); state.archivedActiveRuns = (await ctx.allRuns()).filter(run => ["queued", "running"].includes(run.status)).length; await ctx.input.evidence("identity-company-archived.json", { company: state.archivedCompany, agents: state.archivedAgents, activeRuns: state.archivedActiveRuns }); - await ctx.input.page.goto("/companies", { waitUntil: "domcontentloaded" }); + await ctx.input.page.goto("/companies", { waitUntil: "domcontentloaded", timeout: remaining() }); const company = state.archivedCompany as Entity; const card = ctx.input.page.getByRole("button").filter({ has: ctx.input.page.getByRole("heading", { name: company.name!, exact: true }) }); - await card.getByRole("button").last().click(); - await ctx.input.page.getByRole("menuitem", { name: "Unarchive", exact: true }).click(); - await expect.poll(async () => (await ctx.input.api.get(root)).status).toBe("active"); state.restoredViaList = true; - await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + await card.getByRole("button").last().click({ timeout: remaining() }); + await ctx.input.page.getByRole("menuitem", { name: "Unarchive", exact: true }).click({ timeout: remaining() }); + await expect.poll(async () => (await ctx.input.api.get(root, { timeout: remaining() })).status, { timeout: remaining() }).toBe("active"); state.restoredViaList = true; + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); remaining(); }, async observe(ctx, state) { state.membersAfter = await members(ctx); return observeIdentity(ctx, state); }, }), diff --git a/tests/runner-e2e/chat-catalog-domains/identity-support.ts b/tests/runner-e2e/chat-catalog-domains/identity-support.ts index d69b4bd80d8..66035c62c41 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-support.ts @@ -48,11 +48,18 @@ export function identityDefinition(scenarioId: string, setup: ChatCatalogCase["s }, }; } -export async function waitForReply(ctx: ChatCatalogContext) { +/** Opt-in callers consume the original flow clock, never a fresh outer budget. */ +export function remainingIdentityObservationBudget(ctx: ChatCatalogContext): number { + const deadline = ctx.observationDeadlineAt; + const now = Date.now(); + if (!Number.isFinite(deadline) || deadline! <= now) throw new Error("Original identity observation budget exhausted or missing"); + return deadline! - now; +} +export async function waitForReply(ctx: ChatCatalogContext, observationDeadlineAt?: number) { const { pollUntil } = await import("../api.js"); - await pollUntil({ label: "initial identity response", deadlineAt: Date.now() + 150_000, intervalMs: 500, + await pollUntil({ label: "initial identity response", deadlineAt: observationDeadlineAt ?? Date.now() + 150_000, intervalMs: 500, load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; }, - accept: ({ issue, runs }) => issue?.conversationState === "waiting" && runs.length === 1 && runs.every(run => run.status === "succeeded"), + accept: ({ issue, runs }) => (observationDeadlineAt === undefined || Date.now() < observationDeadlineAt) && issue?.conversationState === "waiting" && runs.length === 1 && runs.every(run => run.status === "succeeded"), reject: ({ runs }) => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Initial response failed" : undefined }); } export async function identityWorker(ctx: ChatCatalogContext, name = "Morgan") { diff --git a/tests/runner-e2e/chat-catalog-flow.ts b/tests/runner-e2e/chat-catalog-flow.ts index dd99da1aa48..fd6557b156c 100644 --- a/tests/runner-e2e/chat-catalog-flow.ts +++ b/tests/runner-e2e/chat-catalog-flow.ts @@ -143,7 +143,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { await input.evidence("chat-catalog-query-rejection.json", rejection); expect(rejection.passed, "Real browser/API query rejection").toBe(true); } - if (implementation.act) await implementation.act(context, state); + if (implementation.act) await implementation.act({ ...context, observationDeadlineAt: deadlineAt }, state); let settledAt: number | undefined; let passedAt: number | undefined; await pollUntil({ label: scenario.id, deadlineAt, intervalMs: 1_000, load: refresh,