test(e2e): share original budget across company followups

This commit is contained in:
Dotta committed 2026-10-28 00:37:55 -05:00
1 parent 6eca7f6bcc
commit fa7304ae5e
5 files changed
+114 -25

No files matched your search

@@ -6,6 +6,8 @@ import type { ChatScenario } from "./chat-feature-cases.js";
export interface ChatCatalogCheck { id: string; passed: boolean; detail: string }
export interface ChatCatalogGrade { passed: boolean; checks: ChatCatalogCheck[] }
export interface ChatCatalogContext {
/** Original flow observation deadline; optional opt-in for bounded human followup acts. */
readonly observationDeadlineAt?: number;
input: ChatFlowInput;
scenario: ChatScenario;
issue(): ChatIssue | undefined;
@@ -0,0 +1,70 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { remainingIdentityObservationBudget, waitForReply } from "./identity-support.js";
import { companyLifecycleIdentityCases } from "./identity-company-lifecycle.js";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
function context(replyAt = 180_000, deadline = 270_000): ChatCatalogContext {
return { observationDeadlineAt: deadline, refreshIssue: async () => {},
issue: () => ({ conversationState: Date.now() >= replyAt ? "waiting" : "active" }),
allRuns: async () => [{ id: "source", status: Date.now() >= replyAt ? "succeeded" : "running" }],
} as unknown as ChatCatalogContext;
}
afterEach(() => vi.useRealTimers());
describe("two company acts share the original observation clock", () => {
it("accepts a genuinely settled reply past150s within the unchanged original deadline", async () => {
vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); let done = false;
const pending = waitForReply(ctx, ctx.observationDeadlineAt).then(() => { done = true; });
await vi.advanceTimersByTimeAsync(151_000); expect(done).toBe(false);
await vi.advanceTimersByTimeAsync(29_000); await pending; expect(done).toBe(true);
expect(remainingIdentityObservationBudget(ctx)).toBe(90_000);
});
it("preserves the default150s behavior for unchanged callers", async () => {
vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context();
const pending = waitForReply(ctx).then(() => "accepted", () => "timed out");
await vi.advanceTimersByTimeAsync(150_500); expect(await pending).toBe("timed out");
});
it("elapsed setup is consumed, not replaced with a new300s clock", async () => {
vi.useFakeTimers(); vi.setSystemTime(200_000); const ctx = context(280_000);
expect(remainingIdentityObservationBudget(ctx)).toBe(70_000);
const pending = waitForReply(ctx, ctx.observationDeadlineAt).then(() => "accepted", () => "timed out");
await vi.advanceTimersByTimeAsync(70_500); expect(await pending).toBe("timed out");
expect(() => remainingIdentityObservationBudget(ctx)).toThrow(/exhausted/);
});
it("cannot accept a still-running native or a late successful read crossing the deadline", async () => {
vi.useFakeTimers(); vi.setSystemTime(269_000); const ctx = context(0);
ctx.refreshIssue = async () => { vi.setSystemTime(271_000); };
const pending = waitForReply(ctx, ctx.observationDeadlineAt).then(() => "accepted", () => "timed out");
await vi.advanceTimersByTimeAsync(500); expect(await pending).toBe("timed out");
vi.setSystemTime(269_000); const running = context(999_000);
const another = waitForReply(running, running.observationDeadlineAt).then(() => "accepted", () => "timed out");
await vi.advanceTimersByTimeAsync(1_500); expect(await another).toBe("timed out");
});
it("rejects actual terminal failure immediately and missing/expired budgets fail closed", async () => {
vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context();
ctx.allRuns = async () => [{ id: "source", status: "failed" }] as never;
await expect(waitForReply(ctx, ctx.observationDeadlineAt)).rejects.toThrow(/Initial response failed/);
for (const deadline of [undefined, NaN, Infinity, 0, -1]) expect(() => remainingIdentityObservationBudget({ observationDeadlineAt: deadline } as never)).toThrow();
});
it.each(["companies.switch-company", "companies.archive-company"])("%s followup cannot receive a fresh budget after a slow action", async id => {
vi.useFakeTimers(); vi.setSystemTime(260_000); const ctx = context(0);
const first = vi.fn(async () => { vi.setSystemTime(271_000); return { status: "active" }; });
const forbidden = vi.fn(() => { throw new Error("must not start another followup"); });
const control = { click: first, filter: () => ({ click: forbidden }) };
ctx.input = { page: { url: () => "https://fixture.test/RUN/chats/ceo", getByRole: () => control },
fixtures: { company: { id: "company" } }, api: { get: first, post: forbidden } } as never;
const act = companyLifecycleIdentityCases.find(row => row.scenarioId === id)!.act!;
await expect(act(ctx, {})).rejects.toThrow(/exhausted/);
expect(first).toHaveBeenCalledOnce(); expect(forbidden).not.toHaveBeenCalled();
});
it.each(["companies.switch-company", "companies.archive-company"])("actual %s act opts in and does not start followup after expiry", async id => {
vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); const followup = vi.fn(() => { throw new Error("real followup boundary"); });
ctx.input = { page: { url: () => "https://fixture.test/RUN/chats/ceo", getByRole: () => ({ click: followup }) },
fixtures: { company: { id: "company" } }, api: { get: followup } } as never;
const act = companyLifecycleIdentityCases.find(row => row.scenarioId === id)!.act!;
const pending = act(ctx, {}).then(() => "accepted", error => String(error));
await vi.advanceTimersByTimeAsync(151_000); expect(followup).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(29_000); expect(await pending).toContain("real followup boundary");
expect(followup).toHaveBeenCalledOnce();
expect(followup.mock.calls[0]).toEqual(id === "companies.switch-company" ? [{ timeout: 90_000 }] : ["/api/companies/company", { timeout: 90_000 }]);
followup.mockClear(); vi.setSystemTime(270_000); await expect(act(ctx, {})).rejects.toThrow(/exhausted/); expect(followup).not.toHaveBeenCalled();
});
});
@@ -2,7 +2,7 @@ import { expect } from "@playwright/test";
import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
import { pollUntil } from "../api.js";
import { checks, entity, project, task, sameTask, type Entity } from "./internal-support.js";
import { identityDefinition, identityObservation, observeIdentity, readAgentResource, probeAgentPermission, waitForReply } from "./identity-support.js";
import { identityDefinition, identityObservation, observeIdentity, readAgentResource, probeAgentPermission, waitForReply, remainingIdentityObservationBudget } from "./identity-support.js";
export function gradeCompanyLifecycle(value: unknown) {
const c = checks(); const o = identityObservation(value);
@@ -55,19 +55,23 @@ export const companyLifecycleIdentityCases: readonly ChatCatalogCase[] = [
}, gradeCompanyLifecycle, {
authentication: "human-session", sourceFiles: sources,
async act(ctx, state) {
await waitForReply(ctx); const source = ctx.input.page.url();
await ctx.input.page.getByRole("button", { name: /^Open .+ (?:organization|workspace|company) switcher$/i }).click();
await ctx.input.page.getByRole("menuitem").filter({ hasText: /^Birch/ }).click();
const remaining = () => remainingIdentityObservationBudget(ctx);
remaining();
await waitForReply(ctx, ctx.observationDeadlineAt);
remaining();
const source = ctx.input.page.url();
await ctx.input.page.getByRole("button", { name: /^Open .+ (?:organization|workspace|company) switcher$/i }).click({ timeout: remaining() });
await ctx.input.page.getByRole("menuitem").filter({ hasText: /^Birch/ }).click({ timeout: remaining() });
const prefix = String(state.foreignPrefix);
await expect(ctx.input.page).toHaveURL(new RegExp(`/${prefix}/`));
await expect(ctx.input.page).toHaveURL(new RegExp(`/${prefix}/`), { timeout: remaining() });
state.switchedCompanyId = entity(state, "otherCompany").id;
await ctx.input.page.goto(`/${prefix}/projects/${entity(state, "foreignProject").id}`, { waitUntil: "domcontentloaded" });
await expect(ctx.input.page.getByText(entity(state, "foreignTask").title!, { exact: true }).first()).toBeVisible();
await ctx.input.page.goto(`/${prefix}/projects/${entity(state, "foreignProject").id}`, { waitUntil: "domcontentloaded", timeout: remaining() });
await expect(ctx.input.page.getByText(entity(state, "foreignTask").title!, { exact: true }).first()).toBeVisible({ timeout: remaining() });
state.foreignTaskVisible = true; state.browserCompanyPrefix = new URL(ctx.input.page.url()).pathname.split("/")[1];
await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
await ctx.input.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() });
state.afterReloadVisible = await ctx.input.page.getByText(entity(state, "foreignTask").title!, { exact: true }).first().isVisible();
await ctx.input.evidence("identity-human-company-switch.json", { companyId: state.switchedCompanyId, prefix: state.browserCompanyPrefix, taskId: entity(state, "foreignTask").id, afterReloadVisible: state.afterReloadVisible });
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); remaining();
},
async observe(ctx, state) { state.foreignAfter = await ctx.input.api.get(`/api/issues/${entity(state, "foreignTask").id}`); return observeIdentity(ctx, state); },
async cleanup(ctx, state) { await ctx.input.api.delete(`/api/companies/${entity(state, "otherCompany").id}`, { allowNotFound: true }); },
@@ -79,24 +83,30 @@ export const companyLifecycleIdentityCases: readonly ChatCatalogCase[] = [
}, gradeCompanyLifecycle, {
authentication: "human-session", sourceFiles: sources,
async act(ctx, state) {
await waitForReply(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
state.beforeHumanStatus = (await ctx.input.api.get<Entity>(root)).status;
const remaining = () => remainingIdentityObservationBudget(ctx);
remaining();
await waitForReply(ctx, ctx.observationDeadlineAt);
remaining();
const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
state.beforeHumanStatus = (await ctx.input.api.get<Entity>(root, { timeout: remaining() })).status;
remaining();
state.agentArchiveStatus = await probeAgentPermission(ctx, `${root}/archive`, "POST", {});
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings`, { waitUntil: "domcontentloaded" });
remaining();
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings`, { waitUntil: "domcontentloaded", timeout: remaining() });
ctx.input.page.once("dialog", dialog => dialog.accept());
await ctx.input.page.getByRole("button", { name: "Archive organization", exact: true }).click();
state.archivedCompany = await pollUntil({ label: "human company archive persisted", deadlineAt: Date.now() + 30_000, load: () => ctx.input.api.get<Entity>(root), accept: row => row.status === "archived" });
state.archivedAgents = await pollUntil({ label: "archive agent gates", deadlineAt: Date.now() + 30_000,
load: () => ctx.input.api.get<Array<{ id: string; status: string; pauseReason: string }>>(`${root}/agents`), accept: rows => rows.length > 0 && rows.every(row => row.status === "paused" && row.pauseReason === "company_archived") });
await ctx.input.page.getByRole("button", { name: "Archive organization", exact: true }).click({ timeout: remaining() });
state.archivedCompany = await pollUntil({ label: "human company archive persisted", deadlineAt: Math.min(ctx.observationDeadlineAt!, Date.now() + 30_000), load: () => ctx.input.api.get<Entity>(root, { timeout: remaining() }), accept: row => row.status === "archived" });
state.archivedAgents = await pollUntil({ label: "archive agent gates", deadlineAt: Math.min(ctx.observationDeadlineAt!, Date.now() + 30_000),
load: () => ctx.input.api.get<Array<{ id: string; status: string; pauseReason: string }>>(`${root}/agents`, { timeout: remaining() }), accept: rows => rows.length > 0 && rows.every(row => row.status === "paused" && row.pauseReason === "company_archived") });
state.archivedActiveRuns = (await ctx.allRuns()).filter(run => ["queued", "running"].includes(run.status)).length;
await ctx.input.evidence("identity-company-archived.json", { company: state.archivedCompany, agents: state.archivedAgents, activeRuns: state.archivedActiveRuns });
await ctx.input.page.goto("/companies", { waitUntil: "domcontentloaded" });
await ctx.input.page.goto("/companies", { waitUntil: "domcontentloaded", timeout: remaining() });
const company = state.archivedCompany as Entity;
const card = ctx.input.page.getByRole("button").filter({ has: ctx.input.page.getByRole("heading", { name: company.name!, exact: true }) });
await card.getByRole("button").last().click();
await ctx.input.page.getByRole("menuitem", { name: "Unarchive", exact: true }).click();
await expect.poll(async () => (await ctx.input.api.get<Entity>(root)).status).toBe("active"); state.restoredViaList = true;
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
await card.getByRole("button").last().click({ timeout: remaining() });
await ctx.input.page.getByRole("menuitem", { name: "Unarchive", exact: true }).click({ timeout: remaining() });
await expect.poll(async () => (await ctx.input.api.get<Entity>(root, { timeout: remaining() })).status, { timeout: remaining() }).toBe("active"); state.restoredViaList = true;
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); remaining();
},
async observe(ctx, state) { state.membersAfter = await members(ctx); return observeIdentity(ctx, state); },
}),
@@ -48,11 +48,18 @@ export function identityDefinition(scenarioId: string, setup: ChatCatalogCase["s
},
};
}
export async function waitForReply(ctx: ChatCatalogContext) {
/** Opt-in callers consume the original flow clock, never a fresh outer budget. */
export function remainingIdentityObservationBudget(ctx: ChatCatalogContext): number {
const deadline = ctx.observationDeadlineAt;
const now = Date.now();
if (!Number.isFinite(deadline) || deadline! <= now) throw new Error("Original identity observation budget exhausted or missing");
return deadline! - now;
}
export async function waitForReply(ctx: ChatCatalogContext, observationDeadlineAt?: number) {
const { pollUntil } = await import("../api.js");
await pollUntil({ label: "initial identity response", deadlineAt: Date.now() + 150_000, intervalMs: 500,
await pollUntil({ label: "initial identity response", deadlineAt: observationDeadlineAt ?? Date.now() + 150_000, intervalMs: 500,
load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; },
accept: ({ issue, runs }) => issue?.conversationState === "waiting" && runs.length === 1 && runs.every(run => run.status === "succeeded"),
accept: ({ issue, runs }) => (observationDeadlineAt === undefined || Date.now() < observationDeadlineAt) && issue?.conversationState === "waiting" && runs.length === 1 && runs.every(run => run.status === "succeeded"),
reject: ({ runs }) => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Initial response failed" : undefined });
}
export async function identityWorker(ctx: ChatCatalogContext, name = "Morgan") {
+1 -1
View File
@@ -143,7 +143,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
await input.evidence("chat-catalog-query-rejection.json", rejection);
expect(rejection.passed, "Real browser/API query rejection").toBe(true);
}
if (implementation.act) await implementation.act(context, state);
if (implementation.act) await implementation.act({ ...context, observationDeadlineAt: deadlineAt }, state);
let settledAt: number | undefined;
let passedAt: number | undefined;
await pollUntil({ label: scenario.id, deadlineAt, intervalMs: 1_000, load: refresh,