mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-05 03:51:54 +01:00
test(e2e): share original budget across company followups
This commit is contained in:
1 parent
6eca7f6bcc
commit
fa7304ae5e
5 files changed
+114
-25
No files matched your search
@@ -6,6 +6,8 @@ import type { ChatScenario } from "./chat-feature-cases.js";
|
||||
export interface ChatCatalogCheck { id: string; passed: boolean; detail: string }
|
||||
export interface ChatCatalogGrade { passed: boolean; checks: ChatCatalogCheck[] }
|
||||
export interface ChatCatalogContext {
|
||||
/** Original flow observation deadline; optional opt-in for bounded human followup acts. */
|
||||
readonly observationDeadlineAt?: number;
|
||||
input: ChatFlowInput;
|
||||
scenario: ChatScenario;
|
||||
issue(): ChatIssue | undefined;
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { remainingIdentityObservationBudget, waitForReply } from "./identity-support.js";
|
||||
import { companyLifecycleIdentityCases } from "./identity-company-lifecycle.js";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
function context(replyAt = 180_000, deadline = 270_000): ChatCatalogContext {
|
||||
return { observationDeadlineAt: deadline, refreshIssue: async () => {},
|
||||
issue: () => ({ conversationState: Date.now() >= replyAt ? "waiting" : "active" }),
|
||||
allRuns: async () => [{ id: "source", status: Date.now() >= replyAt ? "succeeded" : "running" }],
|
||||
} as unknown as ChatCatalogContext;
|
||||
}
|
||||
afterEach(() => vi.useRealTimers());
|
||||
describe("two company acts share the original observation clock", () => {
|
||||
it("accepts a genuinely settled reply past150s within the unchanged original deadline", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); let done = false;
|
||||
const pending = waitForReply(ctx, ctx.observationDeadlineAt).then(() => { done = true; });
|
||||
await vi.advanceTimersByTimeAsync(151_000); expect(done).toBe(false);
|
||||
await vi.advanceTimersByTimeAsync(29_000); await pending; expect(done).toBe(true);
|
||||
expect(remainingIdentityObservationBudget(ctx)).toBe(90_000);
|
||||
});
|
||||
it("preserves the default150s behavior for unchanged callers", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context();
|
||||
const pending = waitForReply(ctx).then(() => "accepted", () => "timed out");
|
||||
await vi.advanceTimersByTimeAsync(150_500); expect(await pending).toBe("timed out");
|
||||
});
|
||||
it("elapsed setup is consumed, not replaced with a new300s clock", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(200_000); const ctx = context(280_000);
|
||||
expect(remainingIdentityObservationBudget(ctx)).toBe(70_000);
|
||||
const pending = waitForReply(ctx, ctx.observationDeadlineAt).then(() => "accepted", () => "timed out");
|
||||
await vi.advanceTimersByTimeAsync(70_500); expect(await pending).toBe("timed out");
|
||||
expect(() => remainingIdentityObservationBudget(ctx)).toThrow(/exhausted/);
|
||||
});
|
||||
it("cannot accept a still-running native or a late successful read crossing the deadline", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(269_000); const ctx = context(0);
|
||||
ctx.refreshIssue = async () => { vi.setSystemTime(271_000); };
|
||||
const pending = waitForReply(ctx, ctx.observationDeadlineAt).then(() => "accepted", () => "timed out");
|
||||
await vi.advanceTimersByTimeAsync(500); expect(await pending).toBe("timed out");
|
||||
vi.setSystemTime(269_000); const running = context(999_000);
|
||||
const another = waitForReply(running, running.observationDeadlineAt).then(() => "accepted", () => "timed out");
|
||||
await vi.advanceTimersByTimeAsync(1_500); expect(await another).toBe("timed out");
|
||||
});
|
||||
it("rejects actual terminal failure immediately and missing/expired budgets fail closed", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context();
|
||||
ctx.allRuns = async () => [{ id: "source", status: "failed" }] as never;
|
||||
await expect(waitForReply(ctx, ctx.observationDeadlineAt)).rejects.toThrow(/Initial response failed/);
|
||||
for (const deadline of [undefined, NaN, Infinity, 0, -1]) expect(() => remainingIdentityObservationBudget({ observationDeadlineAt: deadline } as never)).toThrow();
|
||||
});
|
||||
it.each(["companies.switch-company", "companies.archive-company"])("%s followup cannot receive a fresh budget after a slow action", async id => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(260_000); const ctx = context(0);
|
||||
const first = vi.fn(async () => { vi.setSystemTime(271_000); return { status: "active" }; });
|
||||
const forbidden = vi.fn(() => { throw new Error("must not start another followup"); });
|
||||
const control = { click: first, filter: () => ({ click: forbidden }) };
|
||||
ctx.input = { page: { url: () => "https://fixture.test/RUN/chats/ceo", getByRole: () => control },
|
||||
fixtures: { company: { id: "company" } }, api: { get: first, post: forbidden } } as never;
|
||||
const act = companyLifecycleIdentityCases.find(row => row.scenarioId === id)!.act!;
|
||||
await expect(act(ctx, {})).rejects.toThrow(/exhausted/);
|
||||
expect(first).toHaveBeenCalledOnce(); expect(forbidden).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each(["companies.switch-company", "companies.archive-company"])("actual %s act opts in and does not start followup after expiry", async id => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); const followup = vi.fn(() => { throw new Error("real followup boundary"); });
|
||||
ctx.input = { page: { url: () => "https://fixture.test/RUN/chats/ceo", getByRole: () => ({ click: followup }) },
|
||||
fixtures: { company: { id: "company" } }, api: { get: followup } } as never;
|
||||
const act = companyLifecycleIdentityCases.find(row => row.scenarioId === id)!.act!;
|
||||
const pending = act(ctx, {}).then(() => "accepted", error => String(error));
|
||||
await vi.advanceTimersByTimeAsync(151_000); expect(followup).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(29_000); expect(await pending).toContain("real followup boundary");
|
||||
expect(followup).toHaveBeenCalledOnce();
|
||||
expect(followup.mock.calls[0]).toEqual(id === "companies.switch-company" ? [{ timeout: 90_000 }] : ["/api/companies/company", { timeout: 90_000 }]);
|
||||
followup.mockClear(); vi.setSystemTime(270_000); await expect(act(ctx, {})).rejects.toThrow(/exhausted/); expect(followup).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,7 @@ import { expect } from "@playwright/test";
|
||||
import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
import { pollUntil } from "../api.js";
|
||||
import { checks, entity, project, task, sameTask, type Entity } from "./internal-support.js";
|
||||
import { identityDefinition, identityObservation, observeIdentity, readAgentResource, probeAgentPermission, waitForReply } from "./identity-support.js";
|
||||
import { identityDefinition, identityObservation, observeIdentity, readAgentResource, probeAgentPermission, waitForReply, remainingIdentityObservationBudget } from "./identity-support.js";
|
||||
|
||||
export function gradeCompanyLifecycle(value: unknown) {
|
||||
const c = checks(); const o = identityObservation(value);
|
||||
@@ -55,19 +55,23 @@ export const companyLifecycleIdentityCases: readonly ChatCatalogCase[] = [
|
||||
}, gradeCompanyLifecycle, {
|
||||
authentication: "human-session", sourceFiles: sources,
|
||||
async act(ctx, state) {
|
||||
await waitForReply(ctx); const source = ctx.input.page.url();
|
||||
await ctx.input.page.getByRole("button", { name: /^Open .+ (?:organization|workspace|company) switcher$/i }).click();
|
||||
await ctx.input.page.getByRole("menuitem").filter({ hasText: /^Birch/ }).click();
|
||||
const remaining = () => remainingIdentityObservationBudget(ctx);
|
||||
remaining();
|
||||
await waitForReply(ctx, ctx.observationDeadlineAt);
|
||||
remaining();
|
||||
const source = ctx.input.page.url();
|
||||
await ctx.input.page.getByRole("button", { name: /^Open .+ (?:organization|workspace|company) switcher$/i }).click({ timeout: remaining() });
|
||||
await ctx.input.page.getByRole("menuitem").filter({ hasText: /^Birch/ }).click({ timeout: remaining() });
|
||||
const prefix = String(state.foreignPrefix);
|
||||
await expect(ctx.input.page).toHaveURL(new RegExp(`/${prefix}/`));
|
||||
await expect(ctx.input.page).toHaveURL(new RegExp(`/${prefix}/`), { timeout: remaining() });
|
||||
state.switchedCompanyId = entity(state, "otherCompany").id;
|
||||
await ctx.input.page.goto(`/${prefix}/projects/${entity(state, "foreignProject").id}`, { waitUntil: "domcontentloaded" });
|
||||
await expect(ctx.input.page.getByText(entity(state, "foreignTask").title!, { exact: true }).first()).toBeVisible();
|
||||
await ctx.input.page.goto(`/${prefix}/projects/${entity(state, "foreignProject").id}`, { waitUntil: "domcontentloaded", timeout: remaining() });
|
||||
await expect(ctx.input.page.getByText(entity(state, "foreignTask").title!, { exact: true }).first()).toBeVisible({ timeout: remaining() });
|
||||
state.foreignTaskVisible = true; state.browserCompanyPrefix = new URL(ctx.input.page.url()).pathname.split("/")[1];
|
||||
await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
|
||||
await ctx.input.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() });
|
||||
state.afterReloadVisible = await ctx.input.page.getByText(entity(state, "foreignTask").title!, { exact: true }).first().isVisible();
|
||||
await ctx.input.evidence("identity-human-company-switch.json", { companyId: state.switchedCompanyId, prefix: state.browserCompanyPrefix, taskId: entity(state, "foreignTask").id, afterReloadVisible: state.afterReloadVisible });
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); remaining();
|
||||
},
|
||||
async observe(ctx, state) { state.foreignAfter = await ctx.input.api.get(`/api/issues/${entity(state, "foreignTask").id}`); return observeIdentity(ctx, state); },
|
||||
async cleanup(ctx, state) { await ctx.input.api.delete(`/api/companies/${entity(state, "otherCompany").id}`, { allowNotFound: true }); },
|
||||
@@ -79,24 +83,30 @@ export const companyLifecycleIdentityCases: readonly ChatCatalogCase[] = [
|
||||
}, gradeCompanyLifecycle, {
|
||||
authentication: "human-session", sourceFiles: sources,
|
||||
async act(ctx, state) {
|
||||
await waitForReply(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
|
||||
state.beforeHumanStatus = (await ctx.input.api.get<Entity>(root)).status;
|
||||
const remaining = () => remainingIdentityObservationBudget(ctx);
|
||||
remaining();
|
||||
await waitForReply(ctx, ctx.observationDeadlineAt);
|
||||
remaining();
|
||||
const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
|
||||
state.beforeHumanStatus = (await ctx.input.api.get<Entity>(root, { timeout: remaining() })).status;
|
||||
remaining();
|
||||
state.agentArchiveStatus = await probeAgentPermission(ctx, `${root}/archive`, "POST", {});
|
||||
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings`, { waitUntil: "domcontentloaded" });
|
||||
remaining();
|
||||
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings`, { waitUntil: "domcontentloaded", timeout: remaining() });
|
||||
ctx.input.page.once("dialog", dialog => dialog.accept());
|
||||
await ctx.input.page.getByRole("button", { name: "Archive organization", exact: true }).click();
|
||||
state.archivedCompany = await pollUntil({ label: "human company archive persisted", deadlineAt: Date.now() + 30_000, load: () => ctx.input.api.get<Entity>(root), accept: row => row.status === "archived" });
|
||||
state.archivedAgents = await pollUntil({ label: "archive agent gates", deadlineAt: Date.now() + 30_000,
|
||||
load: () => ctx.input.api.get<Array<{ id: string; status: string; pauseReason: string }>>(`${root}/agents`), accept: rows => rows.length > 0 && rows.every(row => row.status === "paused" && row.pauseReason === "company_archived") });
|
||||
await ctx.input.page.getByRole("button", { name: "Archive organization", exact: true }).click({ timeout: remaining() });
|
||||
state.archivedCompany = await pollUntil({ label: "human company archive persisted", deadlineAt: Math.min(ctx.observationDeadlineAt!, Date.now() + 30_000), load: () => ctx.input.api.get<Entity>(root, { timeout: remaining() }), accept: row => row.status === "archived" });
|
||||
state.archivedAgents = await pollUntil({ label: "archive agent gates", deadlineAt: Math.min(ctx.observationDeadlineAt!, Date.now() + 30_000),
|
||||
load: () => ctx.input.api.get<Array<{ id: string; status: string; pauseReason: string }>>(`${root}/agents`, { timeout: remaining() }), accept: rows => rows.length > 0 && rows.every(row => row.status === "paused" && row.pauseReason === "company_archived") });
|
||||
state.archivedActiveRuns = (await ctx.allRuns()).filter(run => ["queued", "running"].includes(run.status)).length;
|
||||
await ctx.input.evidence("identity-company-archived.json", { company: state.archivedCompany, agents: state.archivedAgents, activeRuns: state.archivedActiveRuns });
|
||||
await ctx.input.page.goto("/companies", { waitUntil: "domcontentloaded" });
|
||||
await ctx.input.page.goto("/companies", { waitUntil: "domcontentloaded", timeout: remaining() });
|
||||
const company = state.archivedCompany as Entity;
|
||||
const card = ctx.input.page.getByRole("button").filter({ has: ctx.input.page.getByRole("heading", { name: company.name!, exact: true }) });
|
||||
await card.getByRole("button").last().click();
|
||||
await ctx.input.page.getByRole("menuitem", { name: "Unarchive", exact: true }).click();
|
||||
await expect.poll(async () => (await ctx.input.api.get<Entity>(root)).status).toBe("active"); state.restoredViaList = true;
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
|
||||
await card.getByRole("button").last().click({ timeout: remaining() });
|
||||
await ctx.input.page.getByRole("menuitem", { name: "Unarchive", exact: true }).click({ timeout: remaining() });
|
||||
await expect.poll(async () => (await ctx.input.api.get<Entity>(root, { timeout: remaining() })).status, { timeout: remaining() }).toBe("active"); state.restoredViaList = true;
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); remaining();
|
||||
},
|
||||
async observe(ctx, state) { state.membersAfter = await members(ctx); return observeIdentity(ctx, state); },
|
||||
}),
|
||||
|
||||
@@ -48,11 +48,18 @@ export function identityDefinition(scenarioId: string, setup: ChatCatalogCase["s
|
||||
},
|
||||
};
|
||||
}
|
||||
export async function waitForReply(ctx: ChatCatalogContext) {
|
||||
/** Opt-in callers consume the original flow clock, never a fresh outer budget. */
|
||||
export function remainingIdentityObservationBudget(ctx: ChatCatalogContext): number {
|
||||
const deadline = ctx.observationDeadlineAt;
|
||||
const now = Date.now();
|
||||
if (!Number.isFinite(deadline) || deadline! <= now) throw new Error("Original identity observation budget exhausted or missing");
|
||||
return deadline! - now;
|
||||
}
|
||||
export async function waitForReply(ctx: ChatCatalogContext, observationDeadlineAt?: number) {
|
||||
const { pollUntil } = await import("../api.js");
|
||||
await pollUntil({ label: "initial identity response", deadlineAt: Date.now() + 150_000, intervalMs: 500,
|
||||
await pollUntil({ label: "initial identity response", deadlineAt: observationDeadlineAt ?? Date.now() + 150_000, intervalMs: 500,
|
||||
load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; },
|
||||
accept: ({ issue, runs }) => issue?.conversationState === "waiting" && runs.length === 1 && runs.every(run => run.status === "succeeded"),
|
||||
accept: ({ issue, runs }) => (observationDeadlineAt === undefined || Date.now() < observationDeadlineAt) && issue?.conversationState === "waiting" && runs.length === 1 && runs.every(run => run.status === "succeeded"),
|
||||
reject: ({ runs }) => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Initial response failed" : undefined });
|
||||
}
|
||||
export async function identityWorker(ctx: ChatCatalogContext, name = "Morgan") {
|
||||
|
||||
@@ -143,7 +143,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
|
||||
await input.evidence("chat-catalog-query-rejection.json", rejection);
|
||||
expect(rejection.passed, "Real browser/API query rejection").toBe(true);
|
||||
}
|
||||
if (implementation.act) await implementation.act(context, state);
|
||||
if (implementation.act) await implementation.act({ ...context, observationDeadlineAt: deadlineAt }, state);
|
||||
let settledAt: number | undefined;
|
||||
let passedAt: number | undefined;
|
||||
await pollUntil({ label: scenario.id, deadlineAt, intervalMs: 1_000, load: refresh,
|
||||
|
||||
Reference in new issue
Block a user