mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-06 14:02:36 +01:00
test(e2e): preserve original approval and session clocks
This commit is contained in:
1 parent
a477b2c5d8
commit
7170ca4b81
7 files changed
+82
-17
No files matched your search
@@ -0,0 +1,33 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import type { ChatCatalogContext } from "./chat-catalog-contract.js";
|
||||
const poll = vi.hoisted(() => vi.fn());
|
||||
vi.mock("./api.js", async original => ({ ...await original<object>(), pollUntil: poll }));
|
||||
vi.mock("../fixtures/connection-review-provider.js", () => ({ startReviewProvider: async () => ({ url: "http://fixture.invalid", captures: [], close: async () => {} }) }));
|
||||
import { toolReviewIdentityCases } from "./chat-catalog-domains/identity-tool-review.js";
|
||||
import { runtimeSessionCases } from "./chat-catalog-domains/runtime-session.js";
|
||||
|
||||
afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); });
|
||||
function context(id: string, deadline: number) {
|
||||
return { observationDeadlineAt: deadline, scenario: { id }, input: { nonce: "unit", privateSecrets: [], execution: { id: `chat-features.runner-opencode.local.${id}` }, fixtures: { agent: { id: "ceo" }, company: { id: "company" } }, api: {
|
||||
get: async () => ({ runtimeConfig: {} }), patch: async () => ({}), post: async () => ({}), putSensitive: async () => ({}),
|
||||
postSensitive: async () => ({ connectionId: "connection", actions: { readOnly: [], canMakeChanges: [] } }),
|
||||
} }, issue: () => ({ id: "chat", conversationState: "waiting" }), refreshIssue: async () => {}, allRuns: async () => [] } as unknown as ChatCatalogContext;
|
||||
}
|
||||
it("actual approval request wait consumes elapsed setup and rejects a late pending card without changing review UI", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(200_000);
|
||||
const ctx = context("questions-and-approvals.approve-tool-once", 570_000), definition = toolReviewIdentityCases[0]!;
|
||||
const state = await definition.setup(ctx); poll.mockRejectedValue(new Error("captured phase"));
|
||||
await expect(definition.act!(ctx, state)).rejects.toThrow("captured phase");
|
||||
const options = poll.mock.calls[0]![0]; expect(options.deadlineAt).toBe(570_000);
|
||||
const pending = [{ status: "pending", payload: { toolAction: { connectionId: "connection" } } }];
|
||||
expect(options.accept(pending)).toBe(true); vi.setSystemTime(570_000); expect(options.accept(pending)).toBe(false);
|
||||
});
|
||||
it("actual reset canonical wait uses the same setup-inclusive clock and retains exact successful-run prerequisites", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(200_000);
|
||||
const ctx = context("runs-adapters.reset-session", 690_000); poll.mockRejectedValue(new Error("captured phase"));
|
||||
await expect(runtimeSessionCases[0]!.act!(ctx, {})).rejects.toThrow("captured phase");
|
||||
const options = poll.mock.calls[0]![0]; expect(options.deadlineAt).toBe(690_000);
|
||||
const valid = { source: { conversationState: "waiting" }, runs: [{ status: "succeeded" }, { status: "succeeded" }] };
|
||||
expect(options.accept(valid)).toBe(true); expect(options.accept({ ...valid, runs: [{ status: "succeeded" }, { status: "running" }] })).toBe(false);
|
||||
vi.setSystemTime(690_000); expect(options.accept(valid)).toBe(false);
|
||||
});
|
||||
@@ -26,6 +26,8 @@ export interface ChatCatalogContext {
|
||||
/** These exact selected OpenCode/local journeys use the original flow clock for
|
||||
* native observation phases. Other cases/lanes keep their authored phase caps. */
|
||||
const originalClockExecutionIds = new Set([
|
||||
"chat-features.runner-opencode.local.questions-and-approvals.approve-tool-once",
|
||||
"chat-features.runner-opencode.local.runs-adapters.reset-session",
|
||||
"chat-features.runner-opencode.local.tasks.website-without-project",
|
||||
"chat-features.runner-opencode.local.inbox-search.archive-partial-failure",
|
||||
"chat-features.runner-opencode.local.cli-operations.feedback-bundle",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { expect, type Page } from "@playwright/test";
|
||||
import { startReviewProvider } from "../../fixtures/connection-review-provider.js";
|
||||
@@ -105,19 +106,21 @@ export const toolReviewIdentityCases = ["approve-tool-once", "decline-tool"].map
|
||||
state.beforeDecisionCalls = calls(provider).length;
|
||||
const review = await openIdentityToolReview(ctx.input.page, card, String(state.connectionId)); state.reviewVisible = true;
|
||||
state.beforeDecisionCommentIds = (await comments(ctx)).map(row => row.id);
|
||||
chatCatalogNativePhaseRemainingBudget(ctx)?.();
|
||||
await review.getByRole("button", { name: id === "decline-tool" ? "Decline" : "Approve & run", exact: true }).click();
|
||||
if (id === "approve-tool-once") {
|
||||
await pollUntil({ label: "first approved synthetic read completed", deadlineAt: Date.now() + 150_000,
|
||||
await pollUntil({ label: "first approved synthetic read completed", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
|
||||
load: async () => { await ctx.refreshIssue(); const rows = await ctx.input.api.get<Interaction[]>(`/api/issues/${ctx.issue()!.id}/interactions`);
|
||||
return { rows, runs: await ctx.allRuns(), source: ctx.issue(), comments: await comments(ctx) }; },
|
||||
accept: value => value.rows.some(row => row.id === card.id && row.result?.toolAction?.status === "executed")
|
||||
accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.rows.some(row => row.id === card.id && row.result?.toolAction?.status === "executed")
|
||||
&& value.source?.conversationState === "waiting" && value.runs.every(run => run.status === "succeeded")
|
||||
&& value.comments.some(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.body.includes(String(state.marker))) });
|
||||
state.firstReply = (await comments(ctx)).filter(row => row.authorAgentId === ctx.input.fixtures.agent.id).map(row => row.body).join("\n");
|
||||
await sendChatMessage(ctx.input.page, "Read the launch notes again using the app's diagnostic query value fail, so we can check its unavailable-service response.");
|
||||
await sendChatMessage(ctx.input.page, "Read the launch notes again using the app's diagnostic query value fail, so we can check its unavailable-service response.", chatCatalogNativePhaseRemainingBudget(ctx));
|
||||
const second = await pendingReview(ctx, state, provider, 1); state.secondDecisionId = second.id; state.beforeSecondDecisionCalls = calls(provider).length;
|
||||
const secondReview = await openIdentityToolReview(ctx.input.page, second, String(state.connectionId)); state.secondReviewVisible = true;
|
||||
state.beforeDecisionCommentIds = (await comments(ctx)).map(row => row.id);
|
||||
chatCatalogNativePhaseRemainingBudget(ctx)?.();
|
||||
await secondReview.getByRole("button", { name: "Approve & run", exact: true }).click();
|
||||
}
|
||||
},
|
||||
@@ -141,9 +144,9 @@ export const toolReviewIdentityCases = ["approve-tool-once", "decline-tool"].map
|
||||
}));
|
||||
async function comments(ctx: ChatCatalogContext) { return ctx.input.api.get<Array<{ id: string; authorAgentId?: string; body: string }>>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); }
|
||||
async function pendingReview(ctx: ChatCatalogContext, state: Record<string, unknown>, provider: Provider, previousCalls = 0) {
|
||||
const rows = await pollUntil({ label: "scoped synthetic service tool review", deadlineAt: Date.now() + 150_000,
|
||||
const rows = await pollUntil({ label: "scoped synthetic service tool review", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
|
||||
load: async () => { await ctx.refreshIssue(); return ctx.issue() ? ctx.input.api.get<Interaction[]>(`/api/issues/${ctx.issue()!.id}/interactions`) : []; },
|
||||
accept: rows => rows.some(row => row.status === "pending" && row.payload.toolAction?.connectionId === state.connectionId),
|
||||
accept: rows => chatCatalogNativePhaseIsTimely(ctx) && rows.some(row => row.status === "pending" && row.payload.toolAction?.connectionId === state.connectionId),
|
||||
reject: () => calls(provider).length > previousCalls ? "Synthetic provider was called before this action's approval" : undefined });
|
||||
return pendingStoryDecision(rows, { kind: "tool", connectionId: String(state.connectionId) });
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
|
||||
import { expect } from "@playwright/test";
|
||||
import { pollUntil } from "../api.js";
|
||||
import { submitTaskReply } from "../user-actions.js";
|
||||
@@ -44,9 +45,9 @@ export const runtimeSessionCases = [runtimeDefinition("runs-adapters.reset-sessi
|
||||
sourceFiles: ["chat-catalog-domains/runtime-session.ts", "chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/runs-adapters.yaml", "../../ui/src/components/AgentActionButtons.tsx", "../../ui/src/pages/AgentDetail.tsx"],
|
||||
expectedRunCount: 3, attemptTimeoutMs: 720_000,
|
||||
async act(ctx, state) {
|
||||
await pollUntil({ label: "Canonical session guidance settled", deadlineAt: Date.now() + 150_000,
|
||||
await pollUntil({ label: "Canonical session guidance settled", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
|
||||
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
|
||||
accept: value => value.source?.conversationState === "waiting" && value.runs.length === 2 && value.runs.every(run => run.status === "succeeded") });
|
||||
accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.source?.conversationState === "waiting" && value.runs.length === 2 && value.runs.every(run => run.status === "succeeded") });
|
||||
const worker = state.worker as RuntimeAgent; const task = state.task as Entity;
|
||||
const query = await observeRuntime(ctx, state); state.queryReply = query.reply;
|
||||
state.agentResetStatus = await probeAgentPermission(ctx, `/api/agents/${worker.id}/runtime-state/reset-session`, "POST", {});
|
||||
@@ -55,6 +56,7 @@ export const runtimeSessionCases = [runtimeDefinition("runs-adapters.reset-sessi
|
||||
const profile = ctx.input.page.locator("#main-content");
|
||||
await expect(profile.getByRole("heading", { name: worker.name!, exact: true })).toBeVisible();
|
||||
await profile.getByRole("button", { name: `Open actions for ${worker.name}`, exact: true }).click();
|
||||
chatCatalogNativePhaseRemainingBudget(ctx)?.();
|
||||
await ctx.input.page.getByRole("button", { name: "Reset Sessions", exact: true }).click();
|
||||
await pollUntil({ label: "Human reset clears the real task session", deadlineAt: Date.now() + 30_000,
|
||||
load: () => ctx.input.api.get<unknown[]>(`/api/agents/${worker.id}/task-sessions`), accept: rows => rows.length === 0 });
|
||||
@@ -62,10 +64,10 @@ export const runtimeSessionCases = [runtimeDefinition("runs-adapters.reset-sessi
|
||||
const beforeRuns = await ctx.allRuns();
|
||||
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
|
||||
const followup = "Please continue this same task using your fresh runtime session. Read the existing website-fixture.txt without changing its bytes, inspect our saved task history, and add a brief task comment confirming it is still readable. Keep existing documents and attachments unchanged, then finish this task. Do not create another task.";
|
||||
await submitTaskReply(ctx.input.page, followup);
|
||||
const future = await pollUntil({ label: "Actual fresh-session continuation", deadlineAt: Date.now() + 180_000,
|
||||
await submitTaskReply(ctx.input.page, followup, chatCatalogNativePhaseRemainingBudget(ctx));
|
||||
const future = await pollUntil({ label: "Actual fresh-session continuation", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000),
|
||||
load: async () => ({ runs: await ctx.allRuns(), task: await ctx.input.api.get<Entity>(`/api/issues/${task.id}`) }),
|
||||
accept: value => value.task.status === "done" && value.runs.length === beforeRuns.length + 1
|
||||
accept: value => chatCatalogNativePhaseIsTimely(ctx) && value.task.status === "done" && value.runs.length === beforeRuns.length + 1
|
||||
&& value.runs.filter(run => !beforeRuns.some(prior => prior.id === run.id)).every(run => run.status === "succeeded"),
|
||||
reject: value => value.runs.some(run => !beforeRuns.some(prior => prior.id === run.id) && ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Fresh-session continuation failed" : undefined });
|
||||
const next = future.runs.find(run => !beforeRuns.some(prior => prior.id === run.id))!;
|
||||
|
||||
@@ -11,7 +11,7 @@ function context(scenarioId = "cli-operations.owned-dev-instance", deadline = 87
|
||||
}
|
||||
afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); });
|
||||
describe("selected native phases share the once-created original flow clock", () => {
|
||||
it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => {
|
||||
it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure", "questions-and-approvals.approve-tool-once", "runs-adapters.reset-session"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario);
|
||||
expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000)).toBe(870_000);
|
||||
vi.setSystemTime(850_000); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toBe(870_000);
|
||||
|
||||
@@ -2,10 +2,28 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
const visibility = vi.hoisted(() => vi.fn(async () => {}));
|
||||
vi.mock("@playwright/test", async importOriginal => ({ ...await importOriginal<object>(), expect: () => ({ toBeVisible: visibility }) }));
|
||||
import { sendChatMessage } from "./chat-flow.js";
|
||||
import { submitTaskReply } from "./user-actions.js";
|
||||
import { verifyInternalResult } from "./chat-catalog-domains/internal-support.js";
|
||||
const remaining = () => { const value = 390_000 - Date.now(); if (value <= 0) throw new Error("original budget exhausted"); return value; };
|
||||
afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); });
|
||||
describe("optional original-clock callbacks reach the real shared UI helpers", () => {
|
||||
it("task continuation cannot send after slow composer work exhausts the original clock", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(389_000);
|
||||
const fill = vi.fn(async () => { vi.setSystemTime(391_000); }); const click = vi.fn();
|
||||
const node: any = { last: () => node, first: () => node, locator: () => node, fill, click };
|
||||
await expect(submitTaskReply({ getByTestId: () => node } as never, "Continue the same task.", remaining)).rejects.toThrow(/exhausted/);
|
||||
expect(fill).toHaveBeenCalledExactlyOnceWith("Continue the same task.", { timeout: 1000 }); expect(click).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([0, -1, NaN, Infinity])("task continuation rejects invalid remaining budget %s before UI work", async budget => {
|
||||
const node: any = { last: () => node }; const prior = visibility.mock.calls.length;
|
||||
await expect(submitTaskReply({ getByTestId: () => node } as never, "Continue.", () => budget)).rejects.toThrow(/exhausted/);
|
||||
expect(visibility.mock.calls.length).toBe(prior);
|
||||
});
|
||||
it("ordinary task reply callers retain existing UI defaults", async () => {
|
||||
const fill = vi.fn(); const click = vi.fn(); const node: any = { last: () => node, first: () => node, locator: () => node, fill, click };
|
||||
await submitTaskReply({ getByTestId: () => node } as never, "Continue.");
|
||||
expect(visibility).toHaveBeenCalledWith({ timeout: 30_000 }); expect(fill).toHaveBeenCalledExactlyOnceWith("Continue."); expect(click).toHaveBeenCalledExactlyOnceWith();
|
||||
});
|
||||
it("slow composer work cannot send a human message after the original deadline", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(389_000);
|
||||
const fill = vi.fn(async () => { vi.setSystemTime(391_000); }); const click = vi.fn();
|
||||
|
||||
@@ -130,14 +130,21 @@ export async function createTaskThroughUi(input: {
|
||||
export async function submitTaskReply(
|
||||
page: Page,
|
||||
body: string,
|
||||
remainingBudget?: () => number,
|
||||
): Promise<number> {
|
||||
const composer = page.getByTestId("task-chat-composer-input").last();
|
||||
await expect(composer).toBeVisible({ timeout: 30_000 });
|
||||
await composer
|
||||
.locator('[contenteditable="true"], textarea')
|
||||
.first()
|
||||
.fill(body);
|
||||
const options = () => {
|
||||
const remaining = remainingBudget!();
|
||||
if (!Number.isFinite(remaining) || remaining <= 0) throw new Error("Task reply original observation budget exhausted");
|
||||
return { timeout: Math.min(30_000, remaining) };
|
||||
};
|
||||
await expect(composer).toBeVisible(remainingBudget ? options() : { timeout: 30_000 });
|
||||
const editor = composer.locator('[contenteditable="true"], textarea').first();
|
||||
if (remainingBudget) await editor.fill(body, options());
|
||||
else await editor.fill(body);
|
||||
const submittedAtMs = Date.now();
|
||||
await page.getByTestId("task-chat-composer-send").last().click();
|
||||
const send = page.getByTestId("task-chat-composer-send").last();
|
||||
if (remainingBudget) await send.click(options());
|
||||
else await send.click();
|
||||
return submittedAtMs;
|
||||
}
|
||||
Reference in new issue
Block a user