mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
test(e2e): keep long-thread measurements within original clock
This commit is contained in:
1 parent
ed24a4a444
commit
8bbf313552
4 files changed
+36
-4
No files matched your search
@@ -26,6 +26,7 @@ export interface ChatCatalogContext {
|
||||
/** These exact selected OpenCode/local journeys use the original flow clock for
|
||||
* native observation phases. Other cases/lanes keep their authored phase caps. */
|
||||
const originalClockExecutionIds = new Set([
|
||||
"chat-features.runner-opencode.local.developer-labs.long-thread-performance",
|
||||
"chat-features.runner-opencode.local.questions-and-approvals.approve-tool-once",
|
||||
"chat-features.runner-opencode.local.runs-adapters.reset-session",
|
||||
"chat-features.runner-opencode.local.tasks.website-without-project",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
|
||||
import { expect } from "@playwright/test";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
@@ -76,10 +77,10 @@ const performance = privilegedDefinition("developer-labs.long-thread-performance
|
||||
} finally { await page.goto(source, { waitUntil: "domcontentloaded" }); }
|
||||
// Declared operator companion step: facts only, no desired conclusion or threshold.
|
||||
const message = `Here is the report from my owned documented long-thread fixture. Please interpret the measured behavior and its limits:\n\n${JSON.stringify(report!)}`;
|
||||
await sendChatMessage(page, message);
|
||||
const final = await pollUntil({ label: "Actual native interpretation of the measured report", deadlineAt: Date.now() + 150_000,
|
||||
await sendChatMessage(page, message, chatCatalogNativePhaseRemainingBudget(ctx));
|
||||
const final = await pollUntil({ label: "Actual native interpretation of the measured report", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
|
||||
load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; },
|
||||
accept: o => o.issue?.conversationState === "waiting" && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded") });
|
||||
accept: o => chatCatalogNativePhaseIsTimely(ctx) && o.issue?.conversationState === "waiting" && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded") });
|
||||
const run = final.runs.find(run => run.id !== first.runs[0]!.id)!;
|
||||
const comments = await ctx.input.api.get<Entity[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`);
|
||||
const human = comments.filter(comment => !comment.authorAgentId && comment.body === message);
|
||||
|
||||
@@ -11,7 +11,7 @@ function context(scenarioId = "cli-operations.owned-dev-instance", deadline = 87
|
||||
}
|
||||
afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); });
|
||||
describe("selected native phases share the once-created original flow clock", () => {
|
||||
it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure", "questions-and-approvals.approve-tool-once", "runs-adapters.reset-session"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => {
|
||||
it.each(["developer-labs.long-thread-performance", "cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure", "questions-and-approvals.approve-tool-once", "runs-adapters.reset-session"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario);
|
||||
expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000)).toBe(870_000);
|
||||
vi.setSystemTime(850_000); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toBe(870_000);
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import type { ChatCatalogContext } from "./chat-catalog-contract.js";
|
||||
const { poll, settle, send } = vi.hoisted(() => ({ poll: vi.fn(), settle: vi.fn(), send: vi.fn() }));
|
||||
vi.mock("./api.js", async original => ({ ...await original<object>(), pollUntil: poll }));
|
||||
vi.mock("./chat-flow.js", async original => ({ ...await original<object>(), sendChatMessage: send }));
|
||||
vi.mock("./chat-catalog-domains/privileged-support.js", async original => ({ ...await original<object>(), settlePrivilegedChat: settle }));
|
||||
vi.mock("@playwright/test", async original => ({ ...await original<object>(), expect: Object.assign(() => ({ toBeVisible: async () => {} }), { poll: () => ({ toBeGreaterThanOrEqual: async () => {} }) }) }));
|
||||
import { privilegedPerformanceCases } from "./chat-catalog-domains/privileged-performance.js";
|
||||
afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); });
|
||||
function context() {
|
||||
const values: Record<string, number> = { "perf-fixture-row-target": 487, "perf-fixture-markdown-rows": 150, "perf-mount-duration": 12.3, "perf-latest-duration": 8.1, "perf-max-duration": 12.3, "perf-total-duration": 20.4, "perf-commit-count": 2 };
|
||||
const page = { url: () => "http://owned.invalid/chat", goto: vi.fn(async () => {}), getByTestId: (id: string) => ({ innerText: async () => String(values[id]) }), evaluate: async () => "unit browser", viewportSize: () => ({ width: 1280, height: 720 }) };
|
||||
return { observationDeadlineAt: 570_000, scenario: { id: "developer-labs.long-thread-performance" }, input: { execution: { id: "chat-features.runner-opencode.local.developer-labs.long-thread-performance" }, page, api: { baseURL: "http://owned.invalid" }, evidence: vi.fn(), capture: vi.fn() }, refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }) } as unknown as ChatCatalogContext;
|
||||
}
|
||||
it("actual measured-report followup and interpretation consume the original clock after browser work", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(200_000); const ctx = context(), state: Record<string, unknown> = {};
|
||||
settle.mockResolvedValue({ runs: [{ id: "first" }] }); send.mockImplementation(async (_page, _text, remaining) => { expect(remaining()).toBe(370_000); }); poll.mockRejectedValue(new Error("captured interpretation"));
|
||||
await expect(privilegedPerformanceCases[0]!.act!(ctx, state)).rejects.toThrow("captured interpretation");
|
||||
expect(send).toHaveBeenCalledExactlyOnceWith(ctx.input.page, expect.stringContaining('"commentCount":469'), expect.any(Function));
|
||||
const options = poll.mock.calls[0]![0]; expect(options.deadlineAt).toBe(570_000);
|
||||
const valid = { issue: { conversationState: "waiting" }, runs: [{ status: "succeeded" }, { status: "succeeded" }] };
|
||||
expect(options.accept(valid)).toBe(true); expect(options.accept({ ...valid, runs: [{ status: "succeeded" }, { status: "running" }] })).toBe(false);
|
||||
vi.setSystemTime(570_000); expect(options.accept(valid)).toBe(false);
|
||||
expect(state.performanceReport).toMatchObject({ commentCount: 469, rowCount: 487, markdownCount: 150, mountMs: 12.3 });
|
||||
});
|
||||
it("elapsed browser measurement cannot reset the report send budget", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(570_000); const ctx = context(); settle.mockResolvedValue({ runs: [{ id: "first" }] }); send.mockImplementation(async (_page, _text, remaining) => remaining());
|
||||
await expect(privilegedPerformanceCases[0]!.act!(ctx, {})).rejects.toThrow(/exhausted/); expect(poll).not.toHaveBeenCalled();
|
||||
expect(ctx.observationDeadlineAt).toBe(570_000);
|
||||
});
|
||||
Reference in new issue
Block a user