mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-05 03:51:54 +01:00
fix(evals): preserve original task and inbox observation budgets
This commit is contained in:
1 parent
efa3382b21
commit
a477b2c5d8
4 files changed
+9
-5
No files matched your search
@@ -26,6 +26,8 @@ export interface ChatCatalogContext {
|
||||
/** These exact selected OpenCode/local journeys use the original flow clock for
|
||||
* native observation phases. Other cases/lanes keep their authored phase caps. */
|
||||
const originalClockExecutionIds = new Set([
|
||||
"chat-features.runner-opencode.local.tasks.website-without-project",
|
||||
"chat-features.runner-opencode.local.inbox-search.archive-partial-failure",
|
||||
"chat-features.runner-opencode.local.cli-operations.feedback-bundle",
|
||||
"chat-features.runner-opencode.local.cli-operations.inspect-run-cli",
|
||||
"chat-features.runner-opencode.local.cli-operations.owned-dev-instance",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { expect } from "@playwright/test";
|
||||
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
|
||||
import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
import { pollUntil } from "../api.js";
|
||||
import { makeDefinition } from "./internal-definition.js";
|
||||
@@ -106,10 +107,10 @@ function archivePartial(): ChatCatalogCase {
|
||||
};
|
||||
return { ...base, coverageGaps: [], authentication: "human-session", expectedRunCount: 2, minimumRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 480000,
|
||||
sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-archive-partial.ts", "chat-catalog-domains/internal-archive-transport.ts", "../e2e/fetch-allowed-port.ts", "chat-archive-fault-control.ts"],
|
||||
async act(ctx, state) { const first = await pollUntil({ label: "Real initial partial archive turn settles", deadlineAt: Date.now() + 150000, load: () => observe(ctx, state), accept: o => o.runs.length === 1 && o.runs[0]?.status === "succeeded" && ctx.issue()?.conversationState === "waiting" });
|
||||
async act(ctx, state) { const first = await pollUntil({ label: "Real initial partial archive turn settles", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150000), load: () => observe(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && o.runs.length === 1 && o.runs[0]?.status === "succeeded" && ctx.issue()?.conversationState === "waiting" });
|
||||
state.firstPartial = { ...first, state: structuredClone(state) }; await ctx.input.evidence("chat-catalog-before-partial-archive-retry.json", state.firstPartial);
|
||||
if (!firstIsValid(first)) throw new Error("Initial partial archive evidence failed before declared retry");
|
||||
await transport(ctx)!.releaseForExplicitRetry(); const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, ctx.scenario.query); },
|
||||
await transport(ctx)!.releaseForExplicitRetry(); const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, ctx.scenario.query, chatCatalogNativePhaseRemainingBudget(ctx)); },
|
||||
observe,
|
||||
async verifyResult(ctx, state) { const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "task").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "task").title!, exact: true }).first()).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } },
|
||||
// Launcher owns the pre-bootstrap transport lifetime; it closes only after server shutdown.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js";
|
||||
import { verifyDistinctTaskFixtures } from "./internal-distinct-fixtures.js";
|
||||
import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
import type { ChatRun, ChatIssue } from "../chat-flow.js";
|
||||
@@ -118,8 +119,8 @@ function creationDefinition(scenarioId: string): ChatCatalogCase {
|
||||
if (scenarioId !== "tasks.website-without-project") return;
|
||||
if (!interception) throw new Error("Lost acknowledgement fixture was not armed");
|
||||
const sent = await interception.committed;
|
||||
const first = await pollUntil({ label: "Committed project and task before lost-ack replay", deadlineAt: Date.now() + 150_000,
|
||||
load: () => observeCreation(ctx, state), accept: o => newTasks(o).length === 1 && o.projects.filter(row => row.name === "Launch").length === 1
|
||||
const first = await pollUntil({ label: "Committed project and task before lost-ack replay", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
|
||||
load: () => observeCreation(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && newTasks(o).length === 1 && o.projects.filter(row => row.name === "Launch").length === 1
|
||||
&& ctx.issue()?.conversationState === "waiting" });
|
||||
await interception.dispose();
|
||||
const replay = await ctx.input.api.post<{ id: string }>(sent.path, sent.data);
|
||||
|
||||
@@ -11,7 +11,7 @@ function context(scenarioId = "cli-operations.owned-dev-instance", deadline = 87
|
||||
}
|
||||
afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); });
|
||||
describe("selected native phases share the once-created original flow clock", () => {
|
||||
it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => {
|
||||
it.each(["cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario);
|
||||
expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000)).toBe(870_000);
|
||||
vi.setSystemTime(850_000); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toBe(870_000);
|
||||
|
||||
Reference in new issue
Block a user