mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
test(evals): give refresh-card's lead the four daily runs its four questions need
The shared chat fixture allows three lead runs a day. refresh-card asks four questions, so the product correctly skipped the final question's wake and the case timed out once the earlier turns became fast enough to reach it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9862ffde3d
commit
f5e02a6ea0
3 files changed
+16
-2
No files matched your search
@@ -5376,7 +5376,7 @@
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-scenarios/status-cards.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.status-card.required-work-failure.v3",
|
||||
"oracleVersion": "paperclip.chat-catalog.status-card.required-work-failure.v4",
|
||||
"expectedRunCount": 7,
|
||||
"expectedHumanMessageCount": 4,
|
||||
"turnTimeoutMs": 120000,
|
||||
|
||||
@@ -219,6 +219,11 @@ it("makes the genuine new blocker visible in the public task title used by bound
|
||||
expect(update).not.toHaveProperty("status"); // No task/card/native status fabricated.
|
||||
expect(JSON.stringify(update)).not.toMatch(/summari[sz]e|write|return|tool|generationIssueId|model/i);
|
||||
});
|
||||
it("allows the refresh lead one run per human message, as the four-turn case requires", async () => {
|
||||
const { REFRESH_CARD_LEAD_TURNS } = await import("./integrator-status-cards.js");
|
||||
const definition = integratorStatusCardsCases.find(row => row.scenarioId === "status-cards.refresh-card")!;
|
||||
expect(REFRESH_CARD_LEAD_TURNS).toBe(definition.expectedHumanMessageCount);
|
||||
});
|
||||
it("declares factual human attachments only for the two actual missing selected referents", () => {
|
||||
for (const id of ["status-cards.explain-card", "status-cards.archive-card"]) {
|
||||
const definition = integratorStatusCardsCases.find(row => row.scenarioId === id)!;
|
||||
|
||||
@@ -229,6 +229,14 @@ export function selectedStatusCardReference(card: StatusCard, sources: Entity[],
|
||||
export function refreshedBlockerTaskInput() {
|
||||
return { title: "Mobile Safari checkout regression blocks launch", description: "Payment approval has been received. The remaining blocker is the mobile Safari checkout regression." };
|
||||
}
|
||||
/** refresh-card asks four questions, so its lead needs four runs; the shared chat fixture allows three a day. */
|
||||
export const REFRESH_CARD_LEAD_TURNS = 4;
|
||||
async function allowLeadTurns(ctx: ChatCatalogContext, turns: number) {
|
||||
const leadPath = `/api/agents/${ctx.input.fixtures.agent.id}`;
|
||||
const lead = await ctx.input.api.get<{ runtimeConfig: { heartbeat?: Record<string, unknown>; [key: string]: unknown } }>(leadPath);
|
||||
await ctx.input.api.patch(leadPath, { runtimeConfig: { ...lead.runtimeConfig, heartbeat: { ...lead.runtimeConfig.heartbeat, maxDailyRuns: turns } } });
|
||||
if ((await ctx.input.api.get<typeof lead>(leadPath)).runtimeConfig.heartbeat?.maxDailyRuns !== turns) throw new Error(`Card fixture requires its declared ${turns}-turn daily lead allowance`);
|
||||
}
|
||||
const experiments = new WeakMap<ChatCatalogContext, boolean>();
|
||||
function define(id: string, seed: "none" | "ceo" | "morgan", extra: Partial<ChatCatalogCase> = {}): ChatCatalogCase {
|
||||
return integratorDefinition("status-cards", id, async ctx => {
|
||||
@@ -242,6 +250,7 @@ function define(id: string, seed: "none" | "ceo" | "morgan", extra: Partial<Chat
|
||||
// the ordinary shared flow applies the same policy before the human query.
|
||||
for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }])
|
||||
await ctx.input.api.post(`${root(ctx)}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
|
||||
if (id === "status-cards.refresh-card") await allowLeadTurns(ctx, REFRESH_CARD_LEAD_TURNS);
|
||||
const cardRecipe = id === "status-cards.refresh-card" ? await prepareCardChecklist(ctx, worker.id, p.id, t) : undefined;
|
||||
const card = await seedCard(ctx, seed === "ceo" ? ctx.input.fixtures.agent.id : worker.id, worker.id, p.id, "Launch", cardRecipe?.interestPrompt);
|
||||
const initialComplete = await pollUntil({ label: "prerequisite summarizer native completion", deadlineAt: Math.min(Date.now() + 45_000, prerequisiteDeadline(ctx, Date.now() + 45_000)),
|
||||
@@ -304,7 +313,7 @@ export const integratorStatusCardsCases: readonly ChatCatalogCase[] = [
|
||||
}),
|
||||
define("status-cards.refresh-card", "ceo", { expectedRunCount: 7, expectedHumanMessageCount: 4, attemptTimeoutMs: 1_200_000,
|
||||
runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 7 } },
|
||||
oracleVersion: "paperclip.chat-catalog.status-card.required-work-failure.v3",
|
||||
oracleVersion: "paperclip.chat-catalog.status-card.required-work-failure.v4",
|
||||
sourceFiles: ["chat-catalog-domains/integrator-status-cards.ts", "chat-catalog-domains/status-card-work-failure.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/status-cards.yaml"],
|
||||
|
||||
async act(ctx, state) {
|
||||
|
||||
Reference in new issue
Block a user