test(evals): give refresh-card's lead the four daily runs its four questions need

The shared chat fixture allows three lead runs a day. refresh-card asks four
questions, so the product correctly skipped the final question's wake and the
case timed out once the earlier turns became fast enough to reach it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-09 05:26:45 -06:00
1 parent 9862ffde3d
commit f5e02a6ea0
3 files changed
+16 -2

No files matched your search

@@ -5376,7 +5376,7 @@
"chat-catalog-domains/integrator-support.ts",
"chat-scenarios/status-cards.yaml"
],
"oracleVersion": "paperclip.chat-catalog.status-card.required-work-failure.v3",
"oracleVersion": "paperclip.chat-catalog.status-card.required-work-failure.v4",
"expectedRunCount": 7,
"expectedHumanMessageCount": 4,
"turnTimeoutMs": 120000,
@@ -219,6 +219,11 @@ it("makes the genuine new blocker visible in the public task title used by bound
expect(update).not.toHaveProperty("status"); // No task/card/native status fabricated.
expect(JSON.stringify(update)).not.toMatch(/summari[sz]e|write|return|tool|generationIssueId|model/i);
});
it("allows the refresh lead one run per human message, as the four-turn case requires", async () => {
const { REFRESH_CARD_LEAD_TURNS } = await import("./integrator-status-cards.js");
const definition = integratorStatusCardsCases.find(row => row.scenarioId === "status-cards.refresh-card")!;
expect(REFRESH_CARD_LEAD_TURNS).toBe(definition.expectedHumanMessageCount);
});
it("declares factual human attachments only for the two actual missing selected referents", () => {
for (const id of ["status-cards.explain-card", "status-cards.archive-card"]) {
const definition = integratorStatusCardsCases.find(row => row.scenarioId === id)!;
@@ -229,6 +229,14 @@ export function selectedStatusCardReference(card: StatusCard, sources: Entity[],
export function refreshedBlockerTaskInput() {
return { title: "Mobile Safari checkout regression blocks launch", description: "Payment approval has been received. The remaining blocker is the mobile Safari checkout regression." };
}
/** refresh-card asks four questions, so its lead needs four runs; the shared chat fixture allows three a day. */
export const REFRESH_CARD_LEAD_TURNS = 4;
async function allowLeadTurns(ctx: ChatCatalogContext, turns: number) {
const leadPath = `/api/agents/${ctx.input.fixtures.agent.id}`;
const lead = await ctx.input.api.get<{ runtimeConfig: { heartbeat?: Record<string, unknown>; [key: string]: unknown } }>(leadPath);
await ctx.input.api.patch(leadPath, { runtimeConfig: { ...lead.runtimeConfig, heartbeat: { ...lead.runtimeConfig.heartbeat, maxDailyRuns: turns } } });
if ((await ctx.input.api.get<typeof lead>(leadPath)).runtimeConfig.heartbeat?.maxDailyRuns !== turns) throw new Error(`Card fixture requires its declared ${turns}-turn daily lead allowance`);
}
const experiments = new WeakMap<ChatCatalogContext, boolean>();
function define(id: string, seed: "none" | "ceo" | "morgan", extra: Partial<ChatCatalogCase> = {}): ChatCatalogCase {
return integratorDefinition("status-cards", id, async ctx => {
@@ -242,6 +250,7 @@ function define(id: string, seed: "none" | "ceo" | "morgan", extra: Partial<Chat
// the ordinary shared flow applies the same policy before the human query.
for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }])
await ctx.input.api.post(`${root(ctx)}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
if (id === "status-cards.refresh-card") await allowLeadTurns(ctx, REFRESH_CARD_LEAD_TURNS);
const cardRecipe = id === "status-cards.refresh-card" ? await prepareCardChecklist(ctx, worker.id, p.id, t) : undefined;
const card = await seedCard(ctx, seed === "ceo" ? ctx.input.fixtures.agent.id : worker.id, worker.id, p.id, "Launch", cardRecipe?.interestPrompt);
const initialComplete = await pollUntil({ label: "prerequisite summarizer native completion", deadlineAt: Math.min(Date.now() + 45_000, prerequisiteDeadline(ctx, Date.now() + 45_000)),
@@ -304,7 +313,7 @@ export const integratorStatusCardsCases: readonly ChatCatalogCase[] = [
}),
define("status-cards.refresh-card", "ceo", { expectedRunCount: 7, expectedHumanMessageCount: 4, attemptTimeoutMs: 1_200_000,
runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 7 } },
oracleVersion: "paperclip.chat-catalog.status-card.required-work-failure.v3",
oracleVersion: "paperclip.chat-catalog.status-card.required-work-failure.v4",
sourceFiles: ["chat-catalog-domains/integrator-status-cards.ts", "chat-catalog-domains/status-card-work-failure.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/status-cards.yaml"],
async act(ctx, state) {