From 0ecd43035a8eeb82e0fa7a8c7cf6ec1af076b0f7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 20:35:39 -0600 Subject: [PATCH] fix(evals): bound status-card setup by original case clock Co-Authored-By: Paperclip --- ...ntegrator-status-card-prerequisite.test.ts | 80 +++++++++++++++++++ .../integrator-status-cards.ts | 22 +++-- 2 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/integrator-status-card-prerequisite.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-status-card-prerequisite.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-status-card-prerequisite.test.ts new file mode 100644 index 00000000000..b58af4adafc --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-status-card-prerequisite.test.ts @@ -0,0 +1,80 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +vi.mock("./internal-definition.js", () => ({ fixtureWorker: async () => ({ id: "worker" }) })); +// This fixture isolates the prerequisite-generation wait; the required-input +// recipe and later feature actions retain their separate owning controls. +vi.mock("./status-card-work-failure.js", async original => ({ ...await original(), prepareCardChecklist: async () => undefined })); +vi.mock("../chat-query-attachments.js", async original => ({ ...await original(), uploadFactualQueryAttachment: async () => ({ id: "fixture-attachment" }) })); +import { integratorStatusCardsCases } from "./integrator-status-cards.js"; +afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.unstubAllGlobals(); }); +const start = 10_000; +const keys = ["status-cards.explain-card", "status-cards.other-authors-card", "status-cards.refresh-card"] as const; +function fixture(key: string, deadline: number | undefined) { + vi.useFakeTimers(); vi.setSystemTime(start); + let readyAt = start + 190_000, cardState = "active", summary = "Payment approval blocks Launch.", generated = true, pending = false, late = false, runStatus = "succeeded", noWork = false, lateNative = false; + const card = () => ({ id: "card", state: Date.now() >= readyAt ? cardState : "generating", summaryBody: Date.now() >= readyAt ? summary : null, + lastGeneratedAt: generated ? "2026-11-03T01:00:00Z" : null, generatingIssueId: pending ? "generation" : null, fingerprint: {} }); + const get = vi.fn(async (path: string) => { + if (path === "/api/instance/settings/experimental") return { enableStatusCards: false }; + if (path === "/api/status-cards/card") { if (late) vi.setSystemTime(deadline! + 1); return card(); } + if (path.endsWith("/updates") || path.endsWith("/summary-revisions")) return []; + throw Error("Unexpected fixture read"); + }); + const post = vi.fn(async (path: string) => { + if (path.endsWith("/projects")) return { id: "project" }; + if (path.endsWith("/issues")) return { id: "source", companyId: "company", title: "Payment approval blocks launch", status: "blocked" }; + if (path.endsWith("/keys")) return { id: "owned-key", token: "synthetic-fixture-key" }; + if (path.endsWith("/budgets/policies")) return {}; + throw Error("Unexpected fixture write"); + }); + const remove = vi.fn(async () => ({})), patch = vi.fn(async () => ({ id: "source", companyId: "company", status: "done" })); + vi.stubGlobal("fetch", vi.fn(async () => ({ ok: true, json: async () => ({ id: "card" }) }))); + const ctx = { scenario: { id: key }, observationDeadlineAt: deadline, + input: { nonce: "synthetic", privateSecrets: [], fixtures: { company: { id: "company", issuePrefix: "ACORN" }, agent: { id: "ceo" } }, + api: { baseURL: "https://fixture.test", get, post, postSensitive: post, patch, delete: remove }, evidence: async () => {} }, + allRuns: async () => { if (lateNative) vi.setSystemTime(deadline! + 1); return noWork ? [] : [{ id: "prerequisite", status: runStatus }]; }, + } as unknown as ChatCatalogContext; + const definition = integratorStatusCardsCases.find(row => row.scenarioId === key)!; + const setup = () => definition.setup(ctx); + const settle = async (ms: number) => { + const result = setup().then(value => ({ value, error: undefined }), error => ({ value: undefined, error: error as Error })); + await vi.dynamicImportSettled(); await vi.advanceTimersByTimeAsync(ms); return result; + }; + return { ctx, definition, setup, settle, get, remove, post, readyAt: (value: number) => { readyAt = value; }, + state: (value: string) => { cardState = value; }, summary: (value: string) => { summary = value; }, + generated: (value: boolean) => { generated = value; }, pending: () => { pending = true; }, late: () => { late = true; }, runStatus: (value: string) => { runStatus = value; }, noWork: () => { noWork = true; }, lateNative: () => { lateNative = true; } }; +} +it.each(keys)("actual %s setup admits genuinely completed prerequisite after190s inside its original clock", async key => { + const cap = key.endsWith("refresh-card") ? 1_200_000 : key.endsWith("explain-card") ? 720_000 : 540_000; + const f = fixture(key, start + cap - 30_000), result = await f.settle(191_000); + expect(result.error).toBeUndefined(); expect(result.value?.card).toMatchObject({ id: "card", state: "active" }); + expect(f.definition.attemptTimeoutMs).toBe(cap); expect(f.ctx.observationDeadlineAt).toBe(start + cap - 30_000); + expect(f.remove).toHaveBeenCalledWith("/api/agents/" + (key.endsWith("other-authors-card") ? "worker" : "ceo") + "/keys/owned-key", { allowNotFound: true }); +}); +it.each([undefined, NaN, Infinity, start, start - 1])("rejects absent/invalid/expired original deadline %s", async deadline => { + const f = fixture(keys[1], deadline); const result = await f.settle(181_000); + expect(result.error).toBeInstanceOf(Error); expect(result.value).toBeUndefined(); +}); +it.each(["error", "missing summary", "missing generation", "still generating", "late await"])("rejects %s rather than accepting a partial prerequisite", async fault => { + const f = fixture(keys[1], start + 1000); f.readyAt(start); + if (fault === "error") f.state("error"); + if (fault === "missing summary") f.summary(""); + if (fault === "missing generation") f.generated(false); + if (fault === "still generating") f.pending(); + if (fault === "late await") f.late(); + const result = await f.settle(3000); expect(result.error).toBeInstanceOf(Error); expect(result.value).toBeUndefined(); +}); +it.each(["failed", "cancelled", "timed_out"])("a completed card cannot hide actual native %s", async status => { + const f = fixture(keys[1], start + 1000); f.readyAt(start); f.runStatus(status); + const result = await f.settle(47_000); expect(result.error).toBeInstanceOf(Error); expect(result.value).toBeUndefined(); +}); +it("does not change the archive-card authored180s prerequisite phase", async () => { + const f = fixture("status-cards.archive-card", start + 510_000); const result = await f.settle(181_000); + expect(result.error?.message).toContain("actual prerequisite card generation"); expect(result.value).toBeUndefined(); +}); + +it.each(["no actual run", "late native await"])("cannot treat %s as completed setup work", async fault => { + const f = fixture(keys[1], start + 1000); f.readyAt(start); + if (fault === "no actual run") f.noWork(); else f.lateNative(); + const result = await f.settle(3000); expect(result.error).toBeInstanceOf(Error); expect(result.value).toBeUndefined(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts index 7b00e73d184..f3a7070ebd9 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts @@ -5,7 +5,7 @@ import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contra import { pollUntil } from "../api.js"; import { checks, observation, project, task, sameTask, hasLink, type Entity, type InternalObservation } from "./internal-support.js"; import { prepareCardChecklist, updateCardChecklistSource, observeCardChecklistSuccess, observeCardWork, failCardChecklist, genuineCardWorkFailure, genuineCardChecklistSuccess, cardAccountingReady, cleanupCardChecklist, type CardRecipe, type CardWorkEvidence } from "./status-card-work-failure.js"; -import { readAgentResource, cleanupIdentityProbes } from "./identity-support.js"; +import { readAgentResource, cleanupIdentityProbes, remainingIdentityObservationBudget } from "./identity-support.js"; import { fixtureWorker } from "./internal-definition.js"; import { groundedStatement, integratorDefinition, inspectResult, resultHref, observeInternal } from "./integrator-support.js"; interface CardsObservation extends InternalObservation { cards: StatusCard[]; updates: Record; summaries: Record; cardsEnabled: boolean; cardWork?: CardWorkEvidence } @@ -120,9 +120,21 @@ async function verifyCard(ctx: ChatCatalogContext, _state: Record row.id === (o.state.card as StatusCard).id) : o.cards[0]); if (!card) throw new Error("Missing saved card"); await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/status/${card.id}`, card.title ?? card.interestPrompt, cardLink(o!, card.id)); } +// These three prerequisite waits consumed a generic180s allowance before the +// query, although the unchanged whole-case clock is installed before setup. +const originalPrerequisiteClock = new Set(["status-cards.explain-card", "status-cards.other-authors-card", "status-cards.refresh-card"]); +function prerequisiteDeadline(ctx: ChatCatalogContext, legacyDeadline: number): number { + if (!originalPrerequisiteClock.has(ctx.scenario.id)) return legacyDeadline; + remainingIdentityObservationBudget(ctx); + return ctx.observationDeadlineAt!; +} +function prerequisiteIsTimely(ctx: ChatCatalogContext): boolean { + return !originalPrerequisiteClock.has(ctx.scenario.id) || Number.isFinite(ctx.observationDeadlineAt) && Date.now() < ctx.observationDeadlineAt!; +} /** Author identity comes from a real revocable agent key. The compile and first * summary are produced by an actual admitted summarizer run, never fixture PUTs. */ async function seedCard(ctx: ChatCatalogContext, authorId: string, summarizerId: string, projectId: string, projectName = "Launch", interestPrompt?: string) { + const deadlineAt = originalPrerequisiteClock.has(ctx.scenario.id) ? prerequisiteDeadline(ctx, 0) : undefined; if (!ctx.input.privateSecrets) throw new Error("Card fixture author key needs private redaction"); const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${authorId}/keys`, { name: `card-prerequisite-${ctx.input.nonce}` }); ctx.input.privateSecrets.push(key.token); @@ -131,8 +143,8 @@ async function seedCard(ctx: ChatCatalogContext, authorId: string, summarizerId: headers: { Authorization: `Bearer ${key.token}`, "content-type": "application/json" }, body: JSON.stringify({ title: `${projectName} blockers`, interestPrompt: interestPrompt ?? `Summarize the biggest blockers for ${projectName} project ${projectId}, citing actual source tasks.`, agentId: summarizerId, refreshPolicy: { mode: "manual" } }) }); if (!response.ok) { await response.body?.cancel(); throw new Error(`Card fixture public creation failed HTTP ${response.status}`); } const created = await response.json() as StatusCard; - return await pollUntil({ label: "actual prerequisite card generation", deadlineAt: Date.now() + 180_000, intervalMs: 1000, - load: () => ctx.input.api.get(`/api/status-cards/${created.id}`), accept: card => card.state === "active" && !!card.summaryBody && !!card.lastGeneratedAt && !card.generatingIssueId, + return await pollUntil({ label: "actual prerequisite card generation", deadlineAt: deadlineAt ?? Date.now() + 180_000, intervalMs: 1000, + load: () => ctx.input.api.get(`/api/status-cards/${created.id}`), accept: card => prerequisiteIsTimely(ctx) && card.state === "active" && !!card.summaryBody && !!card.lastGeneratedAt && !card.generatingIssueId, reject: card => card.state === "error" ? "Prerequisite card generation failed; no synthetic summary substituted" : undefined }); } finally { await ctx.input.api.delete(`/api/agents/${authorId}/keys/${key.id}`, { allowNotFound: true }); } } @@ -165,13 +177,13 @@ function define(id: string, seed: "none" | "ceo" | "morgan", extra: Partial { const runs = await ctx.allRuns(); const update = cardRecipe ? (await ctx.input.api.get(`/api/status-cards/${card.id}/updates`)).find(row => row.kind !== "compile" && row.status === "ok") : undefined; const checklist = cardRecipe && update ? await observeCardChecklistSuccess(ctx, cardRecipe, update) : undefined; return { runs, update, checklist }; - }, accept: value => value.runs.length === 1 && value.runs.every(row => row.status === "succeeded") && (!cardRecipe || !!value.update && genuineCardChecklistSuccess(card.id, cardRecipe, value.update, value.checklist)) }); + }, accept: value => prerequisiteIsTimely(ctx) && value.runs.length === 1 && value.runs.every(row => row.status === "succeeded") && (!cardRecipe || !!value.update && genuineCardChecklistSuccess(card.id, cardRecipe, value.update, value.checklist)) }); const initialChecklist = initialComplete.checklist; if (initialChecklist) await ctx.input.evidence("chat-card-initial-required-checklist.json", initialChecklist); let otherCard: StatusCard | undefined; const sourceTasks = [t];