diff --git a/server/src/__tests__/helpers/recovery-population.ts b/server/src/__tests__/helpers/recovery-population.ts index 16947d39334..15d566d813b 100644 --- a/server/src/__tests__/helpers/recovery-population.ts +++ b/server/src/__tests__/helpers/recovery-population.ts @@ -15,9 +15,11 @@ export type RecoveryPopulation = { export async function seedRecoveryPopulation(db: Db, companyId: string, counts: { agents: number; inReview: number; inProgress?: number; todo?: number; recoveryActions?: number; terminated?: number; + /** False keeps every agent from starting a run, for suites that fake timers. */ + wakeOnDemand?: boolean; }): Promise { const config = { adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() }, - runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true } }, role: "engineer" }; + runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: counts.wakeOnDemand ?? true } }, role: "engineer" }; const [ceo] = await db.insert(agents).values({ ...config, companyId, name: "Chief", status: "idle" }).returning({ id: agents.id }); const managers = await db.insert(agents).values(Array.from({ length: 6 }, (_, index) => ({ ...config, companyId, name: `Manager ${index}`, status: "idle", reportsTo: ceo!.id }))).returning({ id: agents.id }); diff --git a/server/src/__tests__/issue-recovery-idle-backstop.test.ts b/server/src/__tests__/issue-recovery-idle-backstop.test.ts index baf8fd03798..870459fa0d7 100644 --- a/server/src/__tests__/issue-recovery-idle-backstop.test.ts +++ b/server/src/__tests__/issue-recovery-idle-backstop.test.ts @@ -1,7 +1,7 @@ import { setImmediate as immediate } from "node:timers/promises"; import { and, eq, sql } from "drizzle-orm"; import { expect, it, vi } from "vitest"; -import { agents, heartbeatRuns, issues } from "@paperclipai/db"; +import { agentWakeupRequests, agents, heartbeatRuns, issues } from "@paperclipai/db"; import { workSignalDebug } from "@paperclipai/db/work-signal-debug"; import { ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_FULL_BACKSTOP_MS, notifyIssueGraphRecovery } from "../services/issue-recovery-notifications.js"; @@ -14,16 +14,32 @@ import { seedRecoveryPopulation } from "./helpers/recovery-population.js"; vi.hoisted(() => { process.env.PAPERCLIP_DEBUG_WORK_SIGNALS = "stranded-assignments:1"; }); const MINUTE = 60_000; +const STEP_TIMEOUT_MS = 20_000; +const TEST_TIMEOUT_MS = 120_000; +const realSetTimeout = globalThis.setTimeout; +const realClearTimeout = globalThis.clearTimeout; type Recorded = { at: number; name: string; fields: Record }; +/** Fails with a message naming what it waited for instead of hanging. */ +function bounded(work: Promise, waitingFor: () => string): Promise { + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = realSetTimeout(() => reject(new Error(`Timed out after ${STEP_TIMEOUT_MS} ms waiting for ${waitingFor()}`)), STEP_TIMEOUT_MS); + }); + return Promise.race([work, expired]).finally(() => realClearTimeout(timer)); +} + describeIssueRecovery("stranded-task checks at idle for a staging-shaped company", () => { const state = issueRecoveryFixture({ meter: true }); async function advance(ms: number) { const until = Date.now() + ms; while (Date.now() < until) { - await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())); - await Promise.allSettled([...state.passes]); await immediate(); + const at = new Date(Date.now()).toISOString(); + await bounded(vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())), () => `simulated time to advance past ${at}`); + await bounded(Promise.allSettled([...state.passes]), + () => `delivery passes started by simulated ${at} to settle: ${state.pendingPassQueues().join(", ") || "none"}`); + await immediate(); } } @@ -53,21 +69,29 @@ describeIssueRecovery("stranded-task checks at idle for a staging-shaped company const checkedAt = (events: Recorded[], issueId: string) => events.find(event => event.name === "check" && event.fields.issueId === issueId)?.at ?? null; + /** The wakes recovery asked for on a task. No fixture agent takes on-demand + * wakes, so each is recorded as skipped and no real run starts under the + * faked clock (a run there could wait forever on a frozen timer). */ + const wakesFor = (companyId: string, issueId: string) => bounded(state.db.select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`)), () => `the wake requests of task ${issueId} to be read`); + async function seed() { - const f = await state.fixture(); + const f = await bounded(state.fixture(), () => "the company to be created"); const population = await seedRecoveryPopulation(state.db, f.companyId, { agents: 50, inReview: 780, inProgress: 40, todo: 30, - recoveryActions: 12, terminated: 3 }); + recoveryActions: 12, terminated: 3, wakeOnDemand: false }); const [side] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Finished side task", status: "done" }) .returning({ id: issues.id }); - const activity = () => state.db.transaction(tx => notifyIssueGraphRecovery(tx, { companyId: f.companyId, issueId: side!.id })); + const activity = () => bounded(state.db.transaction(tx => notifyIssueGraphRecovery(tx, { companyId: f.companyId, issueId: side!.id })), + () => "the unrelated activity hint to commit"); return { ...f, population, activity }; } async function boot() { - const f = await seed(); + const f = await bounded(seed(), () => "the staging-shaped population to be seeded"); const events = record(); vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); - await state.start({}); + await bounded(state.start({}), () => "the recovery workers to start"); await advance(10 * MINUTE); return { ...f, events, boot: summarize(events.splice(0)) }; } @@ -77,7 +101,7 @@ describeIssueRecovery("stranded-task checks at idle for a staging-shaped company const [held] = await state.db.insert(agents).values({ companyId: f.companyId, name: "Held worker", role: "engineer", status: "paused", lifecycleHolds: ["manual"], adapterType: "process", reportsTo: f.population.agentIds[1], createdAt: old, updatedAt: old, adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() }, - runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true } } }).returning({ id: agents.id }); + runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: false } } }).returning({ id: agents.id }); const [waiting] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Waits for its agent", status: "todo", assigneeAgentId: held!.id, createdAt: old, updatedAt: old }).returning({ id: issues.id }); return { held: held!, waiting: waiting! }; @@ -103,37 +127,34 @@ describeIssueRecovery("stranded-task checks at idle for a staging-shaped company expect(between.totalChecks).toBe(0); expect(between.statements).toBeLessThan(2_000); expect(hour.checks.backstop ?? 0).toBeLessThanOrEqual(f.population.inReview.length + 100); - }, 1_800_000); + }, TEST_TIMEOUT_MS); it("recovers a task whose hint was never sent within the backstop bound", async () => { const f = await boot(); const worker = f.population.agentIds[7]!; const now = new Date(); - const [missed] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Created without a hint", status: "todo", - assigneeAgentId: worker, createdAt: now, updatedAt: now }).returning({ id: issues.id }); + const [missed] = await bounded(state.db.insert(issues).values({ companyId: f.companyId, title: "Created without a hint", status: "todo", + assigneeAgentId: worker, createdAt: now, updatedAt: now }).returning({ id: issues.id }), () => "the task without a hint to be created"); const window = await trickle(f, Math.ceil((ISSUE_RECOVERY_BACKSTOP_MS + MINUTE) / MINUTE) + 1); const checked = checkedAt(window.raw, missed!.id); expect(checked).not.toBeNull(); expect(checked! - now.getTime()).toBeLessThanOrEqual(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE); - const runs = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${missed!.id}`)); - expect(runs.length).toBeGreaterThan(0); - }, 1_800_000); + expect(await wakesFor(f.companyId, missed!.id), "recovery did not ask for a wake of the missed task").not.toHaveLength(0); + }, TEST_TIMEOUT_MS); it("re-checks a task whose only change is outside its own rows within the hourly full pass", async () => { const f = await boot(); - const { held, waiting } = await heldTask(f); - await state.db.execute(sql`update agents set status = 'idle', lifecycle_holds = '[]'::jsonb where id = ${held.id}`); + const { held, waiting } = await bounded(heldTask(f), () => "the held agent and its waiting task to be seeded"); + await bounded(state.db.execute(sql`update agents set status = 'idle', lifecycle_holds = '[]'::jsonb where id = ${held.id}`), + () => "the hold release without a hint to commit"); const changedAt = Date.now(); const window = await trickle(f, Math.ceil((ISSUE_RECOVERY_FULL_BACKSTOP_MS + 2 * MINUTE) / MINUTE)); const checks = window.raw.filter(event => event.name === "check" && event.fields.issueId === waiting.id); expect(checks.length).toBeGreaterThan(0); expect(checks[0]!.at - changedAt).toBeGreaterThan(2 * ISSUE_RECOVERY_BACKSTOP_MS); expect(checks[0]!.at - changedAt).toBeLessThanOrEqual(ISSUE_RECOVERY_FULL_BACKSTOP_MS + 2 * MINUTE); - const runs = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) - .where(and(eq(heartbeatRuns.companyId, f.companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${waiting.id}`)); - expect(runs.length).toBeGreaterThan(0); - }, 1_800_000); + expect(await wakesFor(f.companyId, waiting.id), "recovery did not ask for a wake of the waiting task").not.toHaveLength(0); + }, TEST_TIMEOUT_MS); it("picks up a finished run, a user message, a raised budget and a resumed agent within a minute", async () => { const f = await boot(); @@ -142,25 +163,25 @@ describeIssueRecovery("stranded-task checks at idle for a staging-shaped company const within = async (issueId: string, act: () => Promise) => { f.events.length = 0; const at = Date.now(); - await act(); + await bounded(act(), () => `the change to task ${issueId} to commit`); await advance(60_000); const checked = checkedAt(f.events, issueId); expect(checked, issueId).not.toBeNull(); expect(checked! - at).toBeLessThanOrEqual(60_000); }; - const [run] = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) - .where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${target!}`).limit(1); + const [run] = await bounded(state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${target!}`).limit(1), () => "the finished run to be read"); await within(target!, () => state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: run!.id }))); await advance(31_000); await within(messaged!, () => issueService(state.db).addComment(messaged!, "Please look again", { userId: "recovery-user" })); await advance(31_000); await within(target!, () => state.db.transaction(tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, trigger: "pause_lifted" }))); await advance(31_000); - const { held, waiting } = await heldTask(f); + const { held, waiting } = await bounded(heldTask(f), () => "the held agent and its waiting task to be seeded"); await advance(31_000); await within(waiting.id, async () => { await state.db.update(agents).set({ status: "idle", lifecycleHolds: [], updatedAt: new Date() }).where(eq(agents.id, held.id)); await state.db.transaction(tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, trigger: "pause_lifted" })); }); - }, 1_800_000); + }, TEST_TIMEOUT_MS); }); diff --git a/server/src/__tests__/issue-recovery.test-fixture.ts b/server/src/__tests__/issue-recovery.test-fixture.ts index 8a90d73e492..1cb75fc42f3 100644 --- a/server/src/__tests__/issue-recovery.test-fixture.ts +++ b/server/src/__tests__/issue-recovery.test-fixture.ts @@ -32,6 +32,9 @@ export function issueRecoveryFixture(options: { meter?: boolean } = {}) { const tasks = new Map number | null; hasPending: () => Promise }>(); const errors: unknown[] = []; const passes = new Set>(); + // The queue each pending pass belongs to, so a bounded wait can say which + // queue it was still waiting for. + const passQueues = new Map, string>(); const extraStops: Array<() => Promise> = []; const extraDeadlines: Array<() => number | null> = []; beforeAll(async () => { @@ -61,8 +64,9 @@ export function issueRecoveryFixture(options: { meter?: boolean } = {}) { coordinator.register = (queue, task) => { tasks.set(queue, task); return register(queue, { ...task, run: signal => { - const work = task.run(signal); passes.add(work); - void work.then(() => passes.delete(work), () => passes.delete(work)); + const work = task.run(signal); passes.add(work); passQueues.set(work, queue); + const done = () => { passes.delete(work); passQueues.delete(work); }; + void work.then(done, done); return work; } }); }; @@ -254,5 +258,7 @@ export function issueRecoveryFixture(options: { meter?: boolean } = {}) { set draining(value: typeof draining) { draining = value; }, get errors() { return errors; }, get passes() { return passes; }, + /** The queues of the passes still pending, for a bounded wait's message. */ + pendingPassQueues: () => [...passes].map(pass => passQueues.get(pass) ?? "?"), }; }