diff --git a/server/src/__tests__/issue-recovery-deadlines.test.ts b/server/src/__tests__/issue-recovery-deadlines.test.ts index b2344c7c4fc..f11f3f541f5 100644 --- a/server/src/__tests__/issue-recovery-deadlines.test.ts +++ b/server/src/__tests__/issue-recovery-deadlines.test.ts @@ -240,6 +240,21 @@ describeIssueRecovery("issue recovery production producers and deadlines", () => await state.db.update(agents).set({ status: "idle", updatedAt: new Date(heartbeatAt.getTime() + 1000) }).where(eq(agents.id, f.agentId)); expect(await changed()).toEqual(tasks.map(row => row.id).sort()); }); + it("counts an agent's run start as no change to its other tasks, and a pause during the run as a change", async () => { + const f = await fixture(); + const old = new Date(Date.now() - 60 * 60_000); + await state.db.update(agents).set({ lastHeartbeatAt: old, updatedAt: old }).where(eq(agents.id, f.agentId)); + const tasks = await state.db.insert(issues).values(Array.from({ length: 5 }, (_, index) => ({ companyId: f.companyId, + title: `Settled ${index}`, status: "in_review", assigneeAgentId: f.agentId, createdAt: old, updatedAt: old }))).returning({ id: issues.id }); + const since = new Date(Date.now() - 5 * 60_000); + const changed = async () => (await state.db.select({ id: issues.id }).from(issues) + .where(and(eq(issues.companyId, f.companyId), changedTaskCondition(since)))).map(row => row.id).sort(); + // The write of execution start in heartbeat.ts: a status flip with no heartbeat time. + await state.db.update(agents).set({ status: "running", updatedAt: new Date() }).where(eq(agents.id, f.agentId)); + expect(await changed()).toEqual([]); + await state.db.update(agents).set({ status: "paused", updatedAt: new Date() }).where(eq(agents.id, f.agentId)); + expect(await changed()).toEqual(tasks.map(row => row.id).sort()); + }); it("resolves a run without a task to no tasks and a missing task to the whole company", async () => { const f = await fixture(); const [taskless] = await state.db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running", diff --git a/server/src/__tests__/issue-recovery-live-run-agent-census.test.ts b/server/src/__tests__/issue-recovery-live-run-agent-census.test.ts new file mode 100644 index 00000000000..94032f7f2e8 --- /dev/null +++ b/server/src/__tests__/issue-recovery-live-run-agent-census.test.ts @@ -0,0 +1,169 @@ +import { setImmediate as immediate } from "node:timers/promises"; +import { and, eq, sql } from "drizzle-orm"; +import { expect, it, vi } from "vitest"; +import { agents, heartbeatRuns, issues } from "@paperclipai/db"; +import { workSignalDebug } from "@paperclipai/db/work-signal-debug"; +import { ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_FULL_BACKSTOP_MS, notifyIssueGraphRecovery } + from "../services/issue-recovery-notifications.js"; +import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; +import { seedRecoveryPopulation } from "./helpers/recovery-population.js"; + +vi.hoisted(() => { process.env.PAPERCLIP_DEBUG_WORK_SIGNALS = "stranded-assignments:1,assigned-work-invariant:1"; }); + +const MINUTE = 60_000; +const TOPICS = new Set(["stranded-assignments", "assigned-work-invariant"]); +type Recorded = { at: number; topic: string; name: string; fields: Record }; + +/** + * Staging shape: about 860 settled assigned tasks, most of them held by the + * same few agents, and one agent whose native run stays live for the whole + * window, publishing a run hint every 10 seconds. Nothing about the settled + * tasks changes, so the recovery queues must not re-check them. + */ +describeIssueRecovery("stranded-task checks beside one long live run (census)", () => { + const state = issueRecoveryFixture({ meter: true }); + + async function advance(ms: number, onTick?: () => Promise) { + const until = Date.now() + ms; + let nextTick = Date.now(); + while (Date.now() < until) { + if (onTick && Date.now() >= nextTick) { await onTick(); nextTick = Date.now() + 10_000; } + await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())); + await Promise.allSettled([...state.passes]); await immediate(); + } + } + + function record() { + const events: Recorded[] = []; + vi.spyOn(workSignalDebug, "event").mockImplementation((topic, name, fields = {}) => { + if (TOPICS.has(topic)) events.push({ at: Date.now(), topic, name, fields }); + }); + vi.spyOn(workSignalDebug, "signal").mockImplementation(topic => { events.push({ at: Date.now(), topic, name: "signal", fields: {} }); }); + return events; + } + + const checksOf = (events: Recorded[], topic: string) => events.filter(event => event.topic === topic && event.name === "check"); + function summarize(events: Recorded[]) { + const byWake = (topic: string) => { + const counts: Record = {}; + for (const event of checksOf(events, topic)) counts[String(event.fields.wake)] = (counts[String(event.fields.wake)] ?? 0) + 1; + return counts; + }; + return { stranded: checksOf(events, "stranded-assignments").length, assignedWork: checksOf(events, "assigned-work-invariant").length, + strandedByWake: byWake("stranded-assignments"), assignedWorkByWake: byWake("assigned-work-invariant") }; + } + + async function seed() { + const f = await state.fixture(); + // 12 agents leave 3 workers, so each holds about 266 of the settled reviews. + const population = await seedRecoveryPopulation(state.db, f.companyId, { agents: 12, inReview: 800, inProgress: 30, todo: 30 }); + const liveAgentId = population.agentIds[7]!; + const [liveTask] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Live native task", status: "in_progress", + assigneeAgentId: liveAgentId }).returning({ id: issues.id }); + // A paused agent with one waiting task, both settled long before the window. + const old = new Date(Date.now() - 2 * 60 * MINUTE); + const [held] = await state.db.insert(agents).values({ companyId: f.companyId, name: "Held worker", role: "engineer", status: "paused", + lifecycleHolds: ["manual"], adapterType: "process", reportsTo: population.agentIds[1], createdAt: old, updatedAt: old, + adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true } } }).returning({ id: agents.id }); + const [waiting] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Waits for its agent", status: "todo", + assigneeAgentId: held!.id, createdAt: old, updatedAt: old }).returning({ id: issues.id }); + return { ...f, population, liveAgentId, liveTaskId: liveTask!.id, heldAgentId: held!.id, waitingTaskId: waiting!.id }; + } + + async function boot() { + const f = await seed(); + const events = record(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await state.start({}); + await advance(10 * MINUTE); + events.length = 0; + return { ...f, events }; + } + + /** What execution start writes (heartbeat.ts): the run, then the agent's + * status flip to running without a heartbeat time. */ + async function startLiveRun(f: Awaited>) { + const [run] = await state.db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.liveAgentId, status: "running", + runtimeMode: "native", startedAt: new Date(), contextSnapshot: { issueId: f.liveTaskId }, nativeIssueId: f.liveTaskId }) + .returning({ id: heartbeatRuns.id }); + await state.db.update(issues).set({ executionRunId: run!.id }).where(eq(issues.id, f.liveTaskId)); + await state.db.update(agents).set({ status: "running", updatedAt: new Date() }).where(eq(agents.id, f.liveAgentId)); + const runId = run!.id; + return { + runId, + /** A run event: the run row moves and its writers publish a run hint. */ + tick: () => state.db.transaction(async tx => { + await tx.update(heartbeatRuns).set({ updatedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + await notifyIssueGraphRecovery(tx, { companyId: f.companyId, runId }); + }), + }; + } + + it("does not re-check the live agent's settled tasks while its run stays live", async () => { + const f = await boot(); + const live = await startLiveRun(f); + let ticks = 0; + const before = state.meter.read(); + await advance(30 * MINUTE, () => { ticks += 1; return live.tick(); }); + const after = state.meter.read(); + // Each run event publishes one signal to each of the 7 recovery queues; anything above that is the sweep's own. + const signals = f.events.filter(event => event.name === "signal").length; + const result = { ...summarize(f.events), statements: after.statements - before.statements, ticks, signals, + signalsBeyondRunEvents: signals - ticks * 7 }; + console.info(`live-run census (30 min): ${JSON.stringify(result)}`); + const settled = new Set([...f.population.inReview, ...f.population.inProgress, ...f.population.todo, f.waitingTaskId]); + const settledChecks = f.events.filter(event => event.name === "check" && settled.has(String(event.fields.issueId))); + // Before the fix: 534 settled checks and 540 backstop checks. After: 0 and 6 (the live task itself). + expect(settledChecks.length).toBe(0); + expect(result.strandedByWake.backstop ?? 0).toBeLessThanOrEqual(10); + // The live task's own run hints: about one check per run event. + expect(result.strandedByWake.hint ?? 0).toBeLessThanOrEqual(ticks + 10); + expect(result.assignedWork).toBeLessThanOrEqual(10); + expect(result.signalsBeyondRunEvents).toBe(0); + }, 1_800_000); + + it("still checks a task of the live agent within the backstop bound when its own rows change without a hint", async () => { + const f = await boot(); + const live = await startLiveRun(f); + await advance(6 * MINUTE, live.tick); + f.events.length = 0; + const target = f.population.inReview.find((_, index) => index % 3 === 0)!; + const [current] = await state.db.select({ assignee: issues.assigneeAgentId }).from(issues).where(eq(issues.id, target)); + expect(current!.assignee).toBe(f.liveAgentId); + const changedAt = Date.now(); + // A direct write that sends no hint (for example an edit by an older server). + await state.db.update(issues).set({ status: "todo", updatedAt: new Date() }).where(eq(issues.id, target)); + await advance(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE, live.tick); + const checked = f.events.find(event => event.topic === "stranded-assignments" && event.name === "check" && event.fields.issueId === target); + expect(checked).toBeDefined(); + expect(checked!.at - changedAt).toBeLessThanOrEqual(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE); + }, 1_800_000); + + it("still checks a resumed agent's task within the backstop bound when the resume sends no hint", async () => { + const f = await boot(); + const live = await startLiveRun(f); + await advance(6 * MINUTE, live.tick); + f.events.length = 0; + const waiting = f.waitingTaskId; + const changedAt = Date.now(); + await state.db.update(agents).set({ status: "idle", lifecycleHolds: [], updatedAt: new Date() }) + .where(eq(agents.id, f.heldAgentId)); + await advance(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE, live.tick); + const checked = f.events.find(event => event.topic === "stranded-assignments" && event.name === "check" && event.fields.issueId === waiting); + expect(checked).toBeDefined(); + expect(checked!.at - changedAt).toBeLessThanOrEqual(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE); + }, 1_800_000); + + it("still re-checks every settled task in the hourly full pass", async () => { + const f = await boot(); + const live = await startLiveRun(f); + await advance(ISSUE_RECOVERY_FULL_BACKSTOP_MS + 5 * MINUTE, live.tick); + const checked = new Set(checksOf(f.events, "stranded-assignments").map(event => String(event.fields.issueId))); + const missing = f.population.inReview.filter(id => !checked.has(id)); + expect(missing).toEqual([]); + const runs = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, f.companyId), sql`${heartbeatRuns.status} = 'running'`)); + expect(runs.map(run => run.id)).toContain(live.runId); + }, 1_800_000); +}); diff --git a/server/src/services/recovery/changed-tasks.ts b/server/src/services/recovery/changed-tasks.ts index f28c55bbce8..aecbf30a0e6 100644 --- a/server/src/services/recovery/changed-tasks.ts +++ b/server/src/services/recovery/changed-tasks.ts @@ -1,6 +1,13 @@ import { sql } from "drizzle-orm"; import { issues } from "@paperclipai/db"; +/** Tasks whose own rows changed since `since`, for the backstop between full + * passes. An assignee agent's row counts only for a change that can make its + * tasks need recovery (a pause, resume, hold or configuration change). Run + * lifecycle writes are left out: a run's finish records a heartbeat, and a run's + * start flips the agent to running, which makes none of its other tasks + * stranded. Every run start would otherwise re-check all of the agent's tasks + * for two backstop windows. */ export function changedTaskCondition(since: Date) { const at = sql`${since.toISOString()}::timestamptz`; return sql`(${issues.updatedAt} >= ${at} @@ -15,5 +22,6 @@ export function changedTaskCondition(since: Date) { or exists (select 1 from issue_comments c where c.company_id = ${issues.companyId} and c.issue_id = ${issues.id} and c.created_at >= ${at}) or exists (select 1 from agents ag where ag.id = ${issues.assigneeAgentId} and ag.updated_at >= ${at} + and ag.status <> 'running' and (ag.last_heartbeat_at is null or ag.last_heartbeat_at < ag.updated_at)))`; }