diff --git a/server/src/__tests__/issue-recovery-company-reload-census.test.ts b/server/src/__tests__/issue-recovery-company-reload-census.test.ts index 2137bd0fab9..f4888725852 100644 --- a/server/src/__tests__/issue-recovery-company-reload-census.test.ts +++ b/server/src/__tests__/issue-recovery-company-reload-census.test.ts @@ -11,6 +11,7 @@ import { documentService } from "../services/documents.js"; import { issueService } from "../services/issues.js"; import { createHeartbeatAgentStatus } from "../services/heartbeat/agent-status.js"; import { notifyExecutionRunWork } from "../services/execution-control-notifications.js"; +import { reconcileAbandonedExecutionControl } from "../services/execution-control-reconciliation.js"; import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; vi.hoisted(() => { process.env.PAPERCLIP_DEBUG_WORK_SIGNALS = "stranded-assignments:1,assigned-work-invariant:1"; }); @@ -158,6 +159,67 @@ describeIssueRecovery("company hints and notification reloads (census)", () => { const checksOf = (events: Recorded[], issueId: string, after: number) => events.filter(event => event.topic === "stranded-assignments" && event.name === "check" && event.fields.issueId === issueId && event.at >= after); + /** An agent left "running" with a large settled backlog: one abandoned + * run past its execution-control deadline, or (stranded) no run at all. */ + async function settlingAgent(f: Awaited>, name: string, abandoned: boolean) { + const old = new Date(Date.now() - 2 * HOUR); + const [agent] = await state.db.insert(agents).values({ companyId: f.companyId, name, role: "engineer", status: "running", + adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: false } }, + createdAt: old, updatedAt: old, lastHeartbeatAt: old }).returning({ id: agents.id }); + const tasks = await state.db.insert(issues).values(Array.from({ length: 140 }, (_, index) => ({ companyId: f.companyId, + title: `Settled ${index}`, status: "in_review", assigneeAgentId: agent!.id, createdAt: old, updatedAt: old }))).returning({ id: issues.id }); + let runTask: string | null = null; + if (abandoned) { + const [task] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Abandoned run's task", status: "in_progress", + assigneeAgentId: agent!.id, createdAt: old, updatedAt: old }).returning({ id: issues.id }); + runTask = task!.id; + const [run] = await state.db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent!.id, status: "running", + startedAt: old, updatedAt: old, contextSnapshot: { issueId: runTask }, executionControlDeadlineAt: new Date(Date.now() - MINUTE) }) + .returning({ id: heartbeatRuns.id }); + await state.db.update(issues).set({ executionRunId: run!.id, updatedAt: old }).where(eq(issues.id, runTask)); + } + return { agentId: agent!.id, tasks: tasks.map(row => row.id), runTask }; + } + + it("does not re-check an agent's settled backlog when an abandoned or stranded run settles the agent", async () => { + const f = await boot(); + const abandoned = await bounded(settlingAgent(f, "Abandoned worker", true), () => "the abandoned agent to be seeded"); + const stranded = await bounded(settlingAgent(f, "Stranded worker", false), () => "the stranded agent to be seeded"); + await advance(MINUTE, f.tick); + f.events.length = 0; + const before = state.meter.read(); + await bounded(reconcileAbandonedExecutionControl(state.db, new Date()), () => "the abandoned run to be reconciled"); + await advance(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE, f.tick); + await bounded(createHeartbeatAgentStatus(state.db).reconcileStrandedAgentStatuses(() => new Set(), + { companyId: f.companyId, agentId: stranded.agentId }), () => "the stranded agent status to be reconciled"); + await advance(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE, f.tick); + const [abandonedAgent, strandedAgent] = await Promise.all([abandoned.agentId, stranded.agentId].map(id => + state.db.select({ status: agents.status }).from(agents).where(eq(agents.id, id)).then(rows => rows[0]?.status))); + const backlog = new Set([...abandoned.tasks, ...stranded.tasks]); + const checks = f.events.filter(event => event.name === "check"); + const result = { backlog: backlog.size, abandonedAgent, strandedAgent, + backlogChecks: checks.filter(event => backlog.has(String(event.fields.issueId))).length, + runTaskChecks: checks.filter(event => event.fields.issueId === abandoned.runTask).length, + byWake: byWake(f.events), statements: state.meter.read().statements - before.statements }; + console.info(`agent-settle census: ${JSON.stringify(result)}`); + expect(abandonedAgent).not.toBe("running"); + expect(strandedAgent).not.toBe("running"); + expect(result.backlogChecks).toBe(0); + }, TEST_TIMEOUT_MS); + + it("still checks an abandoned run's task promptly when the run is reconciled", async () => { + const f = await boot(); + const abandoned = await bounded(settlingAgent(f, "Abandoned worker", true), () => "the abandoned agent to be seeded"); + await advance(MINUTE, f.tick); + const reconciledAt = Date.now(); + await bounded(reconcileAbandonedExecutionControl(state.db, new Date()), () => "the abandoned run to be reconciled"); + await advance(2 * MINUTE, f.tick); + const [checked] = checksOf(f.events, abandoned.runTask!, reconciledAt); + expect(checked, "the abandoned run's task was not checked after reconciliation").toBeDefined(); + expect(checked!.at - reconciledAt).toBeLessThanOrEqual(MINUTE); + }, TEST_TIMEOUT_MS); + it("does not re-check settled tasks for company hints, reloads or server replacements", async () => { const f = await boot(); const before = state.meter.read(); diff --git a/server/src/__tests__/issue-recovery-saved-deferrals.test.ts b/server/src/__tests__/issue-recovery-saved-deferrals.test.ts index 15453e20fd2..d6343c09ac1 100644 --- a/server/src/__tests__/issue-recovery-saved-deferrals.test.ts +++ b/server/src/__tests__/issue-recovery-saved-deferrals.test.ts @@ -51,7 +51,8 @@ describeIssueRecovery("saved task recovery passes and deferrals", () => { } /** One server: a coordinator and the recovery queues, with recorded checks. */ - async function server(behavior: { deferUntil: Map; suppressed: () => boolean; checks: Array<{ id: string; at: number }> }) { + async function server(behavior: { deferUntil: Map; suppressed: () => boolean; checks: Array<{ id: string; at: number }>; + rolling?: Set }) { const coordinator = createDeliveryWorkCoordinator({ owner: state.db, canRun: () => true, canReconcile: () => true, onError: error => { throw error; } }); running.add(coordinator); @@ -65,6 +66,7 @@ describeIssueRecovery("saved task recovery passes and deferrals", () => { behavior.checks.push({ id: scope.issueId, at: Date.now() }); const until = behavior.deferUntil.get(scope.issueId); if (until !== undefined && until > Date.now()) scope.onDeferredDeadline(until); + if (behavior.rolling?.has(scope.issueId)) scope.onDeferredDeadline(Date.now() + 30_000); }; const work = registerIssueRecoveryWork(state.db, coordinator, { stranded, orphanBlockers: async () => {}, recoveryActions: async () => {}, @@ -180,6 +182,35 @@ describeIssueRecovery("saved task recovery passes and deferrals", () => { expect(await bounded(saved(), () => "the deferral to be read")).toHaveLength(0); }, TEST_TIMEOUT_MS); + it("writes a task's deferral a few times, not on every check, while it defers every 30 seconds", async () => { + const f = await bounded(seed(10), () => "the fixture to be seeded"); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const target = f.tasks[2]!; + const behavior = { deferUntil: new Map(), rolling: new Set([target]), suppressed: () => false, + checks: [] as Array<{ id: string; at: number }> }; + const statements: string[] = []; + const priorDebug = state.db.$client.options.debug; + state.db.$client.options.debug = (_connection, query) => { statements.push(query); }; + try { + await server(behavior); + await advance(20 * MINUTE); + } finally { state.db.$client.options.debug = priorDebug; } + const checks = behavior.checks.filter(check => check.id === target).length; + const writes = statements.filter(query => /insert into issue_recovery_deferrals/i.test(query)).length; + // The saved deadline still moves forward, so the row of a task that keeps + // deferring never looks stale to the prune (10 minutes past its deadline). + const [row] = await bounded(state.db.select({ dueAt: issueRecoveryDeferrals.dueAt }).from(issueRecoveryDeferrals) + .where(and(eq(issueRecoveryDeferrals.queue, "stranded-assignments"), eq(issueRecoveryDeferrals.issueId, target))), + () => "the deferral to be read"); + const savedBehind = row ? Date.now() - row.dueAt.getTime() : null; + console.info(`rolling-deferral census: ${JSON.stringify({ minutes: 20, checks, deferralWrites: writes, savedBehindMs: savedBehind })}`); + expect(checks).toBeGreaterThanOrEqual(30); + expect(writes).toBeLessThanOrEqual(5); + expect(writes).toBeGreaterThan(1); + expect(savedBehind, "the saved deferral was not kept").not.toBeNull(); + expect(savedBehind!).toBeLessThan(6 * MINUTE); + }, TEST_TIMEOUT_MS); + it("checks every task again after scheduling suppression, even with a recent saved full pass", async () => { const f = await bounded(seed(30), () => "the fixture to be seeded"); vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 3bfc73d77c9..87ca219cfa8 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -9,6 +9,7 @@ import { notifyExecutionRunWork } from "./execution-control-notifications.js"; import { randomUUID } from "node:crypto"; import { logger } from "../middleware/logger.js"; import { and, asc, eq, isNotNull, lte, sql } from "drizzle-orm"; +import { settledHeartbeat } from "./heartbeat/agent-status.js"; import { agents, heartbeatRuns, @@ -292,7 +293,10 @@ export async function reconcileAbandonedExecutionControl( } await tx .update(agents) - .set({ status: "idle", updatedAt: now }) + // A run settling records a heartbeat, as finalizeAgentStatus does, so + // the changed-rows agent rule (recovery/changed-tasks.ts) does not + // read it as a pause or resume and re-check the agent's whole backlog. + .set({ status: "idle", lastHeartbeatAt: settledHeartbeat(now), updatedAt: now }) .where( and( eq(agents.id, run.agentId), diff --git a/server/src/services/heartbeat/agent-status.ts b/server/src/services/heartbeat/agent-status.ts index 0d03f6ac828..511f1da9909 100644 --- a/server/src/services/heartbeat/agent-status.ts +++ b/server/src/services/heartbeat/agent-status.ts @@ -75,6 +75,14 @@ export function settledAgentWorkCondition(liveRunIds: ReadonlySet, now: OR c.receipt ? 'baseline' OR c.receipt->>'cleanupPending' = 'true'))`; } +/** The heartbeat a write records when it settles an agent after its runs end + * without finalizeAgentStatus: the settlement time, so the changed-rows agent + * rule does not re-check the agent's backlog, but never a first heartbeat for + * an agent that has none (that stays the first real one). */ +export function settledHeartbeat(now: Date) { + return sql`case when ${agents.lastHeartbeatAt} is null then null else ${now.toISOString()}::timestamptz end`; +} + export function createHeartbeatAgentStatus(db: Db) { async function finalizeAgentStatus( agentId: string, @@ -153,8 +161,10 @@ export function createHeartbeatAgentStatus(db: Db) { .orderBy(desc(heartbeatRuns.finishedAt), desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); const nextStatus = settledStatus(lastRun?.status); await notifyAttentionRetentionWork(tx, { companyId: candidate.companyId, sourceKind: "agent_error_alert" }); + // Settling the agent's runs records a heartbeat, as finalizeAgentStatus + // does; see the agent rule in recovery/changed-tasks.ts. const [agent] = await tx.update(agents).set({ status: nextStatus, - errorReason: nextStatus === "error" ? errorReason(lastRun?.error) : null, updatedAt: now, + errorReason: nextStatus === "error" ? errorReason(lastRun?.error) : null, lastHeartbeatAt: settledHeartbeat(now), updatedAt: now, }).where(and(eq(agents.id, candidate.id), eq(agents.companyId, candidate.companyId), eq(agents.status, "running"), eq(agents.lifecycleState, "ready"))).returning(); await logActivity(tx as unknown as Db, { diff --git a/server/src/services/recovery/changed-tasks.ts b/server/src/services/recovery/changed-tasks.ts index df6cdbb9a66..0cbdd1e090f 100644 --- a/server/src/services/recovery/changed-tasks.ts +++ b/server/src/services/recovery/changed-tasks.ts @@ -6,7 +6,11 @@ import { issues } from "@paperclipai/db"; * tasks need recovery (a pause, resume, hold or configuration change). Run * lifecycle writes are left out: a run's finish records a heartbeat, and a run's * start flips the agent to running, which makes none of its other tasks - * stranded. Every run start would otherwise re-check all of the agent's tasks + * stranded. Every write that settles an agent's status after its runs end + * (finalizeAgentStatus, abandoned-run and stranded-status reconciliation) + * must set last_heartbeat_at with updated_at (see settledHeartbeat), or this + * clause re-checks all of the agent's tasks. An agent with no heartbeat yet + * keeps none, so its settlement still re-checks its (new) backlog once. Every run start would otherwise re-check all of the agent's tasks * for two backstop windows. */ export function changedTaskCondition(since: Date | SQL) { const at = since instanceof Date ? sql`${since.toISOString()}::timestamptz` : since; diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index 9fc274f2b70..5cff1348a9b 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -19,6 +19,11 @@ type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: numb const PAGE_SIZE = 100; /** How long past its deadline a saved deferral of a task that left its queue is kept. */ const STALE_DEFERRAL_MS = 10 * 60_000; +/** How long past its saved deadline a task that keeps deferring leaves its row as is. + * A boot re-checks every task with a row whatever its deadline, so only the + * stale prune reads it; refreshing well before STALE_DEFERRAL_MS keeps the + * row of a task that is still deferring from ever looking stale. */ +const DEFERRAL_REFRESH_MS = STALE_DEFERRAL_MS / 2; export const resolvedDependencyCandidate = sql`exists (select 1 from ${issueRelations} r where r.company_id = ${issues.companyId} and r.related_issue_id = ${issues.id} and r.type = 'blocks' offset 0) @@ -181,7 +186,9 @@ export function registerIssueRecoveryWork( async deferral(id: string, companyId: string, dueAt: number | null): Promise { const known = deferred.get(id); if (dueAt !== null) { - if (known === dueAt) return true; + // A task that defers again and again (now + 30 s while a run lives) + // keeps its saved row until that row's deadline is well past. + if (known !== undefined && (known === dueAt || Date.now() < known + DEFERRAL_REFRESH_MS)) return true; deferred.delete(id); const saved = await db.execute(sql`insert into issue_recovery_deferrals (queue, issue_id, company_id, due_at) values (${queue}, ${id}::uuid, ${companyId}::uuid, ${at(dueAt)})