From 379dda9e26b0fbbf0cbd17f155120a284faf4b50 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 15 Nov 2026 00:05:27 -0600 Subject: [PATCH] perf(recovery): re-check only changed tasks between hourly full backstop passes Co-Authored-By: Claude Opus 5.5 --- .../__tests__/helpers/recovery-population.ts | 54 ++++++ .../issue-recovery-deadlines.test.ts | 2 +- .../issue-recovery-idle-backstop.test.ts | 166 ++++++++++++++++++ .../__tests__/issue-recovery.test-fixture.ts | 8 +- .../services/issue-recovery-notifications.ts | 1 + .../services/recovery/assigned-work-sweep.ts | 5 +- server/src/services/recovery/changed-tasks.ts | 18 ++ server/src/services/recovery/work.ts | 49 ++++-- 8 files changed, 284 insertions(+), 19 deletions(-) create mode 100644 server/src/__tests__/helpers/recovery-population.ts create mode 100644 server/src/__tests__/issue-recovery-idle-backstop.test.ts create mode 100644 server/src/services/recovery/changed-tasks.ts diff --git a/server/src/__tests__/helpers/recovery-population.ts b/server/src/__tests__/helpers/recovery-population.ts new file mode 100644 index 00000000000..16947d39334 --- /dev/null +++ b/server/src/__tests__/helpers/recovery-population.ts @@ -0,0 +1,54 @@ +import { agents, heartbeatRuns, issueRecoveryActions, issues, type Db } from "@paperclipai/db"; + +const HOUR = 60 * 60_000; + +export type RecoveryPopulation = { + agentIds: string[]; + pausedAgentId: string; + terminatedAgentId: string; + inReview: string[]; + inProgress: string[]; + todo: string[]; + withRecoveryAction: string[]; + terminated: string[]; +}; + +export async function seedRecoveryPopulation(db: Db, companyId: string, counts: { + agents: number; inReview: number; inProgress?: number; todo?: number; recoveryActions?: number; terminated?: number; +}): Promise { + const config = { adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true } }, role: "engineer" }; + const [ceo] = await db.insert(agents).values({ ...config, companyId, name: "Chief", status: "idle" }).returning({ id: agents.id }); + const managers = await db.insert(agents).values(Array.from({ length: 6 }, (_, index) => ({ ...config, companyId, + name: `Manager ${index}`, status: "idle", reportsTo: ceo!.id }))).returning({ id: agents.id }); + const workers = await db.insert(agents).values(Array.from({ length: Math.max(1, counts.agents - 9) }, (_, index) => ({ ...config, + companyId, name: `Worker ${index}`, status: "idle", reportsTo: managers[index % managers.length]!.id }))).returning({ id: agents.id }); + const [paused] = await db.insert(agents).values({ ...config, companyId, name: "Paused worker", status: "paused", + lifecycleHolds: ["manual"], reportsTo: managers[0]!.id }).returning({ id: agents.id }); + const [terminatedAgent] = await db.insert(agents).values({ ...config, companyId, name: "Former worker", status: "terminated", + reportsTo: managers[1]!.id }).returning({ id: agents.id }); + const workerIds = workers.map(row => row.id); + const finishedAt = new Date(Date.now() - HOUR); + + const insertTasks = async (count: number, values: (index: number) => Partial) => count === 0 ? [] + : (await db.insert(issues).values(Array.from({ length: count }, (_, index) => ({ companyId, title: `Task ${index}`, + ...values(index) }))).returning({ id: issues.id, assigneeAgentId: issues.assigneeAgentId })).map(row => row); + + const reviews = await insertTasks(counts.inReview, index => ({ status: "in_review", + assigneeAgentId: workerIds[index % workerIds.length] })); + if (reviews.length) await db.insert(heartbeatRuns).values(reviews.map(task => ({ companyId, agentId: task.assigneeAgentId!, + status: "succeeded", runtimeMode: "native", startedAt: finishedAt, finishedAt, contextSnapshot: { issueId: task.id } }))); + const progressing = await insertTasks(counts.inProgress ?? 0, () => ({ status: "in_progress", assigneeAgentId: paused!.id })); + if (progressing.length) await db.insert(heartbeatRuns).values(progressing.map(task => ({ companyId, agentId: paused!.id, + status: "cancelled", errorCode: "agent_paused", startedAt: finishedAt, finishedAt, contextSnapshot: { issueId: task.id } }))); + const todo = await insertTasks(counts.todo ?? 0, () => ({ status: "todo", assigneeAgentId: paused!.id })); + const terminated = await insertTasks(counts.terminated ?? 0, () => ({ status: "todo", assigneeAgentId: terminatedAgent!.id })); + const withAction = reviews.slice(0, counts.recoveryActions ?? 0); + if (withAction.length) await db.insert(issueRecoveryActions).values(withAction.map((task, index) => ({ companyId, + sourceIssueId: task.id, kind: "missing_disposition", status: "active", ownerType: "agent", ownerAgentId: task.assigneeAgentId, + cause: "seeded", fingerprint: `seeded-${index}`, nextAction: "seeded" }))); + return { agentIds: [ceo!.id, ...managers.map(row => row.id), ...workerIds, paused!.id, terminatedAgent!.id], + pausedAgentId: paused!.id, terminatedAgentId: terminatedAgent!.id, + inReview: reviews.map(row => row.id), inProgress: progressing.map(row => row.id), todo: todo.map(row => row.id), + withRecoveryAction: withAction.map(row => row.id), terminated: terminated.map(row => row.id) }; +} diff --git a/server/src/__tests__/issue-recovery-deadlines.test.ts b/server/src/__tests__/issue-recovery-deadlines.test.ts index f87ab9e9564..09195fc976e 100644 --- a/server/src/__tests__/issue-recovery-deadlines.test.ts +++ b/server/src/__tests__/issue-recovery-deadlines.test.ts @@ -113,7 +113,7 @@ describeIssueRecovery("issue recovery production producers and deadlines", () => const [run] = await state.db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", finishedAt: new Date(), contextSnapshot: { issueId: task.id } }).returning({ id: heartbeatRuns.id }); const backlog = async (count: number) => (await state.db.insert(issues).values(Array.from({ length: count }, (_, index) => ({ - companyId, title: `Backlog ${index}`, status: "todo", assigneeAgentId: other.id, + companyId, title: `Backlog ${index}`, status: "todo", assigneeAgentId: other.id, createdAt: new Date(), updatedAt: new Date(), }))).returning({ id: issues.id })).map(row => row.id); return { runId: run.id, taskId: task.id, siblingId: sibling.id, backlog, affected: [parent.id, task.id, dependent.id].sort() }; } diff --git a/server/src/__tests__/issue-recovery-idle-backstop.test.ts b/server/src/__tests__/issue-recovery-idle-backstop.test.ts new file mode 100644 index 00000000000..baf8fd03798 --- /dev/null +++ b/server/src/__tests__/issue-recovery-idle-backstop.test.ts @@ -0,0 +1,166 @@ +import { setImmediate as immediate } from "node:timers/promises"; +import { and, eq, sql } from "drizzle-orm"; +import { expect, it, vi } from "vitest"; +import { agents, heartbeatRuns, issues } from "@paperclipai/db"; +import { workSignalDebug } from "@paperclipai/db/work-signal-debug"; +import { ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_FULL_BACKSTOP_MS, notifyIssueGraphRecovery } + from "../services/issue-recovery-notifications.js"; +import { issueService } from "../services/issues.js"; +import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js"; +import { promoteDeferredWorkIntents } from "../services/work-intent-outbox.js"; +import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; +import { seedRecoveryPopulation } from "./helpers/recovery-population.js"; + +vi.hoisted(() => { process.env.PAPERCLIP_DEBUG_WORK_SIGNALS = "stranded-assignments:1"; }); + +const MINUTE = 60_000; +type Recorded = { at: number; name: string; fields: Record }; + +describeIssueRecovery("stranded-task checks at idle for a staging-shaped company", () => { + const state = issueRecoveryFixture({ meter: true }); + + async function advance(ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { + await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())); + await Promise.allSettled([...state.passes]); await immediate(); + } + } + + function record() { + const events: Recorded[] = []; + vi.spyOn(workSignalDebug, "event").mockImplementation((topic, name, fields = {}) => { + if (topic === "stranded-assignments") events.push({ at: Date.now(), name, fields }); + }); + vi.spyOn(workSignalDebug, "signal").mockImplementation(() => {}); + return events; + } + + function summarize(events: Recorded[]) { + const checks: Record = {}, loads: Record = {}; + for (const event of events) { + if (event.name === "check") checks[String(event.fields.wake)] = (checks[String(event.fields.wake)] ?? 0) + 1; + if (event.name === "load") { + const scope = String(event.fields.scope); + const kind = scope === "*" ? "reload" : scope.endsWith(",true]") ? "backstop" : /^\["[^"]+",null,null,null,null\]$/.test(scope) + ? "company" : "hint"; + loads[kind] = (loads[kind] ?? 0) + 1; + } + } + return { checks, loads, totalChecks: Object.values(checks).reduce((sum, count) => sum + count, 0) }; + } + + const checkedAt = (events: Recorded[], issueId: string) => + events.find(event => event.name === "check" && event.fields.issueId === issueId)?.at ?? null; + + async function seed() { + const f = await state.fixture(); + const population = await seedRecoveryPopulation(state.db, f.companyId, { agents: 50, inReview: 780, inProgress: 40, todo: 30, + recoveryActions: 12, terminated: 3 }); + const [side] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Finished side task", status: "done" }) + .returning({ id: issues.id }); + const activity = () => state.db.transaction(tx => notifyIssueGraphRecovery(tx, { companyId: f.companyId, issueId: side!.id })); + return { ...f, population, activity }; + } + + async function boot() { + const f = await seed(); + const events = record(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await state.start({}); + await advance(10 * MINUTE); + return { ...f, events, boot: summarize(events.splice(0)) }; + } + + async function heldTask(f: Awaited>) { + const old = new Date(Date.now() - 2 * 60 * MINUTE); + const [held] = await state.db.insert(agents).values({ companyId: f.companyId, name: "Held worker", role: "engineer", status: "paused", + lifecycleHolds: ["manual"], adapterType: "process", reportsTo: f.population.agentIds[1], createdAt: old, updatedAt: old, + adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true } } }).returning({ id: agents.id }); + const [waiting] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Waits for its agent", status: "todo", + assigneeAgentId: held!.id, createdAt: old, updatedAt: old }).returning({ id: issues.id }); + return { held: held!, waiting: waiting! }; + } + + async function trickle(f: Awaited>, minutes: number) { + const before = state.meter.read(); + for (let minute = 0; minute < minutes; minute++) { await f.activity(); await advance(MINUTE); } + const after = state.meter.read(); + const raw = f.events.splice(0); + return { ...summarize(raw), raw, statements: after.statements - before.statements, + bytes: after.bytesReceived - before.bytesReceived }; + } + + it("checks only changed tasks between hourly full passes while unrelated activity continues", async () => { + const f = await boot(); + const between = await trickle(f, 30); + const hour = await trickle(f, 30); + const { raw: _first, ...firstHalfHour } = between, { raw: _second, ...secondHalfHour } = hour; + console.info(JSON.stringify({ boot: f.boot, firstHalfHour, secondHalfHour, + perSecond: { checks: (between.totalChecks + hour.totalChecks) / 3600, statements: (between.statements + hour.statements) / 3600 } })); + expect(f.boot.totalChecks).toBeGreaterThanOrEqual(f.population.inReview.length); + expect(between.totalChecks).toBe(0); + expect(between.statements).toBeLessThan(2_000); + expect(hour.checks.backstop ?? 0).toBeLessThanOrEqual(f.population.inReview.length + 100); + }, 1_800_000); + + it("recovers a task whose hint was never sent within the backstop bound", async () => { + const f = await boot(); + const worker = f.population.agentIds[7]!; + const now = new Date(); + const [missed] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Created without a hint", status: "todo", + assigneeAgentId: worker, createdAt: now, updatedAt: now }).returning({ id: issues.id }); + const window = await trickle(f, Math.ceil((ISSUE_RECOVERY_BACKSTOP_MS + MINUTE) / MINUTE) + 1); + const checked = checkedAt(window.raw, missed!.id); + expect(checked).not.toBeNull(); + expect(checked! - now.getTime()).toBeLessThanOrEqual(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE); + const runs = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${missed!.id}`)); + expect(runs.length).toBeGreaterThan(0); + }, 1_800_000); + + it("re-checks a task whose only change is outside its own rows within the hourly full pass", async () => { + const f = await boot(); + const { held, waiting } = await heldTask(f); + await state.db.execute(sql`update agents set status = 'idle', lifecycle_holds = '[]'::jsonb where id = ${held.id}`); + const changedAt = Date.now(); + const window = await trickle(f, Math.ceil((ISSUE_RECOVERY_FULL_BACKSTOP_MS + 2 * MINUTE) / MINUTE)); + const checks = window.raw.filter(event => event.name === "check" && event.fields.issueId === waiting.id); + expect(checks.length).toBeGreaterThan(0); + expect(checks[0]!.at - changedAt).toBeGreaterThan(2 * ISSUE_RECOVERY_BACKSTOP_MS); + expect(checks[0]!.at - changedAt).toBeLessThanOrEqual(ISSUE_RECOVERY_FULL_BACKSTOP_MS + 2 * MINUTE); + const runs = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, f.companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${waiting.id}`)); + expect(runs.length).toBeGreaterThan(0); + }, 1_800_000); + + it("picks up a finished run, a user message, a raised budget and a resumed agent within a minute", async () => { + const f = await boot(); + await trickle(f, 6); + const [target, messaged] = f.population.inReview; + const within = async (issueId: string, act: () => Promise) => { + f.events.length = 0; + const at = Date.now(); + await act(); + await advance(60_000); + const checked = checkedAt(f.events, issueId); + expect(checked, issueId).not.toBeNull(); + expect(checked! - at).toBeLessThanOrEqual(60_000); + }; + const [run] = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${target!}`).limit(1); + await within(target!, () => state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: run!.id }))); + await advance(31_000); + await within(messaged!, () => issueService(state.db).addComment(messaged!, "Please look again", { userId: "recovery-user" })); + await advance(31_000); + await within(target!, () => state.db.transaction(tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, trigger: "pause_lifted" }))); + await advance(31_000); + const { held, waiting } = await heldTask(f); + await advance(31_000); + await within(waiting.id, async () => { + await state.db.update(agents).set({ status: "idle", lifecycleHolds: [], updatedAt: new Date() }).where(eq(agents.id, held.id)); + await state.db.transaction(tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, trigger: "pause_lifted" })); + }); + }, 1_800_000); +}); diff --git a/server/src/__tests__/issue-recovery.test-fixture.ts b/server/src/__tests__/issue-recovery.test-fixture.ts index da4485f62e4..4aa0b6ac764 100644 --- a/server/src/__tests__/issue-recovery.test-fixture.ts +++ b/server/src/__tests__/issue-recovery.test-fixture.ts @@ -11,6 +11,7 @@ import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js import { instanceSettingsService } from "../services/instance-settings.js"; import { issueService } from "../services/issues.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { meterDatabaseStatements } from "./helpers/database-statement-meter.js"; vi.setConfig({ testTimeout: 90_000 }); @@ -18,7 +19,8 @@ const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSUE_RECOVERY_TEST_D const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); export const describeIssueRecovery: ReturnType = describe.skipIf(!support.supported); -export function issueRecoveryFixture() { +export function issueRecoveryFixture(options: { meter?: boolean } = {}) { + let meter: ReturnType | undefined; let database: Awaited> | undefined; let db: ReturnType; let heartbeat: ReturnType; @@ -35,6 +37,7 @@ export function issueRecoveryFixture() { beforeAll(async () => { if (!external) database = await startEmbeddedPostgresTestDatabase("issue-recovery-work-"); db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + if (options.meter) meter = meterDatabaseStatements(db, external ?? database!.connectionString); await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); }, 90_000); beforeEach(async () => { @@ -49,7 +52,7 @@ export function issueRecoveryFixture() { vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]); }); - afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + afterAll(async () => { meter?.uninstall(); await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); async function start(options: { backstopMs?: number } = { backstopMs: 0 }) { coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); @@ -232,6 +235,7 @@ export function issueRecoveryFixture() { return { start, compete, fixture, completingProvider, settled, quiet, finish, completed, savedConversation, get db() { return db; }, + get meter() { return meter!; }, get heartbeat() { return heartbeat; }, set heartbeat(value: typeof heartbeat) { heartbeat = value; }, get coordinator() { return coordinator; }, diff --git a/server/src/services/issue-recovery-notifications.ts b/server/src/services/issue-recovery-notifications.ts index 08751b36220..5c5038d5425 100644 --- a/server/src/services/issue-recovery-notifications.ts +++ b/server/src/services/issue-recovery-notifications.ts @@ -10,6 +10,7 @@ export type IssueRecoveryScope = { companyId: string; issueId?: string; graphRun export const ISSUE_RECOVERY_MAX_ISSUE_IDS = 100; export const ISSUE_RECOVERY_BACKSTOP_MS = 5 * 60_000; +export const ISSUE_RECOVERY_FULL_BACKSTOP_MS = 60 * 60_000; const UUID_PATTERN = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; const COMPANY_QUEUES: DeliveryQueue[] = [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.taskWatchdogs]; diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index c3ca5f80ba1..fb831e165ab 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -20,6 +20,7 @@ import { import { logger } from "../../middleware/logger.js"; import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; import { evaluateAgentInvokability } from "../agent-invokability.js"; +import { changedTaskCondition } from "./changed-tasks.js"; import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js"; import { unadmittedChatWakeupCondition } from "../durable-chat-wakeup.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; @@ -510,7 +511,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { order by lr.created_at desc limit 1), false)`; } - async function listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; createdAfter?: Date | null }, + async function listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; createdAfter?: Date | null; + changedSince?: Date }, cursor: string | null, limit: number) { const at = now(); const liveRun = sql`exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId} @@ -530,6 +532,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { scope.issueIds ? inArray(issues.id, scope.issueIds) : undefined, cursor ? sql`${issues.id} > ${cursor}` : undefined, scope.createdAfter ? gte(issues.createdAt, scope.createdAfter) : undefined, + scope.changedSince ? changedTaskCondition(scope.changedSince) : undefined, inArray(issues.status, [...OPEN_STATUSES]), sql`${issues.assigneeAgentId} is not null`, isNull(issues.assigneeUserId), diff --git a/server/src/services/recovery/changed-tasks.ts b/server/src/services/recovery/changed-tasks.ts new file mode 100644 index 00000000000..2a0832131aa --- /dev/null +++ b/server/src/services/recovery/changed-tasks.ts @@ -0,0 +1,18 @@ +import { sql } from "drizzle-orm"; +import { issues } from "@paperclipai/db"; + +export function changedTaskCondition(since: Date) { + const at = sql`${since.toISOString()}::timestamptz`; + return sql`(${issues.updatedAt} >= ${at} + or exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId} + and (r.context_snapshot->>'issueId') = ${issues.id}::text and r.updated_at >= ${at}) + or exists (select 1 from issue_recovery_actions a where a.company_id = ${issues.companyId} + and a.source_issue_id = ${issues.id} and a.updated_at >= ${at}) + or exists (select 1 from agent_wakeup_requests w where w.company_id = ${issues.companyId} + and (w.payload->>'issueId') = ${issues.id}::text and w.updated_at >= ${at}) + or exists (select 1 from issue_thread_interactions t where t.company_id = ${issues.companyId} + and t.issue_id = ${issues.id} and t.updated_at >= ${at}) + or exists (select 1 from issue_comments c where c.company_id = ${issues.companyId} + and c.issue_id = ${issues.id} and c.created_at >= ${at}) + or exists (select 1 from agents ag where ag.id = ${issues.assigneeAgentId} and ag.updated_at >= ${at}))`; +} diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index fc7b39b628b..79158470ec2 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -3,13 +3,14 @@ import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql, type SQL } from import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js"; -import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, issueRecoveryBackstopHint, issueRecoveryScopeMatches, resolveIssueGraph, type IssueRecoveryScope } +import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_FULL_BACKSTOP_MS, issueRecoveryBackstopHint, issueRecoveryScopeMatches, resolveIssueGraph, type IssueRecoveryScope } from "../issue-recovery-notifications.js"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; import { workSignalDebug } from "@paperclipai/db/work-signal-debug"; import { recoveryActionSweepCondition } from "./blocker-stall-wakeups.js"; +import { changedTaskCondition } from "./changed-tasks.js"; -type Entry = { id: string; companyId: string; dueAt: number; fromDeadline?: true }; +type Entry = { id: string; companyId: string; dueAt: number; fromDeadline?: true; origin?: "reload" | "backstop" | "company" | "hint" }; type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number, detail?: { unchanged: boolean }) => void }; const PAGE_SIZE = 100; @@ -33,22 +34,35 @@ export function registerIssueRecoveryWork( schedulingSuppressed(): Promise; /** The assigned-work invariant (assigned-work-sweep.ts). */ assignedWork?: { - listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[] }, cursor: string | null, limit: number): + listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; changedSince?: Date }, cursor: string | null, limit: number): Promise>; checkIssue(scope: Scope): Promise; }; }, canRun: () => boolean, - options: { backstopMs?: number } = {}, + options: { backstopMs?: number; fullBackstopMs?: number } = {}, ) { + const backstopMs = options.backstopMs ?? ISSUE_RECOVERY_BACKSTOP_MS; + const fullBackstopMs = options.fullBackstopMs ?? options.backstopMs ?? ISSUE_RECOVERY_FULL_BACKSTOP_MS; function fullPasses() { - const completedAt = new Map(); + const completedAt = new Map(), changedAt = new Map(); + const scans = new Map(); + const lastFull = (companyId: string) => Math.max(completedAt.get("*") ?? 0, completedAt.get(companyId) ?? 0); + const lastLook = (companyId: string) => Math.max(lastFull(companyId), changedAt.get(companyId) ?? 0); return { - backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop - ? Math.max(completedAt.get("*") ?? 0, completedAt.get(scope.companyId) ?? 0) + (options.backstopMs ?? ISSUE_RECOVERY_BACKSTOP_MS) - : null, - loaded(scope: IssueRecoveryScope | undefined, complete: boolean) { - if (complete && !scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) completedAt.set(scope?.companyId ?? "*", Date.now()); + backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop ? lastLook(scope.companyId) + backstopMs : null, + changedSince(scope: IssueRecoveryScope | undefined, key: string, firstPage: boolean): Date | null { + if (!scope?.backstop) return null; + if (firstPage || !scans.has(key)) scans.set(key, Date.now() - lastFull(scope.companyId) >= fullBackstopMs ? null + : new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS)); + return scans.get(key) ?? null; + }, + loaded(scope: IssueRecoveryScope | undefined, key: string, complete: boolean) { + if (!complete) return; + const changed = scans.get(key) ?? null; + scans.delete(key); + if (changed && scope) changedAt.set(scope.companyId, Date.now()); + else if (!scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) completedAt.set(scope?.companyId ?? "*", Date.now()); }, }; } @@ -94,8 +108,10 @@ export function registerIssueRecoveryWork( if (!enabled) { cursors.delete(key); return []; } if (await toCompany(scope, worker)) return { rows: null, complete: true }; const cursor = cursors.get(key); + const changedSince = passes.changedSince(scope, key, !cursor); const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and( scope ? eq(issues.companyId, scope.companyId) : undefined, + changedSince ? changedTaskCondition(changedSince) : undefined, scope?.issueId ? eq(issues.id, scope.issueId) : undefined, scope?.issueIds ? inArray(issues.id, scope.issueIds) : undefined, cursor ? gt(issues.id, cursor) : undefined, @@ -107,13 +123,15 @@ export function registerIssueRecoveryWork( )).orderBy(asc(issues.id)).limit(PAGE_SIZE); const complete = rows.length < PAGE_SIZE; if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); - passes.loaded(scope, complete); - return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete }; + passes.loaded(scope, key, complete); + const origin = !workSignalDebug.enabled ? undefined : !scope ? "reload" as const : scope.backstop ? "backstop" as const + : scope.issueId || isGraph(scope) ? "hint" as const : "company" as const; + return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete }; }, check: async entry => { if (!enabled) return null; if (workSignalDebug.enabled) workSignalDebug.event(queue, "check", { issueId: entry.id, companyId: entry.companyId, - wake: entry.fromDeadline ? "deadline" : "load", dueAt: entry.dueAt }); + wake: entry.fromDeadline ? "deadline" : entry.origin ?? "load", dueAt: entry.dueAt }); let next: number | null = null; await check({ companyId: entry.companyId, issueId: entry.id, onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } }); @@ -145,11 +163,12 @@ export function registerIssueRecoveryWork( const key = scopeKey(scope); if (!enabled) { cursors.delete(key); return []; } if (await toCompany(scope, worker)) return { rows: null, complete: true }; + const changedSince = passes.changedSince(scope, key, !cursors.has(key)); const rows = await sweep.listSuspects(scope ? { companyId: scope.companyId, issueId: scope.issueId, - issueIds: scope.issueIds } : {}, cursors.get(key) ?? null, PAGE_SIZE); + issueIds: scope.issueIds, ...(changedSince ? { changedSince } : {}) } : {}, cursors.get(key) ?? null, PAGE_SIZE); const complete = rows.length < PAGE_SIZE; if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); - passes.loaded(scope, complete); + passes.loaded(scope, key, complete); return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now(), ...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete }; },