From 802a76c153c7fc3e504348fddd02b18897c8d03d Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 15 Nov 2026 00:34:29 -0600 Subject: [PATCH] perf(recovery): keep unloaded deadlines on a changed-tasks pass and start the next window at scan start Co-Authored-By: Claude Opus 5.5 --- .../issue-recovery-deadlines.test.ts | 64 ++++++++++++++++++- server/src/services/recovery/work.ts | 25 +++++--- server/src/services/scoped-deadline-work.ts | 7 +- 3 files changed, 82 insertions(+), 14 deletions(-) diff --git a/server/src/__tests__/issue-recovery-deadlines.test.ts b/server/src/__tests__/issue-recovery-deadlines.test.ts index 09195fc976e..a63b1e3a983 100644 --- a/server/src/__tests__/issue-recovery-deadlines.test.ts +++ b/server/src/__tests__/issue-recovery-deadlines.test.ts @@ -1,7 +1,7 @@ import { setImmediate as immediate } from "node:timers/promises"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { randomUUID } from "node:crypto"; -import { and, eq, sql } from "drizzle-orm"; +import { and, eq, inArray, sql } from "drizzle-orm"; import { expect, it, vi } from "vitest"; import { agents, budgetPolicies, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db"; import { heartbeatService } from "../services/heartbeat.js"; @@ -223,6 +223,68 @@ describeIssueRecovery("issue recovery production producers and deadlines", () => expect(checkedAt.get(later)! - eventAt).toBeLessThanOrEqual(5_000); await quiet(); }); + async function advanceWithoutWaiting(ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())); await immediate(); } + } + it("keeps a saved deadline of a task that a changed-tasks backstop pass does not load", async () => { + const f = await fixture(); + const graph = await runGraph(f.companyId, f.agentId); + const [target] = await graph.backlog(20); + const checkedAt: number[] = []; + await recordedRecovery({ stranded: async scope => { + if (scope.issueId !== target) return; + checkedAt.push(Date.now()); + scope.onDeferredDeadline(Date.now() + 16 * 60_000); + } }); + await settleWork({ backstopPending: true }); + for (let round = 0; round < 2; round++) { + await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000); + await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId })); + await settleWork({ backstopPending: true }); + } + const savedFor = checkedAt.at(-1)! + 16 * 60_000; + checkedAt.length = 0; + await vi.advanceTimersByTimeAsync(savedFor - Date.now() + 2_000); + await settleWork({ backstopPending: true }); + expect(checkedAt.length).toBeGreaterThan(0); + expect(checkedAt[0]! - savedFor).toBeLessThanOrEqual(5_000); + }); + it("starts the next changed-tasks window from the start of a slow backstop scan", async () => { + const f = await fixture(); + const graph = await runGraph(f.companyId, f.agentId); + const backlog = (await graph.backlog(250)).sort(); + const behind = backlog[0]!; + let slow = false, changedBehind = false; + const checkedAt = new Map(); + await recordedRecovery({ stranded: async scope => { + checkedAt.set(scope.issueId, [...checkedAt.get(scope.issueId) ?? [], Date.now()]); + if (!slow) return; + vi.setSystemTime(Date.now() + 20_000); + if (scope.issueId === behind && !changedBehind) { + changedBehind = true; + await state.db.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, behind)); + } + } }, { fullBackstopMs: 24 * 60 * 60_000 }); + await settleWork({ backstopPending: true }); + await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000); + await state.db.update(issues).set({ updatedAt: new Date() }).where(inArray(issues.id, backlog)); + await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000); + slow = true; + checkedAt.clear(); + const scanStartedAt = Date.now(); + await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId })); + await settleWork({ backstopPending: true }); + slow = false; + expect(changedBehind).toBe(true); + expect(backlog.every(id => checkedAt.has(id))).toBe(true); + expect(Date.now() - scanStartedAt).toBeGreaterThan(2 * ISSUE_RECOVERY_BACKSTOP_MS); + checkedAt.clear(); + await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000); + await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId })); + await settleWork({ backstopPending: true }); + expect(checkedAt.has(behind)).toBe(true); + }, 300_000); it("keeps the company pass for hold releases that can admit any task", async () => { const f = await fixture(); const graph = await runGraph(f.companyId, f.agentId); diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index 79158470ec2..0ed478e333f 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -46,23 +46,27 @@ export function registerIssueRecoveryWork( const fullBackstopMs = options.fullBackstopMs ?? options.backstopMs ?? ISSUE_RECOVERY_FULL_BACKSTOP_MS; function fullPasses() { const completedAt = new Map(), changedAt = new Map(); - const scans = new Map(); + const scans = new Map(); const lastFull = (companyId: string) => Math.max(completedAt.get("*") ?? 0, completedAt.get(companyId) ?? 0); const lastLook = (companyId: string) => Math.max(lastFull(companyId), changedAt.get(companyId) ?? 0); return { backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop ? lastLook(scope.companyId) + backstopMs : null, changedSince(scope: IssueRecoveryScope | undefined, key: string, firstPage: boolean): Date | null { - if (!scope?.backstop) return null; - if (firstPage || !scans.has(key)) scans.set(key, Date.now() - lastFull(scope.companyId) >= fullBackstopMs ? null - : new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS)); - return scans.get(key) ?? null; + if (firstPage || !scans.has(key)) scans.set(key, { startedAt: Date.now(), + since: !scope?.backstop || Date.now() - lastFull(scope.companyId) >= fullBackstopMs ? null + : new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS) }); + return scans.get(key)!.since; }, loaded(scope: IssueRecoveryScope | undefined, key: string, complete: boolean) { if (!complete) return; - const changed = scans.get(key) ?? null; + const scan = scans.get(key); scans.delete(key); - if (changed && scope) changedAt.set(scope.companyId, Date.now()); - else if (!scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) completedAt.set(scope?.companyId ?? "*", Date.now()); + const startedAt = scan?.startedAt ?? Date.now(); + if (scan?.since && scope) changedAt.set(scope.companyId, Math.max(changedAt.get(scope.companyId) ?? 0, startedAt)); + else if (!scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) { + const companyKey = scope?.companyId ?? "*"; + completedAt.set(companyKey, Math.max(completedAt.get(companyKey) ?? 0, startedAt)); + } }, }; } @@ -124,9 +128,10 @@ export function registerIssueRecoveryWork( const complete = rows.length < PAGE_SIZE; if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); passes.loaded(scope, key, complete); + const keepUnseen = changedSince ? { keepUnseen: true as const } : {}; const origin = !workSignalDebug.enabled ? undefined : !scope ? "reload" as const : scope.backstop ? "backstop" as const : scope.issueId || isGraph(scope) ? "hint" as const : "company" as const; - return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete }; + return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete, ...keepUnseen }; }, check: async entry => { if (!enabled) return null; @@ -170,7 +175,7 @@ export function registerIssueRecoveryWork( if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); passes.loaded(scope, key, complete); return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now(), - ...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete }; + ...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete, ...(changedSince ? { keepUnseen: true as const } : {}) }; }, check: async entry => { if (!enabled) return null; diff --git a/server/src/services/scoped-deadline-work.ts b/server/src/services/scoped-deadline-work.ts index b6ad70881c6..944f400a343 100644 --- a/server/src/services/scoped-deadline-work.ts +++ b/server/src/services/scoped-deadline-work.ts @@ -36,7 +36,7 @@ export function registerScopedDeadlineWork( * the scopes of its own key (and unkeyed scopes), not against every scope. */ scopeKey?: { entry(entry: Entry): string; scope(scope: Scope): string | undefined }; - load(scope?: Scope, signal?: AbortSignal): Promise; + load(scope?: Scope, signal?: AbortSignal): Promise; // Owners may retain pending queue priority while refreshing eligibility. mergeEntry?(pending: Entry, loaded: Entry): Entry; priority?(entry: Entry): number; @@ -164,7 +164,8 @@ export function registerScopedDeadlineWork( const scan = scans.get(hint) ?? { version: versions.get(hint) ?? 0, seen: new Set() }; scans.set(hint, scan); const loaded = await options.load(scope, signal); - const result = Array.isArray(loaded) ? { rows: loaded, complete: true } : loaded; + const result: { rows: Entry[] | null; complete: boolean; keepUnseen?: boolean } = Array.isArray(loaded) + ? { rows: loaded, complete: true } : loaded; const progressScopes = [...scans].map(([other, progress]) => ({ progress, scope: decode(other) })); for (const entry of result.rows ?? []) { const pending = entries.get(entry.id); @@ -173,7 +174,7 @@ export function registerScopedDeadlineWork( for (const { progress, scope: other } of progressScopes) if (options.matches(entry, other)) progress.seen.add(entry.id); } if (!result.complete) { dirty.add(hint); continue; } - if (result.rows !== null) for (const entry of entriesIn(scope)) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) { + if (result.rows !== null && !result.keepUnseen) for (const entry of entriesIn(scope)) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) { entries.delete(entry.id); failures.delete(entry.id); } scans.delete(hint);