perf(recovery): keep unloaded deadlines on a changed-tasks pass and start the next window at scan start

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-15 00:34:29 -06:00
1 parent 379dda9e26
commit 802a76c153
3 files changed
+82 -14

No files matched your search

@@ -1,7 +1,7 @@
import { setImmediate as immediate } from "node:timers/promises";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { randomUUID } from "node:crypto";
import { and, eq, sql } from "drizzle-orm";
import { and, eq, inArray, sql } from "drizzle-orm";
import { expect, it, vi } from "vitest";
import { agents, budgetPolicies, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db";
import { heartbeatService } from "../services/heartbeat.js";
@@ -223,6 +223,68 @@ describeIssueRecovery("issue recovery production producers and deadlines", () =>
expect(checkedAt.get(later)! - eventAt).toBeLessThanOrEqual(5_000);
await quiet();
});
async function advanceWithoutWaiting(ms: number) {
const until = Date.now() + ms;
while (Date.now() < until) { await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())); await immediate(); }
}
it("keeps a saved deadline of a task that a changed-tasks backstop pass does not load", async () => {
const f = await fixture();
const graph = await runGraph(f.companyId, f.agentId);
const [target] = await graph.backlog(20);
const checkedAt: number[] = [];
await recordedRecovery({ stranded: async scope => {
if (scope.issueId !== target) return;
checkedAt.push(Date.now());
scope.onDeferredDeadline(Date.now() + 16 * 60_000);
} });
await settleWork({ backstopPending: true });
for (let round = 0; round < 2; round++) {
await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000);
await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId }));
await settleWork({ backstopPending: true });
}
const savedFor = checkedAt.at(-1)! + 16 * 60_000;
checkedAt.length = 0;
await vi.advanceTimersByTimeAsync(savedFor - Date.now() + 2_000);
await settleWork({ backstopPending: true });
expect(checkedAt.length).toBeGreaterThan(0);
expect(checkedAt[0]! - savedFor).toBeLessThanOrEqual(5_000);
});
it("starts the next changed-tasks window from the start of a slow backstop scan", async () => {
const f = await fixture();
const graph = await runGraph(f.companyId, f.agentId);
const backlog = (await graph.backlog(250)).sort();
const behind = backlog[0]!;
let slow = false, changedBehind = false;
const checkedAt = new Map<string, number[]>();
await recordedRecovery({ stranded: async scope => {
checkedAt.set(scope.issueId, [...checkedAt.get(scope.issueId) ?? [], Date.now()]);
if (!slow) return;
vi.setSystemTime(Date.now() + 20_000);
if (scope.issueId === behind && !changedBehind) {
changedBehind = true;
await state.db.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, behind));
}
} }, { fullBackstopMs: 24 * 60 * 60_000 });
await settleWork({ backstopPending: true });
await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000);
await state.db.update(issues).set({ updatedAt: new Date() }).where(inArray(issues.id, backlog));
await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000);
slow = true;
checkedAt.clear();
const scanStartedAt = Date.now();
await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId }));
await settleWork({ backstopPending: true });
slow = false;
expect(changedBehind).toBe(true);
expect(backlog.every(id => checkedAt.has(id))).toBe(true);
expect(Date.now() - scanStartedAt).toBeGreaterThan(2 * ISSUE_RECOVERY_BACKSTOP_MS);
checkedAt.clear();
await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000);
await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId }));
await settleWork({ backstopPending: true });
expect(checkedAt.has(behind)).toBe(true);
}, 300_000);
it("keeps the company pass for hold releases that can admit any task", async () => {
const f = await fixture();
const graph = await runGraph(f.companyId, f.agentId);
+15 -10
View File
@@ -46,23 +46,27 @@ export function registerIssueRecoveryWork(
const fullBackstopMs = options.fullBackstopMs ?? options.backstopMs ?? ISSUE_RECOVERY_FULL_BACKSTOP_MS;
function fullPasses() {
const completedAt = new Map<string, number>(), changedAt = new Map<string, number>();
const scans = new Map<string, Date | null>();
const scans = new Map<string, { since: Date | null; startedAt: number }>();
const lastFull = (companyId: string) => Math.max(completedAt.get("*") ?? 0, completedAt.get(companyId) ?? 0);
const lastLook = (companyId: string) => Math.max(lastFull(companyId), changedAt.get(companyId) ?? 0);
return {
backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop ? lastLook(scope.companyId) + backstopMs : null,
changedSince(scope: IssueRecoveryScope | undefined, key: string, firstPage: boolean): Date | null {
if (!scope?.backstop) return null;
if (firstPage || !scans.has(key)) scans.set(key, Date.now() - lastFull(scope.companyId) >= fullBackstopMs ? null
: new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS));
return scans.get(key) ?? null;
if (firstPage || !scans.has(key)) scans.set(key, { startedAt: Date.now(),
since: !scope?.backstop || Date.now() - lastFull(scope.companyId) >= fullBackstopMs ? null
: new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS) });
return scans.get(key)!.since;
},
loaded(scope: IssueRecoveryScope | undefined, key: string, complete: boolean) {
if (!complete) return;
const changed = scans.get(key) ?? null;
const scan = scans.get(key);
scans.delete(key);
if (changed && scope) changedAt.set(scope.companyId, Date.now());
else if (!scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) completedAt.set(scope?.companyId ?? "*", Date.now());
const startedAt = scan?.startedAt ?? Date.now();
if (scan?.since && scope) changedAt.set(scope.companyId, Math.max(changedAt.get(scope.companyId) ?? 0, startedAt));
else if (!scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) {
const companyKey = scope?.companyId ?? "*";
completedAt.set(companyKey, Math.max(completedAt.get(companyKey) ?? 0, startedAt));
}
},
};
}
@@ -124,9 +128,10 @@ export function registerIssueRecoveryWork(
const complete = rows.length < PAGE_SIZE;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
passes.loaded(scope, key, complete);
const keepUnseen = changedSince ? { keepUnseen: true as const } : {};
const origin = !workSignalDebug.enabled ? undefined : !scope ? "reload" as const : scope.backstop ? "backstop" as const
: scope.issueId || isGraph(scope) ? "hint" as const : "company" as const;
return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete };
return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete, ...keepUnseen };
},
check: async entry => {
if (!enabled) return null;
@@ -170,7 +175,7 @@ export function registerIssueRecoveryWork(
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
passes.loaded(scope, key, complete);
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now(),
...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete };
...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete, ...(changedSince ? { keepUnseen: true as const } : {}) };
},
check: async entry => {
if (!enabled) return null;
+4 -3
View File
@@ -36,7 +36,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
* the scopes of its own key (and unkeyed scopes), not against every scope.
*/
scopeKey?: { entry(entry: Entry): string; scope(scope: Scope): string | undefined };
load(scope?: Scope, signal?: AbortSignal): Promise<Entry[] | { rows: Entry[] | null; complete: boolean }>;
load(scope?: Scope, signal?: AbortSignal): Promise<Entry[] | { rows: Entry[] | null; complete: boolean; keepUnseen?: boolean }>;
// Owners may retain pending queue priority while refreshing eligibility.
mergeEntry?(pending: Entry, loaded: Entry): Entry;
priority?(entry: Entry): number;
@@ -164,7 +164,8 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
const scan = scans.get(hint) ?? { version: versions.get(hint) ?? 0, seen: new Set<string>() };
scans.set(hint, scan);
const loaded = await options.load(scope, signal);
const result = Array.isArray(loaded) ? { rows: loaded, complete: true } : loaded;
const result: { rows: Entry[] | null; complete: boolean; keepUnseen?: boolean } = Array.isArray(loaded)
? { rows: loaded, complete: true } : loaded;
const progressScopes = [...scans].map(([other, progress]) => ({ progress, scope: decode(other) }));
for (const entry of result.rows ?? []) {
const pending = entries.get(entry.id);
@@ -173,7 +174,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
for (const { progress, scope: other } of progressScopes) if (options.matches(entry, other)) progress.seen.add(entry.id);
}
if (!result.complete) { dirty.add(hint); continue; }
if (result.rows !== null) for (const entry of entriesIn(scope)) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) {
if (result.rows !== null && !result.keepUnseen) for (const entry of entriesIn(scope)) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) {
entries.delete(entry.id); failures.delete(entry.id);
}
scans.delete(hint);