mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-17 13:01:05 +01:00
perf(recovery): keep unloaded deadlines on a changed-tasks pass and start the next window at scan start
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
379dda9e26
commit
802a76c153
3 files changed
+82
-14
No files matched your search
@@ -1,7 +1,7 @@
|
|||||||
import { setImmediate as immediate } from "node:timers/promises";
|
import { setImmediate as immediate } from "node:timers/promises";
|
||||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||||
import { randomUUID } from "node:crypto";
|
import { randomUUID } from "node:crypto";
|
||||||
import { and, eq, sql } from "drizzle-orm";
|
import { and, eq, inArray, sql } from "drizzle-orm";
|
||||||
import { expect, it, vi } from "vitest";
|
import { expect, it, vi } from "vitest";
|
||||||
import { agents, budgetPolicies, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db";
|
import { agents, budgetPolicies, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db";
|
||||||
import { heartbeatService } from "../services/heartbeat.js";
|
import { heartbeatService } from "../services/heartbeat.js";
|
||||||
@@ -223,6 +223,68 @@ describeIssueRecovery("issue recovery production producers and deadlines", () =>
|
|||||||
expect(checkedAt.get(later)! - eventAt).toBeLessThanOrEqual(5_000);
|
expect(checkedAt.get(later)! - eventAt).toBeLessThanOrEqual(5_000);
|
||||||
await quiet();
|
await quiet();
|
||||||
});
|
});
|
||||||
|
async function advanceWithoutWaiting(ms: number) {
|
||||||
|
const until = Date.now() + ms;
|
||||||
|
while (Date.now() < until) { await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())); await immediate(); }
|
||||||
|
}
|
||||||
|
it("keeps a saved deadline of a task that a changed-tasks backstop pass does not load", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
const graph = await runGraph(f.companyId, f.agentId);
|
||||||
|
const [target] = await graph.backlog(20);
|
||||||
|
const checkedAt: number[] = [];
|
||||||
|
await recordedRecovery({ stranded: async scope => {
|
||||||
|
if (scope.issueId !== target) return;
|
||||||
|
checkedAt.push(Date.now());
|
||||||
|
scope.onDeferredDeadline(Date.now() + 16 * 60_000);
|
||||||
|
} });
|
||||||
|
await settleWork({ backstopPending: true });
|
||||||
|
for (let round = 0; round < 2; round++) {
|
||||||
|
await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000);
|
||||||
|
await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId }));
|
||||||
|
await settleWork({ backstopPending: true });
|
||||||
|
}
|
||||||
|
const savedFor = checkedAt.at(-1)! + 16 * 60_000;
|
||||||
|
checkedAt.length = 0;
|
||||||
|
await vi.advanceTimersByTimeAsync(savedFor - Date.now() + 2_000);
|
||||||
|
await settleWork({ backstopPending: true });
|
||||||
|
expect(checkedAt.length).toBeGreaterThan(0);
|
||||||
|
expect(checkedAt[0]! - savedFor).toBeLessThanOrEqual(5_000);
|
||||||
|
});
|
||||||
|
it("starts the next changed-tasks window from the start of a slow backstop scan", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
const graph = await runGraph(f.companyId, f.agentId);
|
||||||
|
const backlog = (await graph.backlog(250)).sort();
|
||||||
|
const behind = backlog[0]!;
|
||||||
|
let slow = false, changedBehind = false;
|
||||||
|
const checkedAt = new Map<string, number[]>();
|
||||||
|
await recordedRecovery({ stranded: async scope => {
|
||||||
|
checkedAt.set(scope.issueId, [...checkedAt.get(scope.issueId) ?? [], Date.now()]);
|
||||||
|
if (!slow) return;
|
||||||
|
vi.setSystemTime(Date.now() + 20_000);
|
||||||
|
if (scope.issueId === behind && !changedBehind) {
|
||||||
|
changedBehind = true;
|
||||||
|
await state.db.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, behind));
|
||||||
|
}
|
||||||
|
} }, { fullBackstopMs: 24 * 60 * 60_000 });
|
||||||
|
await settleWork({ backstopPending: true });
|
||||||
|
await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000);
|
||||||
|
await state.db.update(issues).set({ updatedAt: new Date() }).where(inArray(issues.id, backlog));
|
||||||
|
await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000);
|
||||||
|
slow = true;
|
||||||
|
checkedAt.clear();
|
||||||
|
const scanStartedAt = Date.now();
|
||||||
|
await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId }));
|
||||||
|
await settleWork({ backstopPending: true });
|
||||||
|
slow = false;
|
||||||
|
expect(changedBehind).toBe(true);
|
||||||
|
expect(backlog.every(id => checkedAt.has(id))).toBe(true);
|
||||||
|
expect(Date.now() - scanStartedAt).toBeGreaterThan(2 * ISSUE_RECOVERY_BACKSTOP_MS);
|
||||||
|
checkedAt.clear();
|
||||||
|
await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000);
|
||||||
|
await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId }));
|
||||||
|
await settleWork({ backstopPending: true });
|
||||||
|
expect(checkedAt.has(behind)).toBe(true);
|
||||||
|
}, 300_000);
|
||||||
it("keeps the company pass for hold releases that can admit any task", async () => {
|
it("keeps the company pass for hold releases that can admit any task", async () => {
|
||||||
const f = await fixture();
|
const f = await fixture();
|
||||||
const graph = await runGraph(f.companyId, f.agentId);
|
const graph = await runGraph(f.companyId, f.agentId);
|
||||||
|
|||||||
@@ -46,23 +46,27 @@ export function registerIssueRecoveryWork(
|
|||||||
const fullBackstopMs = options.fullBackstopMs ?? options.backstopMs ?? ISSUE_RECOVERY_FULL_BACKSTOP_MS;
|
const fullBackstopMs = options.fullBackstopMs ?? options.backstopMs ?? ISSUE_RECOVERY_FULL_BACKSTOP_MS;
|
||||||
function fullPasses() {
|
function fullPasses() {
|
||||||
const completedAt = new Map<string, number>(), changedAt = new Map<string, number>();
|
const completedAt = new Map<string, number>(), changedAt = new Map<string, number>();
|
||||||
const scans = new Map<string, Date | null>();
|
const scans = new Map<string, { since: Date | null; startedAt: number }>();
|
||||||
const lastFull = (companyId: string) => Math.max(completedAt.get("*") ?? 0, completedAt.get(companyId) ?? 0);
|
const lastFull = (companyId: string) => Math.max(completedAt.get("*") ?? 0, completedAt.get(companyId) ?? 0);
|
||||||
const lastLook = (companyId: string) => Math.max(lastFull(companyId), changedAt.get(companyId) ?? 0);
|
const lastLook = (companyId: string) => Math.max(lastFull(companyId), changedAt.get(companyId) ?? 0);
|
||||||
return {
|
return {
|
||||||
backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop ? lastLook(scope.companyId) + backstopMs : null,
|
backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop ? lastLook(scope.companyId) + backstopMs : null,
|
||||||
changedSince(scope: IssueRecoveryScope | undefined, key: string, firstPage: boolean): Date | null {
|
changedSince(scope: IssueRecoveryScope | undefined, key: string, firstPage: boolean): Date | null {
|
||||||
if (!scope?.backstop) return null;
|
if (firstPage || !scans.has(key)) scans.set(key, { startedAt: Date.now(),
|
||||||
if (firstPage || !scans.has(key)) scans.set(key, Date.now() - lastFull(scope.companyId) >= fullBackstopMs ? null
|
since: !scope?.backstop || Date.now() - lastFull(scope.companyId) >= fullBackstopMs ? null
|
||||||
: new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS));
|
: new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS) });
|
||||||
return scans.get(key) ?? null;
|
return scans.get(key)!.since;
|
||||||
},
|
},
|
||||||
loaded(scope: IssueRecoveryScope | undefined, key: string, complete: boolean) {
|
loaded(scope: IssueRecoveryScope | undefined, key: string, complete: boolean) {
|
||||||
if (!complete) return;
|
if (!complete) return;
|
||||||
const changed = scans.get(key) ?? null;
|
const scan = scans.get(key);
|
||||||
scans.delete(key);
|
scans.delete(key);
|
||||||
if (changed && scope) changedAt.set(scope.companyId, Date.now());
|
const startedAt = scan?.startedAt ?? Date.now();
|
||||||
else if (!scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) completedAt.set(scope?.companyId ?? "*", Date.now());
|
if (scan?.since && scope) changedAt.set(scope.companyId, Math.max(changedAt.get(scope.companyId) ?? 0, startedAt));
|
||||||
|
else if (!scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) {
|
||||||
|
const companyKey = scope?.companyId ?? "*";
|
||||||
|
completedAt.set(companyKey, Math.max(completedAt.get(companyKey) ?? 0, startedAt));
|
||||||
|
}
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -124,9 +128,10 @@ export function registerIssueRecoveryWork(
|
|||||||
const complete = rows.length < PAGE_SIZE;
|
const complete = rows.length < PAGE_SIZE;
|
||||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||||
passes.loaded(scope, key, complete);
|
passes.loaded(scope, key, complete);
|
||||||
|
const keepUnseen = changedSince ? { keepUnseen: true as const } : {};
|
||||||
const origin = !workSignalDebug.enabled ? undefined : !scope ? "reload" as const : scope.backstop ? "backstop" as const
|
const origin = !workSignalDebug.enabled ? undefined : !scope ? "reload" as const : scope.backstop ? "backstop" as const
|
||||||
: scope.issueId || isGraph(scope) ? "hint" as const : "company" as const;
|
: scope.issueId || isGraph(scope) ? "hint" as const : "company" as const;
|
||||||
return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete };
|
return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete, ...keepUnseen };
|
||||||
},
|
},
|
||||||
check: async entry => {
|
check: async entry => {
|
||||||
if (!enabled) return null;
|
if (!enabled) return null;
|
||||||
@@ -170,7 +175,7 @@ export function registerIssueRecoveryWork(
|
|||||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||||
passes.loaded(scope, key, complete);
|
passes.loaded(scope, key, complete);
|
||||||
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now(),
|
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now(),
|
||||||
...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete };
|
...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete, ...(changedSince ? { keepUnseen: true as const } : {}) };
|
||||||
},
|
},
|
||||||
check: async entry => {
|
check: async entry => {
|
||||||
if (!enabled) return null;
|
if (!enabled) return null;
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
|||||||
* the scopes of its own key (and unkeyed scopes), not against every scope.
|
* the scopes of its own key (and unkeyed scopes), not against every scope.
|
||||||
*/
|
*/
|
||||||
scopeKey?: { entry(entry: Entry): string; scope(scope: Scope): string | undefined };
|
scopeKey?: { entry(entry: Entry): string; scope(scope: Scope): string | undefined };
|
||||||
load(scope?: Scope, signal?: AbortSignal): Promise<Entry[] | { rows: Entry[] | null; complete: boolean }>;
|
load(scope?: Scope, signal?: AbortSignal): Promise<Entry[] | { rows: Entry[] | null; complete: boolean; keepUnseen?: boolean }>;
|
||||||
// Owners may retain pending queue priority while refreshing eligibility.
|
// Owners may retain pending queue priority while refreshing eligibility.
|
||||||
mergeEntry?(pending: Entry, loaded: Entry): Entry;
|
mergeEntry?(pending: Entry, loaded: Entry): Entry;
|
||||||
priority?(entry: Entry): number;
|
priority?(entry: Entry): number;
|
||||||
@@ -164,7 +164,8 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
|||||||
const scan = scans.get(hint) ?? { version: versions.get(hint) ?? 0, seen: new Set<string>() };
|
const scan = scans.get(hint) ?? { version: versions.get(hint) ?? 0, seen: new Set<string>() };
|
||||||
scans.set(hint, scan);
|
scans.set(hint, scan);
|
||||||
const loaded = await options.load(scope, signal);
|
const loaded = await options.load(scope, signal);
|
||||||
const result = Array.isArray(loaded) ? { rows: loaded, complete: true } : loaded;
|
const result: { rows: Entry[] | null; complete: boolean; keepUnseen?: boolean } = Array.isArray(loaded)
|
||||||
|
? { rows: loaded, complete: true } : loaded;
|
||||||
const progressScopes = [...scans].map(([other, progress]) => ({ progress, scope: decode(other) }));
|
const progressScopes = [...scans].map(([other, progress]) => ({ progress, scope: decode(other) }));
|
||||||
for (const entry of result.rows ?? []) {
|
for (const entry of result.rows ?? []) {
|
||||||
const pending = entries.get(entry.id);
|
const pending = entries.get(entry.id);
|
||||||
@@ -173,7 +174,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
|||||||
for (const { progress, scope: other } of progressScopes) if (options.matches(entry, other)) progress.seen.add(entry.id);
|
for (const { progress, scope: other } of progressScopes) if (options.matches(entry, other)) progress.seen.add(entry.id);
|
||||||
}
|
}
|
||||||
if (!result.complete) { dirty.add(hint); continue; }
|
if (!result.complete) { dirty.add(hint); continue; }
|
||||||
if (result.rows !== null) for (const entry of entriesIn(scope)) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) {
|
if (result.rows !== null && !result.keepUnseen) for (const entry of entriesIn(scope)) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) {
|
||||||
entries.delete(entry.id); failures.delete(entry.id);
|
entries.delete(entry.id); failures.delete(entry.id);
|
||||||
}
|
}
|
||||||
scans.delete(hint);
|
scans.delete(hint);
|
||||||
|
|||||||
Reference in new issue
Block a user