mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
perf(recovery): keep unloaded deadlines on a changed-tasks pass and start the next window at scan start
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
379dda9e26
commit
802a76c153
3 files changed
+82
-14
No files matched your search
@@ -1,7 +1,7 @@
|
||||
import { setImmediate as immediate } from "node:timers/promises";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { and, eq, inArray, sql } from "drizzle-orm";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { agents, budgetPolicies, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
@@ -223,6 +223,68 @@ describeIssueRecovery("issue recovery production producers and deadlines", () =>
|
||||
expect(checkedAt.get(later)! - eventAt).toBeLessThanOrEqual(5_000);
|
||||
await quiet();
|
||||
});
|
||||
async function advanceWithoutWaiting(ms: number) {
|
||||
const until = Date.now() + ms;
|
||||
while (Date.now() < until) { await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())); await immediate(); }
|
||||
}
|
||||
it("keeps a saved deadline of a task that a changed-tasks backstop pass does not load", async () => {
|
||||
const f = await fixture();
|
||||
const graph = await runGraph(f.companyId, f.agentId);
|
||||
const [target] = await graph.backlog(20);
|
||||
const checkedAt: number[] = [];
|
||||
await recordedRecovery({ stranded: async scope => {
|
||||
if (scope.issueId !== target) return;
|
||||
checkedAt.push(Date.now());
|
||||
scope.onDeferredDeadline(Date.now() + 16 * 60_000);
|
||||
} });
|
||||
await settleWork({ backstopPending: true });
|
||||
for (let round = 0; round < 2; round++) {
|
||||
await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000);
|
||||
await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId }));
|
||||
await settleWork({ backstopPending: true });
|
||||
}
|
||||
const savedFor = checkedAt.at(-1)! + 16 * 60_000;
|
||||
checkedAt.length = 0;
|
||||
await vi.advanceTimersByTimeAsync(savedFor - Date.now() + 2_000);
|
||||
await settleWork({ backstopPending: true });
|
||||
expect(checkedAt.length).toBeGreaterThan(0);
|
||||
expect(checkedAt[0]! - savedFor).toBeLessThanOrEqual(5_000);
|
||||
});
|
||||
it("starts the next changed-tasks window from the start of a slow backstop scan", async () => {
|
||||
const f = await fixture();
|
||||
const graph = await runGraph(f.companyId, f.agentId);
|
||||
const backlog = (await graph.backlog(250)).sort();
|
||||
const behind = backlog[0]!;
|
||||
let slow = false, changedBehind = false;
|
||||
const checkedAt = new Map<string, number[]>();
|
||||
await recordedRecovery({ stranded: async scope => {
|
||||
checkedAt.set(scope.issueId, [...checkedAt.get(scope.issueId) ?? [], Date.now()]);
|
||||
if (!slow) return;
|
||||
vi.setSystemTime(Date.now() + 20_000);
|
||||
if (scope.issueId === behind && !changedBehind) {
|
||||
changedBehind = true;
|
||||
await state.db.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, behind));
|
||||
}
|
||||
} }, { fullBackstopMs: 24 * 60 * 60_000 });
|
||||
await settleWork({ backstopPending: true });
|
||||
await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000);
|
||||
await state.db.update(issues).set({ updatedAt: new Date() }).where(inArray(issues.id, backlog));
|
||||
await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000);
|
||||
slow = true;
|
||||
checkedAt.clear();
|
||||
const scanStartedAt = Date.now();
|
||||
await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId }));
|
||||
await settleWork({ backstopPending: true });
|
||||
slow = false;
|
||||
expect(changedBehind).toBe(true);
|
||||
expect(backlog.every(id => checkedAt.has(id))).toBe(true);
|
||||
expect(Date.now() - scanStartedAt).toBeGreaterThan(2 * ISSUE_RECOVERY_BACKSTOP_MS);
|
||||
checkedAt.clear();
|
||||
await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000);
|
||||
await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId }));
|
||||
await settleWork({ backstopPending: true });
|
||||
expect(checkedAt.has(behind)).toBe(true);
|
||||
}, 300_000);
|
||||
it("keeps the company pass for hold releases that can admit any task", async () => {
|
||||
const f = await fixture();
|
||||
const graph = await runGraph(f.companyId, f.agentId);
|
||||
|
||||
@@ -46,23 +46,27 @@ export function registerIssueRecoveryWork(
|
||||
const fullBackstopMs = options.fullBackstopMs ?? options.backstopMs ?? ISSUE_RECOVERY_FULL_BACKSTOP_MS;
|
||||
function fullPasses() {
|
||||
const completedAt = new Map<string, number>(), changedAt = new Map<string, number>();
|
||||
const scans = new Map<string, Date | null>();
|
||||
const scans = new Map<string, { since: Date | null; startedAt: number }>();
|
||||
const lastFull = (companyId: string) => Math.max(completedAt.get("*") ?? 0, completedAt.get(companyId) ?? 0);
|
||||
const lastLook = (companyId: string) => Math.max(lastFull(companyId), changedAt.get(companyId) ?? 0);
|
||||
return {
|
||||
backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop ? lastLook(scope.companyId) + backstopMs : null,
|
||||
changedSince(scope: IssueRecoveryScope | undefined, key: string, firstPage: boolean): Date | null {
|
||||
if (!scope?.backstop) return null;
|
||||
if (firstPage || !scans.has(key)) scans.set(key, Date.now() - lastFull(scope.companyId) >= fullBackstopMs ? null
|
||||
: new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS));
|
||||
return scans.get(key) ?? null;
|
||||
if (firstPage || !scans.has(key)) scans.set(key, { startedAt: Date.now(),
|
||||
since: !scope?.backstop || Date.now() - lastFull(scope.companyId) >= fullBackstopMs ? null
|
||||
: new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS) });
|
||||
return scans.get(key)!.since;
|
||||
},
|
||||
loaded(scope: IssueRecoveryScope | undefined, key: string, complete: boolean) {
|
||||
if (!complete) return;
|
||||
const changed = scans.get(key) ?? null;
|
||||
const scan = scans.get(key);
|
||||
scans.delete(key);
|
||||
if (changed && scope) changedAt.set(scope.companyId, Date.now());
|
||||
else if (!scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) completedAt.set(scope?.companyId ?? "*", Date.now());
|
||||
const startedAt = scan?.startedAt ?? Date.now();
|
||||
if (scan?.since && scope) changedAt.set(scope.companyId, Math.max(changedAt.get(scope.companyId) ?? 0, startedAt));
|
||||
else if (!scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) {
|
||||
const companyKey = scope?.companyId ?? "*";
|
||||
completedAt.set(companyKey, Math.max(completedAt.get(companyKey) ?? 0, startedAt));
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -124,9 +128,10 @@ export function registerIssueRecoveryWork(
|
||||
const complete = rows.length < PAGE_SIZE;
|
||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||
passes.loaded(scope, key, complete);
|
||||
const keepUnseen = changedSince ? { keepUnseen: true as const } : {};
|
||||
const origin = !workSignalDebug.enabled ? undefined : !scope ? "reload" as const : scope.backstop ? "backstop" as const
|
||||
: scope.issueId || isGraph(scope) ? "hint" as const : "company" as const;
|
||||
return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete };
|
||||
return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete, ...keepUnseen };
|
||||
},
|
||||
check: async entry => {
|
||||
if (!enabled) return null;
|
||||
@@ -170,7 +175,7 @@ export function registerIssueRecoveryWork(
|
||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||
passes.loaded(scope, key, complete);
|
||||
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now(),
|
||||
...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete };
|
||||
...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete, ...(changedSince ? { keepUnseen: true as const } : {}) };
|
||||
},
|
||||
check: async entry => {
|
||||
if (!enabled) return null;
|
||||
|
||||
@@ -36,7 +36,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
* the scopes of its own key (and unkeyed scopes), not against every scope.
|
||||
*/
|
||||
scopeKey?: { entry(entry: Entry): string; scope(scope: Scope): string | undefined };
|
||||
load(scope?: Scope, signal?: AbortSignal): Promise<Entry[] | { rows: Entry[] | null; complete: boolean }>;
|
||||
load(scope?: Scope, signal?: AbortSignal): Promise<Entry[] | { rows: Entry[] | null; complete: boolean; keepUnseen?: boolean }>;
|
||||
// Owners may retain pending queue priority while refreshing eligibility.
|
||||
mergeEntry?(pending: Entry, loaded: Entry): Entry;
|
||||
priority?(entry: Entry): number;
|
||||
@@ -164,7 +164,8 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
const scan = scans.get(hint) ?? { version: versions.get(hint) ?? 0, seen: new Set<string>() };
|
||||
scans.set(hint, scan);
|
||||
const loaded = await options.load(scope, signal);
|
||||
const result = Array.isArray(loaded) ? { rows: loaded, complete: true } : loaded;
|
||||
const result: { rows: Entry[] | null; complete: boolean; keepUnseen?: boolean } = Array.isArray(loaded)
|
||||
? { rows: loaded, complete: true } : loaded;
|
||||
const progressScopes = [...scans].map(([other, progress]) => ({ progress, scope: decode(other) }));
|
||||
for (const entry of result.rows ?? []) {
|
||||
const pending = entries.get(entry.id);
|
||||
@@ -173,7 +174,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
for (const { progress, scope: other } of progressScopes) if (options.matches(entry, other)) progress.seen.add(entry.id);
|
||||
}
|
||||
if (!result.complete) { dirty.add(hint); continue; }
|
||||
if (result.rows !== null) for (const entry of entriesIn(scope)) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) {
|
||||
if (result.rows !== null && !result.keepUnseen) for (const entry of entriesIn(scope)) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) {
|
||||
entries.delete(entry.id); failures.delete(entry.id);
|
||||
}
|
||||
scans.delete(hint);
|
||||
|
||||
Reference in new issue
Block a user