From 533d02ff1a9db82e647fa6bf7eaf194f3ab69bdb Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:36:21 -0600 Subject: [PATCH] perf(recovery): match scoped deadline entries only against scopes of their own company (#21467) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Live-run recovery uses the shared scoped deadline worker (`scoped-deadline-work.ts`). Each queue keeps its entries (one for each running run) and the scopes that writes marked dirty (for example one company and one run) > - To schedule and to pick due work, the worker tested every entry against every dirty scope. It also decoded each scope hint with `JSON.parse` again inside those loops, and each reconstructed scope walked all entries to drop vanished ones > - With many runs and many dirty run scopes, this work grows with entries × scopes on the event loop of the one server process that controls all runs > - This pull request lets an owner declare a partition key for its scopes. Live-run recovery uses the company. The worker then tests an entry only against the scopes of its own company, and decodes each hint once > - The benefit is that matching cost grows with entries + scopes, with the same results ## Linked Issues or Issue Description Refs #21444 (the lease renewal change from the same load model). **What existing behavior does this improve?** The scoped deadline worker in `server/src/services/scoped-deadline-work.ts`, which live-run recovery (`server/src/services/heartbeat/live-recovery-work.ts`) uses for its seven queues. **Subsystem affected** Server: background work scheduling for live-run and resource recovery. **Current behavior** For 400 entries in 40 companies and 40 dirty run scopes, one schedule computation and one pass call `matches` 27,830 times and decode each hint many times. **Proposed behavior** The same case calls `matches` fewer than 3,200 times and decodes each hint at most once. Owners that do not declare a key behave exactly as before. **Reason and benefit** The plan for 1,000 concurrent agents needs per-run background work that does not grow with the product of runs and changes. The work runs on the main thread of the controlling process. **Breaking changes** None. `scopeKey` is optional. ## What Changed - `scoped-deadline-work.ts`: new optional `scopeKey` option (`entry(entry)` and `scope(scope)`). The contract: a scope with a defined key matches only entries with the same key. A scope without a key (for example `*`) is tested against every entry, as before. - The worker indexes the dirty scopes by key, so `next()` and the due filter test each entry only against the scopes of its key and the unkeyed scopes. - When a reconstructed keyed scope drops vanished entries, it visits only the entries of its key (an index built once for each pass). - Scope hints are decoded once and the decoded value is reused while the hint is dirty or scanning. - `live-recovery-work.ts`: declares the company as the key. Its `matches` already requires the same company for every defined scope. ## Verification - New test in `scoped-deadline-work.test.ts`: 400 entries in 40 companies, then 40 run-scoped invalidations. It requires fewer than 3,200 `matches` calls and at most 40 decodes, and the next deadline is unchanged. - Red first: with `origin/master` `scoped-deadline-work.ts`, the test failed with 27,830 `matches` calls. - `pnpm exec vitest run src/__tests__/scoped-deadline-work.test.ts`: 22 passed. - The live-run recovery matrix (`live-recovery-work.test.ts`) is covered by CI. ## Risks - If an owner declares a key that its `matches` does not respect, the worker can miss a match. Only live-run recovery declares a key, and its `matches` returns false for any defined scope of another company. - Low risk otherwise: owners without a key keep the old code path. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../__tests__/scoped-deadline-work.test.ts | 73 +++++++++++++++++++ .../services/heartbeat/live-recovery-work.ts | 2 + server/src/services/scoped-deadline-work.ts | 67 +++++++++++++++-- 3 files changed, 134 insertions(+), 8 deletions(-) diff --git a/server/src/__tests__/scoped-deadline-work.test.ts b/server/src/__tests__/scoped-deadline-work.test.ts index 7d7d6a4bc81..a5a430a2410 100644 --- a/server/src/__tests__/scoped-deadline-work.test.ts +++ b/server/src/__tests__/scoped-deadline-work.test.ts @@ -624,3 +624,76 @@ it("keeps partial reconstruction and failed checks uncertified until they finish fail = false; await vi.advanceTimersByTimeAsync(30_000); expect(worker.reconciliationSnapshot().reconciled).toBe(true); }); + +it("tests each entry only against the dirty scopes of its own key, and decodes each hint once", async () => { + vi.useFakeTimers(); + const onError = vi.fn(); + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: vi.fn(async () => undefined) }); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError }); + type Scope = { companyId: string; runId?: string }; + type Row = { id: string; companyId: string; dueAt: number }; + const dueAt = Date.now() + 10 * 60_000; + const rows: Row[] = Array.from({ length: 40 }, (_, company) => Array.from({ length: 10 }, (_, run) => + ({ id: `run-${company}-${run}`, companyId: `company-${company}`, dueAt }))).flat(); + const inScope = (row: Row, scope?: Scope) => !scope + || row.companyId === scope.companyId && (!scope.runId || row.id === scope.runId); + const matches = vi.fn(inScope); + const decodeScope = vi.fn((hint: string): Scope | undefined => hint === "*" ? undefined : JSON.parse(hint)); + const worker = registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.activeExecutionRecovery, + minimumPassIntervalMs: 0, + decodeScope, + matches, + scopeKey: { entry: row => row.companyId, scope: scope => scope.companyId }, + load: async scope => rows.filter(row => inScope(row, scope)), + check: async row => row, + }, () => true); + await worker.ready; + await vi.advanceTimersByTimeAsync(1); + matches.mockClear(); decodeScope.mockClear(); + // One run of each company changes: 40 run-scoped hints at once. + for (let company = 0; company < 40; company++) { + worker.invalidate(JSON.stringify({ companyId: `company-${company}`, runId: `run-${company}-0` })); + } + await vi.advanceTimersByTimeAsync(1); + await vi.advanceTimersByTimeAsync(1); + // Without the key every entry is tested against every dirty scope + // (400 entries x 40 scopes for each schedule computation). + expect(matches.mock.calls.length).toBeLessThan(rows.length * 8); + expect(decodeScope.mock.calls.length).toBeLessThanOrEqual(40); + expect(coordinator.nextWakeAt()).toBe(dueAt); + expect(onError).not.toHaveBeenCalled(); +}); + +it("drops an entry that vanished from a keyed scope and keeps other companies' saved work", async () => { + vi.useFakeTimers(); + const onError = vi.fn(); + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: vi.fn(async () => undefined) }); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError }); + type Scope = { companyId: string; runId?: string }; + type Row = { id: string; companyId: string; dueAt: number }; + const dueAt = Date.now() + 60_000; + let rows: Row[] = ["a", "b"].flatMap(company => [1, 2].map(run => ({ id: `${company}-${run}`, companyId: company, dueAt }))); + const inScope = (row: Row, scope?: Scope) => !scope + || row.companyId === scope.companyId && (!scope.runId || row.id === scope.runId); + const check = vi.fn(async (_row: Row) => null); + const worker = registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.activeExecutionRecovery, + minimumPassIntervalMs: 0, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint), + matches: inScope, + scopeKey: { entry: row => row.companyId, scope: scope => scope.companyId }, + load: async scope => rows.filter(row => inScope(row, scope)), + check, + }, () => true); + await worker.ready; + await vi.advanceTimersByTimeAsync(1); + rows = rows.filter(row => row.id !== "a-1"); + worker.invalidate(JSON.stringify({ companyId: "a" })); + await vi.advanceTimersByTimeAsync(1); + expect(check).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(60_000); + expect(check.mock.calls.map(([row]) => row.id).sort()).toEqual(["a-2", "b-1", "b-2"]); + expect(coordinator.nextWakeAt()).toBeNull(); + expect(onError).not.toHaveBeenCalled(); +}); diff --git a/server/src/services/heartbeat/live-recovery-work.ts b/server/src/services/heartbeat/live-recovery-work.ts index 775ccb57139..4c2ec0d6376 100644 --- a/server/src/services/heartbeat/live-recovery-work.ts +++ b/server/src/services/heartbeat/live-recovery-work.ts @@ -102,6 +102,8 @@ export function registerLiveRecoveryWork(db: Db, coordinator: ReturnType { enabled = await owner.enabled(); }, decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint), + // `matches` below never matches a scope of another company. + scopeKey: { entry: entry => entry.companyId, scope: scope => scope.companyId }, matches: (entry, scope) => !scope || entry.companyId === scope.companyId && ( kind === "agent" ? !scope.agentId || entry.id === scope.agentId : kind === "orphanLease" || kind === "shutdown" ? (!scope.leaseId || entry.id === scope.leaseId) && (!scope.runId || entry.runId === scope.runId) diff --git a/server/src/services/scoped-deadline-work.ts b/server/src/services/scoped-deadline-work.ts index 117632b52c4..b750d3426c4 100644 --- a/server/src/services/scoped-deadline-work.ts +++ b/server/src/services/scoped-deadline-work.ts @@ -29,6 +29,12 @@ export function registerScopedDeadlineWork( beforePass?(): Promise; decodeScope(hint: string): Scope | undefined; matches(entry: Entry, scope?: Scope): boolean; + /** + * Optional partition of `matches`: a scope whose key is defined can match + * only entries with the same key. Then an entry is tested only against + * the scopes of its own key (and unkeyed scopes), not against every scope. + */ + scopeKey?: { entry(entry: Entry): string; scope(scope: Scope): string | undefined }; load(scope?: Scope, signal?: AbortSignal): Promise; // Owners may retain pending queue priority while refreshing eligibility. mergeEntry?(pending: Entry, loaded: Entry): Entry; @@ -52,6 +58,33 @@ export function registerScopedDeadlineWork( const reconstructAfter = new Map(); let earliest = 0; let lastCompany: string | null = null; + // Hints are immutable strings, so a decoded scope can be reused while the + // hint is still referenced (dirty or scanning). + const decodedScopes = new Map(); + function decode(hint: string) { + if (decodedScopes.has(hint)) return decodedScopes.get(hint); + const scope = options.decodeScope(hint); + decodedScopes.set(hint, scope); + return scope; + } + function forgetDecoded() { + for (const hint of decodedScopes.keys()) if (!dirty.has(hint) && !scans.has(hint)) decodedScopes.delete(hint); + } + /** Whether an entry matches any of `scopes`, without testing every scope. */ + function matcher(scopes: Array): (entry: Entry) => boolean { + if (scopes.length === 0) return () => false; + const keyOf = options.scopeKey; + if (!keyOf) return entry => scopes.some(scope => options.matches(entry, scope)); + const unkeyed: Array = []; + const keyed = new Map(); + for (const scope of scopes) { + const key = scope === undefined ? undefined : keyOf.scope(scope); + if (key === undefined) unkeyed.push(scope); + else { const list = keyed.get(key) ?? []; list.push(scope!); keyed.set(key, list); } + } + return entry => unkeyed.some(scope => options.matches(entry, scope)) + || (keyed.get(keyOf.entry(entry))?.some(scope => options.matches(entry, scope)) ?? false); + } function failed(map: typeof failures, id: string) { const count = Math.min((map.get(id)?.count ?? 0) + 1, 5); map.set(id, { count, retryAt: Date.now() + Math.min(300_000, 30_000 * 2 ** (count - 1)) }); @@ -60,15 +93,15 @@ export function registerScopedDeadlineWork( return [...dirty].filter(hint => hint !== "*" || !scopeFailures.has(hint)) .filter(hint => !scans.has(hint) || scopeFailures.has(hint)) .filter(hint => (reconstructAfter.get(hint) ?? 0) <= Date.now() || scopeFailures.has(hint)) - .map(hint => { try { return options.decodeScope(hint); } catch { return undefined; } }); + .map(hint => { try { return decode(hint); } catch { return undefined; } }); } function next() { if (!canRun()) return dirty.size > 0 || entries.size > 0 ? Date.now() + 30_000 : null; let nextAt: number | null = null; const take = (at: number) => { nextAt = nextAt === null ? at : Math.min(nextAt, at); }; for (const hint of dirty) take(Math.max(earliest, reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0)); - const unavailable = dirtyScopes(); - for (const entry of entries.values()) if (!unavailable.some(scope => options.matches(entry, scope))) + const unavailable = matcher(dirtyScopes()); + for (const entry of entries.values()) if (!unavailable(entry)) take(Math.max(earliest, entry.dueAt, failures.get(entry.id)?.retryAt ?? 0)); return nextAt; } @@ -99,24 +132,41 @@ export function registerScopedDeadlineWork( ? ["*", ...ready.filter(hint => hint !== "*")].slice(0, 25) : ready.slice(0, 25); for (const hint of requested) dirty.delete(hint); + // Entries by scope key for this pass, built on first use, so a keyed + // scope visits only its own entries when it drops vanished ones. + let byKey: Map> | null = null; + const indexEntry = (entry: Entry) => { + if (!byKey || !options.scopeKey) return; + const key = options.scopeKey.entry(entry); + const ids = byKey.get(key) ?? new Set(); + ids.add(entry.id); byKey.set(key, ids); + }; + const entriesIn = (scope: Scope | undefined): Entry[] => { + const key = scope === undefined ? undefined : options.scopeKey?.scope(scope); + if (key === undefined) return [...entries.values()]; + if (!byKey) { byKey = new Map(); for (const entry of entries.values()) indexEntry(entry); } + return [...(byKey.get(key) ?? [])].flatMap(id => entries.get(id) ?? []); + }; for (let index = 0; index < requested.length; index++) { if (signal.aborted || !canRun()) { for (const hint of requested.slice(index)) dirty.add(hint); return; } const hint = requested[index]; let scope: Scope | undefined; let decoded = false; try { - scope = options.decodeScope(hint); decoded = true; + scope = decode(hint); decoded = true; const scan = scans.get(hint) ?? { version: versions.get(hint) ?? 0, seen: new Set() }; scans.set(hint, scan); const loaded = await options.load(scope, signal); const result = Array.isArray(loaded) ? { rows: loaded, complete: true } : loaded; + const progressScopes = [...scans].map(([other, progress]) => ({ progress, scope: decode(other) })); for (const entry of result.rows ?? []) { const pending = entries.get(entry.id); entries.set(entry.id, pending && options.mergeEntry ? options.mergeEntry(pending, entry) : entry); - for (const [other, progress] of scans) if (options.matches(entry, options.decodeScope(other))) progress.seen.add(entry.id); + indexEntry(entry); + for (const { progress, scope: other } of progressScopes) if (options.matches(entry, other)) progress.seen.add(entry.id); } if (!result.complete) { dirty.add(hint); continue; } - if (result.rows !== null) for (const entry of entries.values()) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) { + if (result.rows !== null) for (const entry of entriesIn(scope)) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) { entries.delete(entry.id); failures.delete(entry.id); } scans.delete(hint); @@ -133,8 +183,9 @@ export function registerScopedDeadlineWork( dirty.add(retryHint); failed(scopeFailures, retryHint); errors.push(error); } } - const unavailable = dirtyScopes(); - const due = [...entries.values()].filter(entry => !unavailable.some(scope => options.matches(entry, scope)) + forgetDecoded(); + const unavailable = matcher(dirtyScopes()); + const due = [...entries.values()].filter(entry => !unavailable(entry) && Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now()) .sort((a, b) => (options.priority?.(b) ?? 0) - (options.priority?.(a) ?? 0) || a.dueAt - b.dueAt || a.id.localeCompare(b.id));