diff --git a/server/src/__tests__/scoped-deadline-work.test.ts b/server/src/__tests__/scoped-deadline-work.test.ts index 7d7d6a4bc81..a5a430a2410 100644 --- a/server/src/__tests__/scoped-deadline-work.test.ts +++ b/server/src/__tests__/scoped-deadline-work.test.ts @@ -624,3 +624,76 @@ it("keeps partial reconstruction and failed checks uncertified until they finish fail = false; await vi.advanceTimersByTimeAsync(30_000); expect(worker.reconciliationSnapshot().reconciled).toBe(true); }); + +it("tests each entry only against the dirty scopes of its own key, and decodes each hint once", async () => { + vi.useFakeTimers(); + const onError = vi.fn(); + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: vi.fn(async () => undefined) }); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError }); + type Scope = { companyId: string; runId?: string }; + type Row = { id: string; companyId: string; dueAt: number }; + const dueAt = Date.now() + 10 * 60_000; + const rows: Row[] = Array.from({ length: 40 }, (_, company) => Array.from({ length: 10 }, (_, run) => + ({ id: `run-${company}-${run}`, companyId: `company-${company}`, dueAt }))).flat(); + const inScope = (row: Row, scope?: Scope) => !scope + || row.companyId === scope.companyId && (!scope.runId || row.id === scope.runId); + const matches = vi.fn(inScope); + const decodeScope = vi.fn((hint: string): Scope | undefined => hint === "*" ? undefined : JSON.parse(hint)); + const worker = registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.activeExecutionRecovery, + minimumPassIntervalMs: 0, + decodeScope, + matches, + scopeKey: { entry: row => row.companyId, scope: scope => scope.companyId }, + load: async scope => rows.filter(row => inScope(row, scope)), + check: async row => row, + }, () => true); + await worker.ready; + await vi.advanceTimersByTimeAsync(1); + matches.mockClear(); decodeScope.mockClear(); + // One run of each company changes: 40 run-scoped hints at once. + for (let company = 0; company < 40; company++) { + worker.invalidate(JSON.stringify({ companyId: `company-${company}`, runId: `run-${company}-0` })); + } + await vi.advanceTimersByTimeAsync(1); + await vi.advanceTimersByTimeAsync(1); + // Without the key every entry is tested against every dirty scope + // (400 entries x 40 scopes for each schedule computation). + expect(matches.mock.calls.length).toBeLessThan(rows.length * 8); + expect(decodeScope.mock.calls.length).toBeLessThanOrEqual(40); + expect(coordinator.nextWakeAt()).toBe(dueAt); + expect(onError).not.toHaveBeenCalled(); +}); + +it("drops an entry that vanished from a keyed scope and keeps other companies' saved work", async () => { + vi.useFakeTimers(); + const onError = vi.fn(); + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: vi.fn(async () => undefined) }); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError }); + type Scope = { companyId: string; runId?: string }; + type Row = { id: string; companyId: string; dueAt: number }; + const dueAt = Date.now() + 60_000; + let rows: Row[] = ["a", "b"].flatMap(company => [1, 2].map(run => ({ id: `${company}-${run}`, companyId: company, dueAt }))); + const inScope = (row: Row, scope?: Scope) => !scope + || row.companyId === scope.companyId && (!scope.runId || row.id === scope.runId); + const check = vi.fn(async (_row: Row) => null); + const worker = registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.activeExecutionRecovery, + minimumPassIntervalMs: 0, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint), + matches: inScope, + scopeKey: { entry: row => row.companyId, scope: scope => scope.companyId }, + load: async scope => rows.filter(row => inScope(row, scope)), + check, + }, () => true); + await worker.ready; + await vi.advanceTimersByTimeAsync(1); + rows = rows.filter(row => row.id !== "a-1"); + worker.invalidate(JSON.stringify({ companyId: "a" })); + await vi.advanceTimersByTimeAsync(1); + expect(check).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(60_000); + expect(check.mock.calls.map(([row]) => row.id).sort()).toEqual(["a-2", "b-1", "b-2"]); + expect(coordinator.nextWakeAt()).toBeNull(); + expect(onError).not.toHaveBeenCalled(); +}); diff --git a/server/src/services/heartbeat/live-recovery-work.ts b/server/src/services/heartbeat/live-recovery-work.ts index 775ccb57139..4c2ec0d6376 100644 --- a/server/src/services/heartbeat/live-recovery-work.ts +++ b/server/src/services/heartbeat/live-recovery-work.ts @@ -102,6 +102,8 @@ export function registerLiveRecoveryWork(db: Db, coordinator: ReturnType { enabled = await owner.enabled(); }, decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint), + // `matches` below never matches a scope of another company. + scopeKey: { entry: entry => entry.companyId, scope: scope => scope.companyId }, matches: (entry, scope) => !scope || entry.companyId === scope.companyId && ( kind === "agent" ? !scope.agentId || entry.id === scope.agentId : kind === "orphanLease" || kind === "shutdown" ? (!scope.leaseId || entry.id === scope.leaseId) && (!scope.runId || entry.runId === scope.runId) diff --git a/server/src/services/scoped-deadline-work.ts b/server/src/services/scoped-deadline-work.ts index 117632b52c4..b750d3426c4 100644 --- a/server/src/services/scoped-deadline-work.ts +++ b/server/src/services/scoped-deadline-work.ts @@ -29,6 +29,12 @@ export function registerScopedDeadlineWork( beforePass?(): Promise; decodeScope(hint: string): Scope | undefined; matches(entry: Entry, scope?: Scope): boolean; + /** + * Optional partition of `matches`: a scope whose key is defined can match + * only entries with the same key. Then an entry is tested only against + * the scopes of its own key (and unkeyed scopes), not against every scope. + */ + scopeKey?: { entry(entry: Entry): string; scope(scope: Scope): string | undefined }; load(scope?: Scope, signal?: AbortSignal): Promise; // Owners may retain pending queue priority while refreshing eligibility. mergeEntry?(pending: Entry, loaded: Entry): Entry; @@ -52,6 +58,33 @@ export function registerScopedDeadlineWork( const reconstructAfter = new Map(); let earliest = 0; let lastCompany: string | null = null; + // Hints are immutable strings, so a decoded scope can be reused while the + // hint is still referenced (dirty or scanning). + const decodedScopes = new Map(); + function decode(hint: string) { + if (decodedScopes.has(hint)) return decodedScopes.get(hint); + const scope = options.decodeScope(hint); + decodedScopes.set(hint, scope); + return scope; + } + function forgetDecoded() { + for (const hint of decodedScopes.keys()) if (!dirty.has(hint) && !scans.has(hint)) decodedScopes.delete(hint); + } + /** Whether an entry matches any of `scopes`, without testing every scope. */ + function matcher(scopes: Array): (entry: Entry) => boolean { + if (scopes.length === 0) return () => false; + const keyOf = options.scopeKey; + if (!keyOf) return entry => scopes.some(scope => options.matches(entry, scope)); + const unkeyed: Array = []; + const keyed = new Map(); + for (const scope of scopes) { + const key = scope === undefined ? undefined : keyOf.scope(scope); + if (key === undefined) unkeyed.push(scope); + else { const list = keyed.get(key) ?? []; list.push(scope!); keyed.set(key, list); } + } + return entry => unkeyed.some(scope => options.matches(entry, scope)) + || (keyed.get(keyOf.entry(entry))?.some(scope => options.matches(entry, scope)) ?? false); + } function failed(map: typeof failures, id: string) { const count = Math.min((map.get(id)?.count ?? 0) + 1, 5); map.set(id, { count, retryAt: Date.now() + Math.min(300_000, 30_000 * 2 ** (count - 1)) }); @@ -60,15 +93,15 @@ export function registerScopedDeadlineWork( return [...dirty].filter(hint => hint !== "*" || !scopeFailures.has(hint)) .filter(hint => !scans.has(hint) || scopeFailures.has(hint)) .filter(hint => (reconstructAfter.get(hint) ?? 0) <= Date.now() || scopeFailures.has(hint)) - .map(hint => { try { return options.decodeScope(hint); } catch { return undefined; } }); + .map(hint => { try { return decode(hint); } catch { return undefined; } }); } function next() { if (!canRun()) return dirty.size > 0 || entries.size > 0 ? Date.now() + 30_000 : null; let nextAt: number | null = null; const take = (at: number) => { nextAt = nextAt === null ? at : Math.min(nextAt, at); }; for (const hint of dirty) take(Math.max(earliest, reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0)); - const unavailable = dirtyScopes(); - for (const entry of entries.values()) if (!unavailable.some(scope => options.matches(entry, scope))) + const unavailable = matcher(dirtyScopes()); + for (const entry of entries.values()) if (!unavailable(entry)) take(Math.max(earliest, entry.dueAt, failures.get(entry.id)?.retryAt ?? 0)); return nextAt; } @@ -99,24 +132,41 @@ export function registerScopedDeadlineWork( ? ["*", ...ready.filter(hint => hint !== "*")].slice(0, 25) : ready.slice(0, 25); for (const hint of requested) dirty.delete(hint); + // Entries by scope key for this pass, built on first use, so a keyed + // scope visits only its own entries when it drops vanished ones. + let byKey: Map> | null = null; + const indexEntry = (entry: Entry) => { + if (!byKey || !options.scopeKey) return; + const key = options.scopeKey.entry(entry); + const ids = byKey.get(key) ?? new Set(); + ids.add(entry.id); byKey.set(key, ids); + }; + const entriesIn = (scope: Scope | undefined): Entry[] => { + const key = scope === undefined ? undefined : options.scopeKey?.scope(scope); + if (key === undefined) return [...entries.values()]; + if (!byKey) { byKey = new Map(); for (const entry of entries.values()) indexEntry(entry); } + return [...(byKey.get(key) ?? [])].flatMap(id => entries.get(id) ?? []); + }; for (let index = 0; index < requested.length; index++) { if (signal.aborted || !canRun()) { for (const hint of requested.slice(index)) dirty.add(hint); return; } const hint = requested[index]; let scope: Scope | undefined; let decoded = false; try { - scope = options.decodeScope(hint); decoded = true; + scope = decode(hint); decoded = true; const scan = scans.get(hint) ?? { version: versions.get(hint) ?? 0, seen: new Set() }; scans.set(hint, scan); const loaded = await options.load(scope, signal); const result = Array.isArray(loaded) ? { rows: loaded, complete: true } : loaded; + const progressScopes = [...scans].map(([other, progress]) => ({ progress, scope: decode(other) })); for (const entry of result.rows ?? []) { const pending = entries.get(entry.id); entries.set(entry.id, pending && options.mergeEntry ? options.mergeEntry(pending, entry) : entry); - for (const [other, progress] of scans) if (options.matches(entry, options.decodeScope(other))) progress.seen.add(entry.id); + indexEntry(entry); + for (const { progress, scope: other } of progressScopes) if (options.matches(entry, other)) progress.seen.add(entry.id); } if (!result.complete) { dirty.add(hint); continue; } - if (result.rows !== null) for (const entry of entries.values()) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) { + if (result.rows !== null) for (const entry of entriesIn(scope)) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) { entries.delete(entry.id); failures.delete(entry.id); } scans.delete(hint); @@ -133,8 +183,9 @@ export function registerScopedDeadlineWork( dirty.add(retryHint); failed(scopeFailures, retryHint); errors.push(error); } } - const unavailable = dirtyScopes(); - const due = [...entries.values()].filter(entry => !unavailable.some(scope => options.matches(entry, scope)) + forgetDecoded(); + const unavailable = matcher(dirtyScopes()); + const due = [...entries.values()].filter(entry => !unavailable(entry) && Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now()) .sort((a, b) => (options.priority?.(b) ?? 0) - (options.priority?.(a) ?? 0) || a.dueAt - b.dueAt || a.id.localeCompare(b.id));