diff --git a/server/src/services/deadline-work.ts b/server/src/services/deadline-work.ts index 97afd00bdef..543392b07cf 100644 --- a/server/src/services/deadline-work.ts +++ b/server/src/services/deadline-work.ts @@ -12,6 +12,7 @@ export function registerDeadlineWork( canRun: () => boolean; retryMs: number; onSettled?: (hints?: ReadonlySet) => void; + onStop?: () => void; continuationMs?: number; // Optional batching latency for notifications; already-known deadlines still preempt it. notificationCoalesceMs?: number; @@ -28,7 +29,7 @@ export function registerDeadlineWork( const worker = coordinator.register(queue, { retryMs: task.retryMs, pausePassiveMaintenance: true, - onStop: () => { stopped = true; revision++; }, + onStop: () => { stopped = true; revision++; task.onStop?.(); }, onSettled: hints => { revision++; requested++; task.onSettled?.(hints); }, run: async signal => { revision++; running = true; diff --git a/server/src/services/heartbeat/dispatch-work.test.ts b/server/src/services/heartbeat/dispatch-work.test.ts index e929cd79820..daf1fb5e92c 100644 --- a/server/src/services/heartbeat/dispatch-work.test.ts +++ b/server/src/services/heartbeat/dispatch-work.test.ts @@ -1,4 +1,7 @@ import { describe, expect, it, vi } from "vitest"; +import { installDatabaseWorkSignals } from "@paperclipai/db/work-signals"; +import { notifyQueuedRunDispatch } from "../run-dispatch-notifications.js"; +import { isIdleTaskDrainActive, readTaskDrain, startTaskDrain, stopTaskDrain } from "../task-admission.js"; import { createDb } from "@paperclipai/db"; import { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { registerRunDispatchWork } from "./dispatch-work.js"; @@ -35,3 +38,96 @@ describe("queued-run dispatch re-drive", () => { } }, 60_000); }); + + +describe("queued-run dispatch drain resume", () => { + function fixture(options: { canDispatch?: () => boolean; due?: number } = {}) { + const owner = installDatabaseWorkSignals({ + execute: vi.fn(async () => [{ status: "committed" }]), + transaction: async (callback: (tx: object) => Promise) => callback({ execute: async () => [{ xid: "1" }] }), + }); + const errors: unknown[] = []; + const canRun = () => (options.canDispatch?.() ?? true) && readTaskDrain(new Date()) === null; + const coordinator = createDeliveryWorkCoordinator({ owner, canRun, canReconcile: () => true, onError: error => errors.push(error) }); + const resumed: Array = []; + let due = options.due ?? null; + const promote = vi.fn(async () => { if (due !== null && Date.now() >= due) due = null; }); + const work = registerRunDispatchWork(coordinator, { + promoteDueScheduledRetries: promote, + nextScheduledRetryAt: async () => due, + schedulingSuppressed: async () => false, + resumeQueuedRunsAfterDrain: async scope => { resumed.push(scope?.companyId); }, + listUndispatchedQueuedRuns: async () => [], + }, canRun); + const signal = () => owner.transaction(async tx => { await notifyQueuedRunDispatch(tx, { companyId: "company-1" }); }); + return { coordinator, work, signal, resumed, promote, errors }; + } + + it.each(["release", "expiry"] as const)("reconciles the admitted queued-run scope on normal drain %s before the fallback deadline", async mode => { + vi.useFakeTimers(); + const f = fixture(); + try { + await f.work.ready; + expect(f.resumed).toEqual([undefined]); + startTaskDrain({ purpose: "idle", ttlMs: mode === "expiry" ? 2_000 : null }); + await f.signal(); await vi.advanceTimersByTimeAsync(1_000); + expect(f.resumed).toEqual([undefined]); + const fallback = f.coordinator.nextWakeAt(); + expect(fallback).toBeGreaterThan(Date.now()); + expect(fallback).toBeLessThanOrEqual(Date.now() + 30_000); + if (mode === "release") stopTaskDrain(); + else { await vi.advanceTimersByTimeAsync(1_000); expect(isIdleTaskDrainActive()).toBe(false); } + await vi.advanceTimersByTimeAsync(0); + expect(Date.now()).toBeLessThan(fallback!); + expect(f.resumed).toEqual([undefined, "company-1"]); + expect(f.coordinator.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(90_000); + expect(f.resumed).toEqual([undefined, "company-1"]); + expect(f.errors).toEqual([]); + } finally { try { await f.coordinator.stop(); } finally { stopTaskDrain(); vi.useRealTimers(); } } + }, 5_000); + + it("keeps queued reconciliation gated while another drain or dispatch restriction remains", async () => { + vi.useFakeTimers(); let allowed = true; + const f = fixture({ canDispatch: () => allowed }); + try { + await f.work.ready; + startTaskDrain({ purpose: "idle" }); await f.signal(); await vi.advanceTimersByTimeAsync(1_000); + startTaskDrain(); await vi.advanceTimersByTimeAsync(0); + expect(f.resumed).toEqual([undefined]); + allowed = false; stopTaskDrain(); await vi.advanceTimersByTimeAsync(0); + expect(f.resumed).toEqual([undefined]); + allowed = true; startTaskDrain(); stopTaskDrain(); await vi.advanceTimersByTimeAsync(0); + expect(f.resumed).toEqual([undefined, "company-1"]); + expect(f.coordinator.nextWakeAt()).toBeNull(); expect(f.errors).toEqual([]); + } finally { try { await f.coordinator.stop(); } finally { stopTaskDrain(); vi.useRealTimers(); } } + }, 5_000); + + it("unsubscribes pending drain reconciliation when its normal coordinator stops", async () => { + vi.useFakeTimers(); const f = fixture(); + try { + await f.work.ready; + startTaskDrain({ purpose: "idle" }); await f.signal(); await vi.advanceTimersByTimeAsync(1_000); + await f.coordinator.stop(); stopTaskDrain(); await vi.advanceTimersByTimeAsync(60_000); + expect(f.resumed).toEqual([undefined]); expect(f.coordinator.nextWakeAt()).toBeNull(); + expect(f.errors).toEqual([]); + } finally { try { await f.coordinator.stop(); } finally { stopTaskDrain(); vi.useRealTimers(); } } + }, 5_000); + + it("preserves a genuine scheduled retry deadline and adds no work after idle reconciliation", async () => { + vi.useFakeTimers(); const due = Date.now() + 10_000; const f = fixture({ due }); + try { + await f.work.ready; expect(f.promote).toHaveBeenCalledOnce(); + startTaskDrain({ purpose: "idle" }); await f.signal(); await vi.advanceTimersByTimeAsync(1_000); + stopTaskDrain(); await vi.advanceTimersByTimeAsync(0); + expect(f.resumed).toEqual([undefined, "company-1"]); + expect(f.coordinator.nextWakeAt()).toBe(due); expect(f.promote).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(due - Date.now() - 1); expect(f.promote).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); expect(f.promote).toHaveBeenCalledTimes(2); + expect(f.coordinator.nextWakeAt()).toBeNull(); + startTaskDrain({ purpose: "idle" }); stopTaskDrain(); await vi.advanceTimersByTimeAsync(90_000); + expect(f.resumed).toEqual([undefined, "company-1"]); expect(f.promote).toHaveBeenCalledTimes(2); + expect(f.coordinator.nextWakeAt()).toBeNull(); expect(f.errors).toEqual([]); + } finally { try { await f.coordinator.stop(); } finally { stopTaskDrain(); vi.useRealTimers(); } } + }, 5_000); +}); diff --git a/server/src/services/heartbeat/dispatch-work.ts b/server/src/services/heartbeat/dispatch-work.ts index 56c6ee4b7b8..f3ce5834046 100644 --- a/server/src/services/heartbeat/dispatch-work.ts +++ b/server/src/services/heartbeat/dispatch-work.ts @@ -1,3 +1,4 @@ +import { subscribeTaskDrainChanges, subscribeTaskDrainExpiry } from "../task-admission.js"; import type { RunDispatchScope } from "../run-dispatch-notifications.js"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { registerDeadlineWork } from "../deadline-work.js"; @@ -30,11 +31,13 @@ export function registerRunDispatchWork( const scopes = new Set(["*"]); const redrive = createQueuedRunRedriveSchedule(); let dispatched: "all" | Set = new Set(); + let unsubscribeRefresh: (() => void) | undefined; const queue = registerDeadlineWork(coordinator, DELIVERY_QUEUES.queuedRunDispatch, { retryMs: 30_000, notificationCoalesceMs: 1_000, canRun, onSettled: hints => { for (const hint of hints ?? ["*"]) scopes.add(hint); }, + onStop: () => { unsubscribeRefresh?.(); unsubscribeRefresh = undefined; }, run: async signal => { const requested = scopes.has("*") ? ["*"] : [...scopes]; if (requested[0] === "*") scopes.clear(); @@ -78,6 +81,11 @@ export function registerRunDispatchWork( return plan.nextAt; }, }); + const refresh = () => { + if (canRun() && !queue.reconciliationSnapshot().reconciled) queue.wake(); + }; + const unsubscribe = [subscribeTaskDrainChanges(refresh), subscribeTaskDrainExpiry(refresh)]; + unsubscribeRefresh = () => { for (const stop of unsubscribe) stop(); }; return { retries, ready: Promise.all([retries.ready, queue.ready]),